From b2d19b08e57e7d07333d844d2373369e2573b318 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mandiwal Date: Fri, 31 Oct 2025 15:18:25 +0530 Subject: [PATCH 1/8] Added case-insensitive comparison support to is_in_list and is_not_null_and_is_in_list --- docs/dqx/docs/reference/quality_checks.mdx | 4 +- src/databricks/labs/dqx/check_funcs.py | 46 +++++++++++++++++----- tests/integration/test_row_checks.py | 30 +++++++------- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 43c939043..4dac55008 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -28,8 +28,8 @@ You can also define your own custom checks (see [Creating custom checks](#creati | `is_not_null` | Checks whether the values in the input column are not null. | `column`: column to check (can be a string column name or a column expression) | | `is_not_empty` | Checks whether the values in the input column are not empty (but may be null). | `column`: column to check (can be a string column name or a column expression) | | `is_not_null_and_not_empty` | Checks whether the values in the input column are not null and not empty. | `column`: column to check (can be a string column name or a column expression); `trim_strings`: optional boolean flag to trim spaces from strings | -| `is_in_list` | Checks whether the values in the input column are present in the list of allowed values (null values are allowed). This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values | -| `is_not_null_and_is_in_list` | Checks whether the values in the input column are not null and present in the list of allowed values. This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values | +| `is_in_list` | Checks whether the values in the input column are present in the list of allowed values (null values are allowed). We can pass an additional case_sensitive parameter as False for a case insensitive check. This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values | +| `is_not_null_and_is_in_list` | Checks whether the values in the input column are not null and present in the list of allowed values. We can pass an additional case_sensitive parameter as False for a case insensitive check. This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values | | `is_not_null_and_not_empty_array` | Checks whether the values in the array input column are not null and not empty. | `column`: column to check (can be a string column name or a column expression) | | `is_in_range` | Checks whether the values in the input column are in the provided range (inclusive of both boundaries). | `column`: column to check (can be a string column name or a column expression); `min_limit`: min limit as number, date, timestamp, column name or sql expression; `max_limit`: max limit as number, date, timestamp, column name or sql expression | | `is_not_in_range` | Checks whether the values in the input column are outside the provided range (inclusive of both boundaries). | `column`: column to check (can be a string column name or a column expression); `min_limit`: min limit as number, date, timestamp, column name or sql expression; `max_limit`: max limit as number, date, timestamp, column name or sql expression | diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index 962d550a8..a74d02139 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -129,12 +129,13 @@ def is_not_null(column: str | Column) -> Column: @register_rule("row") -def is_not_null_and_is_in_list(column: str | Column, allowed: list) -> Column: +def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) -> Column: """Checks whether the values in the input column are not null and present in the list of allowed values. - + Can optionally perform a case-insensitive comparison. Args: column: column to check; can be a string column name or a column expression allowed: list of allowed values (actual values or Column objects) + case_sensitive: whether to perform a case-sensitive comparison (default: True) Returns: Column object for condition @@ -152,9 +153,21 @@ def is_not_null_and_is_in_list(column: str | Column, allowed: list) -> Column: if not allowed: raise InvalidParameterError("allowed list must not be empty.") - allowed_cols = [item if isinstance(item, Column) else F.lit(item) for item in allowed] + # Keep original values for display in error message + allowed_cols_display = [item if isinstance(item, Column) else F.lit(item) for item in allowed] + col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) - condition = col_expr.isNull() | ~col_expr.isin(*allowed_cols) + + # Create columns for comparison + allowed_cols_compare = allowed_cols_display[:] + col_expr_compare = col_expr + + # Case-insensitive normalization + if not case_sensitive: + col_expr_compare = F.lower(col_expr) + allowed_cols_compare = [F.lower(c) for c in allowed_cols_display] + + condition = col_expr.isNull() | ~col_expr_compare.isin(*allowed_cols_compare) return make_condition( condition, F.concat_ws( @@ -162,7 +175,7 @@ def is_not_null_and_is_in_list(column: str | Column, allowed: list) -> Column: F.lit("Value '"), F.when(col_expr.isNull(), F.lit("null")).otherwise(col_expr.cast("string")), F.lit(f"' in Column '{col_expr_str}' is null or not in the allowed list: ["), - F.concat_ws(", ", *allowed_cols), + F.concat_ws(", ", *allowed_cols_display), F.lit("]"), ), f"{col_str_norm}_is_null_or_is_not_in_the_list", @@ -170,13 +183,14 @@ def is_not_null_and_is_in_list(column: str | Column, allowed: list) -> Column: @register_rule("row") -def is_in_list(column: str | Column, allowed: list) -> Column: +def is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) -> Column: """Checks whether the values in the input column are present in the list of allowed values - (null values are allowed). + (null values are allowed). Can optionally perform a case-insensitive comparison. Args: column: column to check; can be a string column name or a column expression allowed: list of allowed values (actual values or Column objects) + case_sensitive: whether to perform a case-sensitive comparison (default: True) Returns: Column object for condition @@ -194,9 +208,21 @@ def is_in_list(column: str | Column, allowed: list) -> Column: if not allowed: raise InvalidParameterError("allowed list must not be empty.") - allowed_cols = [item if isinstance(item, Column) else F.lit(item) for item in allowed] + # Keep original values for display in error message + allowed_cols_display = [item if isinstance(item, Column) else F.lit(item) for item in allowed] + col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) - condition = ~col_expr.isin(*allowed_cols) + + # Create columns for comparison + allowed_cols_compare = allowed_cols_display[:] + col_expr_compare = col_expr + + # Case-insensitive normalization + if not case_sensitive: + col_expr_compare = F.lower(col_expr) + allowed_cols_compare = [F.lower(c) for c in allowed_cols_display] + + condition = ~col_expr_compare.isin(*allowed_cols_compare) return make_condition( condition, F.concat_ws( @@ -204,7 +230,7 @@ def is_in_list(column: str | Column, allowed: list) -> Column: F.lit("Value '"), F.when(col_expr.isNull(), F.lit("null")).otherwise(col_expr.cast("string")), F.lit(f"' in Column '{col_expr_str}' is not in the allowed list: ["), - F.concat_ws(", ", *allowed_cols), + F.concat_ws(", ", *allowed_cols_display), F.lit("]"), ), f"{col_str_norm}_is_not_in_the_list", diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index 2fc713a03..830ed1985 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -166,10 +166,10 @@ def test_col_is_not_null_and_is_in_list(spark): ) actual = test_df.select( - is_not_null_and_is_in_list("a", ["str1"]), - is_not_null_and_is_in_list("b", [F.lit(3)]), - is_not_null_and_is_in_list(F.col("c").getItem("val"), [F.lit("a")]), - is_not_null_and_is_in_list(F.try_element_at("d", F.lit(2)), ["b"]), + is_not_null_and_is_in_list("a", ["STR1"], case_sensitive=False), + is_not_null_and_is_in_list("b", [F.lit(3)], case_sensitive=True), + is_not_null_and_is_in_list(F.col("c").getItem("val"), [F.lit("A")], case_sensitive=False), + is_not_null_and_is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=True), ) checked_schema = ( @@ -182,15 +182,15 @@ def test_col_is_not_null_and_is_in_list(spark): [ [None, "Value '1' in Column 'b' is null or not in the allowed list: [3]", None, None], [ - "Value 'str2' in Column 'a' is null or not in the allowed list: [str1]", + "Value 'str2' in Column 'a' is null or not in the allowed list: [STR1]", "Value 'null' in Column 'b' is null or not in the allowed list: [3]", - "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", + "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [A]", "Value 'a' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", ], [ - "Value ' ' in Column 'a' is null or not in the allowed list: [str1]", + "Value ' ' in Column 'a' is null or not in the allowed list: [STR1]", None, - "Value ' ' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", + "Value ' ' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [A]", "Value ' ' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", ], ], @@ -212,10 +212,10 @@ def test_col_is_not_in_list(spark): ) actual = test_df.select( - is_in_list("a", ["str1"]), - is_in_list("b", [F.lit(3)]), - is_in_list(F.col("c").getItem("val"), [F.lit("a")]), - is_in_list(F.try_element_at("d", F.lit(2)), ["b"]), + is_in_list("a", ["STR1"], case_sensitive=False), + is_in_list("b", [F.lit(3)], case_sensitive=True), + is_in_list(F.col("c").getItem("val"), [F.lit("A")], case_sensitive=False), + is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=True), ) checked_schema = ( @@ -228,13 +228,13 @@ def test_col_is_not_in_list(spark): [ [None, "Value '1' in Column 'b' is not in the allowed list: [3]", None, None], [ - "Value 'str2' in Column 'a' is not in the allowed list: [str1]", + "Value 'str2' in Column 'a' is not in the allowed list: [STR1]", None, - "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", + "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [A]", "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", ], [ - "Value ' ' in Column 'a' is not in the allowed list: [str1]", + "Value ' ' in Column 'a' is not in the allowed list: [STR1]", None, None, "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", From b24e26a7913c3cd2f525d1ef630ec9c15bf67798 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mandiwal Date: Mon, 3 Nov 2025 21:26:08 +0530 Subject: [PATCH 2/8] fixed documentation and improved integration tests --- docs/dqx/docs/reference/quality_checks.mdx | 4 +- src/databricks/labs/dqx/check_funcs.py | 43 ++++-------- tests/integration/test_row_checks.py | 80 +++++++++++++++++----- 3 files changed, 77 insertions(+), 50 deletions(-) diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 4dac55008..c11f78250 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -28,8 +28,8 @@ You can also define your own custom checks (see [Creating custom checks](#creati | `is_not_null` | Checks whether the values in the input column are not null. | `column`: column to check (can be a string column name or a column expression) | | `is_not_empty` | Checks whether the values in the input column are not empty (but may be null). | `column`: column to check (can be a string column name or a column expression) | | `is_not_null_and_not_empty` | Checks whether the values in the input column are not null and not empty. | `column`: column to check (can be a string column name or a column expression); `trim_strings`: optional boolean flag to trim spaces from strings | -| `is_in_list` | Checks whether the values in the input column are present in the list of allowed values (null values are allowed). We can pass an additional case_sensitive parameter as False for a case insensitive check. This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values | -| `is_not_null_and_is_in_list` | Checks whether the values in the input column are not null and present in the list of allowed values. We can pass an additional case_sensitive parameter as False for a case insensitive check. This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values | +| `is_in_list` | Checks whether the values in the input column are present in the list of allowed values (null values are allowed). This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) | +| `is_not_null_and_is_in_list` | Checks whether the values in the input column are not null and present in the list of allowed values. This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) | | `is_not_null_and_not_empty_array` | Checks whether the values in the array input column are not null and not empty. | `column`: column to check (can be a string column name or a column expression) | | `is_in_range` | Checks whether the values in the input column are in the provided range (inclusive of both boundaries). | `column`: column to check (can be a string column name or a column expression); `min_limit`: min limit as number, date, timestamp, column name or sql expression; `max_limit`: max limit as number, date, timestamp, column name or sql expression | | `is_not_in_range` | Checks whether the values in the input column are outside the provided range (inclusive of both boundaries). | `column`: column to check (can be a string column name or a column expression); `min_limit`: min limit as number, date, timestamp, column name or sql expression; `max_limit`: max limit as number, date, timestamp, column name or sql expression | diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index a74d02139..5723ad0cc 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -131,7 +131,8 @@ def is_not_null(column: str | Column) -> Column: @register_rule("row") def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) -> Column: """Checks whether the values in the input column are not null and present in the list of allowed values. - Can optionally perform a case-insensitive comparison. + Can optionally perform a case-insensitive comparison. + Args: column: column to check; can be a string column name or a column expression allowed: list of allowed values (actual values or Column objects) @@ -146,28 +147,20 @@ def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensiti """ if allowed is None: raise MissingParameterError("allowed list is not provided.") - if not isinstance(allowed, list): raise InvalidParameterError(f"allowed parameter must be a list, got {str(type(allowed))} instead.") - if not allowed: raise InvalidParameterError("allowed list must not be empty.") - # Keep original values for display in error message - allowed_cols_display = [item if isinstance(item, Column) else F.lit(item) for item in allowed] - + allowed_cols = [item if isinstance(item, Column) else F.lit(item) for item in allowed] col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) - # Create columns for comparison - allowed_cols_compare = allowed_cols_display[:] - col_expr_compare = col_expr - - # Case-insensitive normalization - if not case_sensitive: - col_expr_compare = F.lower(col_expr) - allowed_cols_compare = [F.lower(c) for c in allowed_cols_display] + # Apply case-insensitive transformation if needed + col_expr_compare = F.lower(col_expr) if not case_sensitive else col_expr + allowed_cols_compare = [F.lower(c) if not case_sensitive else c for c in allowed_cols] condition = col_expr.isNull() | ~col_expr_compare.isin(*allowed_cols_compare) + return make_condition( condition, F.concat_ws( @@ -175,7 +168,7 @@ def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensiti F.lit("Value '"), F.when(col_expr.isNull(), F.lit("null")).otherwise(col_expr.cast("string")), F.lit(f"' in Column '{col_expr_str}' is null or not in the allowed list: ["), - F.concat_ws(", ", *allowed_cols_display), + F.concat_ws(", ", *allowed_cols), # Use original allowed_cols for display F.lit("]"), ), f"{col_str_norm}_is_null_or_is_not_in_the_list", @@ -201,28 +194,20 @@ def is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) """ if allowed is None: raise MissingParameterError("allowed list is not provided.") - if not isinstance(allowed, list): raise InvalidParameterError(f"allowed parameter must be a list, got {str(type(allowed))} instead.") - if not allowed: raise InvalidParameterError("allowed list must not be empty.") - # Keep original values for display in error message - allowed_cols_display = [item if isinstance(item, Column) else F.lit(item) for item in allowed] - + allowed_cols = [item if isinstance(item, Column) else F.lit(item) for item in allowed] col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) - # Create columns for comparison - allowed_cols_compare = allowed_cols_display[:] - col_expr_compare = col_expr - - # Case-insensitive normalization - if not case_sensitive: - col_expr_compare = F.lower(col_expr) - allowed_cols_compare = [F.lower(c) for c in allowed_cols_display] + # Apply case-insensitive transformation if needed + col_expr_compare = F.lower(col_expr) if not case_sensitive else col_expr + allowed_cols_compare = [F.lower(c) if not case_sensitive else c for c in allowed_cols] condition = ~col_expr_compare.isin(*allowed_cols_compare) + return make_condition( condition, F.concat_ws( @@ -230,7 +215,7 @@ def is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) F.lit("Value '"), F.when(col_expr.isNull(), F.lit("null")).otherwise(col_expr.cast("string")), F.lit(f"' in Column '{col_expr_str}' is not in the allowed list: ["), - F.concat_ws(", ", *allowed_cols_display), + F.concat_ws(", ", *allowed_cols), # Use original allowed_cols for display F.lit("]"), ), f"{col_str_norm}_is_not_in_the_list", diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index 830ed1985..2ae55e1d8 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -161,37 +161,59 @@ def test_col_is_not_null_and_is_in_list(spark): ["str1", 1, {"val": "a"}, ["a", "b"]], ["str2", None, {"val": "str2"}, [None, "a"]], [" ", 3, {"val": " "}, [None, " "]], + ["STR1", 4, {"val": "A"}, ["B", "c"]], ], input_schema, ) actual = test_df.select( - is_not_null_and_is_in_list("a", ["STR1"], case_sensitive=False), - is_not_null_and_is_in_list("b", [F.lit(3)], case_sensitive=True), - is_not_null_and_is_in_list(F.col("c").getItem("val"), [F.lit("A")], case_sensitive=False), - is_not_null_and_is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=True), + is_not_null_and_is_in_list("a", ["str1"]), + is_not_null_and_is_in_list("b", [F.lit(3)]), + is_not_null_and_is_in_list(F.col("c").getItem("val"), [F.lit("a")]), + is_not_null_and_is_in_list(F.try_element_at("d", F.lit(2)), ["b"]), + is_not_null_and_is_in_list("a", ["str1"], case_sensitive=False), + is_not_null_and_is_in_list(F.col("c").getItem("val"), [F.lit("a")], case_sensitive=False), + is_not_null_and_is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=False), ) checked_schema = ( "a_is_null_or_is_not_in_the_list: string, " + "b_is_null_or_is_not_in_the_list: string, " + "unresolvedextractvalue_c_val_is_null_or_is_not_in_the_list: string, " + + "try_element_at_d_2_is_null_or_is_not_in_the_list: string, " + + "a_is_null_or_is_not_in_the_list: string, " + + "unresolvedextractvalue_c_val_is_null_or_is_not_in_the_list: string, " + "try_element_at_d_2_is_null_or_is_not_in_the_list: string" ) expected = spark.createDataFrame( [ - [None, "Value '1' in Column 'b' is null or not in the allowed list: [3]", None, None], + [None, "Value '1' in Column 'b' is null or not in the allowed list: [3]", None, None, None, None, None], [ - "Value 'str2' in Column 'a' is null or not in the allowed list: [STR1]", + "Value 'str2' in Column 'a' is null or not in the allowed list: [str1]", "Value 'null' in Column 'b' is null or not in the allowed list: [3]", - "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [A]", + "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", + "Value 'a' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + "Value 'str2' in Column 'a' is null or not in the allowed list: [str1]", + "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", "Value 'a' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", ], [ - "Value ' ' in Column 'a' is null or not in the allowed list: [STR1]", + "Value ' ' in Column 'a' is null or not in the allowed list: [str1]", None, - "Value ' ' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [A]", + "Value ' ' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", "Value ' ' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + "Value ' ' in Column 'a' is null or not in the allowed list: [str1]", + "Value ' ' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", + "Value ' ' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + ], + [ + "Value 'STR1' in Column 'a' is null or not in the allowed list: [str1]", + "Value '4' in Column 'b' is null or not in the allowed list: [3]", + "Value 'A' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", + "Value 'c' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + None, # STR1 matches str1 (case-insensitive) + None, # A matches a (case-insensitive) + "Value 'c' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", ], ], checked_schema, @@ -207,38 +229,58 @@ def test_col_is_not_in_list(spark): ["str1", 1, {"val": "a"}, ["a", "b"]], ["str2", None, {"val": "str2"}, [None, "a"]], [" ", 3, {"val": None}, [None, "a"]], + ["STR1", 4, {"val": "A"}, ["B", "c"]], ], input_schema, ) - actual = test_df.select( - is_in_list("a", ["STR1"], case_sensitive=False), - is_in_list("b", [F.lit(3)], case_sensitive=True), - is_in_list(F.col("c").getItem("val"), [F.lit("A")], case_sensitive=False), - is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=True), + is_in_list("a", ["str1"]), + is_in_list("b", [F.lit(3)]), + is_in_list(F.col("c").getItem("val"), [F.lit("a")]), + is_in_list(F.try_element_at("d", F.lit(2)), ["b"]), + is_in_list("a", ["str1"], case_sensitive=False), + is_in_list(F.col("c").getItem("val"), [F.lit("a")], case_sensitive=False), + is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=False), ) - checked_schema = ( "a_is_not_in_the_list: string, " + "b_is_not_in_the_list: string, " + "unresolvedextractvalue_c_val_is_not_in_the_list: string, " + + "try_element_at_d_2_is_not_in_the_list: string, " + + "a_is_not_in_the_list: string, " + + "unresolvedextractvalue_c_val_is_not_in_the_list: string, " + "try_element_at_d_2_is_not_in_the_list: string" ) expected = spark.createDataFrame( [ - [None, "Value '1' in Column 'b' is not in the allowed list: [3]", None, None], + [None, "Value '1' in Column 'b' is not in the allowed list: [3]", None, None, None, None, None], [ - "Value 'str2' in Column 'a' is not in the allowed list: [STR1]", + "Value 'str2' in Column 'a' is not in the allowed list: [str1]", None, - "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [A]", + "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", + "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + "Value 'str2' in Column 'a' is not in the allowed list: [str1]", + "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", ], [ - "Value ' ' in Column 'a' is not in the allowed list: [STR1]", + "Value ' ' in Column 'a' is not in the allowed list: [str1]", + None, None, + "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + "Value ' ' in Column 'a' is not in the allowed list: [str1]", None, "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", ], + [ + "Value 'STR1' in Column 'a' is not in the allowed list: [str1]", + "Value '4' in Column 'b' is not in the allowed list: [3]", + "Value 'A' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", + "Value 'c' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + None, + None, + "Value 'c' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + ], ], checked_schema, ) From f70eb140548a08aae34fb3f8335e486b6fa6228d Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Mon, 3 Nov 2025 17:08:48 -0500 Subject: [PATCH 3/8] Fix tests --- tests/unit/test_build_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_build_rules.py b/tests/unit/test_build_rules.py index ca9607554..f6340905a 100644 --- a/tests/unit/test_build_rules.py +++ b/tests/unit/test_build_rules.py @@ -896,7 +896,7 @@ def test_validate_check_func_arguments_too_many_positional(): criticality="error", check_func=is_in_list, column="col1", - check_func_args=[[1, 2], "extra_arg"], + check_func_args=[[1, 2], True, "extra_arg"], ) From 851f6fd41b4633f1bcfcca90f242121b94840f80 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Mon, 3 Nov 2025 17:14:25 -0500 Subject: [PATCH 4/8] Fix tests --- tests/unit/test_build_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_build_rules.py b/tests/unit/test_build_rules.py index f6340905a..00c4f3fe0 100644 --- a/tests/unit/test_build_rules.py +++ b/tests/unit/test_build_rules.py @@ -890,7 +890,7 @@ def test_build_checks_by_metadata_logging_debug_calls(caplog): def test_validate_check_func_arguments_too_many_positional(): - with pytest.raises(TypeError, match="takes 2 positional arguments but 3 were given"): + with pytest.raises(TypeError, match="takes from 2 to 3 positional arguments but 4 were given"): DQRowRule( name="col1_is_not_in_the_list", criticality="error", From cee38120cff63e8c4a0545018c9b66cc5c6d25d7 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mandiwal Date: Thu, 6 Nov 2025 00:55:23 +0530 Subject: [PATCH 5/8] Added lowercase checks for array type columns and fixed documentation --- docs/dqx/docs/reference/quality_checks.mdx | 4 +- src/databricks/labs/dqx/check_funcs.py | 25 +++++++--- src/databricks/labs/dqx/utils.py | 18 +++++++- tests/integration/test_row_checks.py | 53 +++++++++++++++++++--- 4 files changed, 85 insertions(+), 15 deletions(-) diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index c11f78250..c915a97e9 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -28,8 +28,8 @@ You can also define your own custom checks (see [Creating custom checks](#creati | `is_not_null` | Checks whether the values in the input column are not null. | `column`: column to check (can be a string column name or a column expression) | | `is_not_empty` | Checks whether the values in the input column are not empty (but may be null). | `column`: column to check (can be a string column name or a column expression) | | `is_not_null_and_not_empty` | Checks whether the values in the input column are not null and not empty. | `column`: column to check (can be a string column name or a column expression); `trim_strings`: optional boolean flag to trim spaces from strings | -| `is_in_list` | Checks whether the values in the input column are present in the list of allowed values (null values are allowed). This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) | -| `is_not_null_and_is_in_list` | Checks whether the values in the input column are not null and present in the list of allowed values. This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) | +| `is_in_list` | Checks whether the values in the input column are present in the list of allowed values (null values are allowed). This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. This check is not suited for `MapType` or `StructType` columns. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) | +| `is_not_null_and_is_in_list` | Checks whether the values in the input column are not null and present in the list of allowed values. This check is not suited for large lists of allowed values. In such cases, it’s recommended to use the `foreign_key` dataset-level check instead. This check is not suited for `MapType` or `StructType` columns. | `column`: column to check (can be a string column name or a column expression); `allowed`: list of allowed values; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) | | `is_not_null_and_not_empty_array` | Checks whether the values in the array input column are not null and not empty. | `column`: column to check (can be a string column name or a column expression) | | `is_in_range` | Checks whether the values in the input column are in the provided range (inclusive of both boundaries). | `column`: column to check (can be a string column name or a column expression); `min_limit`: min limit as number, date, timestamp, column name or sql expression; `max_limit`: max limit as number, date, timestamp, column name or sql expression | | `is_not_in_range` | Checks whether the values in the input column are outside the provided range (inclusive of both boundaries). | `column`: column to check (can be a string column name or a column expression); `min_limit`: min limit as number, date, timestamp, column name or sql expression; `max_limit`: max limit as number, date, timestamp, column name or sql expression | diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index 5723ad0cc..c718be573 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -18,6 +18,7 @@ is_sql_query_safe, normalize_col_str, get_columns_as_strings, + to_lowercase, ) from databricks.labs.dqx.errors import MissingParameterError, InvalidParameterError, UnsafeSqlQueryError @@ -156,8 +157,14 @@ def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensiti col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) # Apply case-insensitive transformation if needed - col_expr_compare = F.lower(col_expr) if not case_sensitive else col_expr - allowed_cols_compare = [F.lower(c) if not case_sensitive else c for c in allowed_cols] + if not case_sensitive: + has_arrays = any(isinstance(item, (list, tuple)) for item in allowed if not isinstance(item, Column)) + col_expr_compare = to_lowercase(col_expr, is_array=has_arrays) + allowed_cols_compare = [ + to_lowercase(c, is_array=isinstance(allowed[i], (list, tuple))) for i, c in enumerate(allowed_cols) + ] + else: + col_expr_compare, allowed_cols_compare = col_expr, allowed_cols condition = col_expr.isNull() | ~col_expr_compare.isin(*allowed_cols_compare) @@ -168,7 +175,7 @@ def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensiti F.lit("Value '"), F.when(col_expr.isNull(), F.lit("null")).otherwise(col_expr.cast("string")), F.lit(f"' in Column '{col_expr_str}' is null or not in the allowed list: ["), - F.concat_ws(", ", *allowed_cols), # Use original allowed_cols for display + F.concat_ws(", ", *[c.cast("string") for c in allowed_cols]), F.lit("]"), ), f"{col_str_norm}_is_null_or_is_not_in_the_list", @@ -203,8 +210,14 @@ def is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) col_str_norm, col_expr_str, col_expr = _get_normalized_column_and_expr(column) # Apply case-insensitive transformation if needed - col_expr_compare = F.lower(col_expr) if not case_sensitive else col_expr - allowed_cols_compare = [F.lower(c) if not case_sensitive else c for c in allowed_cols] + if not case_sensitive: + has_arrays = any(isinstance(item, (list, tuple)) for item in allowed if not isinstance(item, Column)) + col_expr_compare = to_lowercase(col_expr, is_array=has_arrays) + allowed_cols_compare = [ + to_lowercase(c, is_array=isinstance(allowed[i], (list, tuple))) for i, c in enumerate(allowed_cols) + ] + else: + col_expr_compare, allowed_cols_compare = col_expr, allowed_cols condition = ~col_expr_compare.isin(*allowed_cols_compare) @@ -215,7 +228,7 @@ def is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) F.lit("Value '"), F.when(col_expr.isNull(), F.lit("null")).otherwise(col_expr.cast("string")), F.lit(f"' in Column '{col_expr_str}' is not in the allowed list: ["), - F.concat_ws(", ", *allowed_cols), # Use original allowed_cols for display + F.concat_ws(", ", *[c.cast("string") for c in allowed_cols]), F.lit("]"), ), f"{col_str_norm}_is_not_in_the_list", diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index 62dcba6ab..1d97b267a 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -18,7 +18,7 @@ from databricks.labs.blueprint.limiter import rate_limited from databricks.labs.dqx.errors import InvalidParameterError from databricks.sdk.errors import NotFound - +import pyspark.sql.functions as F logger = logging.getLogger(__name__) @@ -433,3 +433,19 @@ def _match_table_patterns(table: str, patterns: list[str]) -> bool: bool: True if the table name matches any of the patterns, False otherwise. """ return any(fnmatch(table, pattern) for pattern in patterns) + + +def to_lowercase(col_expr: Column, is_array: bool = False) -> Column: + """Converts a column expression to lowercase, handling both scalar and array types. + + Args: + col_expr: Column expression to convert + is_array: Whether the column contains array values + + Returns: + Column expression with lowercase transformation applied + """ + if is_array: + return F.transform(col_expr, lambda x: F.lower(x)) + else: + return F.lower(col_expr) diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index 2ae55e1d8..cc292a32d 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -162,6 +162,7 @@ def test_col_is_not_null_and_is_in_list(spark): ["str2", None, {"val": "str2"}, [None, "a"]], [" ", 3, {"val": " "}, [None, " "]], ["STR1", 4, {"val": "A"}, ["B", "c"]], + ["str5", 5, {"val": "e"}, ["b", "C"]], # New test case for array case-insensitive match ], input_schema, ) @@ -174,6 +175,7 @@ def test_col_is_not_null_and_is_in_list(spark): is_not_null_and_is_in_list("a", ["str1"], case_sensitive=False), is_not_null_and_is_in_list(F.col("c").getItem("val"), [F.lit("a")], case_sensitive=False), is_not_null_and_is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=False), + is_not_null_and_is_in_list("d", [["a", "b"], ["B", "c"]], case_sensitive=False), ) checked_schema = ( @@ -183,11 +185,21 @@ def test_col_is_not_null_and_is_in_list(spark): + "try_element_at_d_2_is_null_or_is_not_in_the_list: string, " + "a_is_null_or_is_not_in_the_list: string, " + "unresolvedextractvalue_c_val_is_null_or_is_not_in_the_list: string, " - + "try_element_at_d_2_is_null_or_is_not_in_the_list: string" + + "try_element_at_d_2_is_null_or_is_not_in_the_list: string, " + + "d_is_null_or_is_not_in_the_list: string" ) expected = spark.createDataFrame( [ - [None, "Value '1' in Column 'b' is null or not in the allowed list: [3]", None, None, None, None, None], + [ + None, + "Value '1' in Column 'b' is null or not in the allowed list: [3]", + None, + None, + None, + None, + None, + None, + ], [ "Value 'str2' in Column 'a' is null or not in the allowed list: [str1]", "Value 'null' in Column 'b' is null or not in the allowed list: [3]", @@ -196,6 +208,7 @@ def test_col_is_not_null_and_is_in_list(spark): "Value 'str2' in Column 'a' is null or not in the allowed list: [str1]", "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", "Value 'a' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + "Value '[null, a]' in Column 'd' is null or not in the allowed list: [[a, b], [B, c]]", ], [ "Value ' ' in Column 'a' is null or not in the allowed list: [str1]", @@ -205,15 +218,27 @@ def test_col_is_not_null_and_is_in_list(spark): "Value ' ' in Column 'a' is null or not in the allowed list: [str1]", "Value ' ' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", "Value ' ' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + "Value '[null, ]' in Column 'd' is null or not in the allowed list: [[a, b], [B, c]]", ], [ "Value 'STR1' in Column 'a' is null or not in the allowed list: [str1]", "Value '4' in Column 'b' is null or not in the allowed list: [3]", "Value 'A' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", "Value 'c' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", - None, # STR1 matches str1 (case-insensitive) - None, # A matches a (case-insensitive) + None, + None, "Value 'c' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + None, + ], + [ + "Value 'str5' in Column 'a' is null or not in the allowed list: [str1]", + "Value '5' in Column 'b' is null or not in the allowed list: [3]", + "Value 'e' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", + "Value 'C' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + "Value 'str5' in Column 'a' is null or not in the allowed list: [str1]", + "Value 'e' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", + "Value 'C' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + None, ], ], checked_schema, @@ -230,6 +255,7 @@ def test_col_is_not_in_list(spark): ["str2", None, {"val": "str2"}, [None, "a"]], [" ", 3, {"val": None}, [None, "a"]], ["STR1", 4, {"val": "A"}, ["B", "c"]], + ["str5", 5, {"val": "e"}, ["b", "C"]], # New test case for array case-insensitive match ], input_schema, ) @@ -241,6 +267,7 @@ def test_col_is_not_in_list(spark): is_in_list("a", ["str1"], case_sensitive=False), is_in_list(F.col("c").getItem("val"), [F.lit("a")], case_sensitive=False), is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=False), + is_in_list("d", [["a", "b"], ["B", "c"]], case_sensitive=False), # New: array case-insensitive ) checked_schema = ( "a_is_not_in_the_list: string, " @@ -249,11 +276,12 @@ def test_col_is_not_in_list(spark): + "try_element_at_d_2_is_not_in_the_list: string, " + "a_is_not_in_the_list: string, " + "unresolvedextractvalue_c_val_is_not_in_the_list: string, " - + "try_element_at_d_2_is_not_in_the_list: string" + + "try_element_at_d_2_is_not_in_the_list: string, " + + "d_is_not_in_the_list: string" ) expected = spark.createDataFrame( [ - [None, "Value '1' in Column 'b' is not in the allowed list: [3]", None, None, None, None, None], + [None, "Value '1' in Column 'b' is not in the allowed list: [3]", None, None, None, None, None, None], [ "Value 'str2' in Column 'a' is not in the allowed list: [str1]", None, @@ -262,6 +290,7 @@ def test_col_is_not_in_list(spark): "Value 'str2' in Column 'a' is not in the allowed list: [str1]", "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + "Value '[null, a]' in Column 'd' is not in the allowed list: [[a, b], [B, c]]", ], [ "Value ' ' in Column 'a' is not in the allowed list: [str1]", @@ -271,6 +300,7 @@ def test_col_is_not_in_list(spark): "Value ' ' in Column 'a' is not in the allowed list: [str1]", None, "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + "Value '[null, a]' in Column 'd' is not in the allowed list: [[a, b], [B, c]]", ], [ "Value 'STR1' in Column 'a' is not in the allowed list: [str1]", @@ -280,6 +310,17 @@ def test_col_is_not_in_list(spark): None, None, "Value 'c' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + None, # ["B", "c"] matches case-insensitively + ], + [ + "Value 'str5' in Column 'a' is not in the allowed list: [str1]", + "Value '5' in Column 'b' is not in the allowed list: [3]", + "Value 'e' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", + "Value 'C' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + "Value 'str5' in Column 'a' is not in the allowed list: [str1]", + "Value 'e' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", + "Value 'C' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", + None, # ["b", "C"] matches ["B", "c"] case-insensitively ], ], checked_schema, From 1dc87a372662843fcbd6b74f675e9c08e380dc49 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Mandiwal Date: Fri, 7 Nov 2025 01:33:27 +0530 Subject: [PATCH 6/8] Added additional tests --- src/databricks/labs/dqx/check_funcs.py | 2 ++ src/databricks/labs/dqx/utils.py | 5 ++--- tests/integration/test_row_checks.py | 25 +++++++++++++++++++------ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index c718be573..6256dc4d0 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -133,6 +133,7 @@ def is_not_null(column: str | Column) -> Column: def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) -> Column: """Checks whether the values in the input column are not null and present in the list of allowed values. Can optionally perform a case-insensitive comparison. + This check is not suited for `MapType` or `StructType` columns. Args: column: column to check; can be a string column name or a column expression @@ -186,6 +187,7 @@ def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensiti def is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) -> Column: """Checks whether the values in the input column are present in the list of allowed values (null values are allowed). Can optionally perform a case-insensitive comparison. + This check is not suited for `MapType` or `StructType` columns. Args: column: column to check; can be a string column name or a column expression diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index 1d97b267a..8a88425c4 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -14,11 +14,11 @@ except ImportError: ConnectColumn = None # type: ignore +import pyspark.sql.functions as F from databricks.sdk import WorkspaceClient from databricks.labs.blueprint.limiter import rate_limited from databricks.labs.dqx.errors import InvalidParameterError from databricks.sdk.errors import NotFound -import pyspark.sql.functions as F logger = logging.getLogger(__name__) @@ -447,5 +447,4 @@ def to_lowercase(col_expr: Column, is_array: bool = False) -> Column: """ if is_array: return F.transform(col_expr, lambda x: F.lower(x)) - else: - return F.lower(col_expr) + return F.lower(col_expr) diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index cc292a32d..65b750553 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -162,7 +162,7 @@ def test_col_is_not_null_and_is_in_list(spark): ["str2", None, {"val": "str2"}, [None, "a"]], [" ", 3, {"val": " "}, [None, " "]], ["STR1", 4, {"val": "A"}, ["B", "c"]], - ["str5", 5, {"val": "e"}, ["b", "C"]], # New test case for array case-insensitive match + ["str5", 5, {"val": "e"}, ["b", "C"]], ], input_schema, ) @@ -175,6 +175,7 @@ def test_col_is_not_null_and_is_in_list(spark): is_not_null_and_is_in_list("a", ["str1"], case_sensitive=False), is_not_null_and_is_in_list(F.col("c").getItem("val"), [F.lit("a")], case_sensitive=False), is_not_null_and_is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=False), + is_not_null_and_is_in_list("d", [["a", "b"], ["B", "c"]]), is_not_null_and_is_in_list("d", [["a", "b"], ["B", "c"]], case_sensitive=False), ) @@ -186,6 +187,7 @@ def test_col_is_not_null_and_is_in_list(spark): + "a_is_null_or_is_not_in_the_list: string, " + "unresolvedextractvalue_c_val_is_null_or_is_not_in_the_list: string, " + "try_element_at_d_2_is_null_or_is_not_in_the_list: string, " + + "d_is_null_or_is_not_in_the_list: string, " + "d_is_null_or_is_not_in_the_list: string" ) expected = spark.createDataFrame( @@ -199,6 +201,7 @@ def test_col_is_not_null_and_is_in_list(spark): None, None, None, + None, ], [ "Value 'str2' in Column 'a' is null or not in the allowed list: [str1]", @@ -209,6 +212,7 @@ def test_col_is_not_null_and_is_in_list(spark): "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", "Value 'a' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", "Value '[null, a]' in Column 'd' is null or not in the allowed list: [[a, b], [B, c]]", + "Value '[null, a]' in Column 'd' is null or not in the allowed list: [[a, b], [B, c]]", ], [ "Value ' ' in Column 'a' is null or not in the allowed list: [str1]", @@ -219,6 +223,7 @@ def test_col_is_not_null_and_is_in_list(spark): "Value ' ' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", "Value ' ' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", "Value '[null, ]' in Column 'd' is null or not in the allowed list: [[a, b], [B, c]]", + "Value '[null, ]' in Column 'd' is null or not in the allowed list: [[a, b], [B, c]]", ], [ "Value 'STR1' in Column 'a' is null or not in the allowed list: [str1]", @@ -229,6 +234,7 @@ def test_col_is_not_null_and_is_in_list(spark): None, "Value 'c' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", None, + None, ], [ "Value 'str5' in Column 'a' is null or not in the allowed list: [str1]", @@ -238,6 +244,7 @@ def test_col_is_not_null_and_is_in_list(spark): "Value 'str5' in Column 'a' is null or not in the allowed list: [str1]", "Value 'e' in Column 'UnresolvedExtractValue(c, val)' is null or not in the allowed list: [a]", "Value 'C' in Column 'try_element_at(d, 2)' is null or not in the allowed list: [b]", + "Value '[b, C]' in Column 'd' is null or not in the allowed list: [[a, b], [B, c]]", None, ], ], @@ -255,7 +262,7 @@ def test_col_is_not_in_list(spark): ["str2", None, {"val": "str2"}, [None, "a"]], [" ", 3, {"val": None}, [None, "a"]], ["STR1", 4, {"val": "A"}, ["B", "c"]], - ["str5", 5, {"val": "e"}, ["b", "C"]], # New test case for array case-insensitive match + ["str5", 5, {"val": "e"}, ["b", "C"]], ], input_schema, ) @@ -267,7 +274,8 @@ def test_col_is_not_in_list(spark): is_in_list("a", ["str1"], case_sensitive=False), is_in_list(F.col("c").getItem("val"), [F.lit("a")], case_sensitive=False), is_in_list(F.try_element_at("d", F.lit(2)), ["b"], case_sensitive=False), - is_in_list("d", [["a", "b"], ["B", "c"]], case_sensitive=False), # New: array case-insensitive + is_in_list("d", [["a", "b"], ["B", "c"]]), + is_in_list("d", [["a", "b"], ["B", "c"]], case_sensitive=False), ) checked_schema = ( "a_is_not_in_the_list: string, " @@ -277,11 +285,12 @@ def test_col_is_not_in_list(spark): + "a_is_not_in_the_list: string, " + "unresolvedextractvalue_c_val_is_not_in_the_list: string, " + "try_element_at_d_2_is_not_in_the_list: string, " + + "d_is_not_in_the_list: string, " + "d_is_not_in_the_list: string" ) expected = spark.createDataFrame( [ - [None, "Value '1' in Column 'b' is not in the allowed list: [3]", None, None, None, None, None, None], + [None, "Value '1' in Column 'b' is not in the allowed list: [3]", None, None, None, None, None, None, None], [ "Value 'str2' in Column 'a' is not in the allowed list: [str1]", None, @@ -291,6 +300,7 @@ def test_col_is_not_in_list(spark): "Value 'str2' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", "Value '[null, a]' in Column 'd' is not in the allowed list: [[a, b], [B, c]]", + "Value '[null, a]' in Column 'd' is not in the allowed list: [[a, b], [B, c]]", ], [ "Value ' ' in Column 'a' is not in the allowed list: [str1]", @@ -301,6 +311,7 @@ def test_col_is_not_in_list(spark): None, "Value 'a' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", "Value '[null, a]' in Column 'd' is not in the allowed list: [[a, b], [B, c]]", + "Value '[null, a]' in Column 'd' is not in the allowed list: [[a, b], [B, c]]", ], [ "Value 'STR1' in Column 'a' is not in the allowed list: [str1]", @@ -310,7 +321,8 @@ def test_col_is_not_in_list(spark): None, None, "Value 'c' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", - None, # ["B", "c"] matches case-insensitively + None, + None, ], [ "Value 'str5' in Column 'a' is not in the allowed list: [str1]", @@ -320,7 +332,8 @@ def test_col_is_not_in_list(spark): "Value 'str5' in Column 'a' is not in the allowed list: [str1]", "Value 'e' in Column 'UnresolvedExtractValue(c, val)' is not in the allowed list: [a]", "Value 'C' in Column 'try_element_at(d, 2)' is not in the allowed list: [b]", - None, # ["b", "C"] matches ["B", "c"] case-insensitively + "Value '[b, C]' in Column 'd' is not in the allowed list: [[a, b], [B, c]]", + None, ], ], checked_schema, From 4f6a43d2ae22929799f56144d73e6c3e698ce4d8 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Tue, 11 Nov 2025 19:56:50 +0100 Subject: [PATCH 7/8] refactor, fmt --- src/databricks/labs/dqx/utils.py | 2 +- tests/integration/test_row_checks.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index 8a88425c4..3ec8d0b58 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -446,5 +446,5 @@ def to_lowercase(col_expr: Column, is_array: bool = False) -> Column: Column expression with lowercase transformation applied """ if is_array: - return F.transform(col_expr, lambda x: F.lower(x)) + return F.transform(col_expr, F.lower) return F.lower(col_expr) diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index 65b750553..0a885bcf3 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -190,6 +190,7 @@ def test_col_is_not_null_and_is_in_list(spark): + "d_is_null_or_is_not_in_the_list: string, " + "d_is_null_or_is_not_in_the_list: string" ) + col_d_null = None expected = spark.createDataFrame( [ [ @@ -201,7 +202,7 @@ def test_col_is_not_null_and_is_in_list(spark): None, None, None, - None, + col_d_null, ], [ "Value 'str2' in Column 'a' is null or not in the allowed list: [str1]", From dca2c8201313f31f09ef9975d26b191e930376b3 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Tue, 11 Nov 2025 20:22:41 +0100 Subject: [PATCH 8/8] added integration tests --- tests/integration/test_row_checks.py | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index 0a885bcf3..da5f0bd81 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -3,6 +3,8 @@ import pytest import pyspark.sql.functions as F from chispa.dataframe_comparer import assert_df_equality # type: ignore +from pyspark.errors import AnalysisException + from databricks.labs.dqx.check_funcs import ( is_equal_to, is_not_equal_to, @@ -343,6 +345,36 @@ def test_col_is_not_in_list(spark): assert_df_equality(actual, expected, ignore_nullable=True) +def test_col_is_not_in_list_mismatch_datatype(spark): + input_schema = "a: string, d: array" + test_df = spark.createDataFrame( + [ + ["str1", ["a", "b"]], + ], + input_schema, + ) + actual = test_df.select( + is_in_list("d", ["a", "b"]), # wrong data type + ) + with pytest.raises(AnalysisException, match="[DATATYPE_MISMATCH.DATA_DIFF_TYPES]"): + actual.count() + + +def test_col_is_not_null_and_is_in_list_mismatch_datatype(spark): + input_schema = "a: string, d: array" + test_df = spark.createDataFrame( + [ + ["str1", ["a", "b"]], + ], + input_schema, + ) + actual = test_df.select( + is_not_null_and_is_in_list("d", ["a", "b"]), # wrong data type + ) + with pytest.raises(AnalysisException, match="[DATATYPE_MISMATCH.DATA_DIFF_TYPES]"): + actual.count() + + def test_col_sql_expression(spark): test_df = spark.createDataFrame([["str1", 1, 1], ["str2", None, None], ["", 2, 3]], SCHEMA + ", c: string")