From 67007b6c905bc93498b45bd508416c837670b45e Mon Sep 17 00:00:00 2001 From: bsr-the-mngrm Date: Sat, 18 Oct 2025 22:00:24 +0200 Subject: [PATCH 01/19] feat(generator): emit temporal checks for date/datetime min/max - Update dq_generate_min_max to pass through datetime.date and datetime.datetime without stringification - Emit is_in_range / is_not_less_than / is_not_greater_than based on provided bounds - Add unit tests for DateType and TimestampType - Preserve existing numeric behavior --- src/databricks/labs/dqx/profiler/generator.py | 50 +++++++++++-------- tests/unit/test_generator_temporal.py | 32 ++++++++++++ 2 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 tests/unit/test_generator_temporal.py diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index f0849b60b..68176c213 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -1,8 +1,8 @@ import logging +import datetime from databricks.labs.dqx.base import DQEngineBase from databricks.labs.dqx.engine import DQEngine -from databricks.labs.dqx.profiler.common import val_maybe_to_str from databricks.labs.dqx.profiler.profiler import DQProfile from databricks.labs.dqx.telemetry import telemetry_logger @@ -70,49 +70,59 @@ def dq_generate_min_max(column: str, level: str = "error", **params: dict): Generates a data quality rule to check if a column's value is within a specified range. Args: - column: The name of the column to check. - level: The criticality level of the rule (default is "error"). - params: Additional parameters, including the minimum and maximum values. + column: The name of the column to check. + level: The criticality level of the rule (default is "error"). + params: Additional parameters, including the minimum and maximum values. Returns: - A dictionary representing the data quality rule, or None if no limits are provided. + A dictionary representing the data quality rule, or None if no limits are provided. """ min_limit = params.get("min") max_limit = params.get("max") - if not isinstance(min_limit, int) or not isinstance(max_limit, int): - return None # TODO handle timestamp and dates: https://github.com/databrickslabs/dqx/issues/71 + if min_limit is None and max_limit is None: + return None + + def _is_num(value): + return isinstance(value, int) + + def _is_temporal(value): + return isinstance(value, (datetime.date, datetime.datetime)) + + def _same_family(value_a, value_b): + # numeric with numeric OR temporal with temporal + if value_a is None or value_b is None: + return True + return (_is_num(value_a) and _is_num(value_b)) or (_is_temporal(value_a) and _is_temporal(value_b)) - if min_limit is not None and max_limit is not None: + # Both bounds + if min_limit is not None and max_limit is not None and _same_family(min_limit, max_limit): return { "check": { "function": "is_in_range", "arguments": { "column": column, - "min_limit": val_maybe_to_str(min_limit, include_sql_quotes=False), - "max_limit": val_maybe_to_str(max_limit, include_sql_quotes=False), + # pass through Python ints or datetime/date without stringification + "min_limit": min_limit, + "max_limit": max_limit, }, }, "name": f"{column}_isnt_in_range", "criticality": level, } - if max_limit is not None: + # Only max + if max_limit is not None and (_is_num(max_limit) or _is_temporal(max_limit)): return { - "check": { - "function": "is_not_greater_than", - "arguments": {"column": column, "limit": val_maybe_to_str(max_limit, include_sql_quotes=False)}, - }, + "check": {"function": "is_not_greater_than", "arguments": {"column": column, "limit": max_limit}}, "name": f"{column}_not_greater_than", "criticality": level, } - if min_limit is not None: + # Only min + if min_limit is not None and (_is_num(min_limit) or _is_temporal(min_limit)): return { - "check": { - "function": "is_not_less_than", - "arguments": {"column": column, "limit": val_maybe_to_str(min_limit, include_sql_quotes=False)}, - }, + "check": {"function": "is_not_less_than", "arguments": {"column": column, "limit": min_limit}}, "name": f"{column}_not_less_than", "criticality": level, } diff --git a/tests/unit/test_generator_temporal.py b/tests/unit/test_generator_temporal.py new file mode 100644 index 000000000..7296ceadb --- /dev/null +++ b/tests/unit/test_generator_temporal.py @@ -0,0 +1,32 @@ +import datetime + +from databricks.labs.dqx.profiler.generator import DQGenerator + + +def test_date_both_bounds_is_in_range(): + result = DQGenerator.dq_generate_min_max( + "dcol", **{"min": datetime.date(2020, 1, 1), "max": datetime.date(2020, 12, 31)} + ) + assert result["check"]["function"] == "is_in_range" + args = result["check"]["arguments"] + assert args["column"] == "dcol" + assert args["min_limit"] == datetime.date(2020, 1, 1) + assert args["max_limit"] == datetime.date(2020, 12, 31) + + +def test_timestamp_only_min_is_not_less_than(): + timestamp = datetime.datetime(2024, 6, 1, 12, 0, 0) + result = DQGenerator.dq_generate_min_max("tscol", **{"min": timestamp, "max": None}) + assert result["check"]["function"] == "is_not_less_than" + args = result["check"]["arguments"] + assert args["column"] == "tscol" + assert args["limit"] == timestamp + + +def test_timestamp_only_max_is_not_greater_than(): + timestamp = datetime.datetime(2024, 6, 30, 23, 59, 59) + result = DQGenerator.dq_generate_min_max("tscol", **{"min": None, "max": timestamp}) + assert result["check"]["function"] == "is_not_greater_than" + args = result["check"]["arguments"] + assert args["column"] == "tscol" + assert args["limit"] == timestamp From f0850699fb9b746ef684a05b57166022f554e214 Mon Sep 17 00:00:00 2001 From: bsr-the-mngrm Date: Sat, 18 Oct 2025 22:36:38 +0200 Subject: [PATCH 02/19] test(integration): include temporal min/max rule in warn expectations and fix logging capture - Update test_generate_dq_rules_warn to expect the temporal is_not_less_than check for product_launch_date when level="warn" - Make test_generate_dq_rules_logging deterministic by adding an unknown rule and capturing the generator logger at INFO --- tests/integration/test_rules_generator.py | 39 ++++++++++++++--------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tests/integration/test_rules_generator.py b/tests/integration/test_rules_generator.py index 8fa49459c..ee8139ec8 100644 --- a/tests/integration/test_rules_generator.py +++ b/tests/integration/test_rules_generator.py @@ -1,5 +1,5 @@ +import logging import datetime -from decimal import Decimal from databricks.labs.dqx.profiler.generator import DQGenerator from databricks.labs.dqx.profiler.profiler import DQProfile @@ -22,19 +22,6 @@ parameters={"min": datetime.date(2020, 1, 1), "max": None}, description="Real min/max values were used", ), - DQProfile( - name="min_max", - column="product_expiry_ts", - parameters={"min": None, "max": datetime.datetime(2020, 1, 1)}, - description="Real min/max values were used", - ), - DQProfile(name="is_random", column="vendor_id", parameters={"in": ["1", "4", "2"]}), - DQProfile( - name='min_max', - column='d1', - description='Real min/max values were used', - parameters={'max': Decimal('333323.00'), 'min': Decimal('1.23')}, - ), ] @@ -71,6 +58,14 @@ def test_generate_dq_rules(ws): "name": "rate_code_id_isnt_in_range", "criticality": "error", }, + { + "check": { + "function": "is_not_less_than", + "arguments": {"column": "product_launch_date", "limit": datetime.date(2020, 1, 1)}, + }, + "name": "product_launch_date_not_less_than", + "criticality": "error", + }, ] assert expectations == expected @@ -108,13 +103,27 @@ def test_generate_dq_rules_warn(ws): "name": "rate_code_id_isnt_in_range", "criticality": "warn", }, + { + "check": { + "function": "is_not_less_than", + "arguments": {"column": "product_launch_date", "limit": datetime.date(2020, 1, 1)}, + }, + "name": "product_launch_date_not_less_than", + "criticality": "warn", + }, ] assert expectations == expected def test_generate_dq_rules_logging(ws, caplog): + # capture INFO from the generator module where the skip log is emitted + caplog.set_level(logging.INFO, logger="databricks.labs.dqx.profiler.generator") + generator = DQGenerator(ws) - generator.generate_dq_rules(test_rules) + # add an unknown rule to trigger the "skipping..." log + unknown_rule = DQProfile(name="is_random", column="vendor_id") + generator.generate_dq_rules(test_rules + [unknown_rule]) + assert "No rule 'is_random' for column 'vendor_id'. skipping..." in caplog.text From 94aa5fc193970908e9740288891979d6bb340672 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 19:55:25 +0100 Subject: [PATCH 03/19] refactor --- src/databricks/labs/dqx/profiler/generator.py | 5 ++- tests/integration/test_generator.py | 38 +++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index 5dca17e49..f29bc6931 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -252,7 +252,10 @@ def _same_family(value_a, value_b): # numeric with numeric OR temporal with temporal if value_a is None or value_b is None: return True - return (_is_num(value_a) and _is_num(value_b)) or (_is_temporal(value_a) and _is_temporal(value_b)) + return any([ + _is_num(value_a) and _is_num(value_b), + _is_temporal(value_a) and _is_temporal(value_b), + ]) # Both bounds if min_limit is not None and max_limit is not None and _same_family(min_limit, max_limit): diff --git a/tests/integration/test_generator.py b/tests/integration/test_generator.py index ba4e3e4dc..26061b01d 100644 --- a/tests/integration/test_generator.py +++ b/tests/integration/test_generator.py @@ -1,5 +1,6 @@ import logging import datetime +from decimal import Decimal from databricks.labs.dqx.profiler.generator import DQGenerator from databricks.labs.dqx.profiler.profiler import DQProfile @@ -22,11 +23,24 @@ parameters={"min": datetime.date(2020, 1, 1), "max": None}, description="Real min/max values were used", ), + DQProfile( + name="min_max", + column="product_expiry_ts", + parameters={"min": None, "max": datetime.datetime(2020, 1, 1)}, + description="Real min/max values were used", + ), + DQProfile(name="is_random", column="vendor_id", parameters={"in": ["1", "4", "2"]}), + DQProfile( + name='min_max', + column='d1', + description='Real min/max values were used', + parameters={'max': Decimal('333323.00'), 'min': Decimal('1.23')}, + ), ] -def test_generate_dq_rules(ws): - generator = DQGenerator(ws) +def test_generate_dq_rules(ws, spark): + generator = DQGenerator(ws, spark) expectations = generator.generate_dq_rules(test_rules) expected = [ { @@ -78,8 +92,8 @@ def test_generate_dq_rules(ws): assert expectations == expected -def test_generate_dq_rules_warn(ws): - generator = DQGenerator(ws) +def test_generate_dq_rules_warn(ws, spark): + generator = DQGenerator(ws, spark) expectations = generator.generate_dq_rules(test_rules, criticality="warn") expected = [ { @@ -131,11 +145,11 @@ def test_generate_dq_rules_warn(ws): assert expectations == expected -def test_generate_dq_rules_logging(ws, caplog): +def test_generate_dq_rules_logging(ws, spark, caplog): # capture INFO from the generator module where the skip log is emitted caplog.set_level(logging.INFO, logger="databricks.labs.dqx.profiler.generator") - generator = DQGenerator(ws) + generator = DQGenerator(ws, spark) # add an unknown rule to trigger the "skipping..." log unknown_rule = DQProfile(name="is_random", column="vendor_id") generator.generate_dq_rules(test_rules + [unknown_rule]) @@ -143,14 +157,14 @@ def test_generate_dq_rules_logging(ws, caplog): assert "No rule 'is_random' for column 'vendor_id'. skipping..." in caplog.text -def test_generate_dq_no_rules(ws): - generator = DQGenerator(ws) +def test_generate_dq_no_rules(ws, spark): + generator = DQGenerator(ws, spark) expectations = generator.generate_dq_rules(None, criticality="warn") assert not expectations -def test_generate_dq_rules_dataframe_filter(ws): - generator = DQGenerator(ws) +def test_generate_dq_rules_dataframe_filter(ws, spark): + generator = DQGenerator(ws, spark) test_rules_filter = [ DQProfile( name="is_not_null", @@ -223,8 +237,8 @@ def test_generate_dq_rules_dataframe_filter(ws): assert expectations == expected -def test_generate_dq_rules_dataframe_filter_none(ws): - generator = DQGenerator(ws) +def test_generate_dq_rules_dataframe_filter_none(ws, spark): + generator = DQGenerator(ws, spark) test_rules_no_filter = [ DQProfile( name="is_not_null", From 6c97e8bfc1ded6deead1a6292f59d1934d30d67e Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 19:58:43 +0100 Subject: [PATCH 04/19] added handling of float --- src/databricks/labs/dqx/profiler/generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index f29bc6931..982b9f0eb 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -3,6 +3,7 @@ import logging import datetime import json +from decimal import Decimal from collections.abc import Callable from pyspark.sql import SparkSession from databricks.sdk import WorkspaceClient @@ -243,7 +244,7 @@ def dq_generate_min_max(column: str, criticality: str = "error", **params: dict) return None def _is_num(value): - return isinstance(value, int) + return isinstance(value, (int, float, Decimal)) def _is_temporal(value): return isinstance(value, (datetime.date, datetime.datetime)) From 7819416aff35590d4aa067e1db536f4511298c9b Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 19:59:59 +0100 Subject: [PATCH 05/19] refactor --- src/databricks/labs/dqx/profiler/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index 982b9f0eb..ffb6d3a5e 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -265,7 +265,7 @@ def _same_family(value_a, value_b): "function": "is_in_range", "arguments": { "column": column, - # pass through Python ints or datetime/date without stringification + # pass through Python num or datetime/date without stringification "min_limit": min_limit, "max_limit": max_limit, }, From df4c96e299530b067818cc23ae4f2f8de0176e5b Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 20:01:12 +0100 Subject: [PATCH 06/19] fmt --- src/databricks/labs/dqx/profiler/generator.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index ffb6d3a5e..9ef523289 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -253,10 +253,12 @@ def _same_family(value_a, value_b): # numeric with numeric OR temporal with temporal if value_a is None or value_b is None: return True - return any([ - _is_num(value_a) and _is_num(value_b), - _is_temporal(value_a) and _is_temporal(value_b), - ]) + return any( + [ + _is_num(value_a) and _is_num(value_b), + _is_temporal(value_a) and _is_temporal(value_b), + ] + ) # Both bounds if min_limit is not None and max_limit is not None and _same_family(min_limit, max_limit): From 4e4758d70afd859b7a406a805929604a51833f27 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 20:41:36 +0100 Subject: [PATCH 07/19] updated tests and functions to support Decimal --- src/databricks/labs/dqx/check_funcs.py | 25 +++++++++--------- tests/integration/test_generator.py | 35 +++++++++++++++++++------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index 29a34fe8c..f7e0d1275 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -4,6 +4,7 @@ import ipaddress import uuid from collections.abc import Callable, Sequence +from decimal import Decimal from enum import Enum from itertools import zip_longest import operator as py_operator @@ -488,7 +489,7 @@ def is_not_in_near_future(column: str | Column, offset: int = 0, curr_timestamp: @register_rule("row") def is_equal_to( - column: str | Column, value: int | float | str | datetime.date | datetime.datetime | Column | None = None + column: str | Column, value: int | float | Decimal | str | datetime.date | datetime.datetime | Column | None = None ) -> Column: """Check whether the values in the input column are equal to the given value. @@ -519,7 +520,7 @@ def is_equal_to( @register_rule("row") def is_not_equal_to( - column: str | Column, value: int | float | str | datetime.date | datetime.datetime | Column | None = None + column: str | Column, value: int | float | Decimal | str | datetime.date | datetime.datetime | Column | None = None ) -> Column: """Check whether the values in the input column are not equal to the given value. @@ -550,13 +551,13 @@ def is_not_equal_to( @register_rule("row") def is_not_less_than( - column: str | Column, limit: int | float | datetime.date | datetime.datetime | str | Column | None = None + column: str | Column, limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None ) -> Column: """Checks whether the values in the input column are not less than the provided limit. Args: column: column to check; can be a string column name or a column expression - limit: limit to use in the condition as number, date, timestamp, column name or sql expression + limit: limit to use in the condition as number, date, timestamp, Decimal, column name or sql expression Returns: new Column @@ -580,13 +581,13 @@ def is_not_less_than( @register_rule("row") def is_not_greater_than( - column: str | Column, limit: int | float | datetime.date | datetime.datetime | str | Column | None = None + column: str | Column, limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None ) -> Column: """Checks whether the values in the input column are not greater than the provided limit. Args: column: column to check; can be a string column name or a column expression - limit: limit to use in the condition as number, date, timestamp, column name or sql expression + limit: limit to use in the condition as number, date, timestamp, Decimal, column name or sql expression Returns: new Column @@ -611,15 +612,15 @@ def is_not_greater_than( @register_rule("row") def is_in_range( column: str | Column, - min_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, - max_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, + min_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, + max_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, ) -> Column: """Checks whether the values in the input column are in the provided limits (inclusive of both boundaries). Args: column: column to check; can be a string column name or a column expression - min_limit: min limit to use in the condition as number, date, timestamp, column name or sql expression - max_limit: max limit to use in the condition as number, date, timestamp, column name or sql expression + min_limit: min limit to use in the condition as number, date, timestamp, Decimal, column name or sql expression + max_limit: max limit to use in the condition as number, date, timestamp, Decimal, column name or sql expression Returns: new Column @@ -649,8 +650,8 @@ def is_in_range( @register_rule("row") def is_not_in_range( column: str | Column, - min_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, - max_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, + min_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, + max_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, ) -> Column: """Checks whether the values in the input column are outside the provided limits (inclusive of both boundaries). diff --git a/tests/integration/test_generator.py b/tests/integration/test_generator.py index 26061b01d..9197d7054 100644 --- a/tests/integration/test_generator.py +++ b/tests/integration/test_generator.py @@ -1,5 +1,6 @@ import logging import datetime +import json from decimal import Decimal from databricks.labs.dqx.profiler.generator import DQGenerator @@ -26,7 +27,7 @@ DQProfile( name="min_max", column="product_expiry_ts", - parameters={"min": None, "max": datetime.datetime(2020, 1, 1)}, + parameters={"min": None, "max": datetime.datetime(2021, 1, 1)}, description="Real min/max values were used", ), DQProfile(name="is_random", column="vendor_id", parameters={"in": ["1", "4", "2"]}), @@ -82,14 +83,22 @@ def test_generate_dq_rules(ws, spark): }, { "check": { - "function": "is_not_less_than", - "arguments": {"column": "product_launch_date", "limit": datetime.date(2020, 1, 1)}, + "function": "is_not_greater_than", + "arguments": {"column": "product_expiry_ts", "limit": datetime.datetime(2021, 1, 1)}, }, - "name": "product_launch_date_not_less_than", + "name": "product_expiry_ts_not_greater_than", + "criticality": "error", + }, + { + "check": { + "function": "is_in_range", + "arguments": {"column": "d1", "min_limit": Decimal('1.23'), "max_limit": Decimal('333323.00')}, + }, + "name": "d1_isnt_in_range", "criticality": "error", }, ] - assert expectations == expected + assert expectations == expected, f"Actual expectations: {json.dumps(expectations, indent=2, default=str)}" def test_generate_dq_rules_warn(ws, spark): @@ -135,14 +144,22 @@ def test_generate_dq_rules_warn(ws, spark): }, { "check": { - "function": "is_not_less_than", - "arguments": {"column": "product_launch_date", "limit": datetime.date(2020, 1, 1)}, + "function": "is_not_greater_than", + "arguments": {"column": "product_expiry_ts", "limit": datetime.datetime(2021, 1, 1)}, }, - "name": "product_launch_date_not_less_than", + "name": "product_expiry_ts_not_greater_than", + "criticality": "warn", + }, + { + "check": { + "function": "is_in_range", + "arguments": {"column": "d1", "min_limit": Decimal('1.23'), "max_limit": Decimal('333323.00')}, + }, + "name": "d1_isnt_in_range", "criticality": "warn", }, ] - assert expectations == expected + assert expectations == expected, f"Actual expectations: {json.dumps(expectations, indent=2, default=str)}" def test_generate_dq_rules_logging(ws, spark, caplog): From 2e1cb8f5270d1cc17796f92fcfb2630127295dcc Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 20:47:47 +0100 Subject: [PATCH 08/19] added support for Decimal --- src/databricks/labs/dqx/check_funcs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index f7e0d1275..d32955005 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -650,8 +650,8 @@ def is_in_range( @register_rule("row") def is_not_in_range( column: str | Column, - min_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, - max_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, + min_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, + max_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, ) -> Column: """Checks whether the values in the input column are outside the provided limits (inclusive of both boundaries). @@ -1403,7 +1403,7 @@ def apply(df: DataFrame, spark: SparkSession, ref_dfs: dict[str, DataFrame]) -> @register_rule("dataset") def is_aggr_not_greater_than( column: str | Column, - limit: int | float | str | Column, + limit: int | float | Decimal | str | Column, aggr_type: str = "count", group_by: list[str | Column] | None = None, row_filter: str | None = None, @@ -1448,7 +1448,7 @@ def is_aggr_not_greater_than( @register_rule("dataset") def is_aggr_not_less_than( column: str | Column, - limit: int | float | str | Column, + limit: int | float | Decimal | str | Column, aggr_type: str = "count", group_by: list[str | Column] | None = None, row_filter: str | None = None, @@ -1493,7 +1493,7 @@ def is_aggr_not_less_than( @register_rule("dataset") def is_aggr_equal( column: str | Column, - limit: int | float | str | Column, + limit: int | float | Decimal | str | Column, aggr_type: str = "count", group_by: list[str | Column] | None = None, row_filter: str | None = None, @@ -1538,7 +1538,7 @@ def is_aggr_equal( @register_rule("dataset") def is_aggr_not_equal( column: str | Column, - limit: int | float | str | Column, + limit: int | float | Decimal | str | Column, aggr_type: str = "count", group_by: list[str | Column] | None = None, row_filter: str | None = None, @@ -2694,7 +2694,7 @@ def _validate_aggregate_return_type( def _is_aggr_compare( column: str | Column, - limit: int | float | str | Column, + limit: int | float | Decimal | str | Column, aggr_type: str, aggr_params: dict[str, Any] | None, group_by: list[str | Column] | None, @@ -2913,7 +2913,7 @@ def _cleanup_alias_name(column: str) -> str: def get_limit_expr( - limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, + limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, ) -> Column: """ Generate a Spark Column expression for a limit value. From e327ade33315b1807c1f4ca50557352fba6fbcde Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 20:49:06 +0100 Subject: [PATCH 09/19] refactor --- tests/integration/test_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_generator.py b/tests/integration/test_generator.py index 9197d7054..0c898ed4e 100644 --- a/tests/integration/test_generator.py +++ b/tests/integration/test_generator.py @@ -98,7 +98,7 @@ def test_generate_dq_rules(ws, spark): "criticality": "error", }, ] - assert expectations == expected, f"Actual expectations: {json.dumps(expectations, indent=2, default=str)}" + assert expectations == expected, f"Actual expectations: {expectations}" def test_generate_dq_rules_warn(ws, spark): @@ -159,7 +159,7 @@ def test_generate_dq_rules_warn(ws, spark): "criticality": "warn", }, ] - assert expectations == expected, f"Actual expectations: {json.dumps(expectations, indent=2, default=str)}" + assert expectations == expected, f"Actual expectations: {expectations}" def test_generate_dq_rules_logging(ws, spark, caplog): From 7f1893c29018e2aecc2a66bcaccd0abf54e72cb1 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 21:48:26 +0100 Subject: [PATCH 10/19] updated tests --- tests/integration/test_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_generator.py b/tests/integration/test_generator.py index 0c898ed4e..157527cf0 100644 --- a/tests/integration/test_generator.py +++ b/tests/integration/test_generator.py @@ -27,7 +27,7 @@ DQProfile( name="min_max", column="product_expiry_ts", - parameters={"min": None, "max": datetime.datetime(2021, 1, 1)}, + parameters={"min": None, "max": datetime.datetime(2020, 1, 1)}, description="Real min/max values were used", ), DQProfile(name="is_random", column="vendor_id", parameters={"in": ["1", "4", "2"]}), @@ -84,7 +84,7 @@ def test_generate_dq_rules(ws, spark): { "check": { "function": "is_not_greater_than", - "arguments": {"column": "product_expiry_ts", "limit": datetime.datetime(2021, 1, 1)}, + "arguments": {"column": "product_expiry_ts", "limit": datetime.datetime(2020, 1, 1)}, }, "name": "product_expiry_ts_not_greater_than", "criticality": "error", @@ -145,7 +145,7 @@ def test_generate_dq_rules_warn(ws, spark): { "check": { "function": "is_not_greater_than", - "arguments": {"column": "product_expiry_ts", "limit": datetime.datetime(2021, 1, 1)}, + "arguments": {"column": "product_expiry_ts", "limit": datetime.datetime(2020, 1, 1)}, }, "name": "product_expiry_ts_not_greater_than", "criticality": "warn", From 2737a86ecd0d2e538d169570c3aa2c239412de9d Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 22:47:24 +0100 Subject: [PATCH 11/19] updated tests --- tests/integration/test_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_generator.py b/tests/integration/test_generator.py index 157527cf0..9f0dbe057 100644 --- a/tests/integration/test_generator.py +++ b/tests/integration/test_generator.py @@ -1,6 +1,5 @@ import logging import datetime -import json from decimal import Decimal from databricks.labs.dqx.profiler.generator import DQGenerator From 81202f9e4a705b6d95fade41154348042c5d078b Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 22:52:43 +0100 Subject: [PATCH 12/19] Update src/databricks/labs/dqx/profiler/generator.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/databricks/labs/dqx/profiler/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index 9ef523289..e9e1c5e60 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -267,7 +267,7 @@ def _same_family(value_a, value_b): "function": "is_in_range", "arguments": { "column": column, - # pass through Python num or datetime/date without stringification + # pass through Python numeric types (int, float, Decimal) or temporal types (datetime/date) without stringification "min_limit": min_limit, "max_limit": max_limit, }, From a089d509fcb81e7d4d8b1275790e262ca7dde7ed Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 23:21:41 +0100 Subject: [PATCH 13/19] refactor --- src/databricks/labs/dqx/check_funcs.py | 29 +++++++++++++------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index d32955005..def804a3a 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -4,7 +4,6 @@ import ipaddress import uuid from collections.abc import Callable, Sequence -from decimal import Decimal from enum import Enum from itertools import zip_longest import operator as py_operator @@ -489,7 +488,7 @@ def is_not_in_near_future(column: str | Column, offset: int = 0, curr_timestamp: @register_rule("row") def is_equal_to( - column: str | Column, value: int | float | Decimal | str | datetime.date | datetime.datetime | Column | None = None + column: str | Column, value: int | float | str | datetime.date | datetime.datetime | Column | None = None ) -> Column: """Check whether the values in the input column are equal to the given value. @@ -520,7 +519,7 @@ def is_equal_to( @register_rule("row") def is_not_equal_to( - column: str | Column, value: int | float | Decimal | str | datetime.date | datetime.datetime | Column | None = None + column: str | Column, value: int | float | str | datetime.date | datetime.datetime | Column | None = None ) -> Column: """Check whether the values in the input column are not equal to the given value. @@ -551,7 +550,7 @@ def is_not_equal_to( @register_rule("row") def is_not_less_than( - column: str | Column, limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None + column: str | Column, limit: int | float | datetime.date | datetime.datetime | str | Column | None = None ) -> Column: """Checks whether the values in the input column are not less than the provided limit. @@ -581,7 +580,7 @@ def is_not_less_than( @register_rule("row") def is_not_greater_than( - column: str | Column, limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None + column: str | Column, limit: int | float | datetime.date | datetime.datetime | str | Column | None = None ) -> Column: """Checks whether the values in the input column are not greater than the provided limit. @@ -612,8 +611,8 @@ def is_not_greater_than( @register_rule("row") def is_in_range( column: str | Column, - min_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, - max_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, + min_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, + max_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, ) -> Column: """Checks whether the values in the input column are in the provided limits (inclusive of both boundaries). @@ -650,8 +649,8 @@ def is_in_range( @register_rule("row") def is_not_in_range( column: str | Column, - min_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, - max_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, + min_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, + max_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, ) -> Column: """Checks whether the values in the input column are outside the provided limits (inclusive of both boundaries). @@ -1403,7 +1402,7 @@ def apply(df: DataFrame, spark: SparkSession, ref_dfs: dict[str, DataFrame]) -> @register_rule("dataset") def is_aggr_not_greater_than( column: str | Column, - limit: int | float | Decimal | str | Column, + limit: int | float | str | Column, aggr_type: str = "count", group_by: list[str | Column] | None = None, row_filter: str | None = None, @@ -1448,7 +1447,7 @@ def is_aggr_not_greater_than( @register_rule("dataset") def is_aggr_not_less_than( column: str | Column, - limit: int | float | Decimal | str | Column, + limit: int | float | str | Column, aggr_type: str = "count", group_by: list[str | Column] | None = None, row_filter: str | None = None, @@ -1493,7 +1492,7 @@ def is_aggr_not_less_than( @register_rule("dataset") def is_aggr_equal( column: str | Column, - limit: int | float | Decimal | str | Column, + limit: int | float | str | Column, aggr_type: str = "count", group_by: list[str | Column] | None = None, row_filter: str | None = None, @@ -1538,7 +1537,7 @@ def is_aggr_equal( @register_rule("dataset") def is_aggr_not_equal( column: str | Column, - limit: int | float | Decimal | str | Column, + limit: int | float | str | Column, aggr_type: str = "count", group_by: list[str | Column] | None = None, row_filter: str | None = None, @@ -2694,7 +2693,7 @@ def _validate_aggregate_return_type( def _is_aggr_compare( column: str | Column, - limit: int | float | Decimal | str | Column, + limit: int | float | str | Column, aggr_type: str, aggr_params: dict[str, Any] | None, group_by: list[str | Column] | None, @@ -2913,7 +2912,7 @@ def _cleanup_alias_name(column: str) -> str: def get_limit_expr( - limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None, + limit: int | float | datetime.date | datetime.datetime | str | Column | None = None, ) -> Column: """ Generate a Spark Column expression for a limit value. From 696cb1c645b595a6aa44aa6784f1c60fa4948bc7 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 23:22:17 +0100 Subject: [PATCH 14/19] refactor --- src/databricks/labs/dqx/check_funcs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index def804a3a..29a34fe8c 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -556,7 +556,7 @@ def is_not_less_than( Args: column: column to check; can be a string column name or a column expression - limit: limit to use in the condition as number, date, timestamp, Decimal, column name or sql expression + limit: limit to use in the condition as number, date, timestamp, column name or sql expression Returns: new Column @@ -586,7 +586,7 @@ def is_not_greater_than( Args: column: column to check; can be a string column name or a column expression - limit: limit to use in the condition as number, date, timestamp, Decimal, column name or sql expression + limit: limit to use in the condition as number, date, timestamp, column name or sql expression Returns: new Column @@ -618,8 +618,8 @@ def is_in_range( Args: column: column to check; can be a string column name or a column expression - min_limit: min limit to use in the condition as number, date, timestamp, Decimal, column name or sql expression - max_limit: max limit to use in the condition as number, date, timestamp, Decimal, column name or sql expression + min_limit: min limit to use in the condition as number, date, timestamp, column name or sql expression + max_limit: max limit to use in the condition as number, date, timestamp, column name or sql expression Returns: new Column From 5eb5b7198aa364f0669651aeacc6cb6bd4f3276b Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 17 Dec 2025 23:26:54 +0100 Subject: [PATCH 15/19] updated tests --- src/databricks/labs/dqx/profiler/generator.py | 3 +-- tests/integration/test_generator.py | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index e9e1c5e60..8ddf57b40 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -3,7 +3,6 @@ import logging import datetime import json -from decimal import Decimal from collections.abc import Callable from pyspark.sql import SparkSession from databricks.sdk import WorkspaceClient @@ -244,7 +243,7 @@ def dq_generate_min_max(column: str, criticality: str = "error", **params: dict) return None def _is_num(value): - return isinstance(value, (int, float, Decimal)) + return isinstance(value, (int, float)) def _is_temporal(value): return isinstance(value, (datetime.date, datetime.datetime)) diff --git a/tests/integration/test_generator.py b/tests/integration/test_generator.py index 9f0dbe057..c76ff3673 100644 --- a/tests/integration/test_generator.py +++ b/tests/integration/test_generator.py @@ -34,7 +34,7 @@ name='min_max', column='d1', description='Real min/max values were used', - parameters={'max': Decimal('333323.00'), 'min': Decimal('1.23')}, + parameters={'max': 333323.00, 'min': 1.23}, ), ] @@ -91,7 +91,7 @@ def test_generate_dq_rules(ws, spark): { "check": { "function": "is_in_range", - "arguments": {"column": "d1", "min_limit": Decimal('1.23'), "max_limit": Decimal('333323.00')}, + "arguments": {"column": "d1", "min_limit": 1.23, "max_limit": 333323.00}, }, "name": "d1_isnt_in_range", "criticality": "error", @@ -152,7 +152,7 @@ def test_generate_dq_rules_warn(ws, spark): { "check": { "function": "is_in_range", - "arguments": {"column": "d1", "min_limit": Decimal('1.23'), "max_limit": Decimal('333323.00')}, + "arguments": {"column": "d1", "min_limit": 1.23, "max_limit": 333323.00}, }, "name": "d1_isnt_in_range", "criticality": "warn", From 12bbe0cd9cdb94123999db4972762754cea953ec Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Thu, 18 Dec 2025 00:00:28 +0100 Subject: [PATCH 16/19] fmt --- tests/integration/test_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_generator.py b/tests/integration/test_generator.py index c76ff3673..3ffea59c0 100644 --- a/tests/integration/test_generator.py +++ b/tests/integration/test_generator.py @@ -1,6 +1,5 @@ import logging import datetime -from decimal import Decimal from databricks.labs.dqx.profiler.generator import DQGenerator from databricks.labs.dqx.profiler.profiler import DQProfile From 560ea00e4cb918ffce6cece9556ee0f1866762b0 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Dec 2025 14:27:16 +0100 Subject: [PATCH 17/19] refactor --- src/databricks/labs/dqx/profiler/generator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index 421a002fd..19a1fb200 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -259,8 +259,6 @@ def _is_temporal(value): def _same_family(value_a, value_b): # numeric with numeric OR temporal with temporal - if value_a is None or value_b is None: - return True return any( [ _is_num(value_a) and _is_num(value_b), From f2749736cf5be224d18405a2cfff8ef515f04484 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Dec 2025 15:55:32 +0100 Subject: [PATCH 18/19] refactor, added temporal stringification --- src/databricks/labs/dqx/profiler/generator.py | 18 +++++++++++++----- tests/integration/test_generator.py | 12 ++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/databricks/labs/dqx/profiler/generator.py b/src/databricks/labs/dqx/profiler/generator.py index 19a1fb200..e26e1a139 100644 --- a/src/databricks/labs/dqx/profiler/generator.py +++ b/src/databricks/labs/dqx/profiler/generator.py @@ -10,6 +10,7 @@ from databricks.labs.dqx.base import DQEngineBase from databricks.labs.dqx.config import LLMModelConfig, InputConfig from databricks.labs.dqx.engine import DQEngine +from databricks.labs.dqx.profiler.common import val_maybe_to_str from databricks.labs.dqx.profiler.profiler import DQProfile from databricks.labs.dqx.telemetry import telemetry_logger from databricks.labs.dqx.errors import MissingParameterError @@ -273,9 +274,10 @@ def _same_family(value_a, value_b): "function": "is_in_range", "arguments": { "column": column, - # pass through Python numeric types (int, float, Decimal) or temporal types (datetime/date) without stringification - "min_limit": min_limit, - "max_limit": max_limit, + # pass through Python numeric types (int, float) without stringification + # except for temporal types (datetime/date) which are not Json serializable + "min_limit": val_maybe_to_str(min_limit, include_sql_quotes=False), + "max_limit": val_maybe_to_str(max_limit, include_sql_quotes=False), }, }, "name": f"{column}_isnt_in_range", @@ -285,7 +287,10 @@ def _same_family(value_a, value_b): # Only max if max_limit is not None and (_is_num(max_limit) or _is_temporal(max_limit)): return { - "check": {"function": "is_not_greater_than", "arguments": {"column": column, "limit": max_limit}}, + "check": { + "function": "is_not_greater_than", + "arguments": {"column": column, "limit": val_maybe_to_str(max_limit, include_sql_quotes=False)}, + }, "name": f"{column}_not_greater_than", "criticality": criticality, } @@ -293,7 +298,10 @@ def _same_family(value_a, value_b): # Only min if min_limit is not None and (_is_num(min_limit) or _is_temporal(min_limit)): return { - "check": {"function": "is_not_less_than", "arguments": {"column": column, "limit": min_limit}}, + "check": { + "function": "is_not_less_than", + "arguments": {"column": column, "limit": val_maybe_to_str(min_limit, include_sql_quotes=False)}, + }, "name": f"{column}_not_less_than", "criticality": criticality, } diff --git a/tests/integration/test_generator.py b/tests/integration/test_generator.py index 3ffea59c0..3b20995a6 100644 --- a/tests/integration/test_generator.py +++ b/tests/integration/test_generator.py @@ -19,13 +19,13 @@ DQProfile( name="min_max", column="product_launch_date", - parameters={"min": datetime.date(2020, 1, 1), "max": None}, + parameters={"min": datetime.date(2020, 1, 2), "max": None}, description="Real min/max values were used", ), DQProfile( name="min_max", column="product_expiry_ts", - parameters={"min": None, "max": datetime.datetime(2020, 1, 1)}, + parameters={"min": None, "max": datetime.datetime(2020, 1, 2, 3, 4, 5)}, description="Real min/max values were used", ), DQProfile(name="is_random", column="vendor_id", parameters={"in": ["1", "4", "2"]}), @@ -74,7 +74,7 @@ def test_generate_dq_rules(ws, spark): { "check": { "function": "is_not_less_than", - "arguments": {"column": "product_launch_date", "limit": datetime.date(2020, 1, 1)}, + "arguments": {"column": "product_launch_date", "limit": "2020-01-02"}, }, "name": "product_launch_date_not_less_than", "criticality": "error", @@ -82,7 +82,7 @@ def test_generate_dq_rules(ws, spark): { "check": { "function": "is_not_greater_than", - "arguments": {"column": "product_expiry_ts", "limit": datetime.datetime(2020, 1, 1)}, + "arguments": {"column": "product_expiry_ts", "limit": "2020-01-02T03:04:05.000000"}, }, "name": "product_expiry_ts_not_greater_than", "criticality": "error", @@ -135,7 +135,7 @@ def test_generate_dq_rules_warn(ws, spark): { "check": { "function": "is_not_less_than", - "arguments": {"column": "product_launch_date", "limit": datetime.date(2020, 1, 1)}, + "arguments": {"column": "product_launch_date", "limit": "2020-01-02"}, }, "name": "product_launch_date_not_less_than", "criticality": "warn", @@ -143,7 +143,7 @@ def test_generate_dq_rules_warn(ws, spark): { "check": { "function": "is_not_greater_than", - "arguments": {"column": "product_expiry_ts", "limit": datetime.datetime(2020, 1, 1)}, + "arguments": {"column": "product_expiry_ts", "limit": "2020-01-02T03:04:05.000000"}, }, "name": "product_expiry_ts_not_greater_than", "criticality": "warn", From 1f4bfd023bcbd0053862efc5d8138f2addd2bc10 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 19 Dec 2025 16:07:56 +0100 Subject: [PATCH 19/19] updated tests --- tests/unit/test_generator_temporal.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_generator_temporal.py b/tests/unit/test_generator_temporal.py index 7296ceadb..a67ca7c98 100644 --- a/tests/unit/test_generator_temporal.py +++ b/tests/unit/test_generator_temporal.py @@ -3,30 +3,30 @@ from databricks.labs.dqx.profiler.generator import DQGenerator -def test_date_both_bounds_is_in_range(): +def test_date_both_bounds_is_in_range(set_utc_timezone): result = DQGenerator.dq_generate_min_max( "dcol", **{"min": datetime.date(2020, 1, 1), "max": datetime.date(2020, 12, 31)} ) assert result["check"]["function"] == "is_in_range" args = result["check"]["arguments"] assert args["column"] == "dcol" - assert args["min_limit"] == datetime.date(2020, 1, 1) - assert args["max_limit"] == datetime.date(2020, 12, 31) + assert args["min_limit"] == "2020-01-01" + assert args["max_limit"] == "2020-12-31" -def test_timestamp_only_min_is_not_less_than(): +def test_timestamp_only_min_is_not_less_than(set_utc_timezone): timestamp = datetime.datetime(2024, 6, 1, 12, 0, 0) result = DQGenerator.dq_generate_min_max("tscol", **{"min": timestamp, "max": None}) assert result["check"]["function"] == "is_not_less_than" args = result["check"]["arguments"] assert args["column"] == "tscol" - assert args["limit"] == timestamp + assert args["limit"] == "2024-06-01T12:00:00.000000" -def test_timestamp_only_max_is_not_greater_than(): +def test_timestamp_only_max_is_not_greater_than(set_utc_timezone): timestamp = datetime.datetime(2024, 6, 30, 23, 59, 59) result = DQGenerator.dq_generate_min_max("tscol", **{"min": None, "max": timestamp}) assert result["check"]["function"] == "is_not_greater_than" args = result["check"]["arguments"] assert args["column"] == "tscol" - assert args["limit"] == timestamp + assert args["limit"] == "2024-06-30T23:59:59.000000"