From fa212f2776480e0dd283a024f89582a655f9ba31 Mon Sep 17 00:00:00 2001 From: bsr-the-mngrm Date: Thu, 7 Aug 2025 21:50:02 +0200 Subject: [PATCH 1/7] fix(profiler.py): avoid unnecessary rounding in _round_datetime when datetime is already midnight --- src/databricks/labs/dqx/profiler/profiler.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/dqx/profiler/profiler.py b/src/databricks/labs/dqx/profiler/profiler.py index 447b5549b..1cef8c63c 100644 --- a/src/databricks/labs/dqx/profiler/profiler.py +++ b/src/databricks/labs/dqx/profiler/profiler.py @@ -681,19 +681,30 @@ def _round_datetime(value: datetime.datetime, direction: str) -> datetime.dateti """ Rounds a datetime value to midnight based on the specified direction. + - "down" → truncate to midnight (00:00:00). + - "up" → return the next midnight unless value is already midnight. + - Raises ValueError for invalid direction. + :param value: The datetime value to round. :param direction: The rounding direction ("up" or "down"). :return: The rounded datetime value. + :raises ValueError: If direction is not 'up' or 'down'. """ + midnight = value.replace(hour=0, minute=0, second=0, microsecond=0) + if direction == "down": - return value.replace(hour=0, minute=0, second=0, microsecond=0) + return midnight + if direction == "up": + if midnight == value: + return value try: - return value.replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1) + return midnight + datetime.timedelta(days=1) except OverflowError: logger.warning("Rounding datetime up caused overflow; returning datetime.max instead.") return datetime.datetime.max - return value + + raise ValueError(f"Invalid rounding direction: {direction}. Use 'up' or 'down'.") @staticmethod def _round_float(value: float, direction: str) -> float: From 122f8f224fa7fad0a135f7f39b35c81d8714cc38 Mon Sep 17 00:00:00 2001 From: bsr-the-mngrm Date: Thu, 7 Aug 2025 22:21:10 +0200 Subject: [PATCH 2/7] test(profiler.py): add integration test for midnight rounding behavior --- tests/integration/test_profiler.py | 98 ++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/integration/test_profiler.py b/tests/integration/test_profiler.py index 733ef05c8..d9dc5cf06 100644 --- a/tests/integration/test_profiler.py +++ b/tests/integration/test_profiler.py @@ -104,6 +104,104 @@ def test_profiler(spark, ws): assert rules == expected_rules +def test_profiler_rounding_midnight_behavior(spark, ws): + inp_schema = T.StructType( + [ + T.StructField("t1", T.IntegerType()), + T.StructField("d1", T.DecimalType(10, 2)), + T.StructField("t2", T.StringType()), + T.StructField( + "s1", + T.StructType( + [ + T.StructField("ns1", T.TimestampType()), + T.StructField( + "s2", + T.StructType([T.StructField("ns2", T.StringType()), T.StructField("ns3", T.DateType())]), + ), + ] + ), + ), + T.StructField("b1", T.ByteType()), + ] + ) + inp_df = spark.createDataFrame( + [ + [ + 1, + Decimal("1.23"), + " test ", + { + "ns1": datetime.fromisoformat("2023-01-08T00:00:00+00:00"), + "s2": {"ns2": "test", "ns3": date.fromisoformat("2023-01-08")}, + }, + 0, + ], + [ + 2, + Decimal("2.41"), + "test2", + { + "ns1": datetime.fromisoformat("2023-01-07T10:00:11+00:00"), + "s2": {"ns2": "test2", "ns3": date.fromisoformat("2023-01-07")}, + }, + 1, + ], + [ + 3, + Decimal("333323.0"), + None, + { + "ns1": datetime.fromisoformat("2023-01-06T10:00:11+00:00"), + "s2": {"ns2": "test", "ns3": date.fromisoformat("2023-01-06")}, + }, + 0, + ], + ], + schema=inp_schema, + ) + + profiler = DQProfiler(ws) + stats, rules = profiler.profile(inp_df, options={"sample_fraction": None}) + + expected_rules = [ + DQProfile(name="is_not_null", column="t1", description=None, parameters=None), + DQProfile( + name="min_max", column="t1", description="Real min/max values were used", parameters={"min": 1, "max": 3} + ), + DQProfile(name='is_not_null', column='d1', description=None, parameters=None), + DQProfile( + name='min_max', + column='d1', + description='Real min/max values were used', + parameters={'max': Decimal('333323.00'), 'min': Decimal('1.23')}, + ), + DQProfile(name='is_not_null_or_empty', column='t2', description=None, parameters={'trim_strings': True}), + DQProfile(name="is_not_null", column="s1.ns1", description=None, parameters=None), + DQProfile( + name="min_max", + column="s1.ns1", + description="Real min/max values were used", + parameters={ + "min": datetime(2023, 1, 6, 0, 0, tzinfo=timezone.utc), + "max": datetime(2023, 1, 8, 0, 0, tzinfo=timezone.utc), + }, + ), + DQProfile(name="is_not_null", column="s1.s2.ns2", description=None, parameters=None), + DQProfile(name="is_not_null", column="s1.s2.ns3", description=None, parameters=None), + DQProfile( + name="min_max", + column="s1.s2.ns3", + description="Real min/max values were used", + parameters={"min": date(2023, 1, 6), "max": date(2023, 1, 8)}, + ), + DQProfile(name="is_not_null", column="b1", description=None, parameters=None), + ] + print(stats) + assert len(stats.keys()) > 0 + assert rules == expected_rules + + def test_profiler_non_default_profile_options(spark, ws): inp_schema = T.StructType( [ From c101762a9c2fb829424afd6433be538e4bed8f79 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 8 Aug 2025 11:15:20 +0200 Subject: [PATCH 3/7] Update tests/integration/test_profiler.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/integration/test_profiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_profiler.py b/tests/integration/test_profiler.py index d9dc5cf06..b3535c038 100644 --- a/tests/integration/test_profiler.py +++ b/tests/integration/test_profiler.py @@ -197,7 +197,6 @@ def test_profiler_rounding_midnight_behavior(spark, ws): ), DQProfile(name="is_not_null", column="b1", description=None, parameters=None), ] - print(stats) assert len(stats.keys()) > 0 assert rules == expected_rules From 33b15c178018820bee5e32bdb104204d1f8d3f93 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 8 Aug 2025 11:16:04 +0200 Subject: [PATCH 4/7] Update tests/integration/test_profiler.py --- tests/integration/test_profiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_profiler.py b/tests/integration/test_profiler.py index b3535c038..45b11cf75 100644 --- a/tests/integration/test_profiler.py +++ b/tests/integration/test_profiler.py @@ -200,7 +200,6 @@ def test_profiler_rounding_midnight_behavior(spark, ws): assert len(stats.keys()) > 0 assert rules == expected_rules - def test_profiler_non_default_profile_options(spark, ws): inp_schema = T.StructType( [ From 5ee2ffde8502634fb4a50f7a1c2f22a4f61fd7cf Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 8 Aug 2025 11:16:10 +0200 Subject: [PATCH 5/7] Update tests/integration/test_profiler.py --- tests/integration/test_profiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_profiler.py b/tests/integration/test_profiler.py index 45b11cf75..2778d1256 100644 --- a/tests/integration/test_profiler.py +++ b/tests/integration/test_profiler.py @@ -199,7 +199,6 @@ def test_profiler_rounding_midnight_behavior(spark, ws): ] assert len(stats.keys()) > 0 assert rules == expected_rules - def test_profiler_non_default_profile_options(spark, ws): inp_schema = T.StructType( [ From 82aa8f78394a66d4a98e6c722acb5570fb147c5c Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 8 Aug 2025 11:16:40 +0200 Subject: [PATCH 6/7] Update tests/integration/test_profiler.py --- tests/integration/test_profiler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_profiler.py b/tests/integration/test_profiler.py index 2778d1256..c842dccc4 100644 --- a/tests/integration/test_profiler.py +++ b/tests/integration/test_profiler.py @@ -199,6 +199,8 @@ def test_profiler_rounding_midnight_behavior(spark, ws): ] assert len(stats.keys()) > 0 assert rules == expected_rules + + def test_profiler_non_default_profile_options(spark, ws): inp_schema = T.StructType( [ From c497073789f31aecb79c0ec7423251d905553f39 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Fri, 8 Aug 2025 11:20:11 +0200 Subject: [PATCH 7/7] Update src/databricks/labs/dqx/profiler/profiler.py --- src/databricks/labs/dqx/profiler/profiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/databricks/labs/dqx/profiler/profiler.py b/src/databricks/labs/dqx/profiler/profiler.py index 1cef8c63c..1cd604610 100644 --- a/src/databricks/labs/dqx/profiler/profiler.py +++ b/src/databricks/labs/dqx/profiler/profiler.py @@ -703,7 +703,6 @@ def _round_datetime(value: datetime.datetime, direction: str) -> datetime.dateti except OverflowError: logger.warning("Rounding datetime up caused overflow; returning datetime.max instead.") return datetime.datetime.max - raise ValueError(f"Invalid rounding direction: {direction}. Use 'up' or 'down'.") @staticmethod