Skip to content

Add Decimal support to min_max generator (#1013) #1017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mwojtyczka merged 14 commits into from
Feb 8, 2026
Merged

Add Decimal support to min_max generator (#1013) #1017

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
63840aa
Add Decimal support to min_max generator
Jgprog117 Jan 31, 2026
3463997
Trigger CI checks
Jgprog117 Jan 31, 2026
0d63ce8
Fix Decimal unit tests and add integration test for min_max generator…
Jgprog117 Feb 1, 2026
66a05a8
Add support for Decimal to checks and change to single-line function …
Jgprog117 Feb 4, 2026
e6d7556
Merge branch 'main' into add-decimal-support-min-max-1013
mwojtyczka Feb 7, 2026
ab9129a
Merge branch 'main' into add-decimal-support-min-max-1013
mwojtyczka Feb 8, 2026
2f996ac
added tests to cover Decimal handling, updated dlt tests, added suppo…
mwojtyczka Feb 8, 2026
2fe6e8f
updated tests and docstrings
mwojtyczka Feb 8, 2026
2ac1bdd
added serialization/deserialization of decimal values
mwojtyczka Feb 8, 2026
a52ea86
added serialization/deserialization of decimal values
mwojtyczka Feb 8, 2026
5928641
refactor
mwojtyczka Feb 8, 2026
720bb73
refactor
mwojtyczka Feb 8, 2026
b0f2e98
updated docstring
mwojtyczka Feb 8, 2026
0bd2966
updated docs
mwojtyczka Feb 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 20 additions & 22 deletions src/databricks/labs/dqx/check_funcs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import warnings
import ipaddress
import uuid
from decimal import Decimal
from collections.abc import Callable, Sequence
from enum import Enum
from itertools import zip_longest
Expand Down Expand Up @@ -609,16 +610,15 @@ def is_not_in_near_future(column: str | Column, offset: int = 0, curr_timestamp:
@register_rule("row")
def is_equal_to(
column: str | Column,
value: int | float | str | datetime.date | datetime.datetime | Column | None = None,
value: int | float | Decimal | str | datetime.date | datetime.datetime | Column | None = None,
abs_tolerance: float | None = None,
rel_tolerance: float | None = None,
) -> Column:
Comment thread
mwojtyczka marked this conversation as resolved.
Comment thread
mwojtyczka marked this conversation as resolved.
"""Check whether the values in the input column are equal to the given value.

Args:
column (str | Column): Column to check. Can be a string column name or a column expression.
value (int | float | str | datetime.date | datetime.datetime | Column | None, optional):
The value to compare with. Can be a literal or a Spark Column. Defaults to None.
value: The value to compare with. Can be a number, date, timestamp literal or a Spark Column. Defaults to None.
abs_tolerance: Values are considered equal if the absolute difference is less than or equal to the tolerance. This is applicable to numeric columns.
Example: abs(a - b) <= tolerance
With tolerance=0.01:
Expand Down Expand Up @@ -647,7 +647,7 @@ def is_equal_to(
col_str_norm, col_expr_str, col_expr = get_normalized_column_and_expr(column)
value_expr = get_limit_expr(value)

if (abs_tolerance > 0.0 or rel_tolerance > 0.0) and isinstance(value, (int, float)):
if (abs_tolerance > 0.0 or rel_tolerance > 0.0) and isinstance(value, (int, float, Decimal)):
# Use tolerance-based comparison for numeric columns
tolerance_match = _match_values_with_tolerance(col_expr, value_expr, abs_tolerance, rel_tolerance)
condition = ~tolerance_match
Expand All @@ -671,16 +671,15 @@ def is_equal_to(
@register_rule("row")
def is_not_equal_to(
column: str | Column,
value: int | float | str | datetime.date | datetime.datetime | Column | None = None,
value: int | float | Decimal | str | datetime.date | datetime.datetime | Column | None = None,
abs_tolerance: float | None = None,
rel_tolerance: float | None = None,
) -> Column:
"""Check whether the values in the input column are not equal to the given value.

Args:
column (str | Column): Column to check. Can be a string column name or a column expression.
value (int | float | str | datetime.date | datetime.datetime | Column | None, optional):
The value to compare with. Can be a literal or a Spark Column. Defaults to None.
value: The value to compare with. Can be a number, date, timestamp literal or a Spark Column. Defaults to None.
abs_tolerance: Values are considered equal if the absolute difference is less than or equal to the tolerance. This is applicable to numeric columns.
Example: abs(a - b) <= tolerance
With tolerance=0.01:
Expand Down Expand Up @@ -710,7 +709,7 @@ def is_not_equal_to(
col_str_norm, col_expr_str, col_expr = get_normalized_column_and_expr(column)
value_expr = get_limit_expr(value)

if (abs_tolerance > 0.0 or rel_tolerance > 0.0) and isinstance(value, (int, float)):
if (abs_tolerance > 0.0 or rel_tolerance > 0.0) and isinstance(value, (int, float, Decimal)):
# Use tolerance-based comparison for numeric columns
tolerance_match = _match_values_with_tolerance(col_expr, value_expr, abs_tolerance, rel_tolerance)
condition = tolerance_match
Expand All @@ -733,7 +732,7 @@ def is_not_equal_to(

@register_rule("row")
def is_not_less_than(
column: str | Column, limit: int | float | datetime.date | datetime.datetime | str | Column | None = None
column: str | Column, limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None
) -> Column:
"""Checks whether the values in the input column are not less than the provided limit.

Expand Down Expand Up @@ -763,7 +762,7 @@ def is_not_less_than(

@register_rule("row")
def is_not_greater_than(
column: str | Column, limit: int | float | datetime.date | datetime.datetime | str | Column | None = None
column: str | Column, limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None
) -> Column:
"""Checks whether the values in the input column are not greater than the provided limit.

Expand Down Expand Up @@ -794,8 +793,8 @@ def is_not_greater_than(
@register_rule("row")
def is_in_range(
column: str | Column,
min_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None,
max_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None,
min_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None,
max_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None,
) -> Column:
"""Checks whether the values in the input column are in the provided limits (inclusive of both boundaries).

Expand Down Expand Up @@ -832,8 +831,8 @@ def is_in_range(
@register_rule("row")
def is_not_in_range(
column: str | Column,
min_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None,
max_limit: int | float | datetime.date | datetime.datetime | str | Column | None = None,
min_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None,
max_limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None,
) -> Column:
"""Checks whether the values in the input column are outside the provided limits (inclusive of both boundaries).

Expand Down Expand Up @@ -1582,7 +1581,7 @@ def apply(df: DataFrame, spark: SparkSession, ref_dfs: dict[str, DataFrame]) ->
@register_rule("dataset")
def is_aggr_not_greater_than(
column: str | Column,
limit: int | float | str | Column,
limit: int | float | Decimal | str | Column,
aggr_type: str = "count",
group_by: list[str | Column] | None = None,
row_filter: str | None = None,
Expand Down Expand Up @@ -1627,7 +1626,7 @@ def is_aggr_not_greater_than(
@register_rule("dataset")
def is_aggr_not_less_than(
column: str | Column,
limit: int | float | str | Column,
limit: int | float | Decimal | str | Column,
aggr_type: str = "count",
group_by: list[str | Column] | None = None,
row_filter: str | None = None,
Expand Down Expand Up @@ -1672,7 +1671,7 @@ def is_aggr_not_less_than(
@register_rule("dataset")
def is_aggr_equal(
column: str | Column,
limit: int | float | str | Column,
limit: int | float | Decimal | str | Column,
aggr_type: str = "count",
group_by: list[str | Column] | None = None,
row_filter: str | None = None,
Expand Down Expand Up @@ -1723,7 +1722,7 @@ def is_aggr_equal(
@register_rule("dataset")
def is_aggr_not_equal(
column: str | Column,
limit: int | float | str | Column,
limit: int | float | Decimal | str | Column,
aggr_type: str = "count",
group_by: list[str | Column] | None = None,
row_filter: str | None = None,
Expand Down Expand Up @@ -2776,7 +2775,6 @@ def _add_column_diffs(
"""
columns_changed = []
if compare_columns:

for col_name in compare_columns:
is_numeric = isinstance(df.schema[col_name].dataType, types.NumericType)

Expand Down Expand Up @@ -2987,7 +2985,7 @@ def _build_aggregate_check_metadata(

def _is_aggr_compare(
column: str | Column,
limit: int | float | str | Column,
limit: int | float | Decimal | str | Column,
aggr_type: str,
aggr_params: dict[str, Any] | None,
group_by: list[str | Column] | None,
Expand Down Expand Up @@ -3214,7 +3212,7 @@ def _cleanup_alias_name(column: str) -> str:


def get_limit_expr(
limit: int | float | datetime.date | datetime.datetime | str | Column | None = None,
limit: int | float | Decimal | datetime.date | datetime.datetime | str | Column | None = None,
) -> Column:
"""
Generate a Spark Column expression for a limit value.
Expand Down Expand Up @@ -3243,7 +3241,7 @@ def get_limit_expr(
parsed_dt = datetime.datetime.fromisoformat(limit)

# Check if the string contains time component
has_time = ':' in limit
has_time = ":" in limit

if has_time:
return F.to_timestamp(F.lit(parsed_dt))
Expand Down
Loading
Loading