From 6cc954fae53ceaa4392c6de70875dd6a8fe01711 Mon Sep 17 00:00:00 2001 From: ianzur Date: Thu, 22 Aug 2019 14:11:15 -0500 Subject: [PATCH 01/13] whatsnew --- doc/source/whatsnew/v0.25.1.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.25.1.rst b/doc/source/whatsnew/v0.25.1.rst index 63dd56f4a3793..86ae7b7169883 100644 --- a/doc/source/whatsnew/v0.25.1.rst +++ b/doc/source/whatsnew/v0.25.1.rst @@ -99,6 +99,7 @@ Reshaping - Bug in :meth:`DataFrame.crosstab` when ``margins`` set to ``True`` and ``normalize`` is not ``False``, an error is raised. (:issue:`27500`) - :meth:`DataFrame.join` now suppresses the ``FutureWarning`` when the sort parameter is specified (:issue:`21952`) - Bug in :meth:`DataFrame.join` raising with readonly arrays (:issue:`27943`) +- Bug :meth:`merge_asof` could not use datetime.timedelta for `tolerance` kwarg (:issue:`28098`) Sparse ^^^^^^ From 96d49f89efafc9fa0c1e39c8956c1e963d70addb Mon Sep 17 00:00:00 2001 From: ianzur Date: Fri, 30 Aug 2019 21:23:46 -0500 Subject: [PATCH 02/13] :bug::wrench: --- pandas/core/reshape/merge.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 225de3f11cf7d..54d8f10ef7037 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -7,6 +7,7 @@ import string import warnings +import datetime import numpy as np from pandas._libs import hashtable as libhashtable, lib @@ -1619,7 +1620,7 @@ def _get_merge_keys(self): ) raise MergeError(msg) - # validate tolerance; must be a Timedelta if we have a DTI + # validate tolerance; datetime.timedelta or Timedelta if we have a DTI if self.tolerance is not None: if self.left_index: @@ -1635,7 +1636,7 @@ def _get_merge_keys(self): ) if is_datetimelike(lt): - if not isinstance(self.tolerance, Timedelta): + if not isinstance(self.tolerance, datetime.timedelta): raise MergeError(msg) if self.tolerance < Timedelta(0): raise MergeError("tolerance must be positive") @@ -1687,6 +1688,7 @@ def flip(xs): # we require sortedness and non-null values in the join keys msg_sorted = "{side} keys must be sorted" msg_missings = "Merge keys contain null values on {side} side" + msg_tolerance = "tolerance of type {t} is unsupported" if not Index(left_values).is_monotonic: if isnull(left_values).any(): @@ -1705,7 +1707,13 @@ def flip(xs): left_values = left_values.view("i8") right_values = right_values.view("i8") if tolerance is not None: - tolerance = tolerance.value + try: + tolerance = tolerance.value + except AttributeError: + # datetime.timedelta(microseconds=1) == minimum resolution -> convert to nano + tolerance = (tolerance / datetime.timedelta(microseconds=1)) * 1000 + else: + raise ValueError(msg_tolerance.format(t=type(tolerance))) # a "by" parameter requires special handling if self.left_by is not None: From 4d983d12b34cf4c38eb61e81569dba663dec6967 Mon Sep 17 00:00:00 2001 From: ianzur Date: Fri, 30 Aug 2019 21:37:54 -0500 Subject: [PATCH 03/13] BUG: convert datetime.timedelta to nanoseconds to fix (#28098) --- pandas/core/reshape/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 54d8f10ef7037..fa9b684e8e383 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -1710,7 +1710,7 @@ def flip(xs): try: tolerance = tolerance.value except AttributeError: - # datetime.timedelta(microseconds=1) == minimum resolution -> convert to nano + # datetime.timedelta(microseconds=1) == min resolution, cvt to nano tolerance = (tolerance / datetime.timedelta(microseconds=1)) * 1000 else: raise ValueError(msg_tolerance.format(t=type(tolerance))) From 03afdf250a5ee3fb2117a252f5f495d2a60bd944 Mon Sep 17 00:00:00 2001 From: ianzur Date: Sat, 31 Aug 2019 08:35:55 -0500 Subject: [PATCH 04/13] BUG: mergeasof() now accepts datetime.timedelta as tolerance kwarg (#28098) --- pandas/core/reshape/merge.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index e00a4b3ca13e9..8d04f65203822 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -1712,8 +1712,6 @@ def flip(xs): except AttributeError: # datetime.timedelta(microseconds=1) == min resolution, cvt to nano tolerance = (tolerance / datetime.timedelta(microseconds=1)) * 1000 - else: - raise ValueError(msg_tolerance.format(t=type(tolerance))) # a "by" parameter requires special handling if self.left_by is not None: From 30596255eb03e917b179568c403772f6eaee86e0 Mon Sep 17 00:00:00 2001 From: ianzur Date: Sat, 31 Aug 2019 08:37:15 -0500 Subject: [PATCH 05/13] BUG: mergeasof() now accepts datetime.timedelta as tolerance kwarg (#28098) --- pandas/core/reshape/merge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 8d04f65203822..3e91008d555c9 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -1688,7 +1688,6 @@ def flip(xs): # we require sortedness and non-null values in the join keys msg_sorted = "{side} keys must be sorted" msg_missings = "Merge keys contain null values on {side} side" - msg_tolerance = "tolerance of type {t} is unsupported" if not Index(left_values).is_monotonic: if isnull(left_values).any(): From 35121e26f03336c145139347ae992ba2cd410a05 Mon Sep 17 00:00:00 2001 From: ianzur Date: Sat, 31 Aug 2019 08:46:47 -0500 Subject: [PATCH 06/13] BUG: mergeasof() now accepts datetime.timedelta as tolerance kwarg (#28098) --- pandas/tests/reshape/merge/test_merge_asof.py | 5 +---- test_fast.sh | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pandas/tests/reshape/merge/test_merge_asof.py b/pandas/tests/reshape/merge/test_merge_asof.py index 7412b1de643a1..f751fd5c5c030 100644 --- a/pandas/tests/reshape/merge/test_merge_asof.py +++ b/pandas/tests/reshape/merge/test_merge_asof.py @@ -594,10 +594,7 @@ def test_non_sorted(self): "tolerance", [ Timedelta("1day"), - pytest.param( - datetime.timedelta(days=1), - marks=pytest.mark.xfail(reason="not implemented", strict=True), - ), + datetime.timedelta(days=1), ], ids=["pd.Timedelta", "datetime.timedelta"], ) diff --git a/test_fast.sh b/test_fast.sh index 0a47f9de600ea..a6205b5308b7d 100755 --- a/test_fast.sh +++ b/test_fast.sh @@ -5,4 +5,4 @@ # https://github.com/pytest-dev/pytest/issues/1075 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') -pytest pandas --skip-slow --skip-network --skip-db -m "not single" -n 4 -r sxX --strict "$@" +pytest pandas --skip-slow --skip-network --skip-db -m "not single" -n 16 -r sxX --strict "$@" From 691edeaaac9ca4a3e9004c1e1e06fd79de131bf8 Mon Sep 17 00:00:00 2001 From: ianzur Date: Sat, 31 Aug 2019 08:47:37 -0500 Subject: [PATCH 07/13] BUG: mergeasof() now accepts datetime.timedelta as tolerance kwarg (#28098) --- pandas/tests/reshape/merge/test_merge_asof.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pandas/tests/reshape/merge/test_merge_asof.py b/pandas/tests/reshape/merge/test_merge_asof.py index f751fd5c5c030..caf2539a9e150 100644 --- a/pandas/tests/reshape/merge/test_merge_asof.py +++ b/pandas/tests/reshape/merge/test_merge_asof.py @@ -592,10 +592,7 @@ def test_non_sorted(self): @pytest.mark.parametrize( "tolerance", - [ - Timedelta("1day"), - datetime.timedelta(days=1), - ], + [Timedelta("1day"), datetime.timedelta(days=1)], ids=["pd.Timedelta", "datetime.timedelta"], ) def test_tolerance(self, tolerance): From 457f0a352e32709714ed7820f8b186f1f32658c4 Mon Sep 17 00:00:00 2001 From: ianzur Date: Sat, 31 Aug 2019 08:50:37 -0500 Subject: [PATCH 08/13] don't change this file --- test_fast.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_fast.sh b/test_fast.sh index a6205b5308b7d..0a47f9de600ea 100755 --- a/test_fast.sh +++ b/test_fast.sh @@ -5,4 +5,4 @@ # https://github.com/pytest-dev/pytest/issues/1075 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') -pytest pandas --skip-slow --skip-network --skip-db -m "not single" -n 16 -r sxX --strict "$@" +pytest pandas --skip-slow --skip-network --skip-db -m "not single" -n 4 -r sxX --strict "$@" From 6ab8140391d1b567c624b705b101b1ee20d69d85 Mon Sep 17 00:00:00 2001 From: ianzur Date: Sat, 31 Aug 2019 09:40:58 -0500 Subject: [PATCH 09/13] CLN: sort imports --- pandas/core/reshape/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 3e91008d555c9..c1e8294b7b218 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -3,11 +3,11 @@ """ import copy +import datetime from functools import partial import string import warnings -import datetime import numpy as np from pandas._libs import hashtable as libhashtable, lib From 1dcaa6360d8703552e8c19e616e434786a032281 Mon Sep 17 00:00:00 2001 From: ianzur Date: Sat, 31 Aug 2019 10:07:33 -0500 Subject: [PATCH 10/13] BUG: (#28098) changes --- pandas/core/reshape/merge.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index c1e8294b7b218..8638abba308c3 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -39,7 +39,15 @@ ) from pandas.core.dtypes.missing import isnull, na_value_for_dtype -from pandas import Categorical, DataFrame, Index, MultiIndex, Series, Timedelta +from pandas import ( + Categorical, + DataFrame, + Index, + MultiIndex, + Series, + Timedelta, + to_timedelta, +) import pandas.core.algorithms as algos from pandas.core.arrays.categorical import _recode_for_categories import pandas.core.common as com @@ -1706,11 +1714,9 @@ def flip(xs): left_values = left_values.view("i8") right_values = right_values.view("i8") if tolerance is not None: - try: - tolerance = tolerance.value - except AttributeError: - # datetime.timedelta(microseconds=1) == min resolution, cvt to nano - tolerance = (tolerance / datetime.timedelta(microseconds=1)) * 1000 + if not isinstance(self.tolerance, Timedelta): + tolerance = to_timedelta(tolerance) + tolerance = tolerance.value # a "by" parameter requires special handling if self.left_by is not None: From b07be3cf8a75b605047b9222b8f3f949ee45a5de Mon Sep 17 00:00:00 2001 From: ianzur Date: Sat, 31 Aug 2019 10:13:14 -0500 Subject: [PATCH 11/13] BUG (#28098) some more changes --- pandas/core/reshape/merge.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 8638abba308c3..8c4e409cb0545 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -39,15 +39,7 @@ ) from pandas.core.dtypes.missing import isnull, na_value_for_dtype -from pandas import ( - Categorical, - DataFrame, - Index, - MultiIndex, - Series, - Timedelta, - to_timedelta, -) +from pandas import Categorical, DataFrame, Index, MultiIndex, Series, Timedelta import pandas.core.algorithms as algos from pandas.core.arrays.categorical import _recode_for_categories import pandas.core.common as com @@ -1715,7 +1707,7 @@ def flip(xs): right_values = right_values.view("i8") if tolerance is not None: if not isinstance(self.tolerance, Timedelta): - tolerance = to_timedelta(tolerance) + tolerance = Timedelta(tolerance) tolerance = tolerance.value # a "by" parameter requires special handling From 877239ca8780beafe03ff8d0a8ca1e11b63a89a2 Mon Sep 17 00:00:00 2001 From: ianzur Date: Tue, 24 Sep 2019 09:32:46 -0500 Subject: [PATCH 12/13] update whatsnew --- doc/source/whatsnew/v0.25.1.rst | 1 - doc/source/whatsnew/v1.0.0.rst | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.1.rst b/doc/source/whatsnew/v0.25.1.rst index 86ae7b7169883..63dd56f4a3793 100644 --- a/doc/source/whatsnew/v0.25.1.rst +++ b/doc/source/whatsnew/v0.25.1.rst @@ -99,7 +99,6 @@ Reshaping - Bug in :meth:`DataFrame.crosstab` when ``margins`` set to ``True`` and ``normalize`` is not ``False``, an error is raised. (:issue:`27500`) - :meth:`DataFrame.join` now suppresses the ``FutureWarning`` when the sort parameter is specified (:issue:`21952`) - Bug in :meth:`DataFrame.join` raising with readonly arrays (:issue:`27943`) -- Bug :meth:`merge_asof` could not use datetime.timedelta for `tolerance` kwarg (:issue:`28098`) Sparse ^^^^^^ diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 7ca93d7d75854..a3d75d69e1e82 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -279,6 +279,7 @@ Reshaping - Bug in :meth:`DataFrame.apply` that caused incorrect output with empty :class:`DataFrame` (:issue:`28202`, :issue:`21959`) - Bug in :meth:`DataFrame.stack` not handling non-unique indexes correctly when creating MultiIndex (:issue: `28301`) +- Bug :func:`merge_asof` could not use :class:`datetime.timedelta` for ``tolerance`` kwarg (:issue:`28098`) Sparse ^^^^^^ From 8de8b081a77a484cd0e4e73dd88abe9de99bfd75 Mon Sep 17 00:00:00 2001 From: ianzur Date: Tue, 24 Sep 2019 09:37:12 -0500 Subject: [PATCH 13/13] BUG: pass timedelta and datetime.timedelta to timedelta --- pandas/core/reshape/merge.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 8c4e409cb0545..62a30073a53fd 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -1706,8 +1706,7 @@ def flip(xs): left_values = left_values.view("i8") right_values = right_values.view("i8") if tolerance is not None: - if not isinstance(self.tolerance, Timedelta): - tolerance = Timedelta(tolerance) + tolerance = Timedelta(tolerance) tolerance = tolerance.value # a "by" parameter requires special handling