From fcb17b0c331048068f3a2860227a19958a39d1ef Mon Sep 17 00:00:00 2001 From: Stefan Scherfke Date: Mon, 24 Apr 2017 15:46:23 +0200 Subject: [PATCH 1/5] Fix issue #15683 --- doc/source/whatsnew/v0.20.0.txt | 1 + pandas/_libs/tslib.pyx | 18 +++++++++--------- pandas/tests/tseries/test_timezones.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/doc/source/whatsnew/v0.20.0.txt b/doc/source/whatsnew/v0.20.0.txt index 945922b5f9ba8..9f748f64dc0c5 100644 --- a/doc/source/whatsnew/v0.20.0.txt +++ b/doc/source/whatsnew/v0.20.0.txt @@ -22,6 +22,7 @@ Highlights include: - Support for S3 handling now uses ``s3fs``, see :ref:`here ` - Google BigQuery support now uses the ``pandas-gbq`` library, see :ref:`here ` - Switched the test framework to use `pytest `__ (:issue:`13097`) +- Fixed an issue with ``Timestamp.replace(tzinfo)`` around DST changes (:issue:`15683`) .. warning:: diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index c471d46262484..c418059eceda5 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -685,14 +685,16 @@ class Timestamp(_Timestamp): cdef: pandas_datetimestruct dts int64_t value - object _tzinfo, result, k, v + object _tzinfo, result, k, v, ts_input _TSObject ts # set to naive if needed _tzinfo = self.tzinfo value = self.value if _tzinfo is not None: - value = tz_convert_single(value, 'UTC', _tzinfo) + value_tz = tz_convert_single(value, _tzinfo, 'UTC') + offset = value - value_tz + value += offset # setup components pandas_datetime_to_datetimestruct(value, PANDAS_FR_ns, &dts) @@ -726,16 +728,14 @@ class Timestamp(_Timestamp): _tzinfo = tzinfo # reconstruct & check bounds - value = pandas_datetimestruct_to_datetime(PANDAS_FR_ns, &dts) + ts_input = datetime(dts.year, dts.month, dts.day, dts.hour, dts.min, + dts.sec, dts.us, tzinfo=_tzinfo) + ts = convert_to_tsobject(ts_input, _tzinfo, None, 0, 0) + value = ts.value + (dts.ps // 1000) if value != NPY_NAT: _check_dts_bounds(&dts) - # set tz if needed - if _tzinfo is not None: - value = tz_convert_single(value, _tzinfo, 'UTC') - - result = create_timestamp_from_ts(value, dts, _tzinfo, self.freq) - return result + return create_timestamp_from_ts(value, dts, _tzinfo, self.freq) def isoformat(self, sep='T'): base = super(_Timestamp, self).isoformat(sep=sep) diff --git a/pandas/tests/tseries/test_timezones.py b/pandas/tests/tseries/test_timezones.py index e3f2c242e3294..c684a5e98b049 100644 --- a/pandas/tests/tseries/test_timezones.py +++ b/pandas/tests/tseries/test_timezones.py @@ -1280,6 +1280,25 @@ def test_ambiguous_compat(self): self.assertEqual(result_pytz.to_pydatetime().tzname(), result_dateutil.to_pydatetime().tzname()) + def test_tzreplace_issue_15683(self): + """Regression test for issue 15683.""" + dt = datetime(2016, 3, 27, 1) + tzinfo = pytz.timezone('CET').localize(dt, is_dst=False).tzinfo + + result_dt = dt.replace(tzinfo=tzinfo) + result_pd = Timestamp(dt).replace(tzinfo=tzinfo) + + self.assertEqual(result_dt.timestamp(), result_pd.timestamp()) + self.assertEqual(result_dt, result_pd) + self.assertEqual(result_dt, result_pd.to_pydatetime()) + + result_dt = dt.replace(tzinfo=tzinfo).replace(tzinfo=None) + result_pd = Timestamp(dt).replace(tzinfo=tzinfo).replace(tzinfo=None) + + self.assertEqual(result_dt.timestamp(), result_pd.timestamp()) + self.assertEqual(result_dt, result_pd) + self.assertEqual(result_dt, result_pd.to_pydatetime()) + def test_index_equals_with_tz(self): left = date_range('1/1/2011', periods=100, freq='H', tz='utc') right = date_range('1/1/2011', periods=100, freq='H', tz='US/Eastern') From b7c2e86e8abcd8715be606a8998b6bde2a1405d0 Mon Sep 17 00:00:00 2001 From: Stefan Scherfke Date: Mon, 24 Apr 2017 17:10:39 +0200 Subject: [PATCH 2/5] Adjustments to coding guidelines --- doc/source/whatsnew/v0.20.0.txt | 2 +- pandas/tests/tseries/test_timezones.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/source/whatsnew/v0.20.0.txt b/doc/source/whatsnew/v0.20.0.txt index 9f748f64dc0c5..87ebe1c2d70a1 100644 --- a/doc/source/whatsnew/v0.20.0.txt +++ b/doc/source/whatsnew/v0.20.0.txt @@ -22,7 +22,6 @@ Highlights include: - Support for S3 handling now uses ``s3fs``, see :ref:`here ` - Google BigQuery support now uses the ``pandas-gbq`` library, see :ref:`here ` - Switched the test framework to use `pytest `__ (:issue:`13097`) -- Fixed an issue with ``Timestamp.replace(tzinfo)`` around DST changes (:issue:`15683`) .. warning:: @@ -1577,6 +1576,7 @@ Conversion - Bug in ``Timestamp.replace`` now raises ``TypeError`` when incorrect argument names are given; previously this raised ``ValueError`` (:issue:`15240`) - Bug in ``Timestamp.replace`` with compat for passing long integers (:issue:`15030`) +- Bug in ``Timestamp.replace`` when replacing ``tzinfo`` around DST changes (:issue:`15683`) - Bug in ``Timestamp`` returning UTC based time/date attributes when a timezone was provided (:issue:`13303`) - Bug in ``Timestamp`` incorrectly localizing timezones during construction (:issue:`11481`, :issue:`15777`) - Bug in ``TimedeltaIndex`` addition where overflow was being allowed without error (:issue:`14816`) diff --git a/pandas/tests/tseries/test_timezones.py b/pandas/tests/tseries/test_timezones.py index c684a5e98b049..dfb6902bc7c81 100644 --- a/pandas/tests/tseries/test_timezones.py +++ b/pandas/tests/tseries/test_timezones.py @@ -1280,24 +1280,24 @@ def test_ambiguous_compat(self): self.assertEqual(result_pytz.to_pydatetime().tzname(), result_dateutil.to_pydatetime().tzname()) - def test_tzreplace_issue_15683(self): - """Regression test for issue 15683.""" + def test_replace_tzinfo(self): + # GH 15683 dt = datetime(2016, 3, 27, 1) tzinfo = pytz.timezone('CET').localize(dt, is_dst=False).tzinfo result_dt = dt.replace(tzinfo=tzinfo) result_pd = Timestamp(dt).replace(tzinfo=tzinfo) - self.assertEqual(result_dt.timestamp(), result_pd.timestamp()) - self.assertEqual(result_dt, result_pd) - self.assertEqual(result_dt, result_pd.to_pydatetime()) + assert result_dt.timestamp() == result_pd.timestamp() + assert result_dt == result_pd + assert result_dt == result_pd.to_pydatetime() result_dt = dt.replace(tzinfo=tzinfo).replace(tzinfo=None) result_pd = Timestamp(dt).replace(tzinfo=tzinfo).replace(tzinfo=None) - self.assertEqual(result_dt.timestamp(), result_pd.timestamp()) - self.assertEqual(result_dt, result_pd) - self.assertEqual(result_dt, result_pd.to_pydatetime()) + assert result_dt.timestamp() == result_pd.timestamp() + assert result_dt == result_pd + assert result_dt == result_pd.to_pydatetime() def test_index_equals_with_tz(self): left = date_range('1/1/2011', periods=100, freq='H', tz='utc') From 5e9bfd2c0d92043c3ad224ea98ee41d94b6066fb Mon Sep 17 00:00:00 2001 From: Stefan Scherfke Date: Wed, 26 Apr 2017 09:32:03 +0200 Subject: [PATCH 3/5] Only perform timestamp() check if meth is available. --- pandas/tests/tseries/test_timezones.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/tseries/test_timezones.py b/pandas/tests/tseries/test_timezones.py index dfb6902bc7c81..ad3b54cf24f8c 100644 --- a/pandas/tests/tseries/test_timezones.py +++ b/pandas/tests/tseries/test_timezones.py @@ -1288,14 +1288,16 @@ def test_replace_tzinfo(self): result_dt = dt.replace(tzinfo=tzinfo) result_pd = Timestamp(dt).replace(tzinfo=tzinfo) - assert result_dt.timestamp() == result_pd.timestamp() + if hasattr(result_dt, 'timestamp'): # New method in Py 3.3 + assert result_dt.timestamp() == result_pd.timestamp() assert result_dt == result_pd assert result_dt == result_pd.to_pydatetime() result_dt = dt.replace(tzinfo=tzinfo).replace(tzinfo=None) result_pd = Timestamp(dt).replace(tzinfo=tzinfo).replace(tzinfo=None) - assert result_dt.timestamp() == result_pd.timestamp() + if hasattr(result_dt, 'timestamp'): # New method in Py 3.3 + assert result_dt.timestamp() == result_pd.timestamp() assert result_dt == result_pd assert result_dt == result_pd.to_pydatetime() From 5b912c231bff1558617c5e4b9550b4f973cbeda8 Mon Sep 17 00:00:00 2001 From: Stefan Scherfke Date: Mon, 21 Aug 2017 14:12:38 +0200 Subject: [PATCH 4/5] Add some typing information --- pandas/_libs/tslib.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 3da242032c0ea..6858d664fa0e1 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -703,7 +703,7 @@ class Timestamp(_Timestamp): cdef: pandas_datetimestruct dts - int64_t value + int64_t value, value_tz, offset object _tzinfo, result, k, v, ts_input _TSObject ts @@ -4216,7 +4216,7 @@ def tz_convert(ndarray[int64_t] vals, object tz1, object tz2): return result -def tz_convert_single(int64_t val, object tz1, object tz2): +cpdef int64_t tz_convert_single(int64_t val, object tz1, object tz2): """ Convert the val (in i8) from timezone1 to timezone2 From ee30db80ae46981d73820a776e053a9b85bd5511 Mon Sep 17 00:00:00 2001 From: Stefan Scherfke Date: Mon, 21 Aug 2017 14:12:48 +0200 Subject: [PATCH 5/5] Update whatsnew entry --- doc/source/whatsnew/v0.21.0.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index c5fe89282bf52..a1c9a0072931b 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -319,6 +319,7 @@ Conversion - Fix :func:`DataFrame.memory_usage` to support PyPy. Objects on PyPy do not have a fixed size, so an approximation is used instead (:issue:`17228`) - Fixed the return type of ``IntervalIndex.is_non_overlapping_monotonic`` to be a Python ``bool`` for consistency with similar attributes/methods. Previously returned a ``numpy.bool_``. (:issue:`17237`) - Bug in ``IntervalIndex.is_non_overlapping_monotonic`` when intervals are closed on both sides and overlap at a point (:issue:`16560`) +- Bug in ``Timestamp.replace`` when replacing ``tzinfo`` around DST changes (:issue:`15683`) Indexing