From 2e5a160ba20168deee7599d7d9815a889c63719f Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 23 Mar 2019 17:57:43 -0400 Subject: [PATCH 01/25] BUG: Fix Timestamp type checks to work with subclassed datetime (#25851) --- doc/source/whatsnew/v0.25.0.rst | 2 +- pandas/_libs/tslib.pyx | 7 +++--- pandas/_libs/tslibs/__init__.py | 2 +- pandas/_libs/tslibs/conversion.pyx | 13 ++++------- pandas/_libs/tslibs/timedeltas.pyx | 9 ++++---- pandas/_libs/tslibs/timestamps.pxd | 16 +++++++++++++ pandas/_libs/tslibs/timestamps.pyx | 9 ++------ pandas/tests/arithmetic/test_datetime64.py | 11 +++++++++ .../tests/scalar/timestamp/test_timestamp.py | 9 ++++++++ pandas/tests/tslibs/test_array_to_datetime.py | 23 +++++++++++++++++++ pandas/tests/tslibs/test_conversion.py | 23 ++++++++++++++++++- pandas/tests/tslibs/test_normalize_date.py | 19 +++++++++++++++ 12 files changed, 117 insertions(+), 26 deletions(-) diff --git a/doc/source/whatsnew/v0.25.0.rst b/doc/source/whatsnew/v0.25.0.rst index ccf5c43280765..e50bee82085ba 100644 --- a/doc/source/whatsnew/v0.25.0.rst +++ b/doc/source/whatsnew/v0.25.0.rst @@ -325,7 +325,7 @@ Sparse Other ^^^^^ -- +- Improved :class:`Timestamp` type checking in various datetime functions to prevent exceptions when using a subclassed `datetime` (:issue:`25851`) - - diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 44ea875f0b49d..f31ecaa4ffffa 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -2,7 +2,6 @@ import cython from cpython.datetime cimport (PyDateTime_Check, PyDate_Check, - PyDateTime_CheckExact, PyDateTime_IMPORT, timedelta, datetime, date, time) # import datetime C API @@ -41,7 +40,8 @@ from pandas._libs.tslibs.nattype cimport ( from pandas._libs.tslibs.offsets cimport to_offset -from pandas._libs.tslibs.timestamps cimport create_timestamp_from_ts +from pandas._libs.tslibs.timestamps cimport ( + create_timestamp_from_ts, _Timestamp) from pandas._libs.tslibs.timestamps import Timestamp @@ -539,8 +539,7 @@ cpdef array_to_datetime(ndarray[object] values, str errors='raise', 'datetime64 unless utc=True') else: iresult[i] = pydatetime_to_dt64(val, &dts) - if not PyDateTime_CheckExact(val): - # i.e. a Timestamp object + if isinstance(val, _Timestamp): iresult[i] += val.nanosecond check_dts_bounds(&dts) diff --git a/pandas/_libs/tslibs/__init__.py b/pandas/_libs/tslibs/__init__.py index a21fdf95559e6..647a46ac8fbf0 100644 --- a/pandas/_libs/tslibs/__init__.py +++ b/pandas/_libs/tslibs/__init__.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # flake8: noqa +from .timestamps import Timestamp # isort:skip from .conversion import normalize_date, localize_pydatetime, tz_convert_single from .nattype import NaT, NaTType, iNaT, is_null_datetimelike from .np_datetime import OutOfBoundsDatetime from .period import Period, IncompatibleFrequency -from .timestamps import Timestamp from .timedeltas import delta_to_nanoseconds, ints_to_pytimedelta, Timedelta diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index d8c3b91d1e460..4a7ac912f6e46 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -13,8 +13,7 @@ from dateutil.tz import tzutc from datetime import time as datetime_time from cpython.datetime cimport (datetime, tzinfo, PyDateTime_Check, PyDate_Check, - PyDateTime_CheckExact, PyDateTime_IMPORT, - PyDelta_Check) + PyDateTime_IMPORT, PyDelta_Check) PyDateTime_IMPORT from pandas._libs.tslibs.ccalendar import DAY_SECONDS, HOUR_SECONDS @@ -31,6 +30,7 @@ from pandas._libs.tslibs.util cimport ( from pandas._libs.tslibs.timedeltas cimport (cast_from_unit, delta_to_nanoseconds) +from pandas._libs.tslibs.timestamps cimport _Timestamp from pandas._libs.tslibs.timezones cimport ( is_utc, is_tzlocal, is_fixed_offset, get_utcoffset, get_dst_info, get_timezone, maybe_get_tz, tz_compare) @@ -379,8 +379,7 @@ cdef _TSObject convert_datetime_to_tsobject(datetime ts, object tz, offset = get_utcoffset(obj.tzinfo, ts) obj.value -= int(offset.total_seconds() * 1e9) - if not PyDateTime_CheckExact(ts): - # datetime instance but not datetime type --> Timestamp + if isinstance(ts, _Timestamp): obj.value += ts.nanosecond obj.dts.ps = ts.nanosecond * 1000 @@ -607,8 +606,7 @@ cpdef inline datetime localize_pydatetime(datetime dt, object tz): """ if tz is None: return dt - elif not PyDateTime_CheckExact(dt): - # i.e. is a Timestamp + elif isinstance(dt, _Timestamp): return dt.tz_localize(tz) elif is_utc(tz): return _localize_pydatetime(dt, tz) @@ -1155,8 +1153,7 @@ def normalize_date(dt: object) -> datetime: TypeError : if input is not datetime.date, datetime.datetime, or Timestamp """ if PyDateTime_Check(dt): - if not PyDateTime_CheckExact(dt): - # i.e. a Timestamp object + if isinstance(dt, _Timestamp): return dt.replace(hour=0, minute=0, second=0, microsecond=0, nanosecond=0) else: diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 77f5001443ae0..ae89626eb3a55 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -16,7 +16,6 @@ from numpy cimport int64_t cnp.import_array() from cpython.datetime cimport (datetime, timedelta, - PyDateTime_CheckExact, PyDateTime_Check, PyDelta_Check, PyDateTime_IMPORT) PyDateTime_IMPORT @@ -37,6 +36,7 @@ from pandas._libs.tslibs.nattype cimport ( checknull_with_nat, NPY_NAT, c_NaT as NaT) from pandas._libs.tslibs.offsets cimport to_offset from pandas._libs.tslibs.offsets import _Tick as Tick +from pandas._libs.tslibs.timestamps cimport _Timestamp # ---------------------------------------------------------------------- # Constants @@ -583,9 +583,10 @@ def _binary_op_method_timedeltalike(op, name): # has-dtype check before then pass - elif is_datetime64_object(other) or PyDateTime_CheckExact(other): - # the PyDateTime_CheckExact case is for a datetime object that - # is specifically *not* a Timestamp, as the Timestamp case will be + elif is_datetime64_object(other) or ( + PyDateTime_Check(other) and not isinstance(other, _Timestamp)): + # this case is for a datetime object that is specifically + # *not* a Timestamp, as the Timestamp case will be # handled after `_validate_ops_compat` returns False below from pandas._libs.tslibs.timestamps import Timestamp return op(self, Timestamp(other)) diff --git a/pandas/_libs/tslibs/timestamps.pxd b/pandas/_libs/tslibs/timestamps.pxd index b7282e02ff117..d19466c83dfff 100644 --- a/pandas/_libs/tslibs/timestamps.pxd +++ b/pandas/_libs/tslibs/timestamps.pxd @@ -1,8 +1,24 @@ # -*- coding: utf-8 -*- +from cpython.datetime cimport datetime + from numpy cimport int64_t from pandas._libs.tslibs.np_datetime cimport npy_datetimestruct cdef object create_timestamp_from_ts(int64_t value, npy_datetimestruct dts, object tz, object freq) + +cdef class _Timestamp(datetime): + cdef readonly: + int64_t value, nanosecond + object freq + list _date_attributes + cpdef bint _get_start_end_field(self, str field) + cpdef _get_date_name_field(self, object field, object locale) + cdef int64_t _maybe_convert_value_to_local(self) + cpdef to_datetime64(self) + cdef _assert_tzawareness_compat(_Timestamp self, datetime other) + cpdef datetime to_pydatetime(_Timestamp self, bint warn=*) + cdef bint _compare_outside_nanorange(_Timestamp self, datetime other, + int op) except -1 diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index c4d47a3c2384a..0d1754b7cc510 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -22,8 +22,6 @@ from pandas._libs.tslibs.util cimport ( cimport pandas._libs.tslibs.ccalendar as ccalendar from pandas._libs.tslibs.ccalendar import DAY_SECONDS -from pandas._libs.tslibs.conversion import ( - tz_localize_to_utc, normalize_i8_timestamps) from pandas._libs.tslibs.conversion cimport ( tz_convert_single, _TSObject, convert_to_tsobject, convert_datetime_to_tsobject) @@ -203,11 +201,6 @@ def round_nsint64(values, mode, freq): # shadows the python class, where we do any heavy lifting. cdef class _Timestamp(datetime): - cdef readonly: - int64_t value, nanosecond - object freq # frequency reference - list _date_attributes - def __hash__(_Timestamp self): if self.nanosecond: return hash(self.value) @@ -1215,6 +1208,7 @@ class Timestamp(_Timestamp): tz = maybe_get_tz(tz) if not is_string_object(ambiguous): ambiguous = [ambiguous] + from pandas._libs.tslibs.conversion import tz_localize_to_utc value = tz_localize_to_utc(np.array([self.value], dtype='i8'), tz, ambiguous=ambiguous, nonexistent=nonexistent)[0] @@ -1409,6 +1403,7 @@ class Timestamp(_Timestamp): DAY_NS = DAY_SECONDS * 1000000000 normalized_value = self.value - (self.value % DAY_NS) return Timestamp(normalized_value).tz_localize(self.tz) + from pandas._libs.tslibs.conversion import normalize_i8_timestamps normalized_value = normalize_i8_timestamps( np.array([self.value], dtype='i8'), tz=self.tz)[0] return Timestamp(normalized_value).tz_localize(self.tz) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 01108c1b8df03..79895f77db8c2 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2351,3 +2351,14 @@ def test_shift_months(years, months): for x in dti] expected = DatetimeIndex(raw) tm.assert_index_equal(actual, expected) + + +def test_dt_subclass_add_timedelta(): + # GH 25851 + class SubDatetime(datetime): + pass + dt = SubDatetime(2000, 1, 1) + td = Timedelta(hours=1) + result = dt + td + expected = SubDatetime(2000, 1, 1, 1) + assert result == expected diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 0466deb4a29a0..7dbff4af22f33 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -604,6 +604,15 @@ def test_dont_convert_dateutil_utc_to_pytz_utc(self): expected = Timestamp(datetime(2018, 1, 1)).tz_localize(tzutc()) assert result == expected + def test_constructor_subclassed_datetime(self): + # GH 25851 + class SubDatetime(datetime): + pass + data = SubDatetime(2000, 1, 1) + result = Timestamp(data) + expected = Timestamp(2000, 1, 1) + assert result == expected + class TestTimestamp(object): diff --git a/pandas/tests/tslibs/test_array_to_datetime.py b/pandas/tests/tslibs/test_array_to_datetime.py index f5b036dde2094..32248a42a4097 100644 --- a/pandas/tests/tslibs/test_array_to_datetime.py +++ b/pandas/tests/tslibs/test_array_to_datetime.py @@ -9,6 +9,7 @@ from pandas._libs import iNaT, tslib from pandas.compat.numpy import np_array_datetime64_compat +from pandas import Timestamp import pandas.util.testing as tm @@ -154,3 +155,25 @@ def test_to_datetime_barely_out_of_bounds(): with pytest.raises(tslib.OutOfBoundsDatetime, match=msg): tslib.array_to_datetime(arr) + + +class SubDatetime(datetime): + pass + + +@pytest.mark.parametrize("data,expected", [ + ([SubDatetime(2000, 1, 1)], + ["2000-01-01T00:00:00.000000000-0000"]), + ([datetime(2000, 1, 1)], + ["2000-01-01T00:00:00.000000000-0000"]), + ([Timestamp(2000, 1, 1)], + ["2000-01-01T00:00:00.000000000-0000"]) +]) +def test_datetime_subclass(data, expected): + # GH 25851 + + arr = np.array(data, dtype=object) + result, _ = tslib.array_to_datetime(arr) + + expected = np_array_datetime64_compat(expected, dtype="M8[ns]") + tm.assert_numpy_array_equal(result, expected) diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index 13398a69b4982..78f755c35e822 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from datetime import datetime + import numpy as np import pytest from pytz import UTC @@ -7,7 +9,7 @@ from pandas._libs.tslib import iNaT from pandas._libs.tslibs import conversion, timezones -from pandas import date_range +from pandas import date_range, Timestamp import pandas.util.testing as tm @@ -66,3 +68,22 @@ def test_length_zero_copy(dtype, copy): arr = np.array([], dtype=dtype) result = conversion.ensure_datetime64ns(arr, copy=copy) assert result.base is (None if copy else arr) + + +class SubDatetime(datetime): + pass + + +@pytest.mark.parametrize("dt, expected", [ + pytest.param(Timestamp("2000-01-01"), + Timestamp("2000-01-01", tz=UTC), id="timestamp"), + pytest.param(datetime(2000, 1, 1), + datetime(2000, 1, 1, tzinfo=UTC), + id="datetime"), + pytest.param(SubDatetime(2000, 1, 1), + SubDatetime(2000, 1, 1, tzinfo=UTC), + id="subdatetime")]) +def test_localize_pydatetime_dt_types(dt, expected): + # GH 25851 + result = conversion.localize_pydatetime(dt, UTC) + assert result == expected diff --git a/pandas/tests/tslibs/test_normalize_date.py b/pandas/tests/tslibs/test_normalize_date.py index 6124121b97186..3c0e5a8a591b6 100644 --- a/pandas/tests/tslibs/test_normalize_date.py +++ b/pandas/tests/tslibs/test_normalize_date.py @@ -7,6 +7,8 @@ from pandas._libs import tslibs +from pandas import Timestamp + @pytest.mark.parametrize("value,expected", [ (date(2012, 9, 7), datetime(2012, 9, 7)), @@ -16,3 +18,20 @@ def test_normalize_date(value, expected): result = tslibs.normalize_date(value) assert result == expected + + +class SubDatetime(datetime): + pass + + +@pytest.mark.parametrize("dt, expected", [ + pytest.param(Timestamp(2000, 1, 1, 1), + Timestamp(2000, 1, 1, 0)), + pytest.param(datetime(2000, 1, 1, 1), + datetime(2000, 1, 1, 0)), + pytest.param(SubDatetime(2000, 1, 1, 1), + SubDatetime(2000, 1, 1, 0))]) +def test_normalize_date_sub_types(dt, expected): + # GH 25851 + result = tslibs.normalize_date(dt) + assert result == expected From 82b242045304abdb279fdbfd594d6274608ae240 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 23 Mar 2019 19:07:57 -0400 Subject: [PATCH 02/25] increase spacing in comment --- pandas/_libs/tslibs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/__init__.py b/pandas/_libs/tslibs/__init__.py index 647a46ac8fbf0..03ac585818002 100644 --- a/pandas/_libs/tslibs/__init__.py +++ b/pandas/_libs/tslibs/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa -from .timestamps import Timestamp # isort:skip +from .timestamps import Timestamp # isort:skip from .conversion import normalize_date, localize_pydatetime, tz_convert_single from .nattype import NaT, NaTType, iNaT, is_null_datetimelike from .np_datetime import OutOfBoundsDatetime From 4447746912d7b661329da6bdf749c9d8c63300f1 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 23 Mar 2019 19:32:03 -0400 Subject: [PATCH 03/25] fix spacing --- pandas/_libs/tslibs/timedeltas.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index ae89626eb3a55..7730bdfa7288d 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -584,8 +584,8 @@ def _binary_op_method_timedeltalike(op, name): pass elif is_datetime64_object(other) or ( - PyDateTime_Check(other) and not isinstance(other, _Timestamp)): - # this case is for a datetime object that is specifically + PyDateTime_Check(other) and not isinstance(other, _Timestamp)): + # this case is for a datetime object that is specifically # *not* a Timestamp, as the Timestamp case will be # handled after `_validate_ops_compat` returns False below from pandas._libs.tslibs.timestamps import Timestamp From 2d2037591cf7ddb0418a319537040c7498810a1d Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 23 Mar 2019 20:47:15 -0400 Subject: [PATCH 04/25] fix import formatting --- pandas/tests/tslibs/test_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index 78f755c35e822..fe381dd411647 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -9,7 +9,7 @@ from pandas._libs.tslib import iNaT from pandas._libs.tslibs import conversion, timezones -from pandas import date_range, Timestamp +from pandas import Timestamp, date_range import pandas.util.testing as tm From 8e8d816d9a1f82c20d8417b7d0a397e1512a17cd Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Wed, 27 Mar 2019 21:40:25 -0400 Subject: [PATCH 05/25] move _Timestamp to separate file to prevent circular import --- pandas/_libs/tslib.pyx | 4 +- pandas/_libs/tslibs/__init__.py | 2 +- pandas/_libs/tslibs/_timestamp.pxd | 19 + pandas/_libs/tslibs/_timestamp.pyx | 409 ++++++++++++++++++ pandas/_libs/tslibs/conversion.pyx | 3 +- pandas/_libs/tslibs/timedeltas.pyx | 3 +- pandas/_libs/tslibs/timestamps.pxd | 16 - pandas/_libs/tslibs/timestamps.pyx | 383 +--------------- pandas/core/arrays/datetimelike.py | 4 +- pandas/tests/arithmetic/test_datetime64.py | 2 + .../tests/scalar/timestamp/test_timestamp.py | 2 + pandas/tests/tslibs/test_api.py | 3 +- pandas/tests/tslibs/test_array_to_datetime.py | 2 + pandas/tests/tslibs/test_conversion.py | 2 + pandas/tests/tslibs/test_normalize_date.py | 2 + setup.py | 6 + 16 files changed, 470 insertions(+), 392 deletions(-) create mode 100644 pandas/_libs/tslibs/_timestamp.pxd create mode 100644 pandas/_libs/tslibs/_timestamp.pyx diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index f31ecaa4ffffa..497202dfccdfb 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -18,6 +18,7 @@ import pytz from pandas._libs.util cimport ( is_integer_object, is_float_object, is_string_object, is_datetime64_object) +from pandas._libs.tslibs._timestamp cimport _Timestamp from pandas._libs.tslibs.np_datetime cimport ( check_dts_bounds, npy_datetimestruct, _string_to_dts, dt64_to_dtstruct, @@ -40,8 +41,7 @@ from pandas._libs.tslibs.nattype cimport ( from pandas._libs.tslibs.offsets cimport to_offset -from pandas._libs.tslibs.timestamps cimport ( - create_timestamp_from_ts, _Timestamp) +from pandas._libs.tslibs.timestamps cimport create_timestamp_from_ts from pandas._libs.tslibs.timestamps import Timestamp diff --git a/pandas/_libs/tslibs/__init__.py b/pandas/_libs/tslibs/__init__.py index 03ac585818002..a21fdf95559e6 100644 --- a/pandas/_libs/tslibs/__init__.py +++ b/pandas/_libs/tslibs/__init__.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # flake8: noqa -from .timestamps import Timestamp # isort:skip from .conversion import normalize_date, localize_pydatetime, tz_convert_single from .nattype import NaT, NaTType, iNaT, is_null_datetimelike from .np_datetime import OutOfBoundsDatetime from .period import Period, IncompatibleFrequency +from .timestamps import Timestamp from .timedeltas import delta_to_nanoseconds, ints_to_pytimedelta, Timedelta diff --git a/pandas/_libs/tslibs/_timestamp.pxd b/pandas/_libs/tslibs/_timestamp.pxd new file mode 100644 index 0000000000000..e41197d0f20a2 --- /dev/null +++ b/pandas/_libs/tslibs/_timestamp.pxd @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from cpython.datetime cimport datetime + +from numpy cimport int64_t + +cdef class _Timestamp(datetime): + cdef readonly: + int64_t value, nanosecond + object freq + list _date_attributes + cpdef bint _get_start_end_field(self, str field) + cpdef _get_date_name_field(self, object field, object locale) + cdef int64_t _maybe_convert_value_to_local(self) + cpdef to_datetime64(self) + cdef _assert_tzawareness_compat(_Timestamp self, datetime other) + cpdef datetime to_pydatetime(_Timestamp self, bint warn=*) + cdef bint _compare_outside_nanorange(_Timestamp self, datetime other, + int op) except -1 diff --git a/pandas/_libs/tslibs/_timestamp.pyx b/pandas/_libs/tslibs/_timestamp.pyx new file mode 100644 index 0000000000000..a68740e7d3ace --- /dev/null +++ b/pandas/_libs/tslibs/_timestamp.pyx @@ -0,0 +1,409 @@ +# -*- coding: utf-8 -*- +import warnings + +from cpython cimport (PyObject_RichCompareBool, PyObject_RichCompare, + Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE) + +import numpy as np +cimport numpy as cnp +from numpy cimport int64_t, int8_t +cnp.import_array() + +from dateutil.tz import tzutc + +from cpython.datetime cimport (datetime, + PyDateTime_Check, PyDelta_Check, + PyDateTime_IMPORT) +PyDateTime_IMPORT + +from pandas._libs.tslibs.util cimport ( + is_datetime64_object, is_timedelta64_object, is_integer_object, + is_array) + +from pandas._libs.tslibs.fields import get_start_end_field, get_date_name_field +from pandas._libs.tslibs.nattype cimport c_NaT as NaT +from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime +from pandas._libs.tslibs.np_datetime cimport ( + reverse_ops, cmp_scalar, npy_datetimestruct, dt64_to_dtstruct) +from pandas._libs.tslibs.timezones cimport ( + get_timezone, get_utcoffset, is_utc, tz_compare) +from pandas._libs.tslibs.timezones import UTC + + +def maybe_integer_op_deprecated(obj): + # GH#22535 add/sub of integers and int-arrays is deprecated + if obj.freq is not None: + warnings.warn("Addition/subtraction of integers and integer-arrays " + "to {cls} is deprecated, will be removed in a future " + "version. Instead of adding/subtracting `n`, use " + "`n * self.freq`" + .format(cls=type(obj).__name__), + FutureWarning) + +# This is PITA. Because we inherit from datetime, which has very specific +# construction requirements, we need to do object instantiation in python +# (see Timestamp class below). This will serve as a C extension type that +# shadows the python class, where we do any heavy lifting. +cdef class _Timestamp(datetime): + + def __hash__(_Timestamp self): + if self.nanosecond: + return hash(self.value) + return datetime.__hash__(self) + + def __richcmp__(_Timestamp self, object other, int op): + cdef: + _Timestamp ots + int ndim + + if isinstance(other, _Timestamp): + ots = other + elif other is NaT: + return op == Py_NE + elif PyDateTime_Check(other): + if self.nanosecond == 0: + val = self.to_pydatetime() + return PyObject_RichCompareBool(val, other, op) + + try: + ots = self.__class__(other) + except ValueError: + return self._compare_outside_nanorange(other, op) + else: + ndim = getattr(other, "ndim", -1) + + if ndim != -1: + if ndim == 0: + if is_datetime64_object(other): + other = self.__class__(other) + else: + if op == Py_EQ: + return False + elif op == Py_NE: + return True + + # only allow ==, != ops + raise TypeError('Cannot compare type %r with type %r' % + (type(self).__name__, + type(other).__name__)) + elif is_array(other): + # avoid recursion error GH#15183 + return PyObject_RichCompare(np.array([self]), other, op) + return PyObject_RichCompare(other, self, reverse_ops[op]) + else: + if op == Py_EQ: + return False + elif op == Py_NE: + return True + raise TypeError('Cannot compare type %r with type %r' % + (type(self).__name__, type(other).__name__)) + + self._assert_tzawareness_compat(other) + return cmp_scalar(self.value, ots.value, op) + + def __reduce_ex__(self, protocol): + # python 3.6 compat + # http://bugs.python.org/issue28730 + # now __reduce_ex__ is defined and higher priority than __reduce__ + return self.__reduce__() + + def __repr__(self): + stamp = self._repr_base + zone = None + + try: + stamp += self.strftime('%z') + if self.tzinfo: + zone = get_timezone(self.tzinfo) + except ValueError: + year2000 = self.replace(year=2000) + stamp += year2000.strftime('%z') + if self.tzinfo: + zone = get_timezone(self.tzinfo) + + try: + stamp += zone.strftime(' %%Z') + except: + pass + + tz = ", tz='{0}'".format(zone) if zone is not None else "" + freq = "" if self.freq is None else ", freq='{0}'".format(self.freqstr) + + return "Timestamp('{stamp}'{tz}{freq})".format(stamp=stamp, + tz=tz, freq=freq) + + cdef bint _compare_outside_nanorange(_Timestamp self, datetime other, + int op) except -1: + cdef: + datetime dtval = self.to_pydatetime() + + self._assert_tzawareness_compat(other) + + if self.nanosecond == 0: + return PyObject_RichCompareBool(dtval, other, op) + else: + if op == Py_EQ: + return False + elif op == Py_NE: + return True + elif op == Py_LT: + return dtval < other + elif op == Py_LE: + return dtval < other + elif op == Py_GT: + return dtval >= other + elif op == Py_GE: + return dtval >= other + + cdef _assert_tzawareness_compat(_Timestamp self, datetime other): + if self.tzinfo is None: + if other.tzinfo is not None: + raise TypeError('Cannot compare tz-naive and tz-aware ' + 'timestamps') + elif other.tzinfo is None: + raise TypeError('Cannot compare tz-naive and tz-aware timestamps') + + cpdef datetime to_pydatetime(_Timestamp self, bint warn=True): + """ + Convert a Timestamp object to a native Python datetime object. + + If warn=True, issue a warning if nanoseconds is nonzero. + """ + if self.nanosecond != 0 and warn: + warnings.warn("Discarding nonzero nanoseconds in conversion", + UserWarning, stacklevel=2) + + return datetime(self.year, self.month, self.day, + self.hour, self.minute, self.second, + self.microsecond, self.tzinfo) + + cpdef to_datetime64(self): + """ + Return a numpy.datetime64 object with 'ns' precision. + """ + return np.datetime64(self.value, 'ns') + + def to_numpy(self, dtype=None, copy=False): + """ + Convert the Timestamp to a NumPy datetime64. + + .. versionadded:: 0.25.0 + + This is an alias method for `Timestamp.to_datetime64()`. The dtype and + copy parameters are available here only for compatibility. Their values + will not affect the return value. + + Returns + ------- + numpy.datetime64 + + See Also + -------- + DatetimeIndex.to_numpy : Similar method for DatetimeIndex. + """ + return self.to_datetime64() + + def __add__(self, other): + cdef: + int64_t other_int, nanos + + if is_timedelta64_object(other): + other_int = other.astype('timedelta64[ns]').view('i8') + return self.__class__(self.value + other_int, + tz=self.tzinfo, freq=self.freq) + + elif is_integer_object(other): + maybe_integer_op_deprecated(self) + + if self is NaT: + # to be compat with Period + return NaT + elif self.freq is None: + raise ValueError("Cannot add integral value to Timestamp " + "without freq.") + return self.__class__((self.freq * other).apply(self), + freq=self.freq) + + elif PyDelta_Check(other) or hasattr(other, 'delta'): + # delta --> offsets.Tick + # logic copied from delta_to_nanoseconds to prevent circular import + if hasattr(other, 'nanos'): + nanos = other.nanos + elif hasattr(other, 'delta'): + nanos = other.delta + elif PyDelta_Check(other): + nanos = (other.days * 24 * 60 * 60 * 1000000 + + other.seconds * 1000000 + + other.microseconds) * 1000 + + result = self.__class__(self.value + nanos, + tz=self.tzinfo, freq=self.freq) + if getattr(other, 'normalize', False): + # DateOffset + result = result.normalize() + return result + + # index/series like + elif hasattr(other, '_typ'): + return NotImplemented + + result = datetime.__add__(self, other) + if PyDateTime_Check(result): + result = self.__class__(result) + result.nanosecond = self.nanosecond + return result + + def __sub__(self, other): + if (is_timedelta64_object(other) or is_integer_object(other) or + PyDelta_Check(other) or hasattr(other, 'delta')): + # `delta` attribute is for offsets.Tick or offsets.Week obj + neg_other = -other + return self + neg_other + + typ = getattr(other, '_typ', None) + + # a Timestamp-DatetimeIndex -> yields a negative TimedeltaIndex + if typ in ('datetimeindex', 'datetimearray'): + # timezone comparison is performed in DatetimeIndex._sub_datelike + return -other.__sub__(self) + + # a Timestamp-TimedeltaIndex -> yields a negative TimedeltaIndex + elif typ in ('timedeltaindex', 'timedeltaarray'): + return (-other).__add__(self) + + elif other is NaT: + return NaT + + # coerce if necessary if we are a Timestamp-like + if (PyDateTime_Check(self) + and (PyDateTime_Check(other) or is_datetime64_object(other))): + if isinstance(self, _Timestamp): + other = self.__class__(other) + else: + self = other.__class__(self) + + # validate tz's + if not tz_compare(self.tzinfo, other.tzinfo): + raise TypeError("Timestamp subtraction must have the " + "same timezones or no timezones") + + # scalar Timestamp/datetime - Timestamp/datetime -> yields a + # Timedelta + try: + return self._create_timedelta(self.value - other.value) + except (OverflowError, OutOfBoundsDatetime): + pass + + # scalar Timestamp/datetime - Timedelta -> yields a Timestamp (with + # same timezone if specified) + return datetime.__sub__(self, other) + + cdef int64_t _maybe_convert_value_to_local(self): + """Convert UTC i8 value to local i8 value if tz exists""" + cdef: + int64_t val + npy_datetimestruct dts + int64_t delta + datetime dt + + val = self.value + if self.tz is not None and not is_utc(self.tz): + # logic copied from _tz_convert_tzlocal_utc + # to prevent importing tslibs.conversion + dt64_to_dtstruct(val, &dts) + dt = datetime(dts.year, dts.month, dts.day, dts.hour, + dts.min, dts.sec, dts.us) + # get_utcoffset (tz.utcoffset under the hood) only makes + # sense if dt is _wall time_, so convert to wall time + dt = dt.replace(tzinfo=tzutc()) + dt = dt.astimezone(self.tz) + delta = (int(get_utcoffset(self.tz, dt).total_seconds()) + * 1000000000) + val = self.value + delta + return val + + cpdef bint _get_start_end_field(self, str field): + cdef: + int64_t val + dict kwds + int8_t out[1] + int month_kw + + freq = self.freq + if freq: + kwds = freq.kwds + month_kw = kwds.get('startingMonth', kwds.get('month', 12)) + freqstr = self.freqstr + else: + month_kw = 12 + freqstr = None + + val = self._maybe_convert_value_to_local() + out = get_start_end_field(np.array([val], dtype=np.int64), + field, freqstr, month_kw) + return out[0] + + cpdef _get_date_name_field(self, object field, object locale): + cdef: + int64_t val + object[:] out + + val = self._maybe_convert_value_to_local() + out = get_date_name_field(np.array([val], dtype=np.int64), + field, locale=locale) + return out[0] + + @property + def _repr_base(self): + return '{date} {time}'.format(date=self._date_repr, + time=self._time_repr) + + @property + def _date_repr(self): + # Ideal here would be self.strftime("%Y-%m-%d"), but + # the datetime strftime() methods require year >= 1900 + return '%d-%.2d-%.2d' % (self.year, self.month, self.day) + + @property + def _time_repr(self): + result = '%.2d:%.2d:%.2d' % (self.hour, self.minute, self.second) + + if self.nanosecond != 0: + result += '.%.9d' % (self.nanosecond + 1000 * self.microsecond) + elif self.microsecond != 0: + result += '.%.6d' % self.microsecond + + return result + + @property + def _short_repr(self): + # format a Timestamp with only _date_repr if possible + # otherwise _repr_base + if (self.hour == 0 and + self.minute == 0 and + self.second == 0 and + self.microsecond == 0 and + self.nanosecond == 0): + return self._date_repr + return self._repr_base + + @property + def asm8(self): + """ + Return numpy datetime64 format in nanoseconds. + """ + return np.datetime64(self.value, 'ns') + + @property + def resolution(self): + """ + Return resolution describing the smallest difference between two + times that can be represented by Timestamp object_state + """ + # GH#21336, GH#21365 + return self._create_timedelta(nanoseconds=1) + + def timestamp(self): + """Return POSIX timestamp as float.""" + # py27 compat, see GH#17329 + return round(self.value / 1e9, 6) diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index 4a7ac912f6e46..39b52dce79547 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -16,6 +16,8 @@ from cpython.datetime cimport (datetime, tzinfo, PyDateTime_IMPORT, PyDelta_Check) PyDateTime_IMPORT +from pandas._libs.tslibs._timestamp cimport _Timestamp + from pandas._libs.tslibs.ccalendar import DAY_SECONDS, HOUR_SECONDS from pandas._libs.tslibs.np_datetime cimport ( @@ -30,7 +32,6 @@ from pandas._libs.tslibs.util cimport ( from pandas._libs.tslibs.timedeltas cimport (cast_from_unit, delta_to_nanoseconds) -from pandas._libs.tslibs.timestamps cimport _Timestamp from pandas._libs.tslibs.timezones cimport ( is_utc, is_tzlocal, is_fixed_offset, get_utcoffset, get_dst_info, get_timezone, maybe_get_tz, tz_compare) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 7730bdfa7288d..376afa09da560 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -26,6 +26,8 @@ from pandas._libs.tslibs.util cimport ( is_timedelta64_object, is_datetime64_object, is_integer_object, is_float_object, is_string_object) +from pandas._libs.tslibs._timestamp cimport _Timestamp + from pandas._libs.tslibs.ccalendar import DAY_SECONDS from pandas._libs.tslibs.np_datetime cimport ( @@ -36,7 +38,6 @@ from pandas._libs.tslibs.nattype cimport ( checknull_with_nat, NPY_NAT, c_NaT as NaT) from pandas._libs.tslibs.offsets cimport to_offset from pandas._libs.tslibs.offsets import _Tick as Tick -from pandas._libs.tslibs.timestamps cimport _Timestamp # ---------------------------------------------------------------------- # Constants diff --git a/pandas/_libs/tslibs/timestamps.pxd b/pandas/_libs/tslibs/timestamps.pxd index d19466c83dfff..b7282e02ff117 100644 --- a/pandas/_libs/tslibs/timestamps.pxd +++ b/pandas/_libs/tslibs/timestamps.pxd @@ -1,24 +1,8 @@ # -*- coding: utf-8 -*- -from cpython.datetime cimport datetime - from numpy cimport int64_t from pandas._libs.tslibs.np_datetime cimport npy_datetimestruct cdef object create_timestamp_from_ts(int64_t value, npy_datetimestruct dts, object tz, object freq) - -cdef class _Timestamp(datetime): - cdef readonly: - int64_t value, nanosecond - object freq - list _date_attributes - cpdef bint _get_start_end_field(self, str field) - cpdef _get_date_name_field(self, object field, object locale) - cdef int64_t _maybe_convert_value_to_local(self) - cpdef to_datetime64(self) - cdef _assert_tzawareness_compat(_Timestamp self, datetime other) - cpdef datetime to_pydatetime(_Timestamp self, bint warn=*) - cdef bint _compare_outside_nanorange(_Timestamp self, datetime other, - int op) except -1 diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 0d1754b7cc510..f8a390afbade1 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -2,40 +2,35 @@ import sys import warnings -from cpython cimport (PyObject_RichCompareBool, PyObject_RichCompare, - Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE) - import numpy as np cimport numpy as cnp -from numpy cimport int64_t, int32_t, int8_t +from numpy cimport int64_t cnp.import_array() from datetime import time as datetime_time, timedelta from cpython.datetime cimport (datetime, - PyDateTime_Check, PyDelta_Check, PyTZInfo_Check, - PyDateTime_IMPORT) + PyTZInfo_Check, PyDateTime_IMPORT) PyDateTime_IMPORT from pandas._libs.tslibs.util cimport ( - is_datetime64_object, is_timedelta64_object, is_integer_object, - is_string_object, is_array, is_offset_object) + is_integer_object, is_string_object, is_offset_object) +from pandas._libs.tslibs._timestamp cimport _Timestamp cimport pandas._libs.tslibs.ccalendar as ccalendar from pandas._libs.tslibs.ccalendar import DAY_SECONDS +from pandas._libs.tslibs.conversion import ( + normalize_i8_timestamps, tz_localize_to_utc) from pandas._libs.tslibs.conversion cimport ( tz_convert_single, _TSObject, convert_to_tsobject, convert_datetime_to_tsobject) from pandas._libs.tslibs.fields import get_start_end_field, get_date_name_field from pandas._libs.tslibs.nattype cimport NPY_NAT, c_NaT as NaT -from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime from pandas._libs.tslibs.np_datetime cimport ( - reverse_ops, cmp_scalar, check_dts_bounds, npy_datetimestruct, - dt64_to_dtstruct) + check_dts_bounds, npy_datetimestruct, dt64_to_dtstruct) from pandas._libs.tslibs.offsets cimport to_offset from pandas._libs.tslibs.timedeltas import Timedelta -from pandas._libs.tslibs.timedeltas cimport delta_to_nanoseconds from pandas._libs.tslibs.timezones cimport ( - get_timezone, is_utc, maybe_get_tz, treat_tz_as_pytz, tz_compare) + is_utc, maybe_get_tz, treat_tz_as_pytz) from pandas._libs.tslibs.timezones import UTC # ---------------------------------------------------------------------- @@ -47,17 +42,6 @@ PY36 = sys.version_info >= (3, 6) # ---------------------------------------------------------------------- -def maybe_integer_op_deprecated(obj): - # GH#22535 add/sub of integers and int-arrays is deprecated - if obj.freq is not None: - warnings.warn("Addition/subtraction of integers and integer-arrays " - "to {cls} is deprecated, will be removed in a future " - "version. Instead of adding/subtracting `n`, use " - "`n * self.freq`" - .format(cls=type(obj).__name__), - FutureWarning) - - cdef inline object create_timestamp_from_ts(int64_t value, npy_datetimestruct dts, object tz, object freq): @@ -195,348 +179,6 @@ def round_nsint64(values, mode, freq): "rounding mode") -# This is PITA. Because we inherit from datetime, which has very specific -# construction requirements, we need to do object instantiation in python -# (see Timestamp class below). This will serve as a C extension type that -# shadows the python class, where we do any heavy lifting. -cdef class _Timestamp(datetime): - - def __hash__(_Timestamp self): - if self.nanosecond: - return hash(self.value) - return datetime.__hash__(self) - - def __richcmp__(_Timestamp self, object other, int op): - cdef: - _Timestamp ots - int ndim - - if isinstance(other, _Timestamp): - ots = other - elif other is NaT: - return op == Py_NE - elif PyDateTime_Check(other): - if self.nanosecond == 0: - val = self.to_pydatetime() - return PyObject_RichCompareBool(val, other, op) - - try: - ots = Timestamp(other) - except ValueError: - return self._compare_outside_nanorange(other, op) - else: - ndim = getattr(other, "ndim", -1) - - if ndim != -1: - if ndim == 0: - if is_datetime64_object(other): - other = Timestamp(other) - else: - if op == Py_EQ: - return False - elif op == Py_NE: - return True - - # only allow ==, != ops - raise TypeError('Cannot compare type %r with type %r' % - (type(self).__name__, - type(other).__name__)) - elif is_array(other): - # avoid recursion error GH#15183 - return PyObject_RichCompare(np.array([self]), other, op) - return PyObject_RichCompare(other, self, reverse_ops[op]) - else: - if op == Py_EQ: - return False - elif op == Py_NE: - return True - raise TypeError('Cannot compare type %r with type %r' % - (type(self).__name__, type(other).__name__)) - - self._assert_tzawareness_compat(other) - return cmp_scalar(self.value, ots.value, op) - - def __reduce_ex__(self, protocol): - # python 3.6 compat - # http://bugs.python.org/issue28730 - # now __reduce_ex__ is defined and higher priority than __reduce__ - return self.__reduce__() - - def __repr__(self): - stamp = self._repr_base - zone = None - - try: - stamp += self.strftime('%z') - if self.tzinfo: - zone = get_timezone(self.tzinfo) - except ValueError: - year2000 = self.replace(year=2000) - stamp += year2000.strftime('%z') - if self.tzinfo: - zone = get_timezone(self.tzinfo) - - try: - stamp += zone.strftime(' %%Z') - except: - pass - - tz = ", tz='{0}'".format(zone) if zone is not None else "" - freq = "" if self.freq is None else ", freq='{0}'".format(self.freqstr) - - return "Timestamp('{stamp}'{tz}{freq})".format(stamp=stamp, - tz=tz, freq=freq) - - cdef bint _compare_outside_nanorange(_Timestamp self, datetime other, - int op) except -1: - cdef: - datetime dtval = self.to_pydatetime() - - self._assert_tzawareness_compat(other) - - if self.nanosecond == 0: - return PyObject_RichCompareBool(dtval, other, op) - else: - if op == Py_EQ: - return False - elif op == Py_NE: - return True - elif op == Py_LT: - return dtval < other - elif op == Py_LE: - return dtval < other - elif op == Py_GT: - return dtval >= other - elif op == Py_GE: - return dtval >= other - - cdef _assert_tzawareness_compat(_Timestamp self, datetime other): - if self.tzinfo is None: - if other.tzinfo is not None: - raise TypeError('Cannot compare tz-naive and tz-aware ' - 'timestamps') - elif other.tzinfo is None: - raise TypeError('Cannot compare tz-naive and tz-aware timestamps') - - cpdef datetime to_pydatetime(_Timestamp self, bint warn=True): - """ - Convert a Timestamp object to a native Python datetime object. - - If warn=True, issue a warning if nanoseconds is nonzero. - """ - if self.nanosecond != 0 and warn: - warnings.warn("Discarding nonzero nanoseconds in conversion", - UserWarning, stacklevel=2) - - return datetime(self.year, self.month, self.day, - self.hour, self.minute, self.second, - self.microsecond, self.tzinfo) - - cpdef to_datetime64(self): - """ - Return a numpy.datetime64 object with 'ns' precision. - """ - return np.datetime64(self.value, 'ns') - - def to_numpy(self, dtype=None, copy=False): - """ - Convert the Timestamp to a NumPy datetime64. - - .. versionadded:: 0.25.0 - - This is an alias method for `Timestamp.to_datetime64()`. The dtype and - copy parameters are available here only for compatibility. Their values - will not affect the return value. - - Returns - ------- - numpy.datetime64 - - See Also - -------- - DatetimeIndex.to_numpy : Similar method for DatetimeIndex. - """ - return self.to_datetime64() - - def __add__(self, other): - cdef: - int64_t other_int, nanos - - if is_timedelta64_object(other): - other_int = other.astype('timedelta64[ns]').view('i8') - return Timestamp(self.value + other_int, - tz=self.tzinfo, freq=self.freq) - - elif is_integer_object(other): - maybe_integer_op_deprecated(self) - - if self is NaT: - # to be compat with Period - return NaT - elif self.freq is None: - raise ValueError("Cannot add integral value to Timestamp " - "without freq.") - return Timestamp((self.freq * other).apply(self), freq=self.freq) - - elif PyDelta_Check(other) or hasattr(other, 'delta'): - # delta --> offsets.Tick - nanos = delta_to_nanoseconds(other) - result = Timestamp(self.value + nanos, - tz=self.tzinfo, freq=self.freq) - if getattr(other, 'normalize', False): - # DateOffset - result = result.normalize() - return result - - # index/series like - elif hasattr(other, '_typ'): - return NotImplemented - - result = datetime.__add__(self, other) - if PyDateTime_Check(result): - result = Timestamp(result) - result.nanosecond = self.nanosecond - return result - - def __sub__(self, other): - if (is_timedelta64_object(other) or is_integer_object(other) or - PyDelta_Check(other) or hasattr(other, 'delta')): - # `delta` attribute is for offsets.Tick or offsets.Week obj - neg_other = -other - return self + neg_other - - typ = getattr(other, '_typ', None) - - # a Timestamp-DatetimeIndex -> yields a negative TimedeltaIndex - if typ in ('datetimeindex', 'datetimearray'): - # timezone comparison is performed in DatetimeIndex._sub_datelike - return -other.__sub__(self) - - # a Timestamp-TimedeltaIndex -> yields a negative TimedeltaIndex - elif typ in ('timedeltaindex', 'timedeltaarray'): - return (-other).__add__(self) - - elif other is NaT: - return NaT - - # coerce if necessary if we are a Timestamp-like - if (PyDateTime_Check(self) - and (PyDateTime_Check(other) or is_datetime64_object(other))): - self = Timestamp(self) - other = Timestamp(other) - - # validate tz's - if not tz_compare(self.tzinfo, other.tzinfo): - raise TypeError("Timestamp subtraction must have the " - "same timezones or no timezones") - - # scalar Timestamp/datetime - Timestamp/datetime -> yields a - # Timedelta - try: - return Timedelta(self.value - other.value) - except (OverflowError, OutOfBoundsDatetime): - pass - - # scalar Timestamp/datetime - Timedelta -> yields a Timestamp (with - # same timezone if specified) - return datetime.__sub__(self, other) - - cdef int64_t _maybe_convert_value_to_local(self): - """Convert UTC i8 value to local i8 value if tz exists""" - cdef: - int64_t val - val = self.value - if self.tz is not None and not is_utc(self.tz): - val = tz_convert_single(self.value, UTC, self.tz) - return val - - cpdef bint _get_start_end_field(self, str field): - cdef: - int64_t val - dict kwds - int8_t out[1] - int month_kw - - freq = self.freq - if freq: - kwds = freq.kwds - month_kw = kwds.get('startingMonth', kwds.get('month', 12)) - freqstr = self.freqstr - else: - month_kw = 12 - freqstr = None - - val = self._maybe_convert_value_to_local() - out = get_start_end_field(np.array([val], dtype=np.int64), - field, freqstr, month_kw) - return out[0] - - cpdef _get_date_name_field(self, object field, object locale): - cdef: - int64_t val - object[:] out - - val = self._maybe_convert_value_to_local() - out = get_date_name_field(np.array([val], dtype=np.int64), - field, locale=locale) - return out[0] - - @property - def _repr_base(self): - return '{date} {time}'.format(date=self._date_repr, - time=self._time_repr) - - @property - def _date_repr(self): - # Ideal here would be self.strftime("%Y-%m-%d"), but - # the datetime strftime() methods require year >= 1900 - return '%d-%.2d-%.2d' % (self.year, self.month, self.day) - - @property - def _time_repr(self): - result = '%.2d:%.2d:%.2d' % (self.hour, self.minute, self.second) - - if self.nanosecond != 0: - result += '.%.9d' % (self.nanosecond + 1000 * self.microsecond) - elif self.microsecond != 0: - result += '.%.6d' % self.microsecond - - return result - - @property - def _short_repr(self): - # format a Timestamp with only _date_repr if possible - # otherwise _repr_base - if (self.hour == 0 and - self.minute == 0 and - self.second == 0 and - self.microsecond == 0 and - self.nanosecond == 0): - return self._date_repr - return self._repr_base - - @property - def asm8(self): - """ - Return numpy datetime64 format in nanoseconds. - """ - return np.datetime64(self.value, 'ns') - - @property - def resolution(self): - """ - Return resolution describing the smallest difference between two - times that can be represented by Timestamp object_state - """ - # GH#21336, GH#21365 - return Timedelta(nanoseconds=1) - - def timestamp(self): - """Return POSIX timestamp as float.""" - # py27 compat, see GH#17329 - return round(self.value / 1e9, 6) - - # ---------------------------------------------------------------------- # Python front end to C extension type _Timestamp @@ -1208,7 +850,6 @@ class Timestamp(_Timestamp): tz = maybe_get_tz(tz) if not is_string_object(ambiguous): ambiguous = [ambiguous] - from pandas._libs.tslibs.conversion import tz_localize_to_utc value = tz_localize_to_utc(np.array([self.value], dtype='i8'), tz, ambiguous=ambiguous, nonexistent=nonexistent)[0] @@ -1403,7 +1044,6 @@ class Timestamp(_Timestamp): DAY_NS = DAY_SECONDS * 1000000000 normalized_value = self.value - (self.value % DAY_NS) return Timestamp(normalized_value).tz_localize(self.tz) - from pandas._libs.tslibs.conversion import normalize_i8_timestamps normalized_value = normalize_i8_timestamps( np.array([self.value], dtype='i8'), tz=self.tz)[0] return Timestamp(normalized_value).tz_localize(self.tz) @@ -1413,6 +1053,13 @@ class Timestamp(_Timestamp): # define it here instead return self + other + def _create_timedelta(self, *args, **kwargs): + """ + Helper to create a Timedelta so that the + Timedelta class doesn't have to be imported elsewhere + """ + return Timedelta(*args, **kwargs) + # Add the min and max fields at the class level cdef int64_t _NS_UPPER_BOUND = np.iinfo(np.int64).max diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 27d7d4f888550..6784c30838ada 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -7,11 +7,11 @@ import numpy as np from pandas._libs import NaT, NaTType, Timestamp, algos, iNaT, lib +from pandas._libs.tslibs._timestamp import maybe_integer_op_deprecated from pandas._libs.tslibs.period import ( DIFFERENT_FREQ, IncompatibleFrequency, Period) from pandas._libs.tslibs.timedeltas import Timedelta, delta_to_nanoseconds -from pandas._libs.tslibs.timestamps import ( - RoundTo, maybe_integer_op_deprecated, round_nsint64) +from pandas._libs.tslibs.timestamps import RoundTo, round_nsint64 import pandas.compat as compat from pandas.compat.numpy import function as nv from pandas.errors import ( diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 79895f77db8c2..95cf1e68d67f3 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2355,6 +2355,8 @@ def test_shift_months(years, months): def test_dt_subclass_add_timedelta(): # GH 25851 + # ensure that subclassed datetime works for + # Timedelta operations class SubDatetime(datetime): pass dt = SubDatetime(2000, 1, 1) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 7dbff4af22f33..9b60182fdc41a 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -606,6 +606,8 @@ def test_dont_convert_dateutil_utc_to_pytz_utc(self): def test_constructor_subclassed_datetime(self): # GH 25851 + # ensure that subclassed datetime works for + # Timestamp creation class SubDatetime(datetime): pass data = SubDatetime(2000, 1, 1) diff --git a/pandas/tests/tslibs/test_api.py b/pandas/tests/tslibs/test_api.py index b2beb93f4b64c..dda32befda51b 100644 --- a/pandas/tests/tslibs/test_api.py +++ b/pandas/tests/tslibs/test_api.py @@ -6,7 +6,8 @@ def test_namespace(): - submodules = ['ccalendar', + submodules = ['_timestamp', + 'ccalendar', 'conversion', 'fields', 'frequencies', diff --git a/pandas/tests/tslibs/test_array_to_datetime.py b/pandas/tests/tslibs/test_array_to_datetime.py index 32248a42a4097..fad778847d3db 100644 --- a/pandas/tests/tslibs/test_array_to_datetime.py +++ b/pandas/tests/tslibs/test_array_to_datetime.py @@ -171,6 +171,8 @@ class SubDatetime(datetime): ]) def test_datetime_subclass(data, expected): # GH 25851 + # ensure that subclassed datetime works with + # array_to_datetime arr = np.array(data, dtype=object) result, _ = tslib.array_to_datetime(arr) diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index fe381dd411647..61ac4269efc67 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -85,5 +85,7 @@ class SubDatetime(datetime): id="subdatetime")]) def test_localize_pydatetime_dt_types(dt, expected): # GH 25851 + # ensure that subclassed datetime works with + # localize_pydatetime result = conversion.localize_pydatetime(dt, UTC) assert result == expected diff --git a/pandas/tests/tslibs/test_normalize_date.py b/pandas/tests/tslibs/test_normalize_date.py index 3c0e5a8a591b6..348578be4f065 100644 --- a/pandas/tests/tslibs/test_normalize_date.py +++ b/pandas/tests/tslibs/test_normalize_date.py @@ -33,5 +33,7 @@ class SubDatetime(datetime): SubDatetime(2000, 1, 1, 0))]) def test_normalize_date_sub_types(dt, expected): # GH 25851 + # ensure that subclassed datetime works with + # normalize_date result = tslibs.normalize_date(dt) assert result == expected diff --git a/setup.py b/setup.py index e5ef0d7bd3aea..46e16bd39a1c3 100755 --- a/setup.py +++ b/setup.py @@ -312,6 +312,7 @@ class CheckSDist(sdist_class): 'pandas/_libs/sparse.pyx', 'pandas/_libs/ops.pyx', 'pandas/_libs/parsers.pyx', + 'pandas/_libs/tslibs/_timestamp.pyx', 'pandas/_libs/tslibs/ccalendar.pyx', 'pandas/_libs/tslibs/period.pyx', 'pandas/_libs/tslibs/strptime.pyx', @@ -590,6 +591,11 @@ def srcpath(name=None, suffix='.pyx', subdir='src'): 'include': ts_include, 'depends': tseries_depends, 'sources': np_datetime_sources}, + '_libs.tslibs._timestamp': { + 'pyxfile': '_libs/tslibs/_timestamp', + 'include': ts_include, + 'depends': tseries_depends, + 'sources': np_datetime_sources}, '_libs.tslibs.ccalendar': { 'pyxfile': '_libs/tslibs/ccalendar', 'include': []}, From 4fd35acce20850361bdf6e90bf079fb282cf247d Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Wed, 27 Mar 2019 22:07:12 -0400 Subject: [PATCH 06/25] fix indenting --- pandas/_libs/tslibs/_timestamp.pyx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/tslibs/_timestamp.pyx b/pandas/_libs/tslibs/_timestamp.pyx index a68740e7d3ace..1ef718cd122b0 100644 --- a/pandas/_libs/tslibs/_timestamp.pyx +++ b/pandas/_libs/tslibs/_timestamp.pyx @@ -46,6 +46,7 @@ def maybe_integer_op_deprecated(obj): # shadows the python class, where we do any heavy lifting. cdef class _Timestamp(datetime): + def __hash__(_Timestamp self): if self.nanosecond: return hash(self.value) @@ -210,7 +211,7 @@ cdef class _Timestamp(datetime): if is_timedelta64_object(other): other_int = other.astype('timedelta64[ns]').view('i8') return self.__class__(self.value + other_int, - tz=self.tzinfo, freq=self.freq) + tz=self.tzinfo, freq=self.freq) elif is_integer_object(other): maybe_integer_op_deprecated(self) @@ -233,11 +234,11 @@ cdef class _Timestamp(datetime): nanos = other.delta elif PyDelta_Check(other): nanos = (other.days * 24 * 60 * 60 * 1000000 + - other.seconds * 1000000 + - other.microseconds) * 1000 + other.seconds * 1000000 + + other.microseconds) * 1000 result = self.__class__(self.value + nanos, - tz=self.tzinfo, freq=self.freq) + tz=self.tzinfo, freq=self.freq) if getattr(other, 'normalize', False): # DateOffset result = result.normalize() From 354d3a3a6a85ed19fdadb7bf653f4f5b24cfed85 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Thu, 28 Mar 2019 06:56:18 -0400 Subject: [PATCH 07/25] fix line spacing --- pandas/_libs/tslibs/_timestamp.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/_timestamp.pyx b/pandas/_libs/tslibs/_timestamp.pyx index 1ef718cd122b0..e867ce58c272d 100644 --- a/pandas/_libs/tslibs/_timestamp.pyx +++ b/pandas/_libs/tslibs/_timestamp.pyx @@ -40,13 +40,13 @@ def maybe_integer_op_deprecated(obj): .format(cls=type(obj).__name__), FutureWarning) + # This is PITA. Because we inherit from datetime, which has very specific # construction requirements, we need to do object instantiation in python # (see Timestamp class below). This will serve as a C extension type that # shadows the python class, where we do any heavy lifting. cdef class _Timestamp(datetime): - def __hash__(_Timestamp self): if self.nanosecond: return hash(self.value) From 600be4f2ca2241c9ef58ef7ea87d729cc7cfd6a9 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Thu, 28 Mar 2019 07:09:39 -0400 Subject: [PATCH 08/25] kickoff new build --- pandas/tests/arithmetic/test_datetime64.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 95cf1e68d67f3..095b0248013cb 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2359,6 +2359,7 @@ def test_dt_subclass_add_timedelta(): # Timedelta operations class SubDatetime(datetime): pass + dt = SubDatetime(2000, 1, 1) td = Timedelta(hours=1) result = dt + td From 7a60942fb0e496f02e72bba5b7463a3bb48ab52f Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 29 Mar 2019 17:26:41 -0400 Subject: [PATCH 09/25] updates for PR review --- pandas/_libs/tslibs/_timestamp.pyx | 8 ++++++++ pandas/tests/tslibs/test_normalize_date.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/_timestamp.pyx b/pandas/_libs/tslibs/_timestamp.pyx index e867ce58c272d..14c3e7278c4a1 100644 --- a/pandas/_libs/tslibs/_timestamp.pyx +++ b/pandas/_libs/tslibs/_timestamp.pyx @@ -1,4 +1,12 @@ # -*- coding: utf-8 -*- +""" +_Timestamp is in its own module to prevent the circular imports +that would happen if it was included in timestamps.pyx + +This allows _Timestamp to be imported in other modules +so that isinstance(obj, _Timestamp) checks can be performed +""" + import warnings from cpython cimport (PyObject_RichCompareBool, PyObject_RichCompare, diff --git a/pandas/tests/tslibs/test_normalize_date.py b/pandas/tests/tslibs/test_normalize_date.py index 348578be4f065..1d89bfcff31bd 100644 --- a/pandas/tests/tslibs/test_normalize_date.py +++ b/pandas/tests/tslibs/test_normalize_date.py @@ -7,7 +7,7 @@ from pandas._libs import tslibs -from pandas import Timestamp +from pandas._libs.tslibs.timestamps import Timestamp @pytest.mark.parametrize("value,expected", [ From 46f306d951677ff416d582c6b2ee9d2253d8f6b8 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 29 Mar 2019 18:41:55 -0400 Subject: [PATCH 10/25] fix isort error --- pandas/tests/tslibs/test_normalize_date.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/tslibs/test_normalize_date.py b/pandas/tests/tslibs/test_normalize_date.py index 1d89bfcff31bd..1bf45fad63dbd 100644 --- a/pandas/tests/tslibs/test_normalize_date.py +++ b/pandas/tests/tslibs/test_normalize_date.py @@ -5,10 +5,10 @@ import pytest -from pandas._libs import tslibs - from pandas._libs.tslibs.timestamps import Timestamp +from pandas._libs import tslibs + @pytest.mark.parametrize("value,expected", [ (date(2012, 9, 7), datetime(2012, 9, 7)), From ff0ddef8ca4146c5b96657071b635f01aff09ce1 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 29 Mar 2019 19:31:01 -0400 Subject: [PATCH 11/25] fix isort error --- pandas/tests/tslibs/test_normalize_date.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/tslibs/test_normalize_date.py b/pandas/tests/tslibs/test_normalize_date.py index 1bf45fad63dbd..36404526ef00a 100644 --- a/pandas/tests/tslibs/test_normalize_date.py +++ b/pandas/tests/tslibs/test_normalize_date.py @@ -5,9 +5,8 @@ import pytest -from pandas._libs.tslibs.timestamps import Timestamp - from pandas._libs import tslibs +from pandas._libs.tslibs.timestamps import Timestamp @pytest.mark.parametrize("value,expected", [ From c54ee5394bef0cbb6580b000c09c0fc66897bcaa Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 30 Mar 2019 08:06:06 -0400 Subject: [PATCH 12/25] force rebuild --- pandas/tests/arithmetic/test_datetime64.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 095b0248013cb..95cf1e68d67f3 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2359,7 +2359,6 @@ def test_dt_subclass_add_timedelta(): # Timedelta operations class SubDatetime(datetime): pass - dt = SubDatetime(2000, 1, 1) td = Timedelta(hours=1) result = dt + td From 6dc805b1b2276f9a26fd0844b4713eabb2021845 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 30 Mar 2019 16:38:18 -0400 Subject: [PATCH 13/25] PR updates --- pandas/_libs/tslibs/_timestamp.pyx | 14 ++++++++------ pandas/tests/arithmetic/test_datetime64.py | 4 ++++ pandas/tests/scalar/timestamp/test_timestamp.py | 1 + 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pandas/_libs/tslibs/_timestamp.pyx b/pandas/_libs/tslibs/_timestamp.pyx index 14c3e7278c4a1..2cf01ce87b92a 100644 --- a/pandas/_libs/tslibs/_timestamp.pyx +++ b/pandas/_libs/tslibs/_timestamp.pyx @@ -1,10 +1,16 @@ # -*- coding: utf-8 -*- """ -_Timestamp is in its own module to prevent the circular imports -that would happen if it was included in timestamps.pyx +_Timestamp is a c-defined subclass of datetime.datetime + +It is separate from timestamps.pyx to prevent circular cimports This allows _Timestamp to be imported in other modules so that isinstance(obj, _Timestamp) checks can be performed + +_Timestamp is PITA. Because we inherit from datetime, which has very specific +construction requirements, we need to do object instantiation in python +(see Timestamp class below). This will serve as a C extension type that +shadows the python class, where we do any heavy lifting. """ import warnings @@ -49,10 +55,6 @@ def maybe_integer_op_deprecated(obj): FutureWarning) -# This is PITA. Because we inherit from datetime, which has very specific -# construction requirements, we need to do object instantiation in python -# (see Timestamp class below). This will serve as a C extension type that -# shadows the python class, where we do any heavy lifting. cdef class _Timestamp(datetime): def __hash__(_Timestamp self): diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index 95cf1e68d67f3..f78405fa8bf91 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2359,8 +2359,12 @@ def test_dt_subclass_add_timedelta(): # Timedelta operations class SubDatetime(datetime): pass + dt = SubDatetime(2000, 1, 1) td = Timedelta(hours=1) result = dt + td expected = SubDatetime(2000, 1, 1, 1) assert result == expected + + result = td + dt + assert result == expected diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 036313f3c7d0a..38dcfefaccbc4 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -609,6 +609,7 @@ def test_constructor_subclassed_datetime(self): # Timestamp creation class SubDatetime(datetime): pass + data = SubDatetime(2000, 1, 1) result = Timestamp(data) expected = Timestamp(2000, 1, 1) From 6a66238627769d4706da05f1190893a99efbf975 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 30 Mar 2019 16:52:58 -0400 Subject: [PATCH 14/25] parameterize test --- pandas/tests/arithmetic/test_datetime64.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pandas/tests/arithmetic/test_datetime64.py b/pandas/tests/arithmetic/test_datetime64.py index f78405fa8bf91..c1452bda4be9d 100644 --- a/pandas/tests/arithmetic/test_datetime64.py +++ b/pandas/tests/arithmetic/test_datetime64.py @@ -2353,18 +2353,20 @@ def test_shift_months(years, months): tm.assert_index_equal(actual, expected) -def test_dt_subclass_add_timedelta(): +class SubDatetime(datetime): + pass + + +@pytest.mark.parametrize("lh,rh", [ + (SubDatetime(2000, 1, 1), + Timedelta(hours=1)), + (Timedelta(hours=1), + SubDatetime(2000, 1, 1)) +]) +def test_dt_subclass_add_timedelta(lh, rh): # GH 25851 # ensure that subclassed datetime works for # Timedelta operations - class SubDatetime(datetime): - pass - - dt = SubDatetime(2000, 1, 1) - td = Timedelta(hours=1) - result = dt + td + result = lh + rh expected = SubDatetime(2000, 1, 1, 1) assert result == expected - - result = td + dt - assert result == expected From 7da7684017372832b66e8d80fd227a4692e1f39a Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 30 Mar 2019 21:07:18 -0400 Subject: [PATCH 15/25] PR updates --- pandas/_libs/tslibs/__init__.py | 1 + pandas/_libs/tslibs/_timestamp.pyx | 12 ++---------- pandas/_libs/tslibs/timestamps.pyx | 16 +++++++++------- pandas/core/arrays/datetimelike.py | 2 +- pandas/tests/tslibs/test_conversion.py | 2 +- pandas/tests/tslibs/test_normalize_date.py | 9 +++------ 6 files changed, 17 insertions(+), 25 deletions(-) diff --git a/pandas/_libs/tslibs/__init__.py b/pandas/_libs/tslibs/__init__.py index a21fdf95559e6..367bce239b1b6 100644 --- a/pandas/_libs/tslibs/__init__.py +++ b/pandas/_libs/tslibs/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa +from ._timestamp import maybe_integer_op_deprecated from .conversion import normalize_date, localize_pydatetime, tz_convert_single from .nattype import NaT, NaTType, iNaT, is_null_datetimelike from .np_datetime import OutOfBoundsDatetime diff --git a/pandas/_libs/tslibs/_timestamp.pyx b/pandas/_libs/tslibs/_timestamp.pyx index 2cf01ce87b92a..eff37e7163a47 100644 --- a/pandas/_libs/tslibs/_timestamp.pyx +++ b/pandas/_libs/tslibs/_timestamp.pyx @@ -300,8 +300,9 @@ cdef class _Timestamp(datetime): # scalar Timestamp/datetime - Timestamp/datetime -> yields a # Timedelta + from pandas._libs.tslibs.timedeltas import Timedelta try: - return self._create_timedelta(self.value - other.value) + return Timedelta(self.value - other.value) except (OverflowError, OutOfBoundsDatetime): pass @@ -405,15 +406,6 @@ cdef class _Timestamp(datetime): """ return np.datetime64(self.value, 'ns') - @property - def resolution(self): - """ - Return resolution describing the smallest difference between two - times that can be represented by Timestamp object_state - """ - # GH#21336, GH#21365 - return self._create_timedelta(nanoseconds=1) - def timestamp(self): """Return POSIX timestamp as float.""" # py27 compat, see GH#17329 diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 625b6e2f77618..88f430692b468 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -751,6 +751,15 @@ class Timestamp(_Timestamp): """ return bool(ccalendar.is_leapyear(self.year)) + @property + def resolution(self): + """ + Return resolution describing the smallest difference between two + times that can be represented by Timestamp object_state + """ + # GH#21336, GH#21365 + return Timedelta(nanoseconds=1) + def tz_localize(self, tz, ambiguous='raise', nonexistent='raise', errors=None): """ @@ -1042,13 +1051,6 @@ class Timestamp(_Timestamp): # define it here instead return self + other - def _create_timedelta(self, *args, **kwargs): - """ - Helper to create a Timedelta so that the - Timedelta class doesn't have to be imported elsewhere - """ - return Timedelta(*args, **kwargs) - # Add the min and max fields at the class level cdef int64_t _NS_UPPER_BOUND = np.iinfo(np.int64).max diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 57382fd714dd2..541ce04046afb 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -7,7 +7,7 @@ import numpy as np from pandas._libs import NaT, NaTType, Timestamp, algos, iNaT, lib -from pandas._libs.tslibs._timestamp import maybe_integer_op_deprecated +from pandas._libs.tslibs import maybe_integer_op_deprecated from pandas._libs.tslibs.period import ( DIFFERENT_FREQ, IncompatibleFrequency, Period) from pandas._libs.tslibs.timedeltas import Timedelta, delta_to_nanoseconds diff --git a/pandas/tests/tslibs/test_conversion.py b/pandas/tests/tslibs/test_conversion.py index 61ac4269efc67..d909c981c3981 100644 --- a/pandas/tests/tslibs/test_conversion.py +++ b/pandas/tests/tslibs/test_conversion.py @@ -82,7 +82,7 @@ class SubDatetime(datetime): id="datetime"), pytest.param(SubDatetime(2000, 1, 1), SubDatetime(2000, 1, 1, tzinfo=UTC), - id="subdatetime")]) + id="subclassed_datetime")]) def test_localize_pydatetime_dt_types(dt, expected): # GH 25851 # ensure that subclassed datetime works with diff --git a/pandas/tests/tslibs/test_normalize_date.py b/pandas/tests/tslibs/test_normalize_date.py index 36404526ef00a..61a07a3f8a4ba 100644 --- a/pandas/tests/tslibs/test_normalize_date.py +++ b/pandas/tests/tslibs/test_normalize_date.py @@ -24,12 +24,9 @@ class SubDatetime(datetime): @pytest.mark.parametrize("dt, expected", [ - pytest.param(Timestamp(2000, 1, 1, 1), - Timestamp(2000, 1, 1, 0)), - pytest.param(datetime(2000, 1, 1, 1), - datetime(2000, 1, 1, 0)), - pytest.param(SubDatetime(2000, 1, 1, 1), - SubDatetime(2000, 1, 1, 0))]) + (Timestamp(2000, 1, 1, 1), Timestamp(2000, 1, 1, 0)), + (datetime(2000, 1, 1, 1), datetime(2000, 1, 1, 0)), + (SubDatetime(2000, 1, 1, 1), SubDatetime(2000, 1, 1, 0))]) def test_normalize_date_sub_types(dt, expected): # GH 25851 # ensure that subclassed datetime works with From dc0f806ee8875aeec32ea7e04271ef188563992b Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sat, 30 Mar 2019 21:35:31 -0400 Subject: [PATCH 16/25] fix namespace test for new api --- pandas/tests/tslibs/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/tslibs/test_api.py b/pandas/tests/tslibs/test_api.py index dda32befda51b..5ab679fb2439e 100644 --- a/pandas/tests/tslibs/test_api.py +++ b/pandas/tests/tslibs/test_api.py @@ -27,6 +27,7 @@ def test_namespace(): 'iNaT', 'is_null_datetimelike', 'OutOfBoundsDatetime', + 'maybe_integer_op_deprecated' 'Period', 'IncompatibleFrequency', 'Timedelta', From 0cad612ff54b0a3bb99d9c45a27567202b4277f3 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Sun, 31 Mar 2019 08:09:20 -0400 Subject: [PATCH 17/25] fix typo --- pandas/tests/tslibs/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/tslibs/test_api.py b/pandas/tests/tslibs/test_api.py index 5ab679fb2439e..76f480353d501 100644 --- a/pandas/tests/tslibs/test_api.py +++ b/pandas/tests/tslibs/test_api.py @@ -27,7 +27,7 @@ def test_namespace(): 'iNaT', 'is_null_datetimelike', 'OutOfBoundsDatetime', - 'maybe_integer_op_deprecated' + 'maybe_integer_op_deprecated', 'Period', 'IncompatibleFrequency', 'Timedelta', From ee9d2d50eb7f059df32c66424eb98181355ec2d5 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Thu, 4 Apr 2019 21:17:37 -0400 Subject: [PATCH 18/25] merge changes into _timestamp --- pandas/_libs/tslibs/_timestamp.pyx | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/pandas/_libs/tslibs/_timestamp.pyx b/pandas/_libs/tslibs/_timestamp.pyx index eff37e7163a47..b6e210a9f8565 100644 --- a/pandas/_libs/tslibs/_timestamp.pyx +++ b/pandas/_libs/tslibs/_timestamp.pyx @@ -88,26 +88,13 @@ cdef class _Timestamp(datetime): if is_datetime64_object(other): other = self.__class__(other) else: - if op == Py_EQ: - return False - elif op == Py_NE: - return True - - # only allow ==, != ops - raise TypeError('Cannot compare type %r with type %r' % - (type(self).__name__, - type(other).__name__)) + return NotImplemented elif is_array(other): # avoid recursion error GH#15183 return PyObject_RichCompare(np.array([self]), other, op) return PyObject_RichCompare(other, self, reverse_ops[op]) else: - if op == Py_EQ: - return False - elif op == Py_NE: - return True - raise TypeError('Cannot compare type %r with type %r' % - (type(self).__name__, type(other).__name__)) + return NotImplemented self._assert_tzawareness_compat(other) return cmp_scalar(self.value, ots.value, op) From 560ef6f593d0143647d3f60cf7d57977fc9df5b8 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Thu, 4 Apr 2019 21:24:49 -0400 Subject: [PATCH 19/25] change _timestamp files to c_timestamp --- pandas/_libs/tslib.pyx | 2 +- pandas/_libs/tslibs/__init__.py | 2 +- pandas/_libs/tslibs/{_timestamp.pxd => c_timestamp.pxd} | 0 pandas/_libs/tslibs/{_timestamp.pyx => c_timestamp.pyx} | 0 pandas/_libs/tslibs/conversion.pyx | 2 +- pandas/_libs/tslibs/timedeltas.pyx | 2 +- pandas/_libs/tslibs/timestamps.pyx | 2 +- pandas/tests/tslibs/test_api.py | 2 +- setup.py | 6 +++--- 9 files changed, 9 insertions(+), 9 deletions(-) rename pandas/_libs/tslibs/{_timestamp.pxd => c_timestamp.pxd} (100%) rename pandas/_libs/tslibs/{_timestamp.pyx => c_timestamp.pyx} (100%) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index c57836894d8ca..111b3bfe5debf 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -18,7 +18,7 @@ import pytz from pandas._libs.util cimport ( is_integer_object, is_float_object, is_datetime64_object) -from pandas._libs.tslibs._timestamp cimport _Timestamp +from pandas._libs.tslibs.c_timestamp cimport _Timestamp from pandas._libs.tslibs.np_datetime cimport ( check_dts_bounds, npy_datetimestruct, _string_to_dts, dt64_to_dtstruct, diff --git a/pandas/_libs/tslibs/__init__.py b/pandas/_libs/tslibs/__init__.py index 367bce239b1b6..d4e9502c23dbb 100644 --- a/pandas/_libs/tslibs/__init__.py +++ b/pandas/_libs/tslibs/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # flake8: noqa -from ._timestamp import maybe_integer_op_deprecated +from .c_timestamp import maybe_integer_op_deprecated from .conversion import normalize_date, localize_pydatetime, tz_convert_single from .nattype import NaT, NaTType, iNaT, is_null_datetimelike from .np_datetime import OutOfBoundsDatetime diff --git a/pandas/_libs/tslibs/_timestamp.pxd b/pandas/_libs/tslibs/c_timestamp.pxd similarity index 100% rename from pandas/_libs/tslibs/_timestamp.pxd rename to pandas/_libs/tslibs/c_timestamp.pxd diff --git a/pandas/_libs/tslibs/_timestamp.pyx b/pandas/_libs/tslibs/c_timestamp.pyx similarity index 100% rename from pandas/_libs/tslibs/_timestamp.pyx rename to pandas/_libs/tslibs/c_timestamp.pyx diff --git a/pandas/_libs/tslibs/conversion.pyx b/pandas/_libs/tslibs/conversion.pyx index db10f1de4f032..76d51726e1342 100644 --- a/pandas/_libs/tslibs/conversion.pyx +++ b/pandas/_libs/tslibs/conversion.pyx @@ -16,7 +16,7 @@ from cpython.datetime cimport (datetime, tzinfo, PyDateTime_IMPORT, PyDelta_Check) PyDateTime_IMPORT -from pandas._libs.tslibs._timestamp cimport _Timestamp +from pandas._libs.tslibs.c_timestamp cimport _Timestamp from pandas._libs.tslibs.ccalendar import DAY_SECONDS, HOUR_SECONDS diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index becbd27140a40..8b71d64db26c6 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -25,7 +25,7 @@ from pandas._libs.tslibs.util cimport ( is_timedelta64_object, is_datetime64_object, is_integer_object, is_float_object) -from pandas._libs.tslibs._timestamp cimport _Timestamp +from pandas._libs.tslibs.c_timestamp cimport _Timestamp from pandas._libs.tslibs.ccalendar import DAY_SECONDS diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 370d03cb33e5b..94a92494ee321 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -15,7 +15,7 @@ PyDateTime_IMPORT from pandas._libs.tslibs.util cimport ( is_integer_object, is_offset_object) -from pandas._libs.tslibs._timestamp cimport _Timestamp +from pandas._libs.tslibs.c_timestamp cimport _Timestamp cimport pandas._libs.tslibs.ccalendar as ccalendar from pandas._libs.tslibs.ccalendar import DAY_SECONDS from pandas._libs.tslibs.conversion import ( diff --git a/pandas/tests/tslibs/test_api.py b/pandas/tests/tslibs/test_api.py index 76f480353d501..f636ea3a4c9ed 100644 --- a/pandas/tests/tslibs/test_api.py +++ b/pandas/tests/tslibs/test_api.py @@ -6,7 +6,7 @@ def test_namespace(): - submodules = ['_timestamp', + submodules = ['c_timestamp', 'ccalendar', 'conversion', 'fields', diff --git a/setup.py b/setup.py index ffb9ce0d5f086..d6b418211a384 100755 --- a/setup.py +++ b/setup.py @@ -312,7 +312,7 @@ class CheckSDist(sdist_class): 'pandas/_libs/sparse.pyx', 'pandas/_libs/ops.pyx', 'pandas/_libs/parsers.pyx', - 'pandas/_libs/tslibs/_timestamp.pyx', + 'pandas/_libs/tslibs/c_timestamp.pyx', 'pandas/_libs/tslibs/ccalendar.pyx', 'pandas/_libs/tslibs/period.pyx', 'pandas/_libs/tslibs/strptime.pyx', @@ -592,8 +592,8 @@ def srcpath(name=None, suffix='.pyx', subdir='src'): 'include': ts_include, 'depends': tseries_depends, 'sources': np_datetime_sources}, - '_libs.tslibs._timestamp': { - 'pyxfile': '_libs/tslibs/_timestamp', + '_libs.tslibs.c_timestamp': { + 'pyxfile': '_libs/tslibs/c_timestamp', 'include': ts_include, 'depends': tseries_depends, 'sources': np_datetime_sources}, From b87aa99c33256b0726705ac08851430b86c25a94 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Thu, 4 Apr 2019 22:41:37 -0400 Subject: [PATCH 20/25] use new tzconversion.tz_convert_single merged function --- pandas/_libs/tslibs/c_timestamp.pyx | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/pandas/_libs/tslibs/c_timestamp.pyx b/pandas/_libs/tslibs/c_timestamp.pyx index b6e210a9f8565..67d9092ceb0d5 100644 --- a/pandas/_libs/tslibs/c_timestamp.pyx +++ b/pandas/_libs/tslibs/c_timestamp.pyx @@ -42,6 +42,7 @@ from pandas._libs.tslibs.np_datetime cimport ( from pandas._libs.tslibs.timezones cimport ( get_timezone, get_utcoffset, is_utc, tz_compare) from pandas._libs.tslibs.timezones import UTC +from pandas._libs.tslibs.tzconversion cimport tz_convert_single def maybe_integer_op_deprecated(obj): @@ -301,24 +302,9 @@ cdef class _Timestamp(datetime): """Convert UTC i8 value to local i8 value if tz exists""" cdef: int64_t val - npy_datetimestruct dts - int64_t delta - datetime dt - val = self.value if self.tz is not None and not is_utc(self.tz): - # logic copied from _tz_convert_tzlocal_utc - # to prevent importing tslibs.conversion - dt64_to_dtstruct(val, &dts) - dt = datetime(dts.year, dts.month, dts.day, dts.hour, - dts.min, dts.sec, dts.us) - # get_utcoffset (tz.utcoffset under the hood) only makes - # sense if dt is _wall time_, so convert to wall time - dt = dt.replace(tzinfo=tzutc()) - dt = dt.astimezone(self.tz) - delta = (int(get_utcoffset(self.tz, dt).total_seconds()) - * 1000000000) - val = self.value + delta + val = tz_convert_single(self.value, UTC, self.tz) return val cpdef bint _get_start_end_field(self, str field): From 11a091ab2491a4b78e5bd0b35a03a8fb5d6a5b3c Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 5 Apr 2019 07:33:57 -0400 Subject: [PATCH 21/25] move imports to fix import exceptions --- pandas/_libs/tslibs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/__init__.py b/pandas/_libs/tslibs/__init__.py index 311ab9419dae4..1b763f5a632b4 100644 --- a/pandas/_libs/tslibs/__init__.py +++ b/pandas/_libs/tslibs/__init__.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa -from .c_timestamp import maybe_integer_op_deprecated from .conversion import normalize_date, localize_pydatetime from .nattype import NaT, NaTType, iNaT, is_null_datetimelike from .np_datetime import OutOfBoundsDatetime @@ -9,3 +8,4 @@ from .timestamps import Timestamp from .timedeltas import delta_to_nanoseconds, ints_to_pytimedelta, Timedelta from .tzconversion import tz_convert_single +from .c_timestamp import maybe_integer_op_deprecated # isort:skip From 48965689219a48a01015f3e705655f71cef04c30 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 5 Apr 2019 10:18:26 -0400 Subject: [PATCH 22/25] change imports --- pandas/_libs/tslibs/__init__.py | 1 - pandas/core/arrays/datetimelike.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/_libs/tslibs/__init__.py b/pandas/_libs/tslibs/__init__.py index 1b763f5a632b4..83f3542b42de1 100644 --- a/pandas/_libs/tslibs/__init__.py +++ b/pandas/_libs/tslibs/__init__.py @@ -8,4 +8,3 @@ from .timestamps import Timestamp from .timedeltas import delta_to_nanoseconds, ints_to_pytimedelta, Timedelta from .tzconversion import tz_convert_single -from .c_timestamp import maybe_integer_op_deprecated # isort:skip diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 541ce04046afb..6225dfcbe5c14 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -7,7 +7,7 @@ import numpy as np from pandas._libs import NaT, NaTType, Timestamp, algos, iNaT, lib -from pandas._libs.tslibs import maybe_integer_op_deprecated +from pandas._libs.tslibs.c_timestamp import maybe_integer_op_deprecated from pandas._libs.tslibs.period import ( DIFFERENT_FREQ, IncompatibleFrequency, Period) from pandas._libs.tslibs.timedeltas import Timedelta, delta_to_nanoseconds From 43ec600b6e4b380216fe11c385304c07709c40f6 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 5 Apr 2019 10:49:39 -0400 Subject: [PATCH 23/25] fix api test --- pandas/tests/tslibs/test_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/tslibs/test_api.py b/pandas/tests/tslibs/test_api.py index 7cc0cfc1892f4..a69b4761dc414 100644 --- a/pandas/tests/tslibs/test_api.py +++ b/pandas/tests/tslibs/test_api.py @@ -28,7 +28,6 @@ def test_namespace(): 'iNaT', 'is_null_datetimelike', 'OutOfBoundsDatetime', - 'maybe_integer_op_deprecated', 'Period', 'IncompatibleFrequency', 'Timedelta', From 9795807701b48f7dc5290c00bcda97be84f5a7c5 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 5 Apr 2019 11:32:49 -0400 Subject: [PATCH 24/25] force rebuild --- pandas/tests/tslibs/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/tslibs/test_api.py b/pandas/tests/tslibs/test_api.py index a69b4761dc414..b8ec16e44a5c6 100644 --- a/pandas/tests/tslibs/test_api.py +++ b/pandas/tests/tslibs/test_api.py @@ -23,6 +23,7 @@ def test_namespace(): 'timezones', 'tzconversion'] + api = ['NaT', 'NaTType', 'iNaT', From ded5e69fba8fe9cf9b2237a73be9f820ab742492 Mon Sep 17 00:00:00 2001 From: ArtificialQualia Date: Fri, 5 Apr 2019 11:33:04 -0400 Subject: [PATCH 25/25] force rebuild --- pandas/tests/tslibs/test_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/tslibs/test_api.py b/pandas/tests/tslibs/test_api.py index b8ec16e44a5c6..a69b4761dc414 100644 --- a/pandas/tests/tslibs/test_api.py +++ b/pandas/tests/tslibs/test_api.py @@ -23,7 +23,6 @@ def test_namespace(): 'timezones', 'tzconversion'] - api = ['NaT', 'NaTType', 'iNaT',