From 1910c7dafb1ac9a6a47346d4867627fc21c97934 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 07:09:38 -0600 Subject: [PATCH 01/18] Apply diff --- pandas/core/arrays/datetimes.py | 17 +++++++++++++---- pandas/core/indexes/datetimes.py | 3 ++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index efa1757a989fc..385ce7c8b1429 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -329,13 +329,15 @@ def _simple_new(cls, values, freq=None, dtype=None): @classmethod def _from_sequence(cls, data, dtype=None, copy=False, tz=None, freq=None, - dayfirst=False, yearfirst=False, ambiguous='raise'): + dayfirst=False, yearfirst=False, ambiguous='raise', + int_as_wall_time=False): freq, freq_infer = dtl.maybe_infer_freq(freq) subarr, tz, inferred_freq = sequence_to_dt64ns( data, dtype=dtype, copy=copy, tz=tz, - dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous) + dayfirst=dayfirst, yearfirst=yearfirst, + ambiguous=ambiguous, int_as_wall_time=int_as_wall_time) freq, freq_infer = dtl.validate_inferred_freq(freq, inferred_freq, freq_infer) @@ -1634,9 +1636,11 @@ def to_julian_date(self): # ------------------------------------------------------------------- # Constructor Helpers + def sequence_to_dt64ns(data, dtype=None, copy=False, tz=None, - dayfirst=False, yearfirst=False, ambiguous='raise'): + dayfirst=False, yearfirst=False, ambiguous='raise', + int_as_wall_time=False): """ Parameters ---------- @@ -1662,7 +1666,6 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, ------ TypeError : PeriodDType data is passed """ - inferred_freq = None dtype = _validate_dt64_dtype(dtype) @@ -1733,6 +1736,12 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, # assume this data are epoch timestamps if data.dtype != _INT64_DTYPE: data = data.astype(np.int64, copy=False) + if int_as_wall_time and tz is not None: + warnings.warn("test", FutureWarning) + tz = timezones.maybe_get_tz(tz) + data = conversion.tz_localize_to_utc(data.view('i8'), tz, + ambiguous=ambiguous) + data = data.view(_NS_DTYPE) result = data.view(_NS_DTYPE) if copy: diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 664ca9c5d2f05..a4e058160e567 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -299,7 +299,8 @@ def __new__(cls, data=None, dtarr = DatetimeArray._from_sequence( data, dtype=dtype, copy=copy, tz=tz, freq=freq, - dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous) + dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous, + int_as_wall_time=True) subarr = cls._simple_new(dtarr, name=name, freq=dtarr.freq, tz=dtarr.tz) From 8c7357f08ff27d3c94b365df776c567b95873cf3 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 08:14:12 -0600 Subject: [PATCH 02/18] Updates --- pandas/core/arrays/datetimes.py | 1 + pandas/tests/indexes/datetimes/test_astype.py | 15 ++++++++++-- .../indexes/datetimes/test_construction.py | 18 +++++++++++---- pandas/tests/indexes/multi/test_integrity.py | 3 ++- pandas/tests/indexes/test_base.py | 23 ++++++++++++------- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 385ce7c8b1429..48fba8bf125d4 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -1707,6 +1707,7 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, data, inferred_tz = objects_to_datetime64ns( data, dayfirst=dayfirst, yearfirst=yearfirst) tz = maybe_infer_tz(tz, inferred_tz) + int_as_wall_time = False # `data` may have originally been a Categorical[datetime64[ns, tz]], # so we need to handle these types. diff --git a/pandas/tests/indexes/datetimes/test_astype.py b/pandas/tests/indexes/datetimes/test_astype.py index 784d1ca6fb82c..a1c2d78f014c7 100644 --- a/pandas/tests/indexes/datetimes/test_astype.py +++ b/pandas/tests/indexes/datetimes/test_astype.py @@ -240,8 +240,19 @@ def _check_rng(rng): def test_integer_index_astype_datetime(self, tz, dtype): # GH 20997, 20964 val = [pd.Timestamp('2018-01-01', tz=tz).value] - result = pd.Index(val).astype(dtype) - expected = pd.DatetimeIndex(['2018-01-01'], tz=tz) + + if tz: + ex_warn = FutureWarning + else: + ex_warn = None + + with tm.assert_produces_warning(ex_warn, check_stacklevel=False): + # XXX: This likely shouldn't warn. + # This raised on 0.24.x, so we can probably do the right thing + # now. + result = pd.Index(val).astype(dtype) + with tm.assert_produces_warning(ex_warn, check_stacklevel=False): + expected = pd.DatetimeIndex(val, tz=tz) tm.assert_index_equal(result, expected) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 07c42afe44b33..e0bb52e4293de 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -118,12 +118,15 @@ def test_construction_with_alt_tz_localize(self, kwargs, tz_aware_fixture): tz = tz_aware_fixture i = pd.date_range('20130101', periods=5, freq='H', tz=tz) kwargs = {key: attrgetter(val)(i) for key, val in kwargs.items()} - result = DatetimeIndex(i.tz_localize(None).asi8, **kwargs) - expected = i.tz_localize(None).tz_localize('UTC').tz_convert(tz) + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + result = DatetimeIndex(i.tz_localize(None).asi8, **kwargs) + expected = DatetimeIndex(i, **kwargs) + # expected = i.tz_localize(None).tz_localize('UTC').tz_convert(tz) tm.assert_index_equal(result, expected) # localize into the provided tz - i2 = DatetimeIndex(i.tz_localize(None).asi8, tz='UTC') + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + i2 = DatetimeIndex(i.tz_localize(None).asi8, tz='UTC') expected = i.tz_localize(None).tz_localize('UTC') tm.assert_index_equal(i2, expected) @@ -559,8 +562,11 @@ def test_constructor_timestamp_near_dst(self): @pytest.mark.parametrize('box', [ np.array, partial(np.array, dtype=object), list]) @pytest.mark.parametrize('tz, dtype', [ - ['US/Pacific', 'datetime64[ns, US/Pacific]'], - [None, 'datetime64[ns]']]) + pytest.param('US/Pacific', 'datetime64[ns, US/Pacific]', + marks=[pytest.mark.xfail(), + pytest.mark.filterwarnings("ignore")]), + [None, 'datetime64[ns]'], + ]) def test_constructor_with_int_tz(self, klass, box, tz, dtype): # GH 20997, 20964 ts = Timestamp('2018-01-01', tz=tz) @@ -568,6 +574,8 @@ def test_constructor_with_int_tz(self, klass, box, tz, dtype): expected = klass([ts]) assert result == expected + @pytest.mark.xfail(reason="TBD", strict=False) # non-strict for tz-naive + @pytest.mark.filterwarnings("ignore::FutureWarning") def test_construction_int_rountrip(self, tz_naive_fixture): # GH 12619 tz = tz_naive_fixture diff --git a/pandas/tests/indexes/multi/test_integrity.py b/pandas/tests/indexes/multi/test_integrity.py index 0efd589902b39..b87c4eeb8d7aa 100644 --- a/pandas/tests/indexes/multi/test_integrity.py +++ b/pandas/tests/indexes/multi/test_integrity.py @@ -50,7 +50,8 @@ def test_values_multiindex_datetimeindex(): # Test to ensure we hit the boxing / nobox part of MI.values ints = np.arange(10 ** 18, 10 ** 18 + 5) naive = pd.DatetimeIndex(ints) - aware = pd.DatetimeIndex(ints, tz='US/Central') + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + aware = pd.DatetimeIndex(ints, tz='US/Central') idx = pd.MultiIndex.from_arrays([naive, aware]) result = idx.values diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 218b1127032f5..f75f0eaa071a2 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -398,27 +398,34 @@ def test_constructor_dtypes_to_timedelta(self, cast_index, vals): @pytest.mark.parametrize("klass", [pd.Index, pd.DatetimeIndex]) def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc, klass): + # TODO: update comment # Test constructing with a datetimetz dtype # .values produces numpy datetimes, so these are considered naive # .asi8 produces integers, so these are considered epoch timestamps index = pd.date_range('2011-01-01', periods=5) arg = getattr(index, attr) - if utc: - index = index.tz_localize('UTC').tz_convert(tz_naive_fixture) - else: - index = index.tz_localize(tz_naive_fixture) + index = index.tz_localize(tz_naive_fixture) dtype = index.dtype - result = klass(arg, tz=tz_naive_fixture) + if tz_naive_fixture and attr == "asi8": + ex_warn = FutureWarning + else: + ex_warn = None + + with tm.assert_produces_warning(ex_warn, check_stacklevel=False): + result = klass(arg, tz=tz_naive_fixture) tm.assert_index_equal(result, index) - result = klass(arg, dtype=dtype) + with tm.assert_produces_warning(ex_warn, check_stacklevel=False): + result = klass(arg, dtype=dtype) tm.assert_index_equal(result, index) - result = klass(list(arg), tz=tz_naive_fixture) + with tm.assert_produces_warning(ex_warn, check_stacklevel=False): + result = klass(list(arg), tz=tz_naive_fixture) tm.assert_index_equal(result, index) - result = klass(list(arg), dtype=dtype) + with tm.assert_produces_warning(ex_warn, check_stacklevel=False): + result = klass(list(arg), dtype=dtype) tm.assert_index_equal(result, index) @pytest.mark.parametrize("attr", ['values', 'asi8']) From 15972209c2357505bd101be3b6b8e693e2327d2b Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 09:36:57 -0600 Subject: [PATCH 03/18] fixup --- pandas/core/reshape/tile.py | 5 ++++- pandas/tests/resample/test_period_index.py | 6 ++++-- pandas/tests/test_base.py | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pandas/core/reshape/tile.py b/pandas/core/reshape/tile.py index 15df0ca2442fa..c107ed51226b0 100644 --- a/pandas/core/reshape/tile.py +++ b/pandas/core/reshape/tile.py @@ -449,7 +449,10 @@ def _convert_bin_to_datelike_type(bins, dtype): bins : Array-like of bins, DatetimeIndex or TimedeltaIndex if dtype is datelike """ - if is_datetime64tz_dtype(dtype) or is_datetime_or_timedelta_dtype(dtype): + if is_datetime64tz_dtype(dtype): + bins = to_datetime(bins.astype(np.int64), + utc=True).tz_convert(dtype.tz) + elif is_datetime_or_timedelta_dtype(dtype): bins = Index(bins.astype(np.int64), dtype=dtype) return bins diff --git a/pandas/tests/resample/test_period_index.py b/pandas/tests/resample/test_period_index.py index 7b57a280c56fc..c2fbb5bbb088c 100644 --- a/pandas/tests/resample/test_period_index.py +++ b/pandas/tests/resample/test_period_index.py @@ -517,8 +517,10 @@ def test_resample_weekly_bug_1726(self): def test_resample_with_dst_time_change(self): # GH 15549 - index = pd.DatetimeIndex([1457537600000000000, 1458059600000000000], - tz='UTC').tz_convert('America/Chicago') + index = ( + pd.DatetimeIndex([1457537600000000000, 1458059600000000000]) + .tz_localize("UTC").tz_convert('America/Chicago') + ) df = pd.DataFrame([1, 2], index=index) result = df.resample('12h', closed='right', label='right').last().ffill() diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index 657f5f193c85e..763be867c86c8 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -1037,6 +1037,7 @@ class TestToIterable(object): lambda x: list(x.__iter__()), ], ids=['tolist', 'to_list', 'list', 'iter']) @pytest.mark.parametrize('typ', [Series, Index]) + @pytest.mark.filterwarnings("ignore::FutureWarning") def test_iterable(self, typ, method, dtype, rdtype): # gh-10904 # gh-13258 From 0eaefb18de178de554062d6a25d56fcd878c8ad4 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 12:27:41 -0600 Subject: [PATCH 04/18] more tests --- pandas/core/arrays/datetimes.py | 19 ++++++++++++++++++- pandas/tests/dtypes/test_common.py | 14 +++++++------- pandas/tests/indexes/datetimes/test_astype.py | 4 ++-- .../indexes/datetimes/test_construction.py | 7 +++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 48fba8bf125d4..7bbf043f59ccf 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -33,6 +33,20 @@ from pandas.tseries.offsets import Day, Tick _midnight = time(0, 0) +_i8_message = ( + "Passing integer-dtype data and a timezone to DatetimeIndex.\n" + "Integer values will be interpreted differently in a future version \n" + "of pandas. Previously, these were viewed as datetime64[ns] values, \n" + "representing the wall time *in the specified timezone*. \n" + "In the future, these will be viewed as datetime64[ns] values \n" + "representing the wall time *in UTC*. This is similar to a \n" + "nanosecond-precision UNIX epoch.\n\n" + "To accept the future behavior, use\n\n" + "\tpd.to_datetime(integer_data, utc=True).tz_convert(tz) " + "\n\n" + "To keep the previous behavior, use \n\n" + "\tpd.to_datetime(integer_data).tz_localize(tz)" +) def tz_to_dtype(tz): @@ -1707,6 +1721,9 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, data, inferred_tz = objects_to_datetime64ns( data, dayfirst=dayfirst, yearfirst=yearfirst) tz = maybe_infer_tz(tz, inferred_tz) + # When a sequence of timestamp objects is passed, we always + # want to treat the (now i8-valued) data as UTC timestamps, + # not wall times. int_as_wall_time = False # `data` may have originally been a Categorical[datetime64[ns, tz]], @@ -1738,8 +1755,8 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, if data.dtype != _INT64_DTYPE: data = data.astype(np.int64, copy=False) if int_as_wall_time and tz is not None: - warnings.warn("test", FutureWarning) tz = timezones.maybe_get_tz(tz) + warnings.warn(_i8_message, FutureWarning, stacklevel=4) data = conversion.tz_localize_to_utc(data.view('i8'), tz, ambiguous=ambiguous) data = data.view(_NS_DTYPE) diff --git a/pandas/tests/dtypes/test_common.py b/pandas/tests/dtypes/test_common.py index f0f77b4977610..62e96fd39a759 100644 --- a/pandas/tests/dtypes/test_common.py +++ b/pandas/tests/dtypes/test_common.py @@ -209,8 +209,8 @@ def test_is_datetime64tz_dtype(): assert not com.is_datetime64tz_dtype(object) assert not com.is_datetime64tz_dtype([1, 2, 3]) assert not com.is_datetime64tz_dtype(pd.DatetimeIndex([1, 2, 3])) - assert com.is_datetime64tz_dtype(pd.DatetimeIndex( - [1, 2, 3], tz="US/Eastern")) + assert com.is_datetime64tz_dtype(pd.DatetimeIndex(['2000'], + tz="US/Eastern")) def test_is_timedelta64_dtype(): @@ -286,7 +286,7 @@ def test_is_datetimelike(): assert com.is_datetimelike(pd.PeriodIndex([], freq="A")) assert com.is_datetimelike(np.array([], dtype=np.datetime64)) assert com.is_datetimelike(pd.Series([], dtype="timedelta64[ns]")) - assert com.is_datetimelike(pd.DatetimeIndex([1, 2, 3], tz="US/Eastern")) + assert com.is_datetimelike(pd.DatetimeIndex(["2000"], tz="US/Eastern")) dtype = DatetimeTZDtype("ns", tz="US/Eastern") s = pd.Series([], dtype=dtype) @@ -480,7 +480,7 @@ def test_needs_i8_conversion(): assert com.needs_i8_conversion(np.datetime64) assert com.needs_i8_conversion(pd.Series([], dtype="timedelta64[ns]")) assert com.needs_i8_conversion(pd.DatetimeIndex( - [1, 2, 3], tz="US/Eastern")) + ["2000"], tz="US/Eastern")) def test_is_numeric_dtype(): @@ -541,7 +541,7 @@ def test_is_extension_type(check_scipy): assert com.is_extension_type(pd.Series(cat)) assert com.is_extension_type(pd.SparseArray([1, 2, 3])) assert com.is_extension_type(pd.SparseSeries([1, 2, 3])) - assert com.is_extension_type(pd.DatetimeIndex([1, 2, 3], tz="US/Eastern")) + assert com.is_extension_type(pd.DatetimeIndex(['2000'], tz="US/Eastern")) dtype = DatetimeTZDtype("ns", tz="US/Eastern") s = pd.Series([], dtype=dtype) @@ -635,8 +635,8 @@ def test__get_dtype_fails(input_param): (pd.DatetimeIndex([1, 2]), np.datetime64), (pd.DatetimeIndex([1, 2]).dtype, np.datetime64), (' Date: Thu, 10 Jan 2019 13:09:39 -0600 Subject: [PATCH 05/18] astype changes --- pandas/core/indexes/base.py | 9 ++++++++- pandas/tests/indexes/datetimes/test_astype.py | 17 +++-------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index c853a30c0de79..40237333ab8ab 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -22,7 +22,8 @@ is_dtype_union_equal, is_extension_array_dtype, is_float, is_float_dtype, is_hashable, is_integer, is_integer_dtype, is_interval_dtype, is_iterator, is_list_like, is_object_dtype, is_period_dtype, is_scalar, - is_signed_integer_dtype, is_timedelta64_dtype, is_unsigned_integer_dtype) + is_signed_integer_dtype, is_timedelta64_dtype, is_unsigned_integer_dtype, + pandas_dtype) import pandas.core.dtypes.concat as _concat from pandas.core.dtypes.generic import ( ABCDataFrame, ABCDateOffset, ABCDatetimeArray, ABCIndexClass, @@ -732,6 +733,12 @@ def astype(self, dtype, copy=True): from .category import CategoricalIndex return CategoricalIndex(self.values, name=self.name, dtype=dtype, copy=copy) + elif is_datetime64tz_dtype(dtype): + # avoid FutureWarning from DatetimeIndex constructor. + from pandas import DatetimeIndex + tz = pandas_dtype(dtype).tz + return (DatetimeIndex(np.asarray(self)) + .tz_localize("UTC").tz_convert(tz)) elif is_extension_array_dtype(dtype): return Index(np.asarray(self), dtype=dtype, copy=copy) diff --git a/pandas/tests/indexes/datetimes/test_astype.py b/pandas/tests/indexes/datetimes/test_astype.py index e76c1f34ed295..ddf6a6ded69f8 100644 --- a/pandas/tests/indexes/datetimes/test_astype.py +++ b/pandas/tests/indexes/datetimes/test_astype.py @@ -238,21 +238,10 @@ def _check_rng(rng): ['US/Pacific', 'datetime64[ns, US/Pacific]'], [None, 'datetime64[ns]']]) def test_integer_index_astype_datetime(self, tz, dtype): - # GH 20997, 20964 + # GH 20997, 20964, 24559 val = [pd.Timestamp('2018-01-01', tz=tz).value] - - if tz: - ex_warn = FutureWarning - else: - ex_warn = None - - with tm.assert_produces_warning(ex_warn): - # XXX: This likely shouldn't warn. - # This raised on 0.24.x, so we can probably do the right thing - # now. - result = pd.Index(val).astype(dtype) - with tm.assert_produces_warning(ex_warn): - expected = pd.DatetimeIndex(val, tz=tz) + result = pd.Index(val).astype(dtype) + expected = pd.DatetimeIndex(["2018-01-01"], tz=tz) tm.assert_index_equal(result, expected) From efe95e4001095e78d888ede875be259378325fbb Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 14:02:02 -0600 Subject: [PATCH 06/18] Updates * message * utc --- pandas/core/arrays/datetimes.py | 36 ++++++++++--------- .../indexes/datetimes/test_construction.py | 21 ++++++++--- pandas/tests/indexes/test_base.py | 3 +- pandas/tests/test_base.py | 3 +- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 7bbf043f59ccf..6dd61c0b40bb5 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -33,20 +33,22 @@ from pandas.tseries.offsets import Day, Tick _midnight = time(0, 0) -_i8_message = ( - "Passing integer-dtype data and a timezone to DatetimeIndex.\n" - "Integer values will be interpreted differently in a future version \n" - "of pandas. Previously, these were viewed as datetime64[ns] values, \n" - "representing the wall time *in the specified timezone*. \n" - "In the future, these will be viewed as datetime64[ns] values \n" - "representing the wall time *in UTC*. This is similar to a \n" - "nanosecond-precision UNIX epoch.\n\n" - "To accept the future behavior, use\n\n" - "\tpd.to_datetime(integer_data, utc=True).tz_convert(tz) " - "\n\n" - "To keep the previous behavior, use \n\n" - "\tpd.to_datetime(integer_data).tz_localize(tz)" -) +_i8_message = """ + Passing integer-dtype data and a timezone to DatetimeIndex. + Integer values will be interpreted differently in a future version + of pandas. Previously, these were viewed as datetime64[ns] values + representing the wall time *in the specified timezone*. + In the future, these will be viewed as datetime64[ns] values + representing the wall time *in UTC*. This is similar to a + nanosecond-precision UNIX epoch. + To accept the future behavior, use + + pd.to_datetime(integer_data, utc=True).tz_convert(tz) + + To keep the previous behavior, use + + pd.to_datetime(integer_data).tz_localize(tz) +""" def tz_to_dtype(tz): @@ -1752,10 +1754,12 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, else: # must be integer dtype otherwise # assume this data are epoch timestamps + if tz: + tz = timezones.maybe_get_tz(tz) + if data.dtype != _INT64_DTYPE: data = data.astype(np.int64, copy=False) - if int_as_wall_time and tz is not None: - tz = timezones.maybe_get_tz(tz) + if int_as_wall_time and tz is not None and not timezones.is_utc(tz): warnings.warn(_i8_message, FutureWarning, stacklevel=4) data = conversion.tz_localize_to_utc(data.view('i8'), tz, ambiguous=ambiguous) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 55c8653943c7e..54180d1274a6f 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -118,15 +118,20 @@ def test_construction_with_alt_tz_localize(self, kwargs, tz_aware_fixture): tz = tz_aware_fixture i = pd.date_range('20130101', periods=5, freq='H', tz=tz) kwargs = {key: attrgetter(val)(i) for key, val in kwargs.items()} - with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + + if str(tz) in ('UTC', 'tzutc()'): + warn = None + else: + warn = FutureWarning + + with tm.assert_produces_warning(warn, check_stacklevel=False): result = DatetimeIndex(i.tz_localize(None).asi8, **kwargs) expected = DatetimeIndex(i, **kwargs) # expected = i.tz_localize(None).tz_localize('UTC').tz_convert(tz) tm.assert_index_equal(result, expected) # localize into the provided tz - with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): - i2 = DatetimeIndex(i.tz_localize(None).asi8, tz='UTC') + i2 = DatetimeIndex(i.tz_localize(None).asi8, tz='UTC') expected = i.tz_localize(None).tz_localize('UTC') tm.assert_index_equal(i2, expected) @@ -387,6 +392,11 @@ def test_integer_values_and_tz_deprecated(self): expected = pd.DatetimeIndex(['2000-01-01T00:00:00'], tz="US/Central") tm.assert_index_equal(result, expected) + # but UTC is *not* deprecated. + with tm.assert_produces_warning(None): + result = DatetimeIndex(values, tz='UTC') + expected = pd.DatetimeIndex(['2000-01-01T00:00:00'], tz="US/Central") + def test_constructor_coverage(self): rng = date_range('1/1/2000', periods=10.5) exp = date_range('1/1/2000', periods=10) @@ -571,7 +581,8 @@ def test_constructor_timestamp_near_dst(self): @pytest.mark.parametrize('tz, dtype', [ pytest.param('US/Pacific', 'datetime64[ns, US/Pacific]', marks=[pytest.mark.xfail(), - pytest.mark.filterwarnings("ignore")]), + pytest.mark.filterwarnings( + "ignore:Passing:FutureWarning")]), [None, 'datetime64[ns]'], ]) def test_constructor_with_int_tz(self, klass, box, tz, dtype): @@ -582,7 +593,7 @@ def test_constructor_with_int_tz(self, klass, box, tz, dtype): assert result == expected @pytest.mark.xfail(reason="TBD", strict=False) # non-strict for tz-naive - @pytest.mark.filterwarnings("ignore::FutureWarning") + @pytest.mark.filterwarnings("ignore:Passing integer:FutureWarning") def test_construction_int_rountrip(self, tz_naive_fixture): # GH 12619 tz = tz_naive_fixture diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index f75f0eaa071a2..e1501cbf69e32 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -407,7 +407,8 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc, index = index.tz_localize(tz_naive_fixture) dtype = index.dtype - if tz_naive_fixture and attr == "asi8": + if (tz_naive_fixture and attr == "asi8" and + str(tz_naive_fixture) not in ('UTC', 'tzutc()')): ex_warn = FutureWarning else: ex_warn = None diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index 763be867c86c8..dfb2847aef706 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -1037,7 +1037,7 @@ class TestToIterable(object): lambda x: list(x.__iter__()), ], ids=['tolist', 'to_list', 'list', 'iter']) @pytest.mark.parametrize('typ', [Series, Index]) - @pytest.mark.filterwarnings("ignore::FutureWarning") + @pytest.mark.filterwarnings("ignore:Passing integer:FutureWarning") def test_iterable(self, typ, method, dtype, rdtype): # gh-10904 # gh-13258 @@ -1090,6 +1090,7 @@ def test_iterable_items(self, dtype, rdtype): ('object', (int, long)), ('category', (int, long))]) @pytest.mark.parametrize('typ', [Series, Index]) + @pytest.mark.filterwarnings("ignore:Passing integer:FutureWarning") def test_iterable_map(self, typ, dtype, rdtype): # gh-13236 # coerce iteration to underlying python / pandas types From e61726137f91ab669f73399bc2f184d5036bc809 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 14:09:45 -0600 Subject: [PATCH 07/18] whatsnew --- doc/source/whatsnew/v0.24.0.rst | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index aee3d78243d2e..41a42eb452a56 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1352,6 +1352,53 @@ the object's ``freq`` attribute (:issue:`21939`, :issue:`23878`). dti + pd.Index([1 * dti.freq, 2 * dti.freq]) +.. _whatsnew_0240.deprecations.integer_tz: + +Passing Integer data and a timezone to DatetimeIndex +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The behavior of :class:`DatetimeIndex` when passed integer data and +a timezone is changing in a future version of pandas. Previously, these +were interpreted as wall times in the desired timezone. In the future, +these will be interpreted as wall times in UTC, which are then converted +to the desired timezone (:issue:`24559`). + +The default behavior remains the same, but issues a warning: + +.. code-block:: ipython + + In [3]: pd.DatetimeIndex([946684800000000000], tz="US/Central") + /bin/ipython:1: FutureWarning: + Passing integer-dtype data and a timezone to DatetimeIndex. + Integer values will be interpreted differently in a future version + of pandas. Previously, these were viewed as datetime64[ns] values + representing the wall time *in the specified timezone*. + In the future, these will be viewed as datetime64[ns] values + representing the wall time *in UTC*. This is similar to a + nanosecond-precision UNIX epoch. + To accept the future behavior, use + + pd.to_datetime(integer_data, utc=True).tz_convert(tz) + + To keep the previous behavior, use + + pd.to_datetime(integer_data).tz_localize(tz) + + #!/bin/python3 + Out[3]: DatetimeIndex(['2000-01-01 00:00:00-06:00'], dtype='datetime64[ns, US/Central]', freq=None) + +As the warning message explains, opt in to the future behavior with + +.. ipython:: python + + pd.to_datetime([946684800000000000], utc=True).tz_convert('US/Central') + +And the old behavior can be retained with + +.. ipython:: python + + pd.to_datetime([946684800000000000]).tz_localize('US/Central') + .. _whatsnew_0240.deprecations.tz_aware_array: Converting Timezone-Aware Series and Index to NumPy Arrays From 74d7320e4516044aa48cab2b3075d1754894a721 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 15:41:15 -0600 Subject: [PATCH 08/18] filter not working? --- pandas/tests/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index dfb2847aef706..0e3f613ea356c 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -1037,7 +1037,7 @@ class TestToIterable(object): lambda x: list(x.__iter__()), ], ids=['tolist', 'to_list', 'list', 'iter']) @pytest.mark.parametrize('typ', [Series, Index]) - @pytest.mark.filterwarnings("ignore:Passing integer:FutureWarning") + @pytest.mark.filterwarnings("ignore::FutureWarning") def test_iterable(self, typ, method, dtype, rdtype): # gh-10904 # gh-13258 @@ -1090,7 +1090,7 @@ def test_iterable_items(self, dtype, rdtype): ('object', (int, long)), ('category', (int, long))]) @pytest.mark.parametrize('typ', [Series, Index]) - @pytest.mark.filterwarnings("ignore:Passing integer:FutureWarning") + @pytest.mark.filterwarnings("ignore::FutureWarning") def test_iterable_map(self, typ, dtype, rdtype): # gh-13236 # coerce iteration to underlying python / pandas types From 35cd7332783bb0a672ce5f41e80534c57b8df306 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 15:55:53 -0600 Subject: [PATCH 09/18] cleanups, fix bad filter --- pandas/tests/indexes/datetimes/test_construction.py | 5 +++-- pandas/tests/indexes/test_base.py | 5 ++++- pandas/tests/test_base.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 54180d1274a6f..e433021602121 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -592,8 +592,9 @@ def test_constructor_with_int_tz(self, klass, box, tz, dtype): expected = klass([ts]) assert result == expected - @pytest.mark.xfail(reason="TBD", strict=False) # non-strict for tz-naive - @pytest.mark.filterwarnings("ignore:Passing integer:FutureWarning") + # This is the desired future behavior + @pytest.mark.xfail(reason="TBD", strict=False) + @pytest.mark.filterwarnings("ignore:\\n Passing:FutureWarning") def test_construction_int_rountrip(self, tz_naive_fixture): # GH 12619 tz = tz_naive_fixture diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index e1501cbf69e32..3422bd05d9573 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -398,10 +398,11 @@ def test_constructor_dtypes_to_timedelta(self, cast_index, vals): @pytest.mark.parametrize("klass", [pd.Index, pd.DatetimeIndex]) def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc, klass): - # TODO: update comment # Test constructing with a datetimetz dtype # .values produces numpy datetimes, so these are considered naive # .asi8 produces integers, so these are considered epoch timestamps + # ^the above will be true in a later version. Right now we `.view` + # the i8 values as NS_DTYPE, effectively treating them as wall times. index = pd.date_range('2011-01-01', periods=5) arg = getattr(index, attr) index = index.tz_localize(tz_naive_fixture) @@ -413,6 +414,8 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc, else: ex_warn = None + # stacklevel is checked elsewhere. We don't do it here since + # Index will have an frame, throwing off the expected. with tm.assert_produces_warning(ex_warn, check_stacklevel=False): result = klass(arg, tz=tz_naive_fixture) tm.assert_index_equal(result, index) diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index 0e3f613ea356c..09986d2834a79 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -1037,7 +1037,7 @@ class TestToIterable(object): lambda x: list(x.__iter__()), ], ids=['tolist', 'to_list', 'list', 'iter']) @pytest.mark.parametrize('typ', [Series, Index]) - @pytest.mark.filterwarnings("ignore::FutureWarning") + @pytest.mark.filterwarnings("ignore:\\n Passing:FutureWarning") def test_iterable(self, typ, method, dtype, rdtype): # gh-10904 # gh-13258 @@ -1090,7 +1090,7 @@ def test_iterable_items(self, dtype, rdtype): ('object', (int, long)), ('category', (int, long))]) @pytest.mark.parametrize('typ', [Series, Index]) - @pytest.mark.filterwarnings("ignore::FutureWarning") + @pytest.mark.filterwarnings("ignore:\\n Passing:FutureWarning") def test_iterable_map(self, typ, dtype, rdtype): # gh-13236 # coerce iteration to underlying python / pandas types From 5411cabb353e1098ecf432cca57f00bcbf24d386 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 16:11:06 -0600 Subject: [PATCH 10/18] reflow --- doc/source/whatsnew/v0.24.0.rst | 23 +++++++++++------------ pandas/core/arrays/datetimes.py | 14 ++++++-------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 41a42eb452a56..17db23e8c5aef 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1369,31 +1369,30 @@ The default behavior remains the same, but issues a warning: In [3]: pd.DatetimeIndex([946684800000000000], tz="US/Central") /bin/ipython:1: FutureWarning: - Passing integer-dtype data and a timezone to DatetimeIndex. - Integer values will be interpreted differently in a future version - of pandas. Previously, these were viewed as datetime64[ns] values - representing the wall time *in the specified timezone*. - In the future, these will be viewed as datetime64[ns] values - representing the wall time *in UTC*. This is similar to a - nanosecond-precision UNIX epoch. - To accept the future behavior, use + Passing integer-dtype data and a timezone to DatetimeIndex. Integer values + will be interpreted differently in a future version of pandas. Previously, + these were viewed as datetime64[ns] values representing the wall time + *in the specified timezone*. In the future, these will be viewed as + datetime64[ns] values representing the wall time *in UTC*. This is similar + to a nanosecond-precision UNIX epoch. To accept the future behavior, use pd.to_datetime(integer_data, utc=True).tz_convert(tz) To keep the previous behavior, use - pd.to_datetime(integer_data).tz_localize(tz) + pd.to_datetime(integer_data).tz_localize(tz) - #!/bin/python3 + #!/bin/python3 Out[3]: DatetimeIndex(['2000-01-01 00:00:00-06:00'], dtype='datetime64[ns, US/Central]', freq=None) -As the warning message explains, opt in to the future behavior with +As the warning message explains, opt in to the future behavior by specifying that +the integer values are UTC, and then converting to the final timezone: .. ipython:: python pd.to_datetime([946684800000000000], utc=True).tz_convert('US/Central') -And the old behavior can be retained with +The old behavior can be retained with by localizing directly to the final timezone: .. ipython:: python diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 6dd61c0b40bb5..6b8a1f22a8a8f 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -34,14 +34,12 @@ _midnight = time(0, 0) _i8_message = """ - Passing integer-dtype data and a timezone to DatetimeIndex. - Integer values will be interpreted differently in a future version - of pandas. Previously, these were viewed as datetime64[ns] values - representing the wall time *in the specified timezone*. - In the future, these will be viewed as datetime64[ns] values - representing the wall time *in UTC*. This is similar to a - nanosecond-precision UNIX epoch. - To accept the future behavior, use + Passing integer-dtype data and a timezone to DatetimeIndex. Integer values + will be interpreted differently in a future version of pandas. Previously, + these were viewed as datetime64[ns] values representing the wall time + *in the specified timezone*. In the future, these will be viewed as + datetime64[ns] values representing the wall time *in UTC*. This is similar + to a nanosecond-precision UNIX epoch. To accept the future behavior, use pd.to_datetime(integer_data, utc=True).tz_convert(tz) From 2e00630b9e34c4ff7027968358f08dd3971c729d Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 16:19:51 -0600 Subject: [PATCH 11/18] another --- pandas/tests/indexes/datetimes/test_construction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index e433021602121..5ed77072d1930 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -582,7 +582,7 @@ def test_constructor_timestamp_near_dst(self): pytest.param('US/Pacific', 'datetime64[ns, US/Pacific]', marks=[pytest.mark.xfail(), pytest.mark.filterwarnings( - "ignore:Passing:FutureWarning")]), + "ignore:\\n Passing:FutureWarning")]), [None, 'datetime64[ns]'], ]) def test_constructor_with_int_tz(self, klass, box, tz, dtype): From 90afdee22835374128e90e3a741817f3a359b6f3 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 20:24:53 -0600 Subject: [PATCH 12/18] maybe? --- doc/source/whatsnew/v0.24.0.rst | 1 - pandas/tests/generic/test_frame.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 17db23e8c5aef..a9c788f2afabc 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1235,7 +1235,6 @@ Datetimelike API Changes - :class:`PeriodIndex` subtraction of another ``PeriodIndex`` will now return an object-dtype :class:`Index` of :class:`DateOffset` objects instead of raising a ``TypeError`` (:issue:`20049`) - :func:`cut` and :func:`qcut` now returns a :class:`DatetimeIndex` or :class:`TimedeltaIndex` bins when the input is datetime or timedelta dtype respectively and ``retbins=True`` (:issue:`19891`) - :meth:`DatetimeIndex.to_period` and :meth:`Timestamp.to_period` will issue a warning when timezone information will be lost (:issue:`21333`) -- :class:`DatetimeIndex` now accepts :class:`Int64Index` arguments as epoch timestamps (:issue:`20997`) - :meth:`PeriodIndex.tz_convert` and :meth:`PeriodIndex.tz_localize` have been removed (:issue:`21781`) .. _whatsnew_0240.api.other: diff --git a/pandas/tests/generic/test_frame.py b/pandas/tests/generic/test_frame.py index 25440702a339b..ef25b79a8e7f1 100644 --- a/pandas/tests/generic/test_frame.py +++ b/pandas/tests/generic/test_frame.py @@ -183,6 +183,7 @@ def test_set_attribute(self): 'StringIndex', 'UnicodeIndex', 'DateIndex', 'PeriodIndex', 'CategoricalIndex', 'TimedeltaIndex']) + @pytest.mark.filterwarnings("ignore") # TODO: Remove. GH-24716 def test_to_xarray_index_types(self, index): from xarray import Dataset @@ -221,6 +222,7 @@ def test_to_xarray_index_types(self, index): check_index_type=False, check_categorical=False) @td.skip_if_no('xarray', min_version='0.7.0') + @pytest.mark.filterwarnings("ignore") # TODO: Remove. GH-24716 def test_to_xarray(self): from xarray import Dataset From 6633344eced0395e7662311919a3b5b3857c2fa1 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 21:04:15 -0600 Subject: [PATCH 13/18] bye bye py2 --- pandas/tests/indexes/datetimes/test_construction.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 5ed77072d1930..36bd30503b5e8 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -11,7 +11,7 @@ import pandas as pd from pandas import ( - DatetimeIndex, Index, Timestamp, date_range, datetime, offsets, + DatetimeIndex, Index, Timestamp, compat, date_range, datetime, offsets, to_datetime) from pandas.core.arrays import DatetimeArray, period_array import pandas.util.testing as tm @@ -581,6 +581,7 @@ def test_constructor_timestamp_near_dst(self): @pytest.mark.parametrize('tz, dtype', [ pytest.param('US/Pacific', 'datetime64[ns, US/Pacific]', marks=[pytest.mark.xfail(), + pytest.mark.skipif(compat.PY2, reason="warn"), pytest.mark.filterwarnings( "ignore:\\n Passing:FutureWarning")]), [None, 'datetime64[ns]'], From e36d99822a332e9eddd1cd8356aef89877e21b65 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 21:07:02 -0600 Subject: [PATCH 14/18] remove xarray filter --- pandas/tests/generic/test_frame.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pandas/tests/generic/test_frame.py b/pandas/tests/generic/test_frame.py index ef25b79a8e7f1..25440702a339b 100644 --- a/pandas/tests/generic/test_frame.py +++ b/pandas/tests/generic/test_frame.py @@ -183,7 +183,6 @@ def test_set_attribute(self): 'StringIndex', 'UnicodeIndex', 'DateIndex', 'PeriodIndex', 'CategoricalIndex', 'TimedeltaIndex']) - @pytest.mark.filterwarnings("ignore") # TODO: Remove. GH-24716 def test_to_xarray_index_types(self, index): from xarray import Dataset @@ -222,7 +221,6 @@ def test_to_xarray_index_types(self, index): check_index_type=False, check_categorical=False) @td.skip_if_no('xarray', min_version='0.7.0') - @pytest.mark.filterwarnings("ignore") # TODO: Remove. GH-24716 def test_to_xarray(self): from xarray import Dataset From f60aa0cce8b58c7e068a7740d01caf3386f445fe Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 22:26:15 -0600 Subject: [PATCH 15/18] maybe fixed --- pandas/tests/indexes/test_base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 3422bd05d9573..9e8a2e7b8ffdb 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -4,6 +4,7 @@ from datetime import datetime, timedelta from decimal import Decimal import math +import sys import numpy as np import pytest @@ -408,6 +409,9 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc, index = index.tz_localize(tz_naive_fixture) dtype = index.dtype + # not sure what this is from. It's Py2 only. + modules = [sys.modules['pandas.core.indexes.base']] + if (tz_naive_fixture and attr == "asi8" and str(tz_naive_fixture) not in ('UTC', 'tzutc()')): ex_warn = FutureWarning @@ -416,7 +420,8 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc, # stacklevel is checked elsewhere. We don't do it here since # Index will have an frame, throwing off the expected. - with tm.assert_produces_warning(ex_warn, check_stacklevel=False): + with tm.assert_produces_warning(ex_warn, check_stacklevel=False, + clear=modules): result = klass(arg, tz=tz_naive_fixture) tm.assert_index_equal(result, index) From 5806ca7c70fff0744cc19028d2bd89c500409891 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 10 Jan 2019 22:56:46 -0600 Subject: [PATCH 16/18] remove skip --- pandas/tests/indexes/datetimes/test_construction.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 36bd30503b5e8..5ed77072d1930 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -11,7 +11,7 @@ import pandas as pd from pandas import ( - DatetimeIndex, Index, Timestamp, compat, date_range, datetime, offsets, + DatetimeIndex, Index, Timestamp, date_range, datetime, offsets, to_datetime) from pandas.core.arrays import DatetimeArray, period_array import pandas.util.testing as tm @@ -581,7 +581,6 @@ def test_constructor_timestamp_near_dst(self): @pytest.mark.parametrize('tz, dtype', [ pytest.param('US/Pacific', 'datetime64[ns, US/Pacific]', marks=[pytest.mark.xfail(), - pytest.mark.skipif(compat.PY2, reason="warn"), pytest.mark.filterwarnings( "ignore:\\n Passing:FutureWarning")]), [None, 'datetime64[ns]'], From b78f82cdb89dd0f1b4f9b4b006a912d35076433c Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 11 Jan 2019 12:11:33 +0100 Subject: [PATCH 17/18] add docstring --- pandas/core/arrays/datetimes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 6b8a1f22a8a8f..931b6323229ab 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -1650,7 +1650,6 @@ def to_julian_date(self): # ------------------------------------------------------------------- # Constructor Helpers - def sequence_to_dt64ns(data, dtype=None, copy=False, tz=None, dayfirst=False, yearfirst=False, ambiguous='raise', @@ -1666,6 +1665,11 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, yearfirst : bool, default False ambiguous : str, bool, or arraylike, default 'raise' See pandas._libs.tslibs.conversion.tz_localize_to_utc + int_as_wall_time : bool, default False + Whether to treat ints as wall time in specified timezone, or as + nanosecond-precision UNIX epoch (wall time in UTC). + This is used in DatetimeIndex.__init__ to deprecated the wall-time + behaviour. Returns ------- @@ -1680,6 +1684,7 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, ------ TypeError : PeriodDType data is passed """ + inferred_freq = None dtype = _validate_dt64_dtype(dtype) From 8332c40626f001577fa585f695cb1e93e05b6c38 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 11 Jan 2019 06:59:33 -0600 Subject: [PATCH 18/18] fixups --- pandas/core/arrays/datetimes.py | 5 ++++- pandas/core/indexes/base.py | 1 + pandas/tests/indexes/datetimes/test_construction.py | 6 ++++-- pandas/tests/indexes/multi/test_integrity.py | 1 + pandas/tests/indexes/test_base.py | 1 + pandas/tests/test_base.py | 2 ++ 6 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index d92aa31789597..a2d67efbecbba 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -33,6 +33,7 @@ from pandas.tseries.offsets import Day, Tick _midnight = time(0, 0) +# TODO(GH-24559): Remove warning, int_as_wall_time parameter. _i8_message = """ Passing integer-dtype data and a timezone to DatetimeIndex. Integer values will be interpreted differently in a future version of pandas. Previously, @@ -1681,9 +1682,11 @@ def sequence_to_dt64ns(data, dtype=None, copy=False, int_as_wall_time : bool, default False Whether to treat ints as wall time in specified timezone, or as nanosecond-precision UNIX epoch (wall time in UTC). - This is used in DatetimeIndex.__init__ to deprecated the wall-time + This is used in DatetimeIndex.__init__ to deprecate the wall-time behaviour. + ..versionadded:: 0.24.0 + Returns ------- result : numpy.ndarray diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 40237333ab8ab..99114b7dcf34d 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -734,6 +734,7 @@ def astype(self, dtype, copy=True): return CategoricalIndex(self.values, name=self.name, dtype=dtype, copy=copy) elif is_datetime64tz_dtype(dtype): + # TODO(GH-24559): Remove this block, use the following elif. # avoid FutureWarning from DatetimeIndex constructor. from pandas import DatetimeIndex tz = pandas_dtype(dtype).tz diff --git a/pandas/tests/indexes/datetimes/test_construction.py b/pandas/tests/indexes/datetimes/test_construction.py index 5ed77072d1930..2768da0316aad 100644 --- a/pandas/tests/indexes/datetimes/test_construction.py +++ b/pandas/tests/indexes/datetimes/test_construction.py @@ -127,7 +127,6 @@ def test_construction_with_alt_tz_localize(self, kwargs, tz_aware_fixture): with tm.assert_produces_warning(warn, check_stacklevel=False): result = DatetimeIndex(i.tz_localize(None).asi8, **kwargs) expected = DatetimeIndex(i, **kwargs) - # expected = i.tz_localize(None).tz_localize('UTC').tz_convert(tz) tm.assert_index_equal(result, expected) # localize into the provided tz @@ -386,6 +385,7 @@ def test_range_kwargs_deprecated(self): DatetimeIndex(start='1/1/2000', end='1/10/2000', freq='D') def test_integer_values_and_tz_deprecated(self): + # GH-24559 values = np.array([946684800000000000]) with tm.assert_produces_warning(FutureWarning): result = DatetimeIndex(values, tz='US/Central') @@ -575,6 +575,7 @@ def test_constructor_timestamp_near_dst(self): ts[1].to_pydatetime()]) tm.assert_index_equal(result, expected) + # TODO(GH-24559): Remove the xfail for the tz-aware case. @pytest.mark.parametrize('klass', [Index, DatetimeIndex]) @pytest.mark.parametrize('box', [ np.array, partial(np.array, dtype=object), list]) @@ -593,10 +594,11 @@ def test_constructor_with_int_tz(self, klass, box, tz, dtype): assert result == expected # This is the desired future behavior - @pytest.mark.xfail(reason="TBD", strict=False) + @pytest.mark.xfail(reason="Future behavior", strict=False) @pytest.mark.filterwarnings("ignore:\\n Passing:FutureWarning") def test_construction_int_rountrip(self, tz_naive_fixture): # GH 12619 + # TODO(GH-24559): Remove xfail tz = tz_naive_fixture result = 1293858000000000000 expected = DatetimeIndex([1293858000000000000], tz=tz).asi8[0] diff --git a/pandas/tests/indexes/multi/test_integrity.py b/pandas/tests/indexes/multi/test_integrity.py index b87c4eeb8d7aa..c1638a9cde660 100644 --- a/pandas/tests/indexes/multi/test_integrity.py +++ b/pandas/tests/indexes/multi/test_integrity.py @@ -50,6 +50,7 @@ def test_values_multiindex_datetimeindex(): # Test to ensure we hit the boxing / nobox part of MI.values ints = np.arange(10 ** 18, 10 ** 18 + 5) naive = pd.DatetimeIndex(ints) + # TODO(GH-24559): Remove the FutureWarning with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): aware = pd.DatetimeIndex(ints, tz='US/Central') diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index 9e8a2e7b8ffdb..d75800b763cb9 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -409,6 +409,7 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc, index = index.tz_localize(tz_naive_fixture) dtype = index.dtype + # TODO(GH-24559): Remove the sys.modules and warnings # not sure what this is from. It's Py2 only. modules = [sys.modules['pandas.core.indexes.base']] diff --git a/pandas/tests/test_base.py b/pandas/tests/test_base.py index 09986d2834a79..ac365eb87d1bc 100644 --- a/pandas/tests/test_base.py +++ b/pandas/tests/test_base.py @@ -1038,6 +1038,7 @@ class TestToIterable(object): ], ids=['tolist', 'to_list', 'list', 'iter']) @pytest.mark.parametrize('typ', [Series, Index]) @pytest.mark.filterwarnings("ignore:\\n Passing:FutureWarning") + # TODO(GH-24559): Remove the filterwarnings def test_iterable(self, typ, method, dtype, rdtype): # gh-10904 # gh-13258 @@ -1091,6 +1092,7 @@ def test_iterable_items(self, dtype, rdtype): ('category', (int, long))]) @pytest.mark.parametrize('typ', [Series, Index]) @pytest.mark.filterwarnings("ignore:\\n Passing:FutureWarning") + # TODO(GH-24559): Remove the filterwarnings def test_iterable_map(self, typ, dtype, rdtype): # gh-13236 # coerce iteration to underlying python / pandas types