Revert handling of i8values to DatetimeIndex

pandas/core/arrays/datetimes.py

@@ -33,6 +33,20 @@
 from pandas.tseries.offsets import Day, Tick
 
 _midnight = time(0, 0)
+_i8_message = (
+    "Passing integer-dtype data and a timezone to DatetimeIndex.\n"
+    "Integer values will be interpreted differently in a future version \n"
+    "of pandas. Previously, these were viewed as datetime64[ns] values, \n"
+    "representing the wall time *in the specified timezone*. \n"
+    "In the future, these will be viewed as datetime64[ns] values \n"
+    "representing the wall time *in UTC*. This is similar to a \n"
+    "nanosecond-precision UNIX epoch.\n\n"
+    "To accept the future behavior, use\n\n"
+    "\tpd.to_datetime(integer_data, utc=True).tz_convert(tz) "
+    "\n\n"
+    "To keep the previous behavior, use \n\n"
+    "\tpd.to_datetime(integer_data).tz_localize(tz)"
+)
 
 
 def tz_to_dtype(tz):
@@ -329,13 +343,15 @@ def _simple_new(cls, values, freq=None, dtype=None):
     @classmethod
     def _from_sequence(cls, data, dtype=None, copy=False,
                        tz=None, freq=None,
-                       dayfirst=False, yearfirst=False, ambiguous='raise'):
+                       dayfirst=False, yearfirst=False, ambiguous='raise',
+                       int_as_wall_time=False):
 
         freq, freq_infer = dtl.maybe_infer_freq(freq)
 
         subarr, tz, inferred_freq = sequence_to_dt64ns(
             data, dtype=dtype, copy=copy, tz=tz,
-            dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous)
+            dayfirst=dayfirst, yearfirst=yearfirst,
+            ambiguous=ambiguous, int_as_wall_time=int_as_wall_time)
 
         freq, freq_infer = dtl.validate_inferred_freq(freq, inferred_freq,
                                                       freq_infer)
@@ -1634,9 +1650,11 @@ def to_julian_date(self):
 # -------------------------------------------------------------------
 # Constructor Helpers
 
+
 def sequence_to_dt64ns(data, dtype=None, copy=False,
                        tz=None,
-                       dayfirst=False, yearfirst=False, ambiguous='raise'):
+                       dayfirst=False, yearfirst=False, ambiguous='raise',
+                       int_as_wall_time=False):
     """
     Parameters
     ----------
@@ -1662,7 +1680,6 @@ def sequence_to_dt64ns(data, dtype=None, copy=False,
     ------
     TypeError : PeriodDType data is passed
     """
-
     inferred_freq = None
 
     dtype = _validate_dt64_dtype(dtype)
@@ -1704,6 +1721,10 @@ def sequence_to_dt64ns(data, dtype=None, copy=False,
             data, inferred_tz = objects_to_datetime64ns(
                 data, dayfirst=dayfirst, yearfirst=yearfirst)
             tz = maybe_infer_tz(tz, inferred_tz)
+            # When a sequence of timestamp objects is passed, we always
+            # want to treat the (now i8-valued) data as UTC timestamps,
+            # not wall times.
+            int_as_wall_time = False
 
     # `data` may have originally been a Categorical[datetime64[ns, tz]],
     # so we need to handle these types.
@@ -1733,6 +1754,12 @@ def sequence_to_dt64ns(data, dtype=None, copy=False,
         # assume this data are epoch timestamps
         if data.dtype != _INT64_DTYPE:
             data = data.astype(np.int64, copy=False)
+        if int_as_wall_time and tz is not None:
+            tz = timezones.maybe_get_tz(tz)
+            warnings.warn(_i8_message, FutureWarning, stacklevel=4)
+            data = conversion.tz_localize_to_utc(data.view('i8'), tz,
+                                                 ambiguous=ambiguous)
+            data = data.view(_NS_DTYPE)
         result = data.view(_NS_DTYPE)
 
     if copy:

pandas/core/indexes/base.py

@@ -22,7 +22,8 @@
     is_dtype_union_equal, is_extension_array_dtype, is_float, is_float_dtype,
     is_hashable, is_integer, is_integer_dtype, is_interval_dtype, is_iterator,
     is_list_like, is_object_dtype, is_period_dtype, is_scalar,
-    is_signed_integer_dtype, is_timedelta64_dtype, is_unsigned_integer_dtype)
+    is_signed_integer_dtype, is_timedelta64_dtype, is_unsigned_integer_dtype,
+    pandas_dtype)
 import pandas.core.dtypes.concat as _concat
 from pandas.core.dtypes.generic import (
     ABCDataFrame, ABCDateOffset, ABCDatetimeArray, ABCIndexClass,
@@ -732,6 +733,12 @@ def astype(self, dtype, copy=True):
             from .category import CategoricalIndex
             return CategoricalIndex(self.values, name=self.name, dtype=dtype,
                                     copy=copy)
+        elif is_datetime64tz_dtype(dtype):
+            # avoid FutureWarning from DatetimeIndex constructor.
+            from pandas import DatetimeIndex
+            tz = pandas_dtype(dtype).tz
+            return (DatetimeIndex(np.asarray(self))
+                    .tz_localize("UTC").tz_convert(tz))
 
         elif is_extension_array_dtype(dtype):
             return Index(np.asarray(self), dtype=dtype, copy=copy)

pandas/core/indexes/datetimes.py

@@ -299,7 +299,8 @@ def __new__(cls, data=None,
 
         dtarr = DatetimeArray._from_sequence(
             data, dtype=dtype, copy=copy, tz=tz, freq=freq,
-            dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous)
+            dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous,
+            int_as_wall_time=True)
 
         subarr = cls._simple_new(dtarr, name=name,
                                  freq=dtarr.freq, tz=dtarr.tz)

pandas/core/reshape/tile.py

@@ -449,7 +449,10 @@ def _convert_bin_to_datelike_type(bins, dtype):
     bins : Array-like of bins, DatetimeIndex or TimedeltaIndex if dtype is
            datelike
     """
-    if is_datetime64tz_dtype(dtype) or is_datetime_or_timedelta_dtype(dtype):
+    if is_datetime64tz_dtype(dtype):
+        bins = to_datetime(bins.astype(np.int64),
+                           utc=True).tz_convert(dtype.tz)
+    elif is_datetime_or_timedelta_dtype(dtype):
         bins = Index(bins.astype(np.int64), dtype=dtype)
     return bins
 

pandas/tests/dtypes/test_common.py

@@ -209,8 +209,8 @@ def test_is_datetime64tz_dtype():
     assert not com.is_datetime64tz_dtype(object)
     assert not com.is_datetime64tz_dtype([1, 2, 3])
     assert not com.is_datetime64tz_dtype(pd.DatetimeIndex([1, 2, 3]))
-    assert com.is_datetime64tz_dtype(pd.DatetimeIndex(
-        [1, 2, 3], tz="US/Eastern"))
+    assert com.is_datetime64tz_dtype(pd.DatetimeIndex(['2000'],
+                                                      tz="US/Eastern"))
 
 
 def test_is_timedelta64_dtype():
@@ -286,7 +286,7 @@ def test_is_datetimelike():
     assert com.is_datetimelike(pd.PeriodIndex([], freq="A"))
     assert com.is_datetimelike(np.array([], dtype=np.datetime64))
     assert com.is_datetimelike(pd.Series([], dtype="timedelta64[ns]"))
-    assert com.is_datetimelike(pd.DatetimeIndex([1, 2, 3], tz="US/Eastern"))
+    assert com.is_datetimelike(pd.DatetimeIndex(["2000"], tz="US/Eastern"))
 
     dtype = DatetimeTZDtype("ns", tz="US/Eastern")
     s = pd.Series([], dtype=dtype)
@@ -480,7 +480,7 @@ def test_needs_i8_conversion():
     assert com.needs_i8_conversion(np.datetime64)
     assert com.needs_i8_conversion(pd.Series([], dtype="timedelta64[ns]"))
     assert com.needs_i8_conversion(pd.DatetimeIndex(
-        [1, 2, 3], tz="US/Eastern"))
+        ["2000"], tz="US/Eastern"))
 
 
 def test_is_numeric_dtype():
@@ -541,7 +541,7 @@ def test_is_extension_type(check_scipy):
     assert com.is_extension_type(pd.Series(cat))
     assert com.is_extension_type(pd.SparseArray([1, 2, 3]))
     assert com.is_extension_type(pd.SparseSeries([1, 2, 3]))
-    assert com.is_extension_type(pd.DatetimeIndex([1, 2, 3], tz="US/Eastern"))
+    assert com.is_extension_type(pd.DatetimeIndex(['2000'], tz="US/Eastern"))
 
     dtype = DatetimeTZDtype("ns", tz="US/Eastern")
     s = pd.Series([], dtype=dtype)
@@ -635,8 +635,8 @@ def test__get_dtype_fails(input_param):
     (pd.DatetimeIndex([1, 2]), np.datetime64),
     (pd.DatetimeIndex([1, 2]).dtype, np.datetime64),
     ('<M8[ns]', np.datetime64),
-    (pd.DatetimeIndex([1, 2], tz='Europe/London'), pd.Timestamp),
-    (pd.DatetimeIndex([1, 2], tz='Europe/London').dtype,
+    (pd.DatetimeIndex(['2000'], tz='Europe/London'), pd.Timestamp),
+    (pd.DatetimeIndex(['2000'], tz='Europe/London').dtype,
      pd.Timestamp),
     ('datetime64[ns, Europe/London]', pd.Timestamp),
     (pd.SparseSeries([1, 2], dtype='int32'), np.int32),

pandas/tests/indexes/datetimes/test_astype.py

@@ -238,10 +238,10 @@ def _check_rng(rng):
         ['US/Pacific', 'datetime64[ns, US/Pacific]'],
         [None, 'datetime64[ns]']])
     def test_integer_index_astype_datetime(self, tz, dtype):
-        # GH 20997, 20964
+        # GH 20997, 20964, 24559
         val = [pd.Timestamp('2018-01-01', tz=tz).value]
         result = pd.Index(val).astype(dtype)
-        expected = pd.DatetimeIndex(['2018-01-01'], tz=tz)
+        expected = pd.DatetimeIndex(["2018-01-01"], tz=tz)
         tm.assert_index_equal(result, expected)
 
 

pandas/tests/indexes/datetimes/test_construction.py

@@ -118,12 +118,15 @@ def test_construction_with_alt_tz_localize(self, kwargs, tz_aware_fixture):
         tz = tz_aware_fixture
         i = pd.date_range('20130101', periods=5, freq='H', tz=tz)
         kwargs = {key: attrgetter(val)(i) for key, val in kwargs.items()}
-        result = DatetimeIndex(i.tz_localize(None).asi8, **kwargs)
-        expected = i.tz_localize(None).tz_localize('UTC').tz_convert(tz)
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            result = DatetimeIndex(i.tz_localize(None).asi8, **kwargs)
+        expected = DatetimeIndex(i, **kwargs)
+        # expected = i.tz_localize(None).tz_localize('UTC').tz_convert(tz)
         tm.assert_index_equal(result, expected)
 
         # localize into the provided tz
-        i2 = DatetimeIndex(i.tz_localize(None).asi8, tz='UTC')
+        with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+            i2 = DatetimeIndex(i.tz_localize(None).asi8, tz='UTC')
         expected = i.tz_localize(None).tz_localize('UTC')
         tm.assert_index_equal(i2, expected)
 
@@ -377,6 +380,13 @@ def test_range_kwargs_deprecated(self):
         with tm.assert_produces_warning(FutureWarning):
             DatetimeIndex(start='1/1/2000', end='1/10/2000', freq='D')
 
+    def test_integer_values_and_tz_deprecated(self):
+        values = np.array([946684800000000000])
+        with tm.assert_produces_warning(FutureWarning):
+            result = DatetimeIndex(values, tz='US/Central')
+        expected = pd.DatetimeIndex(['2000-01-01T00:00:00'], tz="US/Central")
+        tm.assert_index_equal(result, expected)
+
     def test_constructor_coverage(self):
         rng = date_range('1/1/2000', periods=10.5)
         exp = date_range('1/1/2000', periods=10)
@@ -559,15 +569,20 @@ def test_constructor_timestamp_near_dst(self):
     @pytest.mark.parametrize('box', [
         np.array, partial(np.array, dtype=object), list])
     @pytest.mark.parametrize('tz, dtype', [
-        ['US/Pacific', 'datetime64[ns, US/Pacific]'],
-        [None, 'datetime64[ns]']])
+        pytest.param('US/Pacific', 'datetime64[ns, US/Pacific]',
+                     marks=[pytest.mark.xfail(),
+                            pytest.mark.filterwarnings("ignore")]),
+        [None, 'datetime64[ns]'],
+    ])
     def test_constructor_with_int_tz(self, klass, box, tz, dtype):
         # GH 20997, 20964
         ts = Timestamp('2018-01-01', tz=tz)
         result = klass(box([ts.value]), dtype=dtype)
         expected = klass([ts])
         assert result == expected
 
+    @pytest.mark.xfail(reason="TBD", strict=False)  # non-strict for tz-naive
+    @pytest.mark.filterwarnings("ignore::FutureWarning")
     def test_construction_int_rountrip(self, tz_naive_fixture):
         # GH 12619
         tz = tz_naive_fixture

pandas/tests/indexes/multi/test_integrity.py

@@ -50,7 +50,8 @@ def test_values_multiindex_datetimeindex():
     # Test to ensure we hit the boxing / nobox part of MI.values
     ints = np.arange(10 ** 18, 10 ** 18 + 5)
     naive = pd.DatetimeIndex(ints)
-    aware = pd.DatetimeIndex(ints, tz='US/Central')
+    with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+        aware = pd.DatetimeIndex(ints, tz='US/Central')
 
     idx = pd.MultiIndex.from_arrays([naive, aware])
     result = idx.values

pandas/tests/indexes/test_base.py

@@ -398,27 +398,34 @@ def test_constructor_dtypes_to_timedelta(self, cast_index, vals):
     @pytest.mark.parametrize("klass", [pd.Index, pd.DatetimeIndex])
     def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, utc,
                                          klass):
+        # TODO: update comment
         # Test constructing with a datetimetz dtype
         # .values produces numpy datetimes, so these are considered naive
         # .asi8 produces integers, so these are considered epoch timestamps
         index = pd.date_range('2011-01-01', periods=5)
         arg = getattr(index, attr)
-        if utc:
-            index = index.tz_localize('UTC').tz_convert(tz_naive_fixture)
-        else:
-            index = index.tz_localize(tz_naive_fixture)
+        index = index.tz_localize(tz_naive_fixture)
         dtype = index.dtype
 
-        result = klass(arg, tz=tz_naive_fixture)
+        if tz_naive_fixture and attr == "asi8":
+            ex_warn = FutureWarning
+        else:
+            ex_warn = None
+
+        with tm.assert_produces_warning(ex_warn, check_stacklevel=False):
+            result = klass(arg, tz=tz_naive_fixture)
         tm.assert_index_equal(result, index)
 
-        result = klass(arg, dtype=dtype)
+        with tm.assert_produces_warning(ex_warn, check_stacklevel=False):
+            result = klass(arg, dtype=dtype)
         tm.assert_index_equal(result, index)
 
-        result = klass(list(arg), tz=tz_naive_fixture)
+        with tm.assert_produces_warning(ex_warn, check_stacklevel=False):
+            result = klass(list(arg), tz=tz_naive_fixture)
         tm.assert_index_equal(result, index)
 
-        result = klass(list(arg), dtype=dtype)
+        with tm.assert_produces_warning(ex_warn, check_stacklevel=False):
+            result = klass(list(arg), dtype=dtype)
         tm.assert_index_equal(result, index)
 
     @pytest.mark.parametrize("attr", ['values', 'asi8'])

pandas/tests/resample/test_period_index.py

@@ -517,8 +517,10 @@ def test_resample_weekly_bug_1726(self):
 
     def test_resample_with_dst_time_change(self):
         # GH 15549
-        index = pd.DatetimeIndex([1457537600000000000, 1458059600000000000],
-                                 tz='UTC').tz_convert('America/Chicago')
+        index = (
+            pd.DatetimeIndex([1457537600000000000, 1458059600000000000])
+            .tz_localize("UTC").tz_convert('America/Chicago')
+        )
         df = pd.DataFrame([1, 2], index=index)
         result = df.resample('12h', closed='right',
                              label='right').last().ffill()

pandas/tests/test_base.py

@@ -1037,6 +1037,7 @@ class TestToIterable(object):
             lambda x: list(x.__iter__()),
         ], ids=['tolist', 'to_list', 'list', 'iter'])
     @pytest.mark.parametrize('typ', [Series, Index])
+    @pytest.mark.filterwarnings("ignore::FutureWarning")
     def test_iterable(self, typ, method, dtype, rdtype):
         # gh-10904
         # gh-13258