ENH/DEPR: infer date objects to date[pyarrow] dtype

Author: jbrockmendel

pandas/_libs/lib.pyx

@@ -1262,6 +1262,7 @@ cdef class Seen:
         bint period_          # seen_period
         bint interval_        # seen_interval
         bint time_
+        bint date_
 
     def __cinit__(self, bint coerce_numeric=False):
         """
@@ -1289,6 +1290,7 @@ cdef class Seen:
         self.period_ = False
         self.interval_ = False
         self.time_ = False
+        self.date_ = False
         self.coerce_numeric = coerce_numeric
 
     cdef bint check_uint64_conflict(self) except -1:
@@ -2549,6 +2551,11 @@ def maybe_convert_objects(ndarray[object] objects,
             else:
                 seen.object_ = True
                 break
+        elif PyDate_Check(val):
+            if convert_non_numeric:
+                seen.date_ = True
+            else:
+                seen.object_ = True
         elif is_period_object(val):
             if convert_non_numeric:
                 seen.period_ = True
@@ -2672,6 +2679,35 @@ def maybe_convert_objects(ndarray[object] objects,
 
         seen.object_ = True
 
+    elif seen.date_:
+        if is_date_array(objects, skipna=True):
+            opt = get_option("future.infer_date")
+            if opt is True:
+                import pyarrow as pa
+
+                from pandas.core.arrays.arrow import ArrowDtype
+
+                obj = pa.array(objects)
+                dtype = ArrowDtype(obj.type)
+                return dtype.construct_array_type()(obj)
+            elif opt is False:
+                # explicitly set to keep the old behavior and avoid the warning
+                pass
+            else:
+                from pandas.util._exceptions import find_stack_level
+                warnings.warn(
+                    "Pandas type inference with a sequence of `datetime.date` "
+                    "objects is deprecated. In a future version, this will give "
+                    "date32[pyarrow] dtype, which will require pyarrow to be "
+                    "installed. To opt in to the new behavior immediately set "
+                    "`pd.set_option('future.infer_time', True)`. To keep the "
+                    "old behavior pass `dtype=object`.",
+                    FutureWarning,
+                    stacklevel=find_stack_level(),
+                )
+
+        seen.object_ = True
+
     elif seen.nat_:
         if not seen.object_ and not seen.numeric_ and not seen.bool_:
             # all NaT, None, or nan (at least one NaT)

pandas/core/config_init.py

@@ -892,3 +892,12 @@ def register_converter_cb(key) -> None:
         "(at which point this option will be deprecated).",
         validator=is_one_of_factory([True, False, None]),
     )
+
+    cf.register_option(
+        "future.infer_date",
+        None,
+        "Whether to infer sequence of datetime.date objects as pyarrow date "
+        "dtype, which will be the default in pandas 3.0 "
+        "(at which point this option will be deprecated).",
+        validator=is_one_of_factory([True, False, None]),
+    )

pandas/core/construction.py

@@ -389,6 +389,30 @@ def array(
                     stacklevel=find_stack_level(),
                 )
 
+        elif inferred_dtype == "date":
+            opt = get_option("future.infer_date")
+
+            if opt is True:
+                import pyarrow as pa
+
+                obj = pa.array(data)
+                dtype = ArrowDtype(obj.type)
+                return dtype.construct_array_type()(obj)
+            elif opt is False:
+                # explicitly set to keep the old behavior and avoid the warning
+                pass
+            else:
+                warnings.warn(
+                    "Pandas type inference with a sequence of `datetime.date` "
+                    "objects is deprecated. In a future version, this will give "
+                    "date32[pyarrow] dtype, which will require pyarrow to be "
+                    "installed. To opt in to the new behavior immediately set "
+                    "`pd.set_option('future.infer_time', True)`. To keep the "
+                    "old behavior pass `dtype=object`.",
+                    FutureWarning,
+                    stacklevel=find_stack_level(),
+                )
+
     # Pandas overrides NumPy for
     #   1. datetime64[ns,us,ms,s]
     #   2. timedelta64[ns,us,ms,s]

pandas/tests/arrays/categorical/test_constructors.py

@@ -369,7 +369,12 @@ def test_constructor_date_objects(self):
         # we dont cast date objects to timestamps, matching Index constructor
         v = date.today()
 
-        cat = Categorical([v, v])
+        msg = (
+            "Pandas type inference with a sequence of `datetime.date` "
+            "objects is deprecated"
+        )
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            cat = Categorical([v, v])
         assert cat.categories.dtype == object
         assert type(cat.categories[0]) is date
 

pandas/tests/dtypes/test_inference.py

@@ -1059,7 +1059,7 @@ def test_maybe_convert_objects_time(self, future):
 
         with pd.option_context("future.infer_time", future):
             with tm.assert_produces_warning(warn, match=msg):
-                out = lib.maybe_convert_objects(objs, convert_time=True)
+                out = lib.maybe_convert_objects(objs, convert_non_numeric=True)
             with tm.assert_produces_warning(warn, match=msg):
                 ser = Series(objs)
             with tm.assert_produces_warning(warn, match=msg):
@@ -1566,7 +1566,7 @@ def test_other_dtypes_for_array(self, func):
 
     def test_date(self):
         dates = [date(2012, 1, day) for day in range(1, 20)]
-        index = Index(dates)
+        index = Index(dates, dtype=object)
         assert index.inferred_type == "date"
 
         dates = [date(2012, 1, day) for day in range(1, 20)] + [np.nan]

pandas/tests/extension/test_arrow.py

@@ -730,6 +730,13 @@ def test_stack(self, data, columns):
             # FIXME: need to avoid doing inference when calling frame._constructor
             #  in _stack_multi_columns
             warn = FutureWarning
+        if pa.types.is_date(pa_dtype):
+            # FIXME: need to avoid doing inference when calling frame._constructor
+            #  in _stack_multi_columns
+            warn = FutureWarning
+            warn_msg = (
+                "Pandas type inference with a sequence of `datetime.date` objects"
+            )
 
         with tm.assert_produces_warning(warn, match=warn_msg, check_stacklevel=False):
             super().test_stack(data, columns)
@@ -798,9 +805,9 @@ def test_invert(self, data, request):
 class TestBaseMethods(base.BaseMethodsTests):
     def test_hash_pandas_object_works(self, data, as_frame):
         pa_dtype = data.dtype.pyarrow_dtype
-        warn_msg = "Pandas type inference with a sequence of `datetime.time`"
+        warn_msg = "Pandas type inference with a sequence of `datetime.(time|date)`"
         warn = None
-        if pa.types.is_time(pa_dtype):
+        if pa.types.is_time(pa_dtype) or pa.types.is_date(pa_dtype):
             # TODO(#48964) This warning will be avoided by implementing
             #  ArrowExtensionArray.hash_pandas_object
             warn = FutureWarning
@@ -1688,7 +1695,15 @@ def test_pickle_roundtrip(data):
 
 def test_astype_from_non_pyarrow(data):
     # GH49795
-    pd_array = data._pa_array.to_pandas().array
+    msg = (
+        "Pandas type inference with a sequence of `datetime.date` objects is deprecated"
+    )
+    warn = None
+    if pa.types.is_date(data.dtype.pyarrow_dtype):
+        warn = FutureWarning
+
+    with tm.assert_produces_warning(warn, match=msg):
+        pd_array = data._pa_array.to_pandas().array
     result = pd_array.astype(data.dtype)
     assert not isinstance(pd_array.dtype, ArrowDtype)
     assert isinstance(result.dtype, ArrowDtype)

pandas/tests/frame/methods/test_asfreq.py

@@ -186,7 +186,12 @@ def test_asfreq_with_date_object_index(self, frame_or_series):
         ts = frame_or_series(np.random.randn(20), index=rng)
 
         ts2 = ts.copy()
-        ts2.index = [x.date() for x in ts2.index]
+        msg = (
+            "Pandas type inference with a sequence of `datetime.date` "
+            "objects is deprecated"
+        )
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            ts2.index = [x.date() for x in ts2.index]
 
         result = ts2.asfreq("4H", method="ffill")
         expected = ts.asfreq("4H", method="ffill")

pandas/tests/frame/methods/test_join.py

@@ -510,16 +510,26 @@ def test_join_multiindex_dates(self):
         # GH 33692
         date = pd.Timestamp(2000, 1, 1).date()
 
-        df1_index = MultiIndex.from_tuples([(0, date)], names=["index_0", "date"])
+        msg = (
+            "Pandas type inference with a sequence of `datetime.date` "
+            "objects is deprecated"
+        )
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            df1_index = MultiIndex.from_tuples([(0, date)], names=["index_0", "date"])
         df1 = DataFrame({"col1": [0]}, index=df1_index)
-        df2_index = MultiIndex.from_tuples([(0, date)], names=["index_0", "date"])
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            df2_index = MultiIndex.from_tuples([(0, date)], names=["index_0", "date"])
         df2 = DataFrame({"col2": [0]}, index=df2_index)
-        df3_index = MultiIndex.from_tuples([(0, date)], names=["index_0", "date"])
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            df3_index = MultiIndex.from_tuples([(0, date)], names=["index_0", "date"])
         df3 = DataFrame({"col3": [0]}, index=df3_index)
 
         result = df1.join([df2, df3])
 
-        expected_index = MultiIndex.from_tuples([(0, date)], names=["index_0", "date"])
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            expected_index = MultiIndex.from_tuples(
+                [(0, date)], names=["index_0", "date"]
+            )
         expected = DataFrame(
             {"col1": [0], "col2": [0], "col3": [0]}, index=expected_index
         )

pandas/tests/frame/methods/test_reindex.py

@@ -185,7 +185,9 @@ def test_reindex_date_fill_value(self):
         ts = df.iloc[0, 0]
         fv = ts.date()
 
-        res = df.reindex(index=range(4), columns=["A", "B", "C"], fill_value=fv)
+        msg = "type inference with a sequence of `datetime.date` objects is deprecated"
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            res = df.reindex(index=range(4), columns=["A", "B", "C"], fill_value=fv)
 
         expected = DataFrame(
             {"A": df["A"].tolist() + [fv], "B": df["B"].tolist() + [fv], "C": [fv] * 4},

pandas/tests/frame/test_constructors.py

@@ -1895,7 +1895,12 @@ def test_constructor_with_datetimes2(self):
         datetimes = [ts.to_pydatetime() for ts in ind]
         dates = [ts.date() for ts in ind]
         df = DataFrame(datetimes, columns=["datetimes"])
-        df["dates"] = dates
+        msg = (
+            "Pandas type inference with a sequence of `datetime.date` "
+            "objects is deprecated"
+        )
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            df["dates"] = dates
         result = df.dtypes
         expected = Series(
             [np.dtype("datetime64[ns]"), np.dtype("object")],
@@ -2361,7 +2366,12 @@ def test_datetime_date_tuple_columns_from_dict(self):
         # GH 10863
         v = date.today()
         tup = v, v
-        result = DataFrame({tup: Series(range(3), index=range(3))}, columns=[tup])
+        msg = (
+            "Pandas type inference with a sequence of `datetime.date` "
+            "objects is deprecated"
+        )
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            result = DataFrame({tup: Series(range(3), index=range(3))}, columns=[tup])
         expected = DataFrame([0, 1, 2], columns=Index(Series([tup])))
         tm.assert_frame_equal(result, expected)
 

pandas/tests/groupby/aggregate/test_other.py

@@ -68,19 +68,22 @@ def test_agg_datetimes_mixed():
         for row in data
     ]
 
-    df2 = DataFrame(
-        {
-            "key": [x[0] for x in data],
-            "date": [x[1] for x in data],
-            "value": [x[2] for x in data],
-        }
-    )
+    msg = "Pandas type inference with a sequence of `datetime.date` objects"
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        df2 = DataFrame(
+            {
+                "key": [x[0] for x in data],
+                "date": [x[1] for x in data],
+                "value": [x[2] for x in data],
+            }
+        )
 
     df1["weights"] = df1["value"] / df1["value"].sum()
     gb1 = df1.groupby("date").aggregate(np.sum)
 
     df2["weights"] = df1["value"] / df1["value"].sum()
-    gb2 = df2.groupby("date").aggregate(np.sum)
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        gb2 = df2.groupby("date").aggregate(np.sum)
 
     assert len(gb1) == len(gb2)
 
@@ -367,22 +370,25 @@ def test_agg_consistency():
     def P1(a):
         return np.percentile(a.dropna(), q=1)
 
-    df = DataFrame(
-        {
-            "col1": [1, 2, 3, 4],
-            "col2": [10, 25, 26, 31],
-            "date": [
-                dt.date(2013, 2, 10),
-                dt.date(2013, 2, 10),
-                dt.date(2013, 2, 11),
-                dt.date(2013, 2, 11),
-            ],
-        }
-    )
+    msg = "Pandas type inference with a sequence of `datetime.date` objects"
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        df = DataFrame(
+            {
+                "col1": [1, 2, 3, 4],
+                "col2": [10, 25, 26, 31],
+                "date": [
+                    dt.date(2013, 2, 10),
+                    dt.date(2013, 2, 10),
+                    dt.date(2013, 2, 11),
+                    dt.date(2013, 2, 11),
+                ],
+            }
+        )
 
     g = df.groupby("date")
 
-    expected = g.agg([P1])
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        expected = g.agg([P1])
     expected.columns = expected.columns.levels[0]
 
     result = g.agg(P1)