Deprecate is_categorical_dtype (#14274)

This PR deprecates `is_categorical_dtype` to match pandas-dev/pandas#52527 which was introduced in `pandas-2.x`. This PR internalizes the public API since this is a needed utility in our code base.

This PR:
```
= 23835 failed, 5698 passed, 1613 skipped, 288 xfailed, 423 errors in 1976.84s (0:32:56) =
```

On `pandas_2.0_feature_branch`:
```
= 24297 failed, 5115 passed, 1613 skipped, 288 xfailed, 480 errors in 1980.46s (0:33:00) =
```

Author: galipremsagar

python/cudf/cudf/_fuzz_testing/csv.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 
 import logging
 import random
@@ -99,7 +99,7 @@ def set_rand_params(self, params):
                     if dtype_val is not None:
                         dtype_val = {
                             col_name: "category"
-                            if cudf.utils.dtypes.is_categorical_dtype(dtype)
+                            if cudf.utils.dtypes._is_categorical_dtype(dtype)
                             else pandas_dtypes_to_np_dtypes[dtype]
                             for col_name, dtype in dtype_val.items()
                         }

python/cudf/cudf/_fuzz_testing/json.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 
 import logging
 import random
@@ -27,7 +27,7 @@ def _get_dtype_param_value(dtype_val):
     if dtype_val is not None and isinstance(dtype_val, abc.Mapping):
         processed_dtypes = {}
         for col_name, dtype in dtype_val.items():
-            if cudf.utils.dtypes.is_categorical_dtype(dtype):
+            if cudf.utils.dtypes._is_categorical_dtype(dtype):
                 processed_dtypes[col_name] = "category"
             else:
                 processed_dtypes[col_name] = str(

python/cudf/cudf/_lib/column.pyx

@@ -11,7 +11,7 @@ import rmm
 import cudf
 import cudf._lib as libcudf
 from cudf._lib import pylibcudf
-from cudf.api.types import is_categorical_dtype, is_datetime64tz_dtype
+from cudf.api.types import _is_categorical_dtype, is_datetime64tz_dtype
 from cudf.core.buffer import (
     Buffer,
     ExposureTrackedBuffer,
@@ -331,7 +331,7 @@ cdef class Column:
             )
 
     cdef mutable_column_view mutable_view(self) except *:
-        if is_categorical_dtype(self.dtype):
+        if _is_categorical_dtype(self.dtype):
             col = self.base_children[0]
             data_dtype = col.dtype
         elif is_datetime64tz_dtype(self.dtype):
@@ -394,7 +394,7 @@ cdef class Column:
         return self._view(c_null_count)
 
     cdef column_view _view(self, libcudf_types.size_type null_count) except *:
-        if is_categorical_dtype(self.dtype):
+        if _is_categorical_dtype(self.dtype):
             col = self.base_children[0]
             data_dtype = col.dtype
         elif is_datetime64tz_dtype(self.dtype):
@@ -469,7 +469,7 @@ cdef class Column:
         # categoricals because cudf supports ordered and unordered categoricals
         # while libcudf supports only unordered categoricals (see
         # https://github.com/rapidsai/cudf/pull/8567).
-        if is_categorical_dtype(self.dtype):
+        if _is_categorical_dtype(self.dtype):
             col = self.base_children[0]
         else:
             col = self
@@ -635,7 +635,7 @@ cdef class Column:
         """
         column_owner = isinstance(owner, Column)
         mask_owner = owner
-        if column_owner and is_categorical_dtype(owner.dtype):
+        if column_owner and _is_categorical_dtype(owner.dtype):
             owner = owner.base_children[0]
 
         size = cv.size()

python/cudf/cudf/_lib/csv.pyx

@@ -434,19 +434,19 @@ def read_csv(
     if dtype is not None:
         if isinstance(dtype, abc.Mapping):
             for k, v in dtype.items():
-                if cudf.api.types.is_categorical_dtype(v):
+                if cudf.api.types._is_categorical_dtype(v):
                     df._data[str(k)] = df._data[str(k)].astype(v)
         elif (
             cudf.api.types.is_scalar(dtype) or
             isinstance(dtype, (
                 np.dtype, pd.api.extensions.ExtensionDtype, type
             ))
         ):
-            if cudf.api.types.is_categorical_dtype(dtype):
+            if cudf.api.types._is_categorical_dtype(dtype):
                 df = df.astype(dtype)
         elif isinstance(dtype, abc.Collection):
             for index, col_dtype in enumerate(dtype):
-                if cudf.api.types.is_categorical_dtype(col_dtype):
+                if cudf.api.types._is_categorical_dtype(col_dtype):
                     col_name = df._data.names[index]
                     df._data[col_name] = df._data[col_name].astype(col_dtype)
 
@@ -547,7 +547,7 @@ cdef data_type _get_cudf_data_type_from_dtype(object dtype) except *:
     # TODO: Remove this work-around Dictionary types
     # in libcudf are fully mapped to categorical columns:
     # https://github.com/rapidsai/cudf/issues/3960
-    if cudf.api.types.is_categorical_dtype(dtype):
+    if cudf.api.types._is_categorical_dtype(dtype):
         if isinstance(dtype, str):
             dtype = "str"
         else:

python/cudf/cudf/_lib/groupby.pyx

@@ -3,7 +3,7 @@
 from pandas.core.groupby.groupby import DataError
 
 from cudf.api.types import (
-    is_categorical_dtype,
+    _is_categorical_dtype,
     is_decimal_dtype,
     is_interval_dtype,
     is_list_dtype,
@@ -189,7 +189,7 @@ cdef class GroupBy:
             valid_aggregations = (
                 _LIST_AGGS if is_list_dtype(dtype)
                 else _STRING_AGGS if is_string_dtype(dtype)
-                else _CATEGORICAL_AGGS if is_categorical_dtype(dtype)
+                else _CATEGORICAL_AGGS if _is_categorical_dtype(dtype)
                 else _STRUCT_AGGS if is_struct_dtype(dtype)
                 else _INTERVAL_AGGS if is_interval_dtype(dtype)
                 else _DECIMAL_AGGS if is_decimal_dtype(dtype)
@@ -260,7 +260,7 @@ cdef class GroupBy:
             valid_aggregations = (
                 _LIST_AGGS if is_list_dtype(dtype)
                 else _STRING_AGGS if is_string_dtype(dtype)
-                else _CATEGORICAL_AGGS if is_categorical_dtype(dtype)
+                else _CATEGORICAL_AGGS if _is_categorical_dtype(dtype)
                 else _STRUCT_AGGS if is_struct_dtype(dtype)
                 else _INTERVAL_AGGS if is_interval_dtype(dtype)
                 else _DECIMAL_AGGS if is_decimal_dtype(dtype)

python/cudf/cudf/_lib/json.pyx

@@ -214,7 +214,7 @@ def write_json(
 cdef schema_element _get_cudf_schema_element_from_dtype(object dtype) except *:
     cdef schema_element s_element
     cdef data_type lib_type
-    if cudf.api.types.is_categorical_dtype(dtype):
+    if cudf.api.types._is_categorical_dtype(dtype):
         raise NotImplementedError(
             "CategoricalDtype as dtype is not yet "
             "supported in JSON reader"
@@ -237,7 +237,7 @@ cdef schema_element _get_cudf_schema_element_from_dtype(object dtype) except *:
 
 
 cdef data_type _get_cudf_data_type_from_dtype(object dtype) except *:
-    if cudf.api.types.is_categorical_dtype(dtype):
+    if cudf.api.types._is_categorical_dtype(dtype):
         raise NotImplementedError(
             "CategoricalDtype as dtype is not yet "
             "supported in JSON reader"

python/cudf/cudf/_lib/utils.pyx

@@ -23,7 +23,7 @@ except ImportError:
     import json
 
 from cudf.api.types import (
-    is_categorical_dtype,
+    _is_categorical_dtype,
     is_decimal_dtype,
     is_list_dtype,
     is_struct_dtype,
@@ -92,7 +92,7 @@ cpdef generate_pandas_metadata(table, index):
     # Columns
     for name, col in table._data.items():
         col_names.append(name)
-        if is_categorical_dtype(col):
+        if _is_categorical_dtype(col):
             raise ValueError(
                 "'category' column dtypes are currently not "
                 + "supported by the gpu accelerated parquet writer"
@@ -147,7 +147,7 @@ cpdef generate_pandas_metadata(table, index):
                         level=level,
                         column_names=col_names
                     )
-                if is_categorical_dtype(idx):
+                if _is_categorical_dtype(idx):
                     raise ValueError(
                         "'category' column dtypes are currently not "
                         + "supported by the gpu accelerated parquet writer"

python/cudf/cudf/api/types.py

@@ -18,7 +18,7 @@
 from cudf.core.dtypes import (  # noqa: F401
     _BaseDtype,
     dtype,
-    is_categorical_dtype,
+    _is_categorical_dtype,
     is_decimal32_dtype,
     is_decimal64_dtype,
     is_decimal128_dtype,
@@ -112,7 +112,7 @@ def is_string_dtype(obj):
         or (
             pd.api.types.is_string_dtype(obj)
             # Reject all cudf extension types.
-            and not is_categorical_dtype(obj)
+            and not _is_categorical_dtype(obj)
             and not is_decimal_dtype(obj)
             and not is_list_dtype(obj)
             and not is_struct_dtype(obj)
@@ -486,6 +486,7 @@ def is_any_real_numeric_dtype(arr_or_dtype) -> bool:
 is_iterator = pd_types.is_iterator
 is_bool = pd_types.is_bool
 is_categorical = pd_types.is_categorical_dtype
+# TODO
 is_complex = pd_types.is_complex
 is_float = pd_types.is_float
 is_hashable = pd_types.is_hashable

python/cudf/cudf/core/_internals/where.py

@@ -8,8 +8,8 @@
 import cudf
 from cudf._typing import ScalarLike
 from cudf.api.types import (
+    _is_categorical_dtype,
     _is_non_decimal_numeric_dtype,
-    is_categorical_dtype,
     is_scalar,
 )
 from cudf.core.column import ColumnBase
@@ -45,7 +45,7 @@ def _check_and_cast_columns_with_other(
 ) -> Tuple[ColumnBase, Union[ScalarLike, ColumnBase]]:
     # Returns type-casted `source_col` & `other` based on `inplace`.
     source_dtype = source_col.dtype
-    if is_categorical_dtype(source_dtype):
+    if _is_categorical_dtype(source_dtype):
         return _normalize_categorical(source_col, other)
 
     other_is_scalar = is_scalar(other)

python/cudf/cudf/core/column/categorical.py

@@ -16,7 +16,7 @@
 from cudf import _lib as libcudf
 from cudf._lib.transform import bools_to_mask
 from cudf._typing import ColumnBinaryOperand, ColumnLike, Dtype, ScalarLike
-from cudf.api.types import is_categorical_dtype, is_interval_dtype
+from cudf.api.types import _is_categorical_dtype, is_interval_dtype
 from cudf.core.buffer import Buffer
 from cudf.core.column import column
 from cudf.core.column.methods import ColumnMethods
@@ -98,7 +98,7 @@ class CategoricalAccessor(ColumnMethods):
     _column: CategoricalColumn
 
     def __init__(self, parent: SeriesOrSingleColumnIndex):
-        if not is_categorical_dtype(parent.dtype):
+        if not _is_categorical_dtype(parent.dtype):
             raise AttributeError(
                 "Can only use .cat accessor with a 'category' dtype"
             )

python/cudf/cudf/core/column/column.py

@@ -49,10 +49,10 @@
 from cudf._lib.types import size_type_dtype
 from cudf._typing import ColumnLike, Dtype, ScalarLike
 from cudf.api.types import (
+    _is_categorical_dtype,
     _is_non_decimal_numeric_dtype,
     infer_dtype,
     is_bool_dtype,
-    is_categorical_dtype,
     is_datetime64_dtype,
     is_datetime64tz_dtype,
     is_decimal32_dtype,
@@ -977,7 +977,7 @@ def can_cast_safely(self, to_dtype: Dtype) -> bool:
     def astype(self, dtype: Dtype, **kwargs) -> ColumnBase:
         if self.dtype == dtype:
             return self
-        if is_categorical_dtype(dtype):
+        if _is_categorical_dtype(dtype):
             return self.as_categorical_column(dtype, **kwargs)
 
         dtype = (
@@ -987,7 +987,7 @@ def astype(self, dtype: Dtype, **kwargs) -> ColumnBase:
         )
         if _is_non_decimal_numeric_dtype(dtype):
             return self.as_numerical_column(dtype, **kwargs)
-        elif is_categorical_dtype(dtype):
+        elif _is_categorical_dtype(dtype):
             return self.as_categorical_column(dtype, **kwargs)
         elif cudf.dtype(dtype).type in {
             np.str_,
@@ -1423,7 +1423,7 @@ def column_empty_like(
 
     if (
         hasattr(column, "dtype")
-        and is_categorical_dtype(column.dtype)
+        and _is_categorical_dtype(column.dtype)
         and dtype == column.dtype
     ):
         catcolumn = cast("cudf.core.column.CategoricalColumn", column)
@@ -1476,7 +1476,7 @@ def column_empty(
             full(row_count + 1, 0, dtype=libcudf.types.size_type_dtype),
             column_empty(row_count, dtype=dtype.element_type),
         )
-    elif is_categorical_dtype(dtype):
+    elif _is_categorical_dtype(dtype):
         data = None
         children = (
             build_column(
@@ -1553,7 +1553,7 @@ def build_column(
             offset=offset,
             null_count=null_count,
         )
-    if is_categorical_dtype(dtype):
+    if _is_categorical_dtype(dtype):
         if not len(children) == 1:
             raise ValueError(
                 "Must specify exactly one child column for CategoricalColumn"
@@ -2037,7 +2037,7 @@ def as_column(
                     f"{arbitrary.dtype} is not supported. Convert first to "
                     f"{arbitrary.dtype.subtype}."
                 )
-        if is_categorical_dtype(arbitrary.dtype):
+        if _is_categorical_dtype(arbitrary.dtype):
             if isinstance(
                 arbitrary.dtype.categories.dtype, pd.DatetimeTZDtype
             ):
@@ -2219,7 +2219,7 @@ def as_column(
             data = data.astype(cudf.dtype(dtype))
 
     elif isinstance(arbitrary, NumpyExtensionArray):
-        if is_categorical_dtype(arbitrary.dtype):
+        if _is_categorical_dtype(arbitrary.dtype):
             arb_dtype = arbitrary.dtype
         else:
             if arbitrary.dtype == pd.StringDtype():
@@ -2347,7 +2347,9 @@ def as_column(
             np_type = None
             try:
                 if dtype is not None:
-                    if is_categorical_dtype(dtype) or is_interval_dtype(dtype):
+                    if _is_categorical_dtype(dtype) or is_interval_dtype(
+                        dtype
+                    ):
                         raise TypeError
                     if is_datetime64tz_dtype(dtype):
                         raise NotImplementedError(
@@ -2491,7 +2493,7 @@ def as_column(
             except (pa.ArrowInvalid, pa.ArrowTypeError, TypeError) as e:
                 if isinstance(e, MixedTypeError):
                     raise TypeError(str(e))
-                if is_categorical_dtype(dtype):
+                if _is_categorical_dtype(dtype):
                     sr = pd.Series(arbitrary, dtype="category")
                     data = as_column(sr, nan_as_null=nan_as_null, dtype=dtype)
                 elif np_type == np.str_:
@@ -2774,7 +2776,7 @@ def concat_columns(objs: "MutableSequence[ColumnBase]") -> ColumnBase:
     # ColumnBase._concat so that all subclasses can override necessary
     # behavior. However, at the moment it's not clear what that API should look
     # like, so CategoricalColumn simply implements a minimal working API.
-    if all(is_categorical_dtype(o.dtype) for o in objs):
+    if all(_is_categorical_dtype(o.dtype) for o in objs):
         return cudf.core.column.categorical.CategoricalColumn._concat(
             cast(
                 MutableSequence[

python/cudf/cudf/core/column/interval.py

@@ -5,7 +5,7 @@
 import pyarrow as pa
 
 import cudf
-from cudf.api.types import is_categorical_dtype, is_interval_dtype
+from cudf.api.types import _is_categorical_dtype, is_interval_dtype
 from cudf.core.column import StructColumn
 from cudf.core.dtypes import IntervalDtype
 
@@ -102,7 +102,7 @@ def copy(self, deep=True):
 
     def as_interval_column(self, dtype, **kwargs):
         if is_interval_dtype(dtype):
-            if is_categorical_dtype(self):
+            if _is_categorical_dtype(self):
                 new_struct = self._get_decategorized_column()
                 return IntervalColumn.from_struct_column(new_struct)
             if is_interval_dtype(dtype):