Set correct missing value indicator in astype for categorical (#45012)

Author: phofl

pandas/core/arrays/categorical.py

@@ -108,7 +108,6 @@
 )
 import pandas.core.common as com
 from pandas.core.construction import (
-    ensure_wrapped_if_datetimelike,
     extract_array,
     sanitize_array,
 )
@@ -539,14 +538,15 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
 
         else:
             # GH8628 (PERF): astype category codes instead of astyping array
-            if is_datetime64_dtype(self.categories):
-                new_cats = ensure_wrapped_if_datetimelike(self.categories._values)
-            else:
-                new_cats = np.asarray(self.categories)
+            new_cats = self.categories._values
 
             try:
                 new_cats = new_cats.astype(dtype=dtype, copy=copy)
-                fill_value = lib.item_from_zerodim(np.array(np.nan).astype(dtype))
+                fill_value = self.categories._na_value
+                if not is_valid_na_for_dtype(fill_value, dtype):
+                    fill_value = lib.item_from_zerodim(
+                        np.array(self.categories._na_value).astype(dtype)
+                    )
             except (
                 TypeError,  # downstream error msg for CategoricalIndex is misleading
                 ValueError,

pandas/core/dtypes/missing.py

@@ -59,6 +59,8 @@
 
 nan_checker = np.isnan
 INF_AS_NA = False
+_dtype_object = np.dtype("object")
+_dtype_str = np.dtype(str)
 
 
 def isna(obj):
@@ -647,7 +649,11 @@ def is_valid_na_for_dtype(obj, dtype: DtypeObj) -> bool:
         # Numeric
         return obj is not NaT and not isinstance(obj, (np.datetime64, np.timedelta64))
 
-    elif dtype == np.dtype("object"):
+    elif dtype == _dtype_str:
+        # numpy string dtypes to avoid float np.nan
+        return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal, float))
+
+    elif dtype == _dtype_object:
         # This is needed for Categorical, but is kind of weird
         return True
 

pandas/tests/arrays/categorical/test_dtypes.py

@@ -182,7 +182,7 @@ def test_astype_object_datetime_categories(self):
         # GH#40754
         cat = Categorical(to_datetime(["2021-03-27", NaT]))
         result = cat.astype(object)
-        expected = np.array([Timestamp("2021-03-27 00:00:00"), np.nan], dtype="object")
+        expected = np.array([Timestamp("2021-03-27 00:00:00"), NaT], dtype="object")
         tm.assert_numpy_array_equal(result, expected)
 
     def test_astype_object_timestamp_categories(self):