diff --git a/doc/source/whatsnew/v1.1.5.rst b/doc/source/whatsnew/v1.1.5.rst index a8bbf692a72e5..29b0e99a3a356 100644 --- a/doc/source/whatsnew/v1.1.5.rst +++ b/doc/source/whatsnew/v1.1.5.rst @@ -21,6 +21,7 @@ Fixed regressions - Fixed regression in metadata propagation for ``groupby`` iterator (:issue:`37343`) - Fixed regression in indexing on a :class:`Series` with ``CategoricalDtype`` after unpickling (:issue:`37631`) - Fixed regression in ``df.groupby(..).rolling(..)`` with the resulting :class:`MultiIndex` when grouping by a label that is in the index (:issue:`37641`) +- Fixed regression in :meth:`DataFrame.fillna` not filling ``NaN`` after other operations such as :meth:`DataFrame.pivot` (:issue:`36495`). .. --------------------------------------------------------------------------- diff --git a/pandas/core/generic.py b/pandas/core/generic.py index be85ab251c0c3..1c6248ad71b62 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3484,6 +3484,8 @@ class animal locomotion if axis == 1: return self[key] + self._consolidate_inplace() + index = self.index if isinstance(index, MultiIndex): loc, new_index = self.index.get_loc_level(key, drop_level=drop_level) @@ -6011,6 +6013,8 @@ def fillna( inplace = validate_bool_kwarg(inplace, "inplace") value, method = validate_fillna_kwargs(value, method) + self._consolidate_inplace() + # set the default here, so functions examining the signaure # can detect if something was set (e.g. in groupby) (GH9221) if axis is None: @@ -6449,6 +6453,8 @@ def replace( if not is_bool(regex) and to_replace is not None: raise AssertionError("'to_replace' must be 'None' if 'regex' is not a bool") + self._consolidate_inplace() + if value is None: # passing a single value that is scalar like # when value is None (GH5319), for compat diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 4c52343d08513..67bf2584bb84e 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -417,6 +417,7 @@ def apply(self: T, f, align_keys=None, **kwargs) -> T: def quantile( self, axis: int = 0, + consolidate: bool = True, transposed: bool = False, interpolation="linear", qs=None, @@ -430,6 +431,8 @@ def quantile( Parameters ---------- axis: reduction axis, default 0 + consolidate: bool, default True. Join together blocks having same + dtype transposed: bool, default False we are holding transposed data interpolation : type of interpolation, default 'linear' @@ -444,6 +447,9 @@ def quantile( # simplify some of the code here and in the blocks assert self.ndim >= 2 + if consolidate: + self._consolidate_inplace() + def get_axe(block, qs, axes): # Because Series dispatches to DataFrame, we will always have # block.ndim == 2 diff --git a/pandas/tests/frame/test_missing.py b/pandas/tests/frame/test_missing.py index b4f91590e09d1..0f5048dde3250 100644 --- a/pandas/tests/frame/test_missing.py +++ b/pandas/tests/frame/test_missing.py @@ -717,3 +717,14 @@ def test_fill_corner(self, float_frame, float_string_frame): # TODO(wesm): unused? result = empty_float.fillna(value=0) # noqa + + +def test_fillna_nonconsolidated_frame(): + # https://github.com/pandas-dev/pandas/issues/36495 + df = DataFrame( + [[1, 1, 1, 1.0], [2, 2, 2, 2.0], [3, 3, 3, 3.0]], + columns=["i1", "i2", "i3", "f1"], + ) + df_nonconsol = df.pivot("i1", "i2") + result = df_nonconsol.fillna(0) + assert result.isna().sum().sum() == 0