Backport PR #34407 on branch 1.1.x: REGR: revert "CLN: _consolidate_inplace less" / fix regression in fillna() (#38115)

Co-authored-by: Joris Van den Bossche <[email protected]>

Author: simonjayhawkins

doc/source/whatsnew/v1.1.5.rst

@@ -21,6 +21,7 @@ Fixed regressions
 - Fixed regression in metadata propagation for ``groupby`` iterator (:issue:`37343`)
 - Fixed regression in indexing on a :class:`Series` with ``CategoricalDtype`` after unpickling (:issue:`37631`)
 - Fixed regression in ``df.groupby(..).rolling(..)`` with the resulting :class:`MultiIndex` when grouping by a label that is in the index (:issue:`37641`)
+- Fixed regression in :meth:`DataFrame.fillna` not filling ``NaN`` after other operations such as :meth:`DataFrame.pivot` (:issue:`36495`).
 
 .. ---------------------------------------------------------------------------
 

pandas/core/generic.py

@@ -3484,6 +3484,8 @@ class   animal   locomotion
         if axis == 1:
             return self[key]
 
+        self._consolidate_inplace()
+
         index = self.index
         if isinstance(index, MultiIndex):
             loc, new_index = self.index.get_loc_level(key, drop_level=drop_level)
@@ -6011,6 +6013,8 @@ def fillna(
         inplace = validate_bool_kwarg(inplace, "inplace")
         value, method = validate_fillna_kwargs(value, method)
 
+        self._consolidate_inplace()
+
         # set the default here, so functions examining the signaure
         # can detect if something was set (e.g. in groupby) (GH9221)
         if axis is None:
@@ -6449,6 +6453,8 @@ def replace(
         if not is_bool(regex) and to_replace is not None:
             raise AssertionError("'to_replace' must be 'None' if 'regex' is not a bool")
 
+        self._consolidate_inplace()
+
         if value is None:
             # passing a single value that is scalar like
             # when value is None (GH5319), for compat

pandas/core/internals/managers.py

@@ -417,6 +417,7 @@ def apply(self: T, f, align_keys=None, **kwargs) -> T:
     def quantile(
         self,
         axis: int = 0,
+        consolidate: bool = True,
         transposed: bool = False,
         interpolation="linear",
         qs=None,
@@ -430,6 +431,8 @@ def quantile(
         Parameters
         ----------
         axis: reduction axis, default 0
+        consolidate: bool, default True. Join together blocks having same
+            dtype
         transposed: bool, default False
             we are holding transposed data
         interpolation : type of interpolation, default 'linear'
@@ -444,6 +447,9 @@ def quantile(
         #  simplify some of the code here and in the blocks
         assert self.ndim >= 2
 
+        if consolidate:
+            self._consolidate_inplace()
+
         def get_axe(block, qs, axes):
             # Because Series dispatches to DataFrame, we will always have
             #  block.ndim == 2

pandas/tests/frame/test_missing.py

@@ -717,3 +717,14 @@ def test_fill_corner(self, float_frame, float_string_frame):
 
         # TODO(wesm): unused?
         result = empty_float.fillna(value=0)  # noqa
+
+
+def test_fillna_nonconsolidated_frame():
+    # https://github.com/pandas-dev/pandas/issues/36495
+    df = DataFrame(
+        [[1, 1, 1, 1.0], [2, 2, 2, 2.0], [3, 3, 3, 3.0]],
+        columns=["i1", "i2", "i3", "f1"],
+    )
+    df_nonconsol = df.pivot("i1", "i2")
+    result = df_nonconsol.fillna(0)
+    assert result.isna().sum().sum() == 0