Backport PR #64092 on branch 3.0.x (BUG: DataFrame.loc fills b'' instead of NaN when adding new row and columns simultaneously (#58316)) (#64164)

Co-authored-by: roeimed0 <roeimed01@gmail.com>

Author: meeseeksmachine

doc/source/whatsnew/v3.0.1.rst

@@ -32,6 +32,7 @@ Bug fixes
 - Fixed a bug in the :func:`comparison_op` raising a ``TypeError`` for zerodim
   subclasses of ``np.ndarray`` (:issue:`63205`)
 - Added additional typing aliases in :py:mod:`pandas.api.typing.aliases` (:issue:`64098`)
+- Fixed bug in :meth:`DataFrame.loc` when setting new row and new columns simultaneously filling existing columns with ``b''`` instead of ``NaN`` (:issue:`58316`)
 - Fixed thread safety issues in :class:`DataFrame` internals on the free-threaded build (:issue:`63685`).
 - Prevent buffer overflow in :meth:`Rolling.corr` and :meth:`Rolling.cov` with variable windows when passing ``other`` with a longer index than the original window. This now raises ``ValueError`` (:issue:`62937`)
 

pandas/core/internals/managers.py

@@ -845,16 +845,24 @@ def reindex_indexer(
                 )
             )
         else:
-            new_blocks = [
-                blk.take_nd(
-                    indexer,
-                    axis=1,
-                    fill_value=(
-                        fill_value if fill_value is not None else blk.fill_value
-                    ),
-                )
-                for blk in self.blocks
-            ]
+            new_blocks = []
+            for blk in self.blocks:
+                if blk.dtype == np.void:
+                    # GH#58316: np.void placeholders cast to b'' when
+                    # reindexed; preserve np.void so _setitem_single_column
+                    # can later infer the correct dtype
+                    vals = np.empty((blk.values.shape[0], len(indexer)), dtype=np.void)
+                    new_blocks.append(NumpyBlock(vals, blk.mgr_locs, ndim=2))
+                else:
+                    new_blocks.append(
+                        blk.take_nd(
+                            indexer,
+                            axis=1,
+                            fill_value=(
+                                fill_value if fill_value is not None else blk.fill_value
+                            ),
+                        )
+                    )
 
         new_axes = list(self.axes)
         new_axes[axis] = new_axis

pandas/tests/indexing/test_loc.py

@@ -2228,6 +2228,21 @@ def test_loc_setitem_ea_not_full_column(self):
         assert expected.dtypes["B"] == val.dtype
         tm.assert_frame_equal(df, expected)
 
+    def test_loc_setitem_with_expansion_new_row_and_new_columns(self):
+        # GH#58316
+        df = DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+        df.loc["x", ["C", "D"]] = 91
+        expected = DataFrame(
+            {
+                "A": [1.0, 2.0, 3.0, np.nan],
+                "B": [4.0, 5.0, 6.0, np.nan],
+                "C": [np.nan, np.nan, np.nan, 91.0],
+                "D": [np.nan, np.nan, np.nan, 91.0],
+            },
+            index=Index([0, 1, 2, "x"]),
+        )
+        tm.assert_frame_equal(df, expected)
+
 
 class TestLocCallable:
     def test_frame_loc_getitem_callable(self):