From 339dddf946d1bbf0921be053d17569e301cabadd Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Mon, 12 Jun 2023 18:37:40 -0400 Subject: [PATCH 1/8] DEPR: Add new implementation of DataFrame.stack and deprecate old --- doc/source/conf.py | 8 + .../comparison/comparison_with_r.rst | 2 +- doc/source/user_guide/10min.rst | 2 +- doc/source/user_guide/cookbook.rst | 4 +- doc/source/user_guide/groupby.rst | 2 +- doc/source/user_guide/reshaping.rst | 20 +- doc/source/whatsnew/v2.1.0.rst | 34 +- pandas/core/frame.py | 63 ++- pandas/core/groupby/generic.py | 2 +- pandas/core/indexes/multi.py | 4 + pandas/core/resample.py | 2 +- pandas/core/reshape/pivot.py | 2 +- pandas/core/reshape/reshape.py | 116 +++- pandas/tests/extension/base/reshaping.py | 7 +- pandas/tests/extension/json/test_json.py | 2 +- .../tests/frame/methods/test_reset_index.py | 6 +- pandas/tests/frame/test_stack_unstack.py | 498 +++++++++++------- pandas/tests/frame/test_subclass.py | 14 +- pandas/tests/groupby/aggregate/test_cython.py | 2 +- pandas/tests/groupby/test_categorical.py | 12 +- pandas/tests/groupby/test_function.py | 2 +- .../indexes/datetimes/test_partial_slicing.py | 2 +- pandas/tests/indexes/multi/test_indexing.py | 10 +- pandas/tests/indexes/multi/test_integrity.py | 2 +- pandas/tests/io/json/test_pandas.py | 4 +- pandas/tests/io/pytables/test_append.py | 2 +- .../tests/series/methods/test_reset_index.py | 2 +- pandas/tests/series/methods/test_unstack.py | 2 +- 28 files changed, 568 insertions(+), 260 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 66fca61c2c6e5..88e0d883b5fb4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -376,6 +376,14 @@ os.chdir(r'{os.path.dirname(os.path.dirname(__file__))}') """ +warnings.filterwarnings( + "ignore", + category=UserWarning, + message=( + "Matplotlib is currently using agg, which is a " + "non-GUI backend, so cannot show the figure." + ), +) html_context = { "redirects": dict(moved_api_pages), diff --git a/doc/source/getting_started/comparison/comparison_with_r.rst b/doc/source/getting_started/comparison/comparison_with_r.rst index 767779b0f58a8..55fe323988959 100644 --- a/doc/source/getting_started/comparison/comparison_with_r.rst +++ b/doc/source/getting_started/comparison/comparison_with_r.rst @@ -438,7 +438,7 @@ In Python, the :meth:`~pandas.melt` method is the R equivalent: ) pd.melt(cheese, id_vars=["first", "last"]) - cheese.set_index(["first", "last"]).stack() # alternative way + cheese.set_index(["first", "last"]).stack(v3=True) # alternative way For more details and examples see :ref:`the reshaping documentation `. diff --git a/doc/source/user_guide/10min.rst b/doc/source/user_guide/10min.rst index 7c98c99fecd5b..3e6613fb3f0c1 100644 --- a/doc/source/user_guide/10min.rst +++ b/doc/source/user_guide/10min.rst @@ -569,7 +569,7 @@ columns: .. ipython:: python - stacked = df2.stack() + stacked = df2.stack(v3=True) stacked With a "stacked" DataFrame or Series (having a :class:`MultiIndex` as the diff --git a/doc/source/user_guide/cookbook.rst b/doc/source/user_guide/cookbook.rst index c7278c604ca02..2bb221a9f9809 100644 --- a/doc/source/user_guide/cookbook.rst +++ b/doc/source/user_guide/cookbook.rst @@ -311,7 +311,7 @@ The :ref:`multindexing ` docs. df.columns = pd.MultiIndex.from_tuples([tuple(c.split("_")) for c in df.columns]) df # Now stack & Reset - df = df.stack(0).reset_index(1) + df = df.stack(0, v3=True).reset_index(1) df # And fix the labels (Notice the label 'level_1' got added automatically) df.columns = ["Sample", "All_X", "All_Y"] @@ -688,7 +688,7 @@ The :ref:`Pivot ` docs. aggfunc=np.sum, margins=True, ) - table.stack("City") + table.stack("City", v3=True) `Frequency table like plyr in R `__ diff --git a/doc/source/user_guide/groupby.rst b/doc/source/user_guide/groupby.rst index 6f4008853f161..5bb260afb8189 100644 --- a/doc/source/user_guide/groupby.rst +++ b/doc/source/user_guide/groupby.rst @@ -1726,4 +1726,4 @@ column index name will be used as the name of the inserted column: result - result.stack() + result.stack(v3=True) diff --git a/doc/source/user_guide/reshaping.rst b/doc/source/user_guide/reshaping.rst index 8d0f1048f6e77..814039b1ba6c3 100644 --- a/doc/source/user_guide/reshaping.rst +++ b/doc/source/user_guide/reshaping.rst @@ -127,7 +127,7 @@ stacked level becomes the new lowest level in a :class:`MultiIndex` on the colum .. ipython:: python - stacked = df2.stack() + stacked = df2.stack(v3=True) stacked With a "stacked" :class:`DataFrame` or :class:`Series` (having a :class:`MultiIndex` as the @@ -163,7 +163,7 @@ will result in a **sorted** copy of the original :class:`DataFrame` or :class:`S index = pd.MultiIndex.from_product([[2, 1], ["a", "b"]]) df = pd.DataFrame(np.random.randn(4), index=index, columns=["A"]) df - all(df.unstack().stack() == df.sort_index()) + all(df.unstack().stack(v3=True) == df.sort_index()) The above code will raise a ``TypeError`` if the call to :meth:`~DataFrame.sort_index` is removed. @@ -191,16 +191,16 @@ processed individually. df = pd.DataFrame(np.random.randn(4, 4), columns=columns) df - df.stack(level=["animal", "hair_length"]) + df.stack(level=["animal", "hair_length"], v3=True) The list of levels can contain either level names or level numbers (but not a mixture of the two). .. ipython:: python - # df.stack(level=['animal', 'hair_length']) + # df.stack(level=['animal', 'hair_length'], v3=True) # from above is equivalent to: - df.stack(level=[1, 2]) + df.stack(level=[1, 2], v3=True) Missing data ~~~~~~~~~~~~ @@ -233,8 +233,8 @@ which level in the columns to stack: .. ipython:: python - df2.stack("exp") - df2.stack("animal") + df2.stack("exp", v3=True) + df2.stack("animal", v3=True) Unstacking can result in missing values if subgroups do not have the same set of labels. By default, missing values will be replaced with the default @@ -345,12 +345,12 @@ some very expressive and fast data manipulations. .. ipython:: python df - df.stack().mean(1).unstack() + df.stack(v3=True).mean(1).unstack() # same result, another way df.T.groupby(level=1).mean() - df.stack().groupby(level=1).mean() + df.stack(v3=True).groupby(level=1).mean() df.mean().unstack(0) @@ -460,7 +460,7 @@ as having a multi-level index: .. ipython:: python - table.stack() + table.stack(v3=True) .. _reshaping.crosstabulations: diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 34b91823abc09..20f039cdd52ae 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -26,7 +26,7 @@ Copy-on-Write improvements of Index objects and specifying ``copy=False``, will now use a lazy copy of those Index objects for the columns of the DataFrame (:issue:`52947`) -.. _whatsnew_210.enhancements.enhancement2: +.. _whatsnew_210.enhancements.map_na_action: ``map(func, na_action="ignore")`` now works for all array types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -76,6 +76,38 @@ Also, note that :meth:`Categorical.map` implicitly has had its ``na_action`` set This has been deprecated and will :meth:`Categorical.map` in the future change the default to ``na_action=None``, like for all the other array types. +.. _whatsnew_210.enhancements.new_stack: + +New implementation of :meth:`DataFrame.stack` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +pandas has reimplemented :meth:`DataFrame.stack`. To use the new implementation, pass the argument ``v3=True``. This will become the only option in pandas 3.0. The previous implementation would unnecessarily introduce NA values into the result which could be removed with ``dropna=True``. The new implementation will no longer unnecessarily introduce NA values when stacking multiple levels. As such, the argument ``dropna`` must remain unspecified when using ``v3=True`` and will be removed in the next major release. + +.. ipython:: python + + columns = pd.MultiIndex.from_tuples([("A", "c"), ("B", "d")]) + df = pd.DataFrame([[0, 2], [1, 3]], columns=columns) + df + +In the previous version (``v3=False``), the default of ``dropna=True`` would remove unnecessarily introduced NA values but still coerce the dtype to ``float64`` in the process. In the new version, no NAs are introduced and so there is no coercion of the dtype. + +.. ipython:: python + :okwarning: + + df.stack([0, 1], v3=False, dropna=True) + df.stack([0, 1], v3=True) + +If the input contains NA values, the previous version would drop those as well with ``dropna=True`` or introduce new NA values with ``dropna=False``. The new version persists all values from the input. + +.. ipython:: python + :okwarning: + + df = pd.DataFrame([[0, 2], [np.nan, np.nan]], columns=columns) + df + df.stack([0, 1], v3=False, dropna=True) + df.stack([0, 1], v3=False, dropna=False) + df.stack([0, 1], v3=True) + .. _whatsnew_210.enhancements.other: Other enhancements diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 72d586964b524..482d422e99405 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -9017,7 +9017,12 @@ def pivot_table( sort=sort, ) - def stack(self, level: IndexLabel = -1, dropna: bool = True): + def stack( + self, + level: IndexLabel = -1, + dropna: bool | lib.NoDefault = lib.no_default, + v3: bool = False, + ): """ Stack the prescribed level(s) from columns to index. @@ -9043,6 +9048,11 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True): axis can create combinations of index and column values that are missing from the original dataframe. See Examples section. + v3: bool, default False + Whether to use the new implementation that will replace the current + implementation in pandas 3.0. When True, dropna and sort have no impact + on the result and must remain unspecified. See :ref:`pandas 2.1.0 Release + notes ` for more details. Returns ------- @@ -9082,7 +9092,7 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True): weight height cat 0 1 dog 2 3 - >>> df_single_level_cols.stack() + >>> df_single_level_cols.stack(v3=True) cat weight 0 height 1 dog weight 2 @@ -9104,7 +9114,7 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True): kg pounds cat 1 2 dog 2 4 - >>> df_multi_level_cols1.stack() + >>> df_multi_level_cols1.stack(v3=True) weight cat kg 1 pounds 2 @@ -9129,7 +9139,7 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True): kg m cat 1.0 2.0 dog 3.0 4.0 - >>> df_multi_level_cols2.stack() + >>> df_multi_level_cols2.stack(v3=True) weight height cat kg 1.0 NaN m NaN 2.0 @@ -9140,13 +9150,13 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True): The first parameter controls which level or levels are stacked: - >>> df_multi_level_cols2.stack(0) + >>> df_multi_level_cols2.stack(0, v3=True) kg m cat weight 1.0 NaN height NaN 2.0 dog weight 3.0 NaN height NaN 4.0 - >>> df_multi_level_cols2.stack([0, 1]) + >>> df_multi_level_cols2.stack([0, 1], v3=True) cat weight kg 1.0 height m 2.0 dog weight kg 3.0 @@ -9180,15 +9190,42 @@ def stack(self, level: IndexLabel = -1, dropna: bool = True): dog kg 2.0 NaN m NaN 3.0 """ - from pandas.core.reshape.reshape import ( - stack, - stack_multiple, - ) + if not v3: + from pandas.core.reshape.reshape import ( + stack, + stack_multiple, + ) - if isinstance(level, (tuple, list)): - result = stack_multiple(self, level, dropna=dropna) + if dropna is lib.no_default: + dropna = True + + if isinstance(level, (tuple, list)): + result = stack_multiple(self, level, dropna=dropna) + else: + result = stack(self, level, dropna=dropna) else: - result = stack(self, level, dropna=dropna) + from pandas.core.reshape.reshape import stack_v3 + + if dropna is not lib.no_default: + raise ValueError( + "Cannot specify dropna with v3=True, this argument will be " + "removed in a future version of pandas" + ) + + if ( + isinstance(level, (tuple, list)) + and not all(lev in self.columns.names for lev in level) + and not all(isinstance(lev, int) for lev in level) + ): + raise ValueError( + "level should contain all level names or all level " + "numbers, not a mixture of the two." + ) + + if not isinstance(level, (tuple, list)): + level = [level] + level = [self.columns._get_level_number(lev) for lev in level] + result = stack_v3(self, level) return result.__finalize__(self, method="stack") diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 78f8448cc7cb3..5f28fd75fe2bb 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -410,7 +410,7 @@ def _wrap_applied_output( res_df = self._reindex_output(res_df) # if self.observed is False, # keep all-NaN rows created while re-indexing - res_ser = res_df.stack(dropna=self.observed) + res_ser = res_df.stack(v3=True) res_ser.name = self.obj.name return res_ser elif isinstance(values[0], (Series, DataFrame)): diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 5181623c0c327..6ccd8b9c850cd 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -2386,6 +2386,10 @@ def reorder_levels(self, order) -> MultiIndex: names=['y', 'x']) """ order = [self._get_level_number(i) for i in order] + result = self._reorder_ilevels(order) + return result + + def _reorder_ilevels(self, order) -> MultiIndex: if len(order) != self.nlevels: raise AssertionError( f"Length of order must be same as number of levels ({self.nlevels}), " diff --git a/pandas/core/resample.py b/pandas/core/resample.py index 9c8a9a0d63a14..9e164df161574 100644 --- a/pandas/core/resample.py +++ b/pandas/core/resample.py @@ -1292,7 +1292,7 @@ def size(self): # If the result is a non-empty DataFrame we stack to get a Series # GH 46826 if isinstance(result, ABCDataFrame) and not result.empty: - result = result.stack() + result = result.stack(v3=True) if not len(self.ax): from pandas import Series diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 2852ca8cf576a..c32b018b6fcce 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -439,7 +439,7 @@ def _all_key(key): if len(cols) > 0: row_margin = data[cols + values].groupby(cols, observed=observed).agg(aggfunc) - row_margin = row_margin.stack() + row_margin = row_margin.stack(v3=True) # slight hack new_order = [len(cols)] + list(range(len(cols))) diff --git a/pandas/core/reshape/reshape.py b/pandas/core/reshape/reshape.py index b0c74745511c4..a8792ad7262ac 100644 --- a/pandas/core/reshape/reshape.py +++ b/pandas/core/reshape/reshape.py @@ -27,14 +27,19 @@ from pandas.core.dtypes.missing import notna import pandas.core.algorithms as algos -from pandas.core.algorithms import unique +from pandas.core.algorithms import ( + factorize, + unique, +) from pandas.core.arrays.categorical import factorize_from_iterable from pandas.core.construction import ensure_wrapped_if_datetimelike from pandas.core.frame import DataFrame from pandas.core.indexes.api import ( Index, MultiIndex, + RangeIndex, ) +from pandas.core.reshape.concat import concat from pandas.core.series import Series from pandas.core.sorting import ( compress_group_index, @@ -498,7 +503,7 @@ def unstack(obj: Series | DataFrame, level, fill_value=None, sort: bool = True): if isinstance(obj.index, MultiIndex): return _unstack_frame(obj, level, fill_value=fill_value, sort=sort) else: - return obj.T.stack(dropna=False) + return obj.T.stack(v3=True) elif not isinstance(obj.index, MultiIndex): # GH 36113 # Give nicer error messages when unstack a Series whose @@ -874,3 +879,110 @@ def _reorder_for_extension_array_stack( # c0r1, c1r1, c2r1, ...] idx = np.arange(n_rows * n_columns).reshape(n_columns, n_rows).T.ravel() return arr.take(idx) + + +def stack_v3(frame: DataFrame, level: list[int]) -> Series | DataFrame: + if frame.columns.nunique() != len(frame.columns): + raise ValueError("Columns with duplicate values are not supported in stack") + + # If we need to drop `level` from columns, it needs to be in descending order + drop_levnums = sorted(level)[::-1] + stack_cols = frame.columns._drop_level_numbers( + [k for k in range(frame.columns.nlevels) if k not in level][::-1] + ) + if len(level) > 1: + # Arrange columns in the order we want to take them, e.g. level=[2, 0, 1] + sorter = np.argsort(level) + ordered_stack_cols = stack_cols._reorder_ilevels(sorter) + else: + ordered_stack_cols = stack_cols + + stack_cols_unique = stack_cols.unique() + ordered_stack_cols_unique = ordered_stack_cols.unique() + + # Grab data for each unique index to be stacked + buf = [] + for idx in stack_cols_unique: + if len(frame.columns) == 1: + data = frame.copy() + else: + # Take the data from frame corresponding to this idx value + if not isinstance(idx, tuple): + idx = (idx,) + gen = iter(idx) + column_indexer = tuple( + next(gen) if k in level else slice(None) + for k in range(frame.columns.nlevels) + ) + data = frame.loc[:, column_indexer] + + if len(level) < frame.columns.nlevels: + data.columns = data.columns._drop_level_numbers(drop_levnums) + elif stack_cols.nlevels == 1: + if data.ndim == 1: + data.name = 0 + else: + data.columns = RangeIndex(len(data.columns)) + buf.append(data) + + result: Series | DataFrame + if len(buf) > 0 and not frame.empty: + result = concat(buf) + ratio = len(result) // len(frame) + else: + # input is empty + if len(level) < frame.columns.nlevels: + # concat column order may be different from dropping the levels + new_columns = frame.columns._drop_level_numbers(drop_levnums).unique() + else: + new_columns = [0] + result = DataFrame(columns=new_columns, dtype=frame._values.dtype) + ratio = 0 + + if len(level) < frame.columns.nlevels: + # concat column order may be different from dropping the levels + desired_columns = frame.columns._drop_level_numbers(drop_levnums).unique() + if not result.columns.equals(desired_columns): + result = result[desired_columns] + + # Construct the correct MultiIndex by combining the frame's index and + # stacked columns. + index_levels: list | FrozenList + if isinstance(frame.index, MultiIndex): + index_levels = frame.index.levels + index_codes = list(np.tile(frame.index.codes, (1, ratio))) + else: + index_levels = [frame.index.unique()] + codes = factorize(frame.index)[0] + index_codes = list(np.tile(codes, (1, ratio))) + if isinstance(stack_cols, MultiIndex): + column_levels = ordered_stack_cols.levels + column_codes = ordered_stack_cols.drop_duplicates().codes + else: + column_levels = [ordered_stack_cols.unique()] + column_codes = [factorize(ordered_stack_cols_unique, use_na_sentinel=False)[0]] + column_codes = [np.repeat(codes, len(frame)) for codes in column_codes] + result.index = MultiIndex( + levels=index_levels + column_levels, + codes=index_codes + column_codes, + names=frame.index.names + list(ordered_stack_cols.names), + verify_integrity=False, + ) + + # sort result, but faster than calling sort_index since we know the order we need + len_df = len(frame) + n_uniques = len(ordered_stack_cols_unique) + indexer = np.arange(n_uniques) + idxs = np.tile(len_df * indexer, len_df) + np.repeat(np.arange(len_df), n_uniques) + result = result.take(idxs) + + # Reshape/rename if needed and dropna + if result.ndim == 2 and frame.columns.nlevels == len(level): + if len(result.columns) == 0: + result = Series(index=result.index) + else: + result = result.iloc[:, 0] + if result.ndim == 1: + result.name = None + + return result diff --git a/pandas/tests/extension/base/reshaping.py b/pandas/tests/extension/base/reshaping.py index cc970c690529d..5a6c20af35f5f 100644 --- a/pandas/tests/extension/base/reshaping.py +++ b/pandas/tests/extension/base/reshaping.py @@ -252,11 +252,12 @@ def test_merge_on_extension_array_duplicates(self, data): ), ], ) - def test_stack(self, data, columns): + @pytest.mark.parametrize("v3", [True, False]) + def test_stack(self, data, columns, v3): df = pd.DataFrame({"A": data[:5], "B": data[:5]}) df.columns = columns - result = df.stack() - expected = df.astype(object).stack() + result = df.stack(v3=v3) + expected = df.astype(object).stack(v3=v3) # we need a second astype(object), in case the constructor inferred # object -> specialized, as is done for period. expected = expected.astype(object) diff --git a/pandas/tests/extension/json/test_json.py b/pandas/tests/extension/json/test_json.py index 0920e70142446..970fef9cfb4ab 100644 --- a/pandas/tests/extension/json/test_json.py +++ b/pandas/tests/extension/json/test_json.py @@ -199,7 +199,7 @@ class TestReshaping(BaseJSON, base.BaseReshapingTests): @pytest.mark.xfail(reason="Different definitions of NA") def test_stack(self): """ - The test does .astype(object).stack(). If we happen to have + The test does .astype(object).stack(v3=True). If we happen to have any missing values in `data`, then we'll end up with different rows since we consider `{}` NA, but `.astype(object)` doesn't. """ diff --git a/pandas/tests/frame/methods/test_reset_index.py b/pandas/tests/frame/methods/test_reset_index.py index a2ab02c5a6038..f844b49f22e26 100644 --- a/pandas/tests/frame/methods/test_reset_index.py +++ b/pandas/tests/frame/methods/test_reset_index.py @@ -112,7 +112,7 @@ def test_reset_index_with_intervals(self): tm.assert_frame_equal(result2, original) def test_reset_index(self, float_frame): - stacked = float_frame.stack()[::2] + stacked = float_frame.stack(v3=True)[::2] stacked = DataFrame({"foo": stacked, "bar": stacked}) names = ["first", "second"] @@ -749,7 +749,7 @@ def test_reset_index_rename(float_frame): def test_reset_index_rename_multiindex(float_frame): # GH 6878 - stacked_df = float_frame.stack()[::2] + stacked_df = float_frame.stack(v3=True)[::2] stacked_df = DataFrame({"foo": stacked_df, "bar": stacked_df}) names = ["first", "second"] @@ -763,7 +763,7 @@ def test_reset_index_rename_multiindex(float_frame): def test_errorreset_index_rename(float_frame): # GH 6878 - stacked_df = float_frame.stack()[::2] + stacked_df = float_frame.stack(v3=True)[::2] stacked_df = DataFrame({"first": stacked_df, "second": stacked_df}) with pytest.raises( diff --git a/pandas/tests/frame/test_stack_unstack.py b/pandas/tests/frame/test_stack_unstack.py index ffdcb06ee2847..6d3a02e67d82a 100644 --- a/pandas/tests/frame/test_stack_unstack.py +++ b/pandas/tests/frame/test_stack_unstack.py @@ -6,6 +6,7 @@ import numpy as np import pytest +from pandas._libs import lib from pandas.errors import PerformanceWarning import pandas as pd @@ -22,12 +23,17 @@ from pandas.core.reshape import reshape as reshape_lib +@pytest.fixture(params=[True, False]) +def v3(request): + return request.param + + class TestDataFrameReshape: - def test_stack_unstack(self, float_frame): + def test_stack_unstack(self, float_frame, v3): df = float_frame.copy() df[:] = np.arange(np.prod(df.shape)).reshape(df.shape) - stacked = df.stack() + stacked = df.stack(v3=v3) stacked_df = DataFrame({"foo": stacked, "bar": stacked}) unstacked = stacked.unstack() @@ -41,26 +47,26 @@ def test_stack_unstack(self, float_frame): tm.assert_frame_equal(unstacked_cols.T, df) tm.assert_frame_equal(unstacked_cols_df["bar"].T, df) - def test_stack_mixed_level(self): + def test_stack_mixed_level(self, v3): # GH 18310 levels = [range(3), [3, "a", "b"], [1, 2]] # flat columns: df = DataFrame(1, index=levels[0], columns=levels[1]) - result = df.stack() + result = df.stack(v3=v3) expected = Series(1, index=MultiIndex.from_product(levels[:2])) tm.assert_series_equal(result, expected) # MultiIndex columns: df = DataFrame(1, index=levels[0], columns=MultiIndex.from_product(levels[1:])) - result = df.stack(1) + result = df.stack(1, v3=v3) expected = DataFrame( 1, index=MultiIndex.from_product([levels[0], levels[2]]), columns=levels[1] ) tm.assert_frame_equal(result, expected) # as above, but used labels in level are actually of homogeneous type - result = df[["a", "b"]].stack(1) + result = df[["a", "b"]].stack(1, v3=v3) expected = expected[["a", "b"]] tm.assert_frame_equal(result, expected) @@ -76,7 +82,7 @@ def test_unstack_not_consolidated(self, using_array_manager): expected = df.unstack() tm.assert_series_equal(res, expected) - def test_unstack_fill(self): + def test_unstack_fill(self, v3): # GH #9746: fill_value keyword argument for Series # and DataFrame unstack @@ -109,7 +115,7 @@ def test_unstack_fill(self): result = Series([0, 0, 2], index=unstacked.index, name=key) tm.assert_series_equal(result, expected) - stacked = unstacked.stack(["x", "y"]) + stacked = unstacked.stack(["x", "y"], v3=v3) stacked.index = stacked.index.reorder_levels(df.index.names) # Workaround for GH #17886 (unnecessarily casts to float): stacked = stacked.astype(np.int64) @@ -382,13 +388,17 @@ def unstack_and_compare(df, column_name): s = df1["A"] unstack_and_compare(s, "index") - def test_stack_ints(self): + def test_stack_ints(self, v3): columns = MultiIndex.from_tuples(list(itertools.product(range(3), repeat=3))) df = DataFrame(np.random.randn(30, 27), columns=columns) - tm.assert_frame_equal(df.stack(level=[1, 2]), df.stack(level=1).stack(level=1)) tm.assert_frame_equal( - df.stack(level=[-2, -1]), df.stack(level=1).stack(level=1) + df.stack(level=[1, 2], v3=v3), + df.stack(level=1, v3=v3).stack(level=1, v3=v3), + ) + tm.assert_frame_equal( + df.stack(level=[-2, -1], v3=v3), + df.stack(level=1, v3=v3).stack(level=1, v3=v3), ) df_named = df.copy() @@ -396,10 +406,11 @@ def test_stack_ints(self): assert return_value is None tm.assert_frame_equal( - df_named.stack(level=[1, 2]), df_named.stack(level=1).stack(level=1) + df_named.stack(level=[1, 2], v3=v3), + df_named.stack(level=1, v3=v3).stack(level=1, v3=v3), ) - def test_stack_mixed_levels(self): + def test_stack_mixed_levels(self, v3): columns = MultiIndex.from_tuples( [ ("A", "cat", "long"), @@ -411,8 +422,8 @@ def test_stack_mixed_levels(self): ) df = DataFrame(np.random.randn(4, 4), columns=columns) - animal_hair_stacked = df.stack(level=["animal", "hair_length"]) - exp_hair_stacked = df.stack(level=["exp", "hair_length"]) + animal_hair_stacked = df.stack(level=["animal", "hair_length"], v3=v3) + exp_hair_stacked = df.stack(level=["exp", "hair_length"], v3=v3) # GH #8584: Need to check that stacking works when a number # is passed that is both a level name and in the range of @@ -420,10 +431,14 @@ def test_stack_mixed_levels(self): df2 = df.copy() df2.columns.names = ["exp", "animal", 1] tm.assert_frame_equal( - df2.stack(level=["animal", 1]), animal_hair_stacked, check_names=False + df2.stack(level=["animal", 1], v3=v3), + animal_hair_stacked, + check_names=False, ) tm.assert_frame_equal( - df2.stack(level=["exp", 1]), exp_hair_stacked, check_names=False + df2.stack(level=["exp", 1], v3=v3), + exp_hair_stacked, + check_names=False, ) # When mixed types are passed and the ints are not level @@ -433,17 +448,19 @@ def test_stack_mixed_levels(self): "a mixture of the two" ) with pytest.raises(ValueError, match=msg): - df2.stack(level=["animal", 0]) + df2.stack(level=["animal", 0], v3=v3) # GH #8584: Having 0 in the level names could raise a # strange error about lexsort depth df3 = df.copy() df3.columns.names = ["exp", "animal", 0] tm.assert_frame_equal( - df3.stack(level=["animal", 0]), animal_hair_stacked, check_names=False + df3.stack(level=["animal", 0], v3=v3), + animal_hair_stacked, + check_names=False, ) - def test_stack_int_level_names(self): + def test_stack_int_level_names(self, v3): columns = MultiIndex.from_tuples( [ ("A", "cat", "long"), @@ -455,33 +472,45 @@ def test_stack_int_level_names(self): ) df = DataFrame(np.random.randn(4, 4), columns=columns) - exp_animal_stacked = df.stack(level=["exp", "animal"]) - animal_hair_stacked = df.stack(level=["animal", "hair_length"]) - exp_hair_stacked = df.stack(level=["exp", "hair_length"]) + exp_animal_stacked = df.stack(level=["exp", "animal"], v3=v3) + animal_hair_stacked = df.stack(level=["animal", "hair_length"], v3=v3) + exp_hair_stacked = df.stack(level=["exp", "hair_length"], v3=v3) df2 = df.copy() df2.columns.names = [0, 1, 2] tm.assert_frame_equal( - df2.stack(level=[1, 2]), animal_hair_stacked, check_names=False + df2.stack(level=[1, 2], v3=v3), + animal_hair_stacked, + check_names=False, ) tm.assert_frame_equal( - df2.stack(level=[0, 1]), exp_animal_stacked, check_names=False + df2.stack(level=[0, 1], v3=v3), + exp_animal_stacked, + check_names=False, ) tm.assert_frame_equal( - df2.stack(level=[0, 2]), exp_hair_stacked, check_names=False + df2.stack(level=[0, 2], v3=v3), + exp_hair_stacked, + check_names=False, ) # Out-of-order int column names df3 = df.copy() df3.columns.names = [2, 0, 1] tm.assert_frame_equal( - df3.stack(level=[0, 1]), animal_hair_stacked, check_names=False + df3.stack(level=[0, 1], v3=v3), + animal_hair_stacked, + check_names=False, ) tm.assert_frame_equal( - df3.stack(level=[2, 0]), exp_animal_stacked, check_names=False + df3.stack(level=[2, 0], v3=v3), + exp_animal_stacked, + check_names=False, ) tm.assert_frame_equal( - df3.stack(level=[2, 1]), exp_hair_stacked, check_names=False + df3.stack(level=[2, 1], v3=v3), + exp_hair_stacked, + check_names=False, ) def test_unstack_bool(self): @@ -498,7 +527,7 @@ def test_unstack_bool(self): ) tm.assert_frame_equal(rs, xp) - def test_unstack_level_binding(self): + def test_unstack_level_binding(self, v3): # GH9856 mi = MultiIndex( levels=[["foo", "bar"], ["one", "two"], ["a", "b"]], @@ -506,7 +535,7 @@ def test_unstack_level_binding(self): names=["first", "second", "third"], ) s = Series(0, index=mi) - result = s.unstack([1, 2]).stack(0) + result = s.unstack([1, 2]).stack(0, v3=v3) expected_mi = MultiIndex( levels=[["foo", "bar"], ["one", "two"]], @@ -625,7 +654,7 @@ def test_unstack_dtypes_mixed_date(self, c, d): assert left.shape == (3, 2) tm.assert_frame_equal(left, right) - def test_unstack_non_unique_index_names(self): + def test_unstack_non_unique_index_names(self, v3): idx = MultiIndex.from_tuples([("a", "b"), ("c", "d")], names=["c1", "c1"]) df = DataFrame([1, 2], index=idx) msg = "The name c1 occurs multiple times, use a level number" @@ -633,7 +662,7 @@ def test_unstack_non_unique_index_names(self): df.unstack("c1") with pytest.raises(ValueError, match=msg): - df.T.stack("c1") + df.T.stack("c1", v3=v3) def test_unstack_unused_levels(self): # GH 17845: unused codes in index make unstack() cast int to float @@ -989,11 +1018,11 @@ def test_unstack_nan_index5(self): key = r["1st"], (col, r["2nd"], r["3rd"]) assert r[col] == left.loc[key] - def test_stack_datetime_column_multiIndex(self): + def test_stack_datetime_column_multiIndex(self, v3): # GH 8039 t = datetime(2014, 1, 1) df = DataFrame([1, 2, 3, 4], columns=MultiIndex.from_tuples([(t, "A", "B")])) - result = df.stack() + result = df.stack(v3=v3) eidx = MultiIndex.from_product([(0, 1, 2, 3), ("B",)]) ecols = MultiIndex.from_tuples([(t, "A")]) @@ -1027,8 +1056,9 @@ def test_stack_datetime_column_multiIndex(self): ], ) @pytest.mark.parametrize("level", (-1, 0, 1, [0, 1], [1, 0])) - def test_stack_partial_multiIndex(self, multiindex_columns, level): + def test_stack_partial_multiIndex(self, multiindex_columns, level, v3): # GH 8844 + dropna = False if not v3 else lib.no_default full_multiindex = MultiIndex.from_tuples( [("B", "x"), ("B", "z"), ("A", "y"), ("C", "x"), ("C", "u")], names=["Upper", "Lower"], @@ -1038,13 +1068,13 @@ def test_stack_partial_multiIndex(self, multiindex_columns, level): np.arange(3 * len(multiindex)).reshape(3, len(multiindex)), columns=multiindex, ) - result = df.stack(level=level, dropna=False) + result = df.stack(level=level, dropna=dropna, v3=v3) - if isinstance(level, int): + if isinstance(level, int) and not v3: # Stacking a single level should not make any all-NaN rows, # so df.stack(level=level, dropna=False) should be the same # as df.stack(level=level, dropna=True). - expected = df.stack(level=level, dropna=True) + expected = df.stack(level=level, dropna=True, v3=v3) if isinstance(expected, Series): tm.assert_series_equal(result, expected) else: @@ -1053,20 +1083,21 @@ def test_stack_partial_multiIndex(self, multiindex_columns, level): df.columns = MultiIndex.from_tuples( df.columns.to_numpy(), names=df.columns.names ) - expected = df.stack(level=level, dropna=False) + expected = df.stack(level=level, dropna=dropna, v3=v3) if isinstance(expected, Series): tm.assert_series_equal(result, expected) else: tm.assert_frame_equal(result, expected) - def test_stack_full_multiIndex(self): + def test_stack_full_multiIndex(self, v3): # GH 8844 full_multiindex = MultiIndex.from_tuples( [("B", "x"), ("B", "z"), ("A", "y"), ("C", "x"), ("C", "u")], names=["Upper", "Lower"], ) df = DataFrame(np.arange(6).reshape(2, 3), columns=full_multiindex[[0, 1, 3]]) - result = df.stack(dropna=False) + dropna = False if not v3 else lib.no_default + result = df.stack(dropna=dropna, v3=v3) expected = DataFrame( [[0, 2], [1, np.nan], [3, 5], [4, np.nan]], index=MultiIndex( @@ -1080,12 +1111,11 @@ def test_stack_full_multiIndex(self): tm.assert_frame_equal(result, expected) @pytest.mark.parametrize("ordered", [False, True]) - @pytest.mark.parametrize("labels", [list("yxz"), list("yxy")]) - def test_stack_preserve_categorical_dtype(self, ordered, labels): + def test_stack_preserve_categorical_dtype(self, ordered, v3): # GH13854 - cidx = pd.CategoricalIndex(labels, categories=list("xyz"), ordered=ordered) + cidx = pd.CategoricalIndex(list("yxz"), categories=list("xyz"), ordered=ordered) df = DataFrame([[10, 11, 12]], columns=cidx) - result = df.stack() + result = df.stack(v3=v3) # `MultiIndex.from_product` preserves categorical dtype - # it's tested elsewhere. @@ -1102,24 +1132,24 @@ def test_stack_preserve_categorical_dtype(self, ordered, labels): (list("zyx"), [10, 11, 12, 13, 14, 15]), ], ) - def test_stack_multi_preserve_categorical_dtype(self, ordered, labels, data): + def test_stack_multi_preserve_categorical_dtype(self, ordered, labels, data, v3): # GH-36991 cidx = pd.CategoricalIndex(labels, categories=sorted(labels), ordered=ordered) cidx2 = pd.CategoricalIndex(["u", "v"], ordered=ordered) midx = MultiIndex.from_product([cidx, cidx2]) df = DataFrame([data], columns=midx) - result = df.stack([0, 1]) + result = df.stack([0, 1], v3=v3) s_cidx = pd.CategoricalIndex(labels, ordered=ordered) expected = Series(data, index=MultiIndex.from_product([[0], s_cidx, cidx2])) tm.assert_series_equal(result, expected) - def test_stack_preserve_categorical_dtype_values(self): + def test_stack_preserve_categorical_dtype_values(self, v3): # GH-23077 cat = pd.Categorical(["a", "a", "b", "c"]) df = DataFrame({"A": cat, "B": cat}) - result = df.stack() + result = df.stack(v3=v3) index = MultiIndex.from_product([[0, 1, 2, 3], ["A", "B"]]) expected = Series( pd.Categorical(["a", "a", "a", "a", "b", "b", "c", "c"]), index=index @@ -1134,10 +1164,10 @@ def test_stack_preserve_categorical_dtype_values(self): ([0, 1, 2, 3], MultiIndex.from_product([[1, 2], ["a", "b"]])), ], ) - def test_stack_multi_columns_non_unique_index(self, index, columns): + def test_stack_multi_columns_non_unique_index(self, index, columns, v3): # GH-28301 df = DataFrame(index=index, columns=columns).fillna(1) - stacked = df.stack() + stacked = df.stack(v3=v3) new_index = MultiIndex.from_tuples(stacked.index.to_numpy()) expected = DataFrame( stacked.to_numpy(), index=new_index, columns=stacked.columns @@ -1155,7 +1185,7 @@ def test_stack_multi_columns_non_unique_index(self, index, columns): ], ) def test_stack_multi_columns_mixed_extension_types( - self, vals1, vals2, dtype1, dtype2, expected_dtype + self, vals1, vals2, dtype1, dtype2, expected_dtype, v3 ): # GH45740 df = DataFrame( @@ -1164,8 +1194,8 @@ def test_stack_multi_columns_mixed_extension_types( ("A", 2): Series(vals2, dtype=dtype2), } ) - result = df.stack() - expected = df.astype(object).stack().astype(expected_dtype) + result = df.stack(v3=v3) + expected = df.astype(object).stack(v3=v3).astype(expected_dtype) tm.assert_frame_equal(result, expected) @pytest.mark.parametrize("level", [0, 1]) @@ -1295,11 +1325,11 @@ def test_unstack_timezone_aware_values(): tm.assert_frame_equal(result, expected) -def test_stack_timezone_aware_values(): +def test_stack_timezone_aware_values(v3): # GH 19420 ts = date_range(freq="D", start="20180101", end="20180103", tz="America/New_York") df = DataFrame({"A": ts}, index=["a", "b", "c"]) - result = df.stack() + result = df.stack(v3=v3) expected = Series( ts, index=MultiIndex(levels=[["a", "b", "c"], ["A"]], codes=[[0, 1, 2], [0, 0, 0]]), @@ -1307,24 +1337,36 @@ def test_stack_timezone_aware_values(): tm.assert_series_equal(result, expected) -@pytest.mark.parametrize("dropna", [True, False]) -def test_stack_empty_frame(dropna): +@pytest.mark.parametrize("dropna", [True, False, lib.no_default]) +def test_stack_empty_frame(dropna, v3): # GH 36113 levels = [np.array([], dtype=np.int64), np.array([], dtype=np.int64)] expected = Series(dtype=np.float64, index=MultiIndex(levels=levels, codes=[[], []])) - result = DataFrame(dtype=np.float64).stack(dropna=dropna) - tm.assert_series_equal(result, expected) + if v3 and dropna is not lib.no_default: + with pytest.raises(ValueError, match="Cannot specify dropna"): + DataFrame(dtype=np.float64).stack(dropna=dropna, v3=v3) + else: + result = DataFrame(dtype=np.float64).stack(dropna=dropna, v3=v3) + tm.assert_series_equal(result, expected) -@pytest.mark.parametrize("dropna", [True, False]) +@pytest.mark.parametrize("dropna", [True, False, lib.no_default]) @pytest.mark.parametrize("fill_value", [None, 0]) -def test_stack_unstack_empty_frame(dropna, fill_value): +def test_stack_unstack_empty_frame(dropna, fill_value, v3): # GH 36113 - result = ( - DataFrame(dtype=np.int64).stack(dropna=dropna).unstack(fill_value=fill_value) - ) - expected = DataFrame(dtype=np.int64) - tm.assert_frame_equal(result, expected) + if v3 and dropna is not lib.no_default: + with pytest.raises(ValueError, match="Cannot specify dropna"): + DataFrame(dtype=np.int64).stack(dropna=dropna, v3=v3).unstack( + fill_value=fill_value + ) + else: + result = ( + DataFrame(dtype=np.int64) + .stack(dropna=dropna, v3=v3) + .unstack(fill_value=fill_value) + ) + expected = DataFrame(dtype=np.int64) + tm.assert_frame_equal(result, expected) def test_unstack_single_index_series(): @@ -1365,11 +1407,11 @@ def test_unstacking_multi_index_df(): tm.assert_frame_equal(result, expected) -def test_stack_positional_level_duplicate_column_names(): +def test_stack_positional_level_duplicate_column_names(v3): # https://github.com/pandas-dev/pandas/issues/36353 columns = MultiIndex.from_product([("x", "y"), ("y", "z")], names=["a", "a"]) df = DataFrame([[1, 1, 1, 1]], columns=columns) - result = df.stack(0) + result = df.stack(0, v3=v3) new_columns = Index(["y", "z"], name="a") new_index = MultiIndex.from_tuples([(0, "x"), (0, "y")], names=[None, "a"]) @@ -1400,7 +1442,7 @@ def test_unstack_non_slice_like_blocks(using_array_manager): tm.assert_frame_equal(res, expected) -def test_stack_nosort(): +def test_stack_nosort(v3): # GH 15105, GH 53825 data = [[1, 2, 3.0, 4.0], [2, 3, 4.0, 5.0], [3, 4, np.nan, np.nan]] df = DataFrame( @@ -1409,11 +1451,22 @@ def test_stack_nosort(): levels=[["B", "A"], ["x", "y"]], codes=[[0, 0, 1, 1], [0, 1, 0, 1]] ), ) - result = df.stack(level=0) - expected = DataFrame( - {"x": [1.0, 3.0, 2.0, 4.0, 3.0], "y": [2.0, 4.0, 3.0, 5.0, 4.0]}, - index=MultiIndex.from_arrays([[0, 0, 1, 1, 2], ["B", "A", "B", "A", "B"]]), - ) + result = df.stack(level=0, v3=v3) + if v3: + expected = DataFrame( + { + "x": [1.0, 3.0, 2.0, 4.0, 3.0, np.nan], + "y": [2.0, 4.0, 3.0, 5.0, 4.0, np.nan], + }, + index=MultiIndex.from_arrays( + [[0, 0, 1, 1, 2, 2], ["B", "A", "B", "A", "B", "A"]] + ), + ) + else: + expected = DataFrame( + {"x": [1.0, 3.0, 2.0, 4.0, 3.0], "y": [2.0, 4.0, 3.0, 5.0, 4.0]}, + index=MultiIndex.from_arrays([[0, 0, 1, 1, 2], ["B", "A", "B", "A", "B"]]), + ) tm.assert_frame_equal(result, expected) # Codes sorted in this call @@ -1421,15 +1474,15 @@ def test_stack_nosort(): data, columns=MultiIndex.from_arrays([["B", "B", "A", "A"], ["x", "y", "x", "y"]]), ) - result = df.stack(level=0) + result = df.stack(level=0, v3=v3) tm.assert_frame_equal(result, expected) -def test_stack_nosort_multi_level(): +def test_stack_nosort_multi_level(v3): # GH 15105, GH 53825 idx = MultiIndex.from_tuples([("weight", "kg"), ("height", "m")]) df = DataFrame([[1.0, 2.0], [3.0, 4.0]], index=["cat", "dog"], columns=idx) - result = df.stack([0, 1]) + result = df.stack([0, 1], v3=v3) expected_index = MultiIndex.from_tuples( [ ("cat", "weight", "kg"), @@ -1510,75 +1563,83 @@ def test_unstack_multiple_no_empty_columns(self): expected = unstacked.dropna(axis=1, how="all") tm.assert_frame_equal(unstacked, expected) - def test_stack(self, multiindex_year_month_day_dataframe_random_data): + def test_stack(self, multiindex_year_month_day_dataframe_random_data, v3): ymd = multiindex_year_month_day_dataframe_random_data # regular roundtrip unstacked = ymd.unstack() - restacked = unstacked.stack() + restacked = unstacked.stack(v3=v3) + if v3: + # NA values in unstacked persist to restacked in version 3 + restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked, ymd) unlexsorted = ymd.sort_index(level=2) unstacked = unlexsorted.unstack(2) - restacked = unstacked.stack() + restacked = unstacked.stack(v3=v3) + if v3: + # NA values in unstacked persist to restacked in version 3 + restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked.sort_index(level=0), ymd) unlexsorted = unlexsorted[::-1] unstacked = unlexsorted.unstack(1) - restacked = unstacked.stack().swaplevel(1, 2) + restacked = unstacked.stack(v3=v3).swaplevel(1, 2) + if v3: + # NA values in unstacked persist to restacked in version 3 + restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked.sort_index(level=0), ymd) unlexsorted = unlexsorted.swaplevel(0, 1) unstacked = unlexsorted.unstack(0).swaplevel(0, 1, axis=1) - restacked = unstacked.stack(0).swaplevel(1, 2) + restacked = unstacked.stack(0, v3=v3).swaplevel(1, 2) + if v3: + # NA values in unstacked persist to restacked in version 3 + restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked.sort_index(level=0), ymd) # columns unsorted unstacked = ymd.unstack() - restacked = unstacked.stack() + restacked = unstacked.stack(v3=v3) + if v3: + # NA values in unstacked persist to restacked in version 3 + restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked, ymd) # more than 2 levels in the columns unstacked = ymd.unstack(1).unstack(1) - result = unstacked.stack(1) + result = unstacked.stack(1, v3=v3) expected = ymd.unstack() tm.assert_frame_equal(result, expected) - result = unstacked.stack(2) + result = unstacked.stack(2, v3=v3) expected = ymd.unstack(1) tm.assert_frame_equal(result, expected) - result = unstacked.stack(0) - expected = ymd.stack().unstack(1).unstack(1) + result = unstacked.stack(0, v3=v3) + expected = ymd.stack(v3=v3).unstack(1).unstack(1) tm.assert_frame_equal(result, expected) # not all levels present in each echelon unstacked = ymd.unstack(2).loc[:, ::3] - stacked = unstacked.stack().stack() - ymd_stacked = ymd.stack() + stacked = unstacked.stack(v3=v3).stack(v3=v3) + ymd_stacked = ymd.stack(v3=v3) + if v3: + # NA values in unstacked persist to restacked in version 3 + stacked = stacked.dropna(how="all") + ymd_stacked = ymd_stacked.dropna(how="all") tm.assert_series_equal(stacked, ymd_stacked.reindex(stacked.index)) # stack with negative number - result = ymd.unstack(0).stack(-2) - expected = ymd.unstack(0).stack(0) + result = ymd.unstack(0).stack(-2, v3=v3) + expected = ymd.unstack(0).stack(0, v3=v3) tm.assert_equal(result, expected) @pytest.mark.parametrize( "idx, columns, exp_idx", [ - [ - list("abab"), - ["1st", "2nd", "3rd"], - MultiIndex( - levels=[["a", "b"], ["1st", "2nd", "3rd"]], - codes=[ - np.tile(np.arange(2).repeat(3), 2), - np.tile(np.arange(3), 4), - ], - ), - ], [ list("abab"), ["1st", "2nd", "1st"], @@ -1601,21 +1662,26 @@ def test_stack(self, multiindex_year_month_day_dataframe_random_data): ], ], ) - def test_stack_duplicate_index(self, idx, columns, exp_idx): + def test_stack_duplicate_index(self, idx, columns, exp_idx, v3): # GH10417 df = DataFrame( np.arange(12).reshape(4, 3), index=idx, columns=columns, ) - result = df.stack() - expected = Series(np.arange(12), index=exp_idx) - tm.assert_series_equal(result, expected) - assert result.index.is_unique is False - li, ri = result.index, expected.index - tm.assert_index_equal(li, ri) + if v3: + msg = "Columns with duplicate values are not supported in stack" + with pytest.raises(ValueError, match=msg): + df.stack(v3=v3) + else: + result = df.stack(v3=v3) + expected = Series(np.arange(12), index=exp_idx) + tm.assert_series_equal(result, expected) + assert result.index.is_unique is False + li, ri = result.index, expected.index + tm.assert_index_equal(li, ri) - def test_unstack_odd_failure(self): + def test_unstack_odd_failure(self, v3): data = """day,time,smoker,sum,len Fri,Dinner,No,8.25,3. Fri,Dinner,Yes,27.03,9 @@ -1634,23 +1700,26 @@ def test_unstack_odd_failure(self): # it works, #2100 result = df.unstack(2) - recons = result.stack() + recons = result.stack(v3=v3) + if v3: + # NA values in unstacked persist to restacked in version 3 + recons = recons.dropna(how="all") tm.assert_frame_equal(recons, df) - def test_stack_mixed_dtype(self, multiindex_dataframe_random_data): + def test_stack_mixed_dtype(self, multiindex_dataframe_random_data, v3): frame = multiindex_dataframe_random_data df = frame.T df["foo", "four"] = "foo" df = df.sort_index(level=1, axis=1) - stacked = df.stack() - result = df["foo"].stack().sort_index() + stacked = df.stack(v3=v3) + result = df["foo"].stack(v3=v3).sort_index() tm.assert_series_equal(stacked["foo"], result, check_names=False) assert result.name is None assert stacked["bar"].dtype == np.float_ - def test_unstack_bug(self): + def test_unstack_bug(self, v3): df = DataFrame( { "state": ["naive", "naive", "naive", "active", "active", "active"], @@ -1664,22 +1733,22 @@ def test_unstack_bug(self): result = df.groupby(["state", "exp", "barcode", "v"]).apply(len) unstacked = result.unstack() - restacked = unstacked.stack() + restacked = unstacked.stack(v3=v3) tm.assert_series_equal(restacked, result.reindex(restacked.index).astype(float)) - def test_stack_unstack_preserve_names(self, multiindex_dataframe_random_data): + def test_stack_unstack_preserve_names(self, multiindex_dataframe_random_data, v3): frame = multiindex_dataframe_random_data unstacked = frame.unstack() assert unstacked.index.name == "first" assert unstacked.columns.names == ["exp", "second"] - restacked = unstacked.stack() + restacked = unstacked.stack(v3=v3) assert restacked.index.names == frame.index.names @pytest.mark.parametrize("method", ["stack", "unstack"]) def test_stack_unstack_wrong_level_name( - self, method, multiindex_dataframe_random_data + self, method, multiindex_dataframe_random_data, v3 ): # GH 18303 - wrong level name should raise frame = multiindex_dataframe_random_data @@ -1687,14 +1756,15 @@ def test_stack_unstack_wrong_level_name( # A DataFrame with flat axes: df = frame.loc["foo"] + kwargs = {"v3": v3} if method == "stack" else {} with pytest.raises(KeyError, match="does not match index name"): - getattr(df, method)("mistake") + getattr(df, method)("mistake", **kwargs) if method == "unstack": # Same on a Series: s = df.iloc[:, 0] with pytest.raises(KeyError, match="does not match index name"): - getattr(s, method)("mistake") + getattr(s, method)("mistake", **kwargs) def test_unstack_level_name(self, multiindex_dataframe_random_data): frame = multiindex_dataframe_random_data @@ -1703,20 +1773,20 @@ def test_unstack_level_name(self, multiindex_dataframe_random_data): expected = frame.unstack(level=1) tm.assert_frame_equal(result, expected) - def test_stack_level_name(self, multiindex_dataframe_random_data): + def test_stack_level_name(self, multiindex_dataframe_random_data, v3): frame = multiindex_dataframe_random_data unstacked = frame.unstack("second") - result = unstacked.stack("exp") - expected = frame.unstack().stack(0) + result = unstacked.stack("exp", v3=v3) + expected = frame.unstack().stack(0, v3=v3) tm.assert_frame_equal(result, expected) - result = frame.stack("exp") - expected = frame.stack() + result = frame.stack("exp", v3=v3) + expected = frame.stack(v3=v3) tm.assert_series_equal(result, expected) def test_stack_unstack_multiple( - self, multiindex_year_month_day_dataframe_random_data + self, multiindex_year_month_day_dataframe_random_data, v3 ): ymd = multiindex_year_month_day_dataframe_random_data @@ -1730,7 +1800,10 @@ def test_stack_unstack_multiple( s_unstacked = s.unstack(["year", "month"]) tm.assert_frame_equal(s_unstacked, expected["A"]) - restacked = unstacked.stack(["year", "month"]) + restacked = unstacked.stack(["year", "month"], v3=v3) + if v3: + # NA values in unstacked persist to restacked in version 3 + restacked = restacked.dropna(how="all") restacked = restacked.swaplevel(0, 1).swaplevel(1, 2) restacked = restacked.sort_index(level=0) @@ -1747,7 +1820,7 @@ def test_stack_unstack_multiple( tm.assert_frame_equal(unstacked, expected.loc[:, unstacked.columns]) def test_stack_names_and_numbers( - self, multiindex_year_month_day_dataframe_random_data + self, multiindex_year_month_day_dataframe_random_data, v3 ): ymd = multiindex_year_month_day_dataframe_random_data @@ -1755,10 +1828,10 @@ def test_stack_names_and_numbers( # Can't use mixture of names and numbers to stack with pytest.raises(ValueError, match="level should contain"): - unstacked.stack([0, "month"]) + unstacked.stack([0, "month"], v3=v3) def test_stack_multiple_out_of_bounds( - self, multiindex_year_month_day_dataframe_random_data + self, multiindex_year_month_day_dataframe_random_data, v3 ): # nlevels == 3 ymd = multiindex_year_month_day_dataframe_random_data @@ -1766,9 +1839,9 @@ def test_stack_multiple_out_of_bounds( unstacked = ymd.unstack(["year", "month"]) with pytest.raises(IndexError, match="Too many levels"): - unstacked.stack([2, 3]) + unstacked.stack([2, 3], v3=v3) with pytest.raises(IndexError, match="not a valid level number"): - unstacked.stack([-4, -3]) + unstacked.stack([-4, -3], v3=v3) def test_unstack_period_series(self): # GH4342 @@ -1886,7 +1959,7 @@ def test_unstack_period_frame(self): tm.assert_frame_equal(result3, expected) - def test_stack_multiple_bug(self): + def test_stack_multiple_bug(self, v3): # bug when some uniques are not present in the data GH#3170 id_col = ([1] * 3) + ([2] * 3) name = (["a"] * 3) + (["b"] * 3) @@ -1901,23 +1974,28 @@ def test_stack_multiple_bug(self): with pytest.raises(TypeError, match=msg): unst.resample("W-THU").mean() down = unst.resample("W-THU").mean(numeric_only=True) - rs = down.stack("ID") - xp = unst.loc[:, ["VAR1"]].resample("W-THU").mean().stack("ID") + rs = down.stack("ID", v3=v3) + xp = unst.loc[:, ["VAR1"]].resample("W-THU").mean().stack("ID", v3=v3) xp.columns.name = "Params" tm.assert_frame_equal(rs, xp) - def test_stack_dropna(self): + def test_stack_dropna(self, v3): # GH#3997 df = DataFrame({"A": ["a1", "a2"], "B": ["b1", "b2"], "C": [1, 1]}) df = df.set_index(["A", "B"]) - stacked = df.unstack().stack(dropna=False) + dropna = False if not v3 else lib.no_default + stacked = df.unstack().stack(dropna=dropna, v3=v3) assert len(stacked) > len(stacked.dropna()) - stacked = df.unstack().stack(dropna=True) - tm.assert_frame_equal(stacked, stacked.dropna()) + if v3: + with pytest.raises(ValueError, match="Cannot specify dropna"): + df.unstack().stack(dropna=True, v3=v3) + else: + stacked = df.unstack().stack(dropna=True, v3=v3) + tm.assert_frame_equal(stacked, stacked.dropna()) - def test_unstack_multiple_hierarchical(self): + def test_unstack_multiple_hierarchical(self, v3): df = DataFrame( index=[ [0, 0, 0, 0, 1, 1, 1, 1], @@ -1954,7 +2032,7 @@ def test_unstack_sparse_keyspace(self): # it works! is sufficient idf.unstack("E") - def test_unstack_unobserved_keys(self): + def test_unstack_unobserved_keys(self, v3): # related to GH#2278 refactoring levels = [[0, 1], [0, 1, 2, 3]] codes = [[0, 0, 1, 1], [0, 2, 0, 2]] @@ -1966,7 +2044,7 @@ def test_unstack_unobserved_keys(self): result = df.unstack() assert len(result.columns) == 4 - recons = result.stack() + recons = result.stack(v3=v3) tm.assert_frame_equal(recons, df) @pytest.mark.slow @@ -1999,12 +2077,12 @@ def __init__(self, *args, **kwargs) -> None: ), ) @pytest.mark.parametrize("stack_lev", range(2)) - def test_stack_order_with_unsorted_levels(self, levels, stack_lev): + def test_stack_order_with_unsorted_levels(self, levels, stack_lev, v3): # GH#16323 # deep check for 1-row case columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]]) df = DataFrame(columns=columns, data=[range(4)]) - df_stacked = df.stack(stack_lev) + df_stacked = df.stack(stack_lev, v3=v3) for row in df.index: for col in df.columns: expected = df.loc[row, col] @@ -2013,7 +2091,7 @@ def test_stack_order_with_unsorted_levels(self, levels, stack_lev): result = df_stacked.loc[result_row, result_col] assert result == expected - def test_stack_order_with_unsorted_levels_multi_row(self): + def test_stack_order_with_unsorted_levels_multi_row(self, v3): # GH#16323 # check multi-row case @@ -2025,18 +2103,18 @@ def test_stack_order_with_unsorted_levels_multi_row(self): columns=mi, index=range(5), data=np.arange(5 * len(mi)).reshape(5, -1) ) assert all( - df.loc[row, col] == df.stack(0).loc[(row, col[0]), col[1]] + df.loc[row, col] == df.stack(0, v3=v3).loc[(row, col[0]), col[1]] for row in df.index for col in df.columns ) - def test_stack_order_with_unsorted_levels_multi_row_2(self): + def test_stack_order_with_unsorted_levels_multi_row_2(self, v3): # GH#53636 levels = ((0, 1), (1, 0)) stack_lev = 1 columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]]) df = DataFrame(columns=columns, data=[range(4)], index=[1, 0, 2, 3]) - result = df.stack(stack_lev) + result = df.stack(stack_lev, v3=v3) expected_index = MultiIndex( levels=[[0, 1, 2, 3], [0, 1]], codes=[[1, 1, 0, 0, 2, 2, 3, 3], [1, 0, 1, 0, 1, 0, 1, 0]], @@ -2050,7 +2128,7 @@ def test_stack_order_with_unsorted_levels_multi_row_2(self): ) tm.assert_frame_equal(result, expected) - def test_stack_unstack_unordered_multiindex(self): + def test_stack_unstack_unordered_multiindex(self, v3): # GH# 18265 values = np.arange(5) data = np.vstack( @@ -2065,7 +2143,7 @@ def test_stack_unstack_unordered_multiindex(self): multi_level_df = pd.concat(second_level_dict, axis=1) multi_level_df.columns.names = ["second", "first"] df = multi_level_df.reindex(sorted(multi_level_df.columns), axis=1) - result = df.stack(["first", "second"]).unstack(["first", "second"]) + result = df.stack(["first", "second"], v3=v3).unstack(["first", "second"]) expected = DataFrame( [["a0", "b0"], ["a1", "b1"], ["a2", "b2"], ["a3", "b3"], ["a4", "b4"]], index=[0, 1, 2, 3, 4], @@ -2088,7 +2166,7 @@ def test_unstack_preserve_types( assert unstacked["E", 1].dtype == np.object_ assert unstacked["F", 1].dtype == np.float64 - def test_unstack_group_index_overflow(self): + def test_unstack_group_index_overflow(self, v3): codes = np.tile(np.arange(500), 2) level = np.arange(500) @@ -2102,7 +2180,7 @@ def test_unstack_group_index_overflow(self): assert result.shape == (500, 2) # test roundtrip - stacked = result.stack() + stacked = result.stack(v3=v3) tm.assert_series_equal(s, stacked.reindex(s.index)) # put it at beginning @@ -2181,7 +2259,7 @@ def test_unstack_with_level_has_nan(self): tm.assert_index_equal(result, expected) - def test_stack_nan_in_multiindex_columns(self): + def test_stack_nan_in_multiindex_columns(self, v3): # GH#39481 df = DataFrame( np.zeros([1, 5]), @@ -2195,15 +2273,21 @@ def test_stack_nan_in_multiindex_columns(self): ], ), ) - result = df.stack(2) + result = df.stack(2, v3=v3) + if v3: + index = MultiIndex(levels=[[0], [0.0, 1.0]], codes=[[0, 0, 0], [-1, 0, 1]]) + columns = MultiIndex(levels=[[0], [2, 3]], codes=[[0, 0, 0], [-1, 0, 1]]) + else: + index = Index([(0, None), (0, 0), (0, 1)]) + columns = Index([(0, None), (0, 2), (0, 3)]) expected = DataFrame( [[0.0, np.nan, np.nan], [np.nan, 0.0, 0.0], [np.nan, 0.0, 0.0]], - index=Index([(0, None), (0, 0), (0, 1)]), - columns=Index([(0, None), (0, 2), (0, 3)]), + index=index, + columns=columns, ) tm.assert_frame_equal(result, expected) - def test_multi_level_stack_categorical(self): + def test_multi_level_stack_categorical(self, v3): # GH 15239 midx = MultiIndex.from_arrays( [ @@ -2213,30 +2297,52 @@ def test_multi_level_stack_categorical(self): ] ) df = DataFrame(np.arange(8).reshape(2, 4), columns=midx) - result = df.stack([1, 2]) - expected = DataFrame( - [ - [0, np.nan], - [np.nan, 2], - [1, np.nan], - [np.nan, 3], - [4, np.nan], - [np.nan, 6], - [5, np.nan], - [np.nan, 7], - ], - columns=["A", "B"], - index=MultiIndex.from_arrays( + result = df.stack([1, 2], v3=v3) + if v3: + expected = DataFrame( [ - [0] * 4 + [1] * 4, - pd.Categorical(list("aabbaabb")), - pd.Categorical(list("cdcdcdcd")), - ] - ), - ) + [0, np.nan], + [1, np.nan], + [np.nan, 2], + [np.nan, 3], + [4, np.nan], + [5, np.nan], + [np.nan, 6], + [np.nan, 7], + ], + columns=["A", "B"], + index=MultiIndex.from_arrays( + [ + [0] * 4 + [1] * 4, + pd.Categorical(list("abababab")), + pd.Categorical(list("ccddccdd")), + ] + ), + ) + else: + expected = DataFrame( + [ + [0, np.nan], + [np.nan, 2], + [1, np.nan], + [np.nan, 3], + [4, np.nan], + [np.nan, 6], + [5, np.nan], + [np.nan, 7], + ], + columns=["A", "B"], + index=MultiIndex.from_arrays( + [ + [0] * 4 + [1] * 4, + pd.Categorical(list("aabbaabb")), + pd.Categorical(list("cdcdcdcd")), + ] + ), + ) tm.assert_frame_equal(result, expected) - def test_stack_nan_level(self): + def test_stack_nan_level(self, v3): # GH 9406 df_nan = DataFrame( np.arange(4).reshape(2, 2), @@ -2246,13 +2352,21 @@ def test_stack_nan_level(self): index=Index([0, 1], name="Num"), dtype=np.float64, ) - result = df_nan.stack() + result = df_nan.stack(v3=v3) + if v3: + index = MultiIndex( + levels=[[0, 1], [np.nan, "b"]], + codes=[[0, 0, 1, 1], [0, 1, 0, 1]], + names=["Num", "Lower"], + ) + else: + index = MultiIndex.from_tuples( + [(0, np.nan), (0, "b"), (1, np.nan), (1, "b")], names=["Num", "Lower"] + ) expected = DataFrame( [[0.0, np.nan], [np.nan, 1], [2.0, np.nan], [np.nan, 3.0]], columns=Index(["A", "B"], name="Upper"), - index=MultiIndex.from_tuples( - [(0, np.nan), (0, "b"), (1, np.nan), (1, "b")], names=["Num", "Lower"] - ), + index=index, ) tm.assert_frame_equal(result, expected) @@ -2271,7 +2385,7 @@ def test_unstack_categorical_columns(self): expected.columns = MultiIndex.from_tuples([("cat", 0), ("cat", 1)]) tm.assert_frame_equal(result, expected) - def test_stack_unsorted(self): + def test_stack_unsorted(self, v3): # GH 16925 PAE = ["ITA", "FRA"] VAR = ["A1", "A2"] @@ -2285,11 +2399,11 @@ def test_stack_unsorted(self): DF.columns = DF.columns.droplevel(0) DF.loc[:, ("A0", "NET")] = 9999 - result = DF.stack(["VAR", "TYP"]).sort_index() - expected = DF.sort_index(axis=1).stack(["VAR", "TYP"]).sort_index() + result = DF.stack(["VAR", "TYP"], v3=v3).sort_index() + expected = DF.sort_index(axis=1).stack(["VAR", "TYP"], v3=v3).sort_index() tm.assert_series_equal(result, expected) - def test_stack_nullable_dtype(self): + def test_stack_nullable_dtype(self, v3): # GH#43561 columns = MultiIndex.from_product( [["54511", "54515"], ["r", "t_mean"]], names=["station", "element"] @@ -2299,14 +2413,14 @@ def test_stack_nullable_dtype(self): arr = np.array([[50, 226, 10, 215], [10, 215, 9, 220], [305, 232, 111, 220]]) df = DataFrame(arr, columns=columns, index=index, dtype=pd.Int64Dtype()) - result = df.stack("station") + result = df.stack("station", v3=v3) - expected = df.astype(np.int64).stack("station").astype(pd.Int64Dtype()) + expected = df.astype(np.int64).stack("station", v3=v3).astype(pd.Int64Dtype()) tm.assert_frame_equal(result, expected) # non-homogeneous case df[df.columns[0]] = df[df.columns[0]].astype(pd.Float64Dtype()) - result = df.stack("station") + result = df.stack("station", v3=v3) expected = DataFrame( { diff --git a/pandas/tests/frame/test_subclass.py b/pandas/tests/frame/test_subclass.py index 3d1e9d26c1ea6..a7c8373933feb 100644 --- a/pandas/tests/frame/test_subclass.py +++ b/pandas/tests/frame/test_subclass.py @@ -216,7 +216,7 @@ def test_subclass_stack(self): columns=["X", "Y", "Z"], ) - res = df.stack() + res = df.stack(v3=True) exp = tm.SubclassedSeries( [1, 2, 3, 4, 5, 6, 7, 8, 9], index=[list("aaabbbccc"), list("XYZXYZXYZ")] ) @@ -253,10 +253,10 @@ def test_subclass_stack_multi(self): columns=Index(["W", "X"], name="www"), ) - res = df.stack() + res = df.stack(v3=True) tm.assert_frame_equal(res, exp) - res = df.stack("yyy") + res = df.stack("yyy", v3=True) tm.assert_frame_equal(res, exp) exp = tm.SubclassedDataFrame( @@ -277,7 +277,7 @@ def test_subclass_stack_multi(self): columns=Index(["y", "z"], name="yyy"), ) - res = df.stack("www") + res = df.stack("www", v3=True) tm.assert_frame_equal(res, exp) def test_subclass_stack_multi_mixed(self): @@ -315,10 +315,10 @@ def test_subclass_stack_multi_mixed(self): columns=Index(["W", "X"], name="www"), ) - res = df.stack() + res = df.stack(v3=True) tm.assert_frame_equal(res, exp) - res = df.stack("yyy") + res = df.stack("yyy", v3=True) tm.assert_frame_equal(res, exp) exp = tm.SubclassedDataFrame( @@ -339,7 +339,7 @@ def test_subclass_stack_multi_mixed(self): columns=Index(["y", "z"], name="yyy"), ) - res = df.stack("www") + res = df.stack("www", v3=True) tm.assert_frame_equal(res, exp) def test_subclass_unstack(self): diff --git a/pandas/tests/groupby/aggregate/test_cython.py b/pandas/tests/groupby/aggregate/test_cython.py index 2fb7c8eb03bb0..690d7258536c5 100644 --- a/pandas/tests/groupby/aggregate/test_cython.py +++ b/pandas/tests/groupby/aggregate/test_cython.py @@ -67,7 +67,7 @@ def test_cythonized_aggers(op_name): expd = {} for (cat1, cat2), group in grouped: expd.setdefault(cat1, {})[cat2] = op(group["C"]) - exp = DataFrame(expd).T.stack(dropna=False) + exp = DataFrame(expd).T.stack(v3=True) exp.index.names = ["A", "B"] exp.name = "C" diff --git a/pandas/tests/groupby/test_categorical.py b/pandas/tests/groupby/test_categorical.py index b25950192018d..64bf7550f2d39 100644 --- a/pandas/tests/groupby/test_categorical.py +++ b/pandas/tests/groupby/test_categorical.py @@ -218,9 +218,9 @@ def f(x): # GH 10460 expc = Categorical.from_codes(np.arange(4).repeat(8), levels, ordered=True) exp = CategoricalIndex(expc) - tm.assert_index_equal((desc_result.stack().index.get_level_values(0)), exp) + tm.assert_index_equal((desc_result.stack(v3=True).index.get_level_values(0)), exp) exp = Index(["count", "mean", "std", "min", "25%", "50%", "75%", "max"] * 4) - tm.assert_index_equal((desc_result.stack().index.get_level_values(1)), exp) + tm.assert_index_equal((desc_result.stack(v3=True).index.get_level_values(1)), exp) def test_level_get_group(observed): @@ -651,9 +651,9 @@ def test_datetime(): # GH 10460 expc = Categorical.from_codes(np.arange(4).repeat(8), levels, ordered=True) exp = CategoricalIndex(expc) - tm.assert_index_equal((desc_result.stack().index.get_level_values(0)), exp) + tm.assert_index_equal((desc_result.stack(v3=True).index.get_level_values(0)), exp) exp = Index(["count", "mean", "std", "min", "25%", "50%", "75%", "max"] * 4) - tm.assert_index_equal((desc_result.stack().index.get_level_values(1)), exp) + tm.assert_index_equal((desc_result.stack(v3=True).index.get_level_values(1)), exp) def test_categorical_index(): @@ -691,8 +691,8 @@ def test_describe_categorical_columns(): df = DataFrame(np.random.randn(20, 4), columns=cats) result = df.groupby([1, 2, 3, 4] * 5).describe() - tm.assert_index_equal(result.stack().columns, cats) - tm.assert_categorical_equal(result.stack().columns.values, cats.values) + tm.assert_index_equal(result.stack(v3=True).columns, cats) + tm.assert_categorical_equal(result.stack(v3=True).columns.values, cats.values) def test_unstack_categorical(): diff --git a/pandas/tests/groupby/test_function.py b/pandas/tests/groupby/test_function.py index b97afe8ae9524..7dde736597db7 100644 --- a/pandas/tests/groupby/test_function.py +++ b/pandas/tests/groupby/test_function.py @@ -1098,7 +1098,7 @@ def test_series_describe_single(): ts = tm.makeTimeSeries() grouped = ts.groupby(lambda x: x.month) result = grouped.apply(lambda x: x.describe()) - expected = grouped.describe().stack() + expected = grouped.describe().stack(v3=True) tm.assert_series_equal(result, expected) diff --git a/pandas/tests/indexes/datetimes/test_partial_slicing.py b/pandas/tests/indexes/datetimes/test_partial_slicing.py index 28c9c07a9c9ef..fe90cee5a1922 100644 --- a/pandas/tests/indexes/datetimes/test_partial_slicing.py +++ b/pandas/tests/indexes/datetimes/test_partial_slicing.py @@ -359,7 +359,7 @@ def test_partial_slicing_with_multiindex_series(self): # partial slice on a series mi ser = DataFrame( np.random.rand(1000, 1000), index=date_range("2000-1-1", periods=1000) - ).stack() + ).stack(v3=True) s2 = ser[:-1].copy() expected = s2["2000-1-4"] diff --git a/pandas/tests/indexes/multi/test_indexing.py b/pandas/tests/indexes/multi/test_indexing.py index 2b75efd130aa2..50bcbaf1b3702 100644 --- a/pandas/tests/indexes/multi/test_indexing.py +++ b/pandas/tests/indexes/multi/test_indexing.py @@ -37,12 +37,12 @@ def test_slice_locs_partial(self, idx): def test_slice_locs(self): df = tm.makeTimeDataFrame() - stacked = df.stack() + stacked = df.stack(v3=True) idx = stacked.index slob = slice(*idx.slice_locs(df.index[5], df.index[15])) sliced = stacked[slob] - expected = df[5:16].stack() + expected = df[5:16].stack(v3=True) tm.assert_almost_equal(sliced.values, expected.values) slob = slice( @@ -52,19 +52,19 @@ def test_slice_locs(self): ) ) sliced = stacked[slob] - expected = df[6:15].stack() + expected = df[6:15].stack(v3=True) tm.assert_almost_equal(sliced.values, expected.values) def test_slice_locs_with_type_mismatch(self): df = tm.makeTimeDataFrame() - stacked = df.stack() + stacked = df.stack(v3=True) idx = stacked.index with pytest.raises(TypeError, match="^Level type mismatch"): idx.slice_locs((1, 3)) with pytest.raises(TypeError, match="^Level type mismatch"): idx.slice_locs(df.index[5] + timedelta(seconds=30), (5, 2)) df = tm.makeCustomDataframe(5, 5) - stacked = df.stack() + stacked = df.stack(v3=True) idx = stacked.index with pytest.raises(TypeError, match="^Level type mismatch"): idx.slice_locs(timedelta(seconds=30)) diff --git a/pandas/tests/indexes/multi/test_integrity.py b/pandas/tests/indexes/multi/test_integrity.py index 72b6754542fa6..e15435a0f6068 100644 --- a/pandas/tests/indexes/multi/test_integrity.py +++ b/pandas/tests/indexes/multi/test_integrity.py @@ -235,7 +235,7 @@ def test_rangeindex_fallback_coercion_bug(): # GH 12893 df1 = pd.DataFrame(np.arange(100).reshape((10, 10))) df2 = pd.DataFrame(np.arange(100).reshape((10, 10))) - df = pd.concat({"df1": df1.stack(), "df2": df2.stack()}, axis=1) + df = pd.concat({"df1": df1.stack(v3=True), "df2": df2.stack(v3=True)}, axis=1) df.index.names = ["fizz", "buzz"] str(df) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 11909bf56f05c..339f466e4be88 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1840,7 +1840,7 @@ def test_frame_int_overflow(self): ], ) def test_json_multiindex(self, dataframe, expected): - series = dataframe.stack() + series = dataframe.stack(v3=True) result = series.to_json(orient="index") assert result == expected @@ -1879,7 +1879,7 @@ def test_to_json_multiindex_escape(self): True, index=pd.date_range("2017-01-20", "2017-01-23"), columns=["foo", "bar"], - ).stack() + ).stack(v3=True) result = df.to_json() expected = ( "{\"(Timestamp('2017-01-20 00:00:00'), 'foo')\":true," diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py index b31a520924d5f..fe49748791baf 100644 --- a/pandas/tests/io/pytables/test_append.py +++ b/pandas/tests/io/pytables/test_append.py @@ -131,7 +131,7 @@ def test_append_series(setup_path): mi["C"] = "foo" mi.loc[3:5, "C"] = "bar" mi.set_index(["C", "B"], inplace=True) - s = mi.stack() + s = mi.stack(v3=True) s.index = s.index.droplevel(2) store.append("mi", s) tm.assert_series_equal(store["mi"], s, check_index_type=True) diff --git a/pandas/tests/series/methods/test_reset_index.py b/pandas/tests/series/methods/test_reset_index.py index ec38c5b8b744b..91ad8de76766b 100644 --- a/pandas/tests/series/methods/test_reset_index.py +++ b/pandas/tests/series/methods/test_reset_index.py @@ -34,7 +34,7 @@ def test_reset_index_dti_round_trip(self): def test_reset_index(self): df = tm.makeDataFrame()[:5] - ser = df.stack() + ser = df.stack(v3=True) ser.index.names = ["hash", "category"] ser.name = "value" diff --git a/pandas/tests/series/methods/test_unstack.py b/pandas/tests/series/methods/test_unstack.py index 6e1c76bd170c6..e79df357abdde 100644 --- a/pandas/tests/series/methods/test_unstack.py +++ b/pandas/tests/series/methods/test_unstack.py @@ -133,7 +133,7 @@ def test_unstack_mixed_type_name_in_multiindex( def test_unstack_multi_index_categorical_values(): - mi = tm.makeTimeDataFrame().stack().index.rename(["major", "minor"]) + mi = tm.makeTimeDataFrame().stack(v3=True).index.rename(["major", "minor"]) ser = Series(["foo"] * len(mi), index=mi, name="category", dtype="category") result = ser.unstack() From 5d0b9e6d3e50db8b225d5cc40b04f8b85aa3a85c Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Thu, 13 Jul 2023 17:20:02 -0400 Subject: [PATCH 2/8] Merge cleanup --- doc/source/whatsnew/v2.1.0.rst | 13 ++++++++++--- pandas/core/frame.py | 6 ++++++ pandas/tests/extension/test_sparse.py | 5 +++-- pandas/tests/frame/test_stack_unstack.py | 23 ++++++++++++++++------- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 1bd9bfe28fcbf..c5fc32931de83 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -131,12 +131,19 @@ to ``na_action=None``, like for all the other array types. New implementation of :meth:`DataFrame.stack` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -pandas has reimplemented :meth:`DataFrame.stack`. To use the new implementation, pass the argument ``v3=True``. This will become the only option in pandas 3.0. The previous implementation would unnecessarily introduce NA values into the result which could be removed with ``dropna=True``. The new implementation will no longer unnecessarily introduce NA values when stacking multiple levels. As such, the argument ``dropna`` must remain unspecified when using ``v3=True`` and will be removed in the next major release. +pandas has reimplemented :meth:`DataFrame.stack`. To use the new implementation, pass the argument ``v3=True``. This will become the only option in pandas 3.0. + +The previous implementation had two main behavioral downsides. + + 1. The previous implementation would unnecessarily introduce NA values into the result. The user could have NA values automatically removed by passing ``dropna=True`` (the default), but doing this could also remove NA values from the result that existed in the input. See the examples below. + 2. The previous implementation with ``sort=True`` (the default) would sometimes sort part of the resulting index, and sometimes not. If the input's columns are *not* a :class:`MultiIndex`, then the resulting index would never be sorted. If the columns are a :class:`MultiIndex`, then in most cases the level(s) in the resulting index that come from stacking the column level(s) would be sorted. In rare cases such level(s) would be sorted in a non-standard order, depending on how the columns were created. + +The new implementation (``v3=True``) will no longer unnecessarily introduce NA values when stacking multiple levels and will never sort. As such, the arguments ``dropna`` and ``sort`` are not utilized and must remain unspecified when using ``v3=True``. These arguments will be removed in the next major release. .. ipython:: python - columns = pd.MultiIndex.from_tuples([("A", "c"), ("B", "d")]) - df = pd.DataFrame([[0, 2], [1, 3]], columns=columns) + columns = pd.MultiIndex.from_tuples([("B", "d"), ("A", "c")]) + df = pd.DataFrame([[0, 2], [1, 3]], index=["z", "y"], columns=columns) df In the previous version (``v3=False``), the default of ``dropna=True`` would remove unnecessarily introduced NA values but still coerce the dtype to ``float64`` in the process. In the new version, no NAs are introduced and so there is no coercion of the dtype. diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 57fd7227dd03e..f49fa567aa3cd 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -9229,6 +9229,12 @@ def stack( "removed in a future version of pandas" ) + if sort is not lib.no_default: + raise ValueError( + "Cannot specify sort with v3=True, this argument will be " + "removed in a future version of pandas" + ) + if ( isinstance(level, (tuple, list)) and not all(lev in self.columns.names for lev in level) diff --git a/pandas/tests/extension/test_sparse.py b/pandas/tests/extension/test_sparse.py index d8bab71b76df4..274880b13129f 100644 --- a/pandas/tests/extension/test_sparse.py +++ b/pandas/tests/extension/test_sparse.py @@ -156,8 +156,9 @@ def test_concat_mixed_dtypes(self, data): ), ], ) - def test_stack(self, data, columns): - super().test_stack(data, columns) + @pytest.mark.parametrize("v3", [True, False]) + def test_stack(self, data, columns, v3): + super().test_stack(data, columns, v3) def test_concat_columns(self, data, na_value): self._check_unsupported(data) diff --git a/pandas/tests/frame/test_stack_unstack.py b/pandas/tests/frame/test_stack_unstack.py index 1884f6dfc9352..130e8fb98be4e 100644 --- a/pandas/tests/frame/test_stack_unstack.py +++ b/pandas/tests/frame/test_stack_unstack.py @@ -1140,8 +1140,12 @@ def test_stack_multi_preserve_categorical_dtype(self, ordered, labels, data, v3) df = DataFrame([sorted(data)], columns=midx) result = df.stack([0, 1], v3=v3) - s_cidx = pd.CategoricalIndex(sorted(labels), ordered=ordered) - expected = Series(data, index=MultiIndex.from_product([[0], s_cidx, cidx2])) + labels = labels if v3 else sorted(labels) + s_cidx = pd.CategoricalIndex(labels, ordered=ordered) + expected_data = sorted(data) if v3 else data + expected = Series( + expected_data, index=MultiIndex.from_product([[0], s_cidx, cidx2]) + ) tm.assert_series_equal(result, expected) @@ -1451,7 +1455,8 @@ def test_stack_sort_false(v3): levels=[["B", "A"], ["x", "y"]], codes=[[0, 0, 1, 1], [0, 1, 0, 1]] ), ) - result = df.stack(level=0, sort=False, v3=v3) + kwargs = {} if v3 else {"sort": False} + result = df.stack(level=0, v3=v3, **kwargs) if v3: expected = DataFrame( { @@ -1474,7 +1479,8 @@ def test_stack_sort_false(v3): data, columns=MultiIndex.from_arrays([["B", "B", "A", "A"], ["x", "y", "x", "y"]]), ) - result = df.stack(level=0, sort=False, v3=v3) + kwargs = {} if v3 else {"sort": False} + result = df.stack(level=0, v3=v3, **kwargs) tm.assert_frame_equal(result, expected) @@ -1482,7 +1488,8 @@ def test_stack_sort_false_multi_level(v3): # GH 15105 idx = MultiIndex.from_tuples([("weight", "kg"), ("height", "m")]) df = DataFrame([[1.0, 2.0], [3.0, 4.0]], index=["cat", "dog"], columns=idx) - result = df.stack([0, 1], sort=False, v3=v3) + kwargs = {} if v3 else {"sort": False} + result = df.stack([0, 1], v3=v3, **kwargs) expected_index = MultiIndex.from_tuples( [ ("cat", "weight", "kg"), @@ -2083,7 +2090,8 @@ def test_stack_order_with_unsorted_levels(self, levels, stack_lev, sort, v3): # deep check for 1-row case columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]]) df = DataFrame(columns=columns, data=[range(4)]) - df_stacked = df.stack(stack_lev, sort=sort, v3=v3) + kwargs = {} if v3 else {"sort": sort} + df_stacked = df.stack(stack_lev, v3=v3, **kwargs) for row in df.index: for col in df.columns: expected = df.loc[row, col] @@ -2115,7 +2123,8 @@ def test_stack_order_with_unsorted_levels_multi_row_2(self, v3): stack_lev = 1 columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]]) df = DataFrame(columns=columns, data=[range(4)], index=[1, 0, 2, 3]) - result = df.stack(stack_lev, sort=True, v3=v3) + kwargs = {} if v3 else {"sort": True} + result = df.stack(stack_lev, v3=v3, **kwargs) expected_index = MultiIndex( levels=[[0, 1, 2, 3], [0, 1]], codes=[[1, 1, 0, 0, 2, 2, 3, 3], [1, 0, 1, 0, 1, 0, 1, 0]], From 5f08fbe39ba7eaf6fca5c0dab60a97841ae058e3 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Thu, 13 Jul 2023 17:22:14 -0400 Subject: [PATCH 3/8] Revert filterwarnings in conf.py --- doc/source/conf.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 4da138558198d..71bc05f6fd6e1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -376,14 +376,6 @@ os.chdir(r'{os.path.dirname(os.path.dirname(__file__))}') """ -warnings.filterwarnings( - "ignore", - category=UserWarning, - message=( - "Matplotlib is currently using agg, which is a " - "non-GUI backend, so cannot show the figure." - ), -) html_context = { "redirects": dict(moved_api_pages), From 991e05d829ad8262dcbe47019a3a71b6f0d06411 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Sun, 16 Jul 2023 09:47:41 -0400 Subject: [PATCH 4/8] Merge fixup --- pandas/core/frame.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index f49fa567aa3cd..cd5d5802d4bea 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -9215,6 +9215,8 @@ def stack( if dropna is lib.no_default: dropna = True + if sort is lib.no_default: + sort = True if isinstance(level, (tuple, list)): result = stack_multiple(self, level, dropna=dropna, sort=sort) From ccb901a853914debf272f066035a87144447a97d Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Wed, 19 Jul 2023 16:55:27 -0400 Subject: [PATCH 5/8] Rename inner function --- pandas/core/reshape/reshape.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/reshape/reshape.py b/pandas/core/reshape/reshape.py index 19064e1174e6e..87f100c8c557d 100644 --- a/pandas/core/reshape/reshape.py +++ b/pandas/core/reshape/reshape.py @@ -587,7 +587,7 @@ def stack(frame: DataFrame, level=-1, dropna: bool = True, sort: bool = True): stacked : Series or DataFrame """ - def factorize(index): + def stack_factorize(index): if index.is_unique: return index, np.arange(len(index)) codes, categories = factorize_from_iterable(index) @@ -606,7 +606,7 @@ def factorize(index): new_levels = list(frame.index.levels) new_codes = [lab.repeat(K) for lab in frame.index.codes] - clev, clab = factorize(frame.columns) + clev, clab = stack_factorize(frame.columns) new_levels.append(clev) new_codes.append(np.tile(clab, N).ravel()) From 37f2ca88ccde002b6e975bf8ab9d109f3c1ce87f Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Mon, 31 Jul 2023 17:14:44 -0400 Subject: [PATCH 6/8] v3->future_stack; other refinements --- .../comparison/comparison_with_r.rst | 2 +- doc/source/user_guide/10min.rst | 2 +- doc/source/user_guide/cookbook.rst | 4 +- doc/source/user_guide/groupby.rst | 2 +- doc/source/user_guide/reshaping.rst | 20 +- doc/source/whatsnew/v2.1.0.rst | 16 +- pandas/core/frame.py | 26 +- pandas/core/groupby/generic.py | 2 +- pandas/core/resample.py | 2 +- pandas/core/reshape/pivot.py | 2 +- pandas/core/reshape/reshape.py | 6 +- pandas/tests/extension/base/reshaping.py | 8 +- pandas/tests/extension/json/test_json.py | 2 +- pandas/tests/extension/test_sparse.py | 6 +- .../tests/frame/methods/test_reset_index.py | 6 +- pandas/tests/frame/test_stack_unstack.py | 396 ++++++++++-------- pandas/tests/frame/test_subclass.py | 14 +- pandas/tests/groupby/aggregate/test_cython.py | 2 +- pandas/tests/groupby/test_categorical.py | 22 +- pandas/tests/groupby/test_function.py | 2 +- .../indexes/datetimes/test_partial_slicing.py | 2 +- pandas/tests/indexes/multi/test_indexing.py | 10 +- pandas/tests/indexes/multi/test_integrity.py | 5 +- pandas/tests/io/json/test_pandas.py | 4 +- pandas/tests/io/pytables/test_append.py | 2 +- .../tests/series/methods/test_reset_index.py | 2 +- pandas/tests/series/methods/test_unstack.py | 4 +- 27 files changed, 316 insertions(+), 255 deletions(-) diff --git a/doc/source/getting_started/comparison/comparison_with_r.rst b/doc/source/getting_started/comparison/comparison_with_r.rst index 3d14b9e242394..a6cfcd4614984 100644 --- a/doc/source/getting_started/comparison/comparison_with_r.rst +++ b/doc/source/getting_started/comparison/comparison_with_r.rst @@ -438,7 +438,7 @@ In Python, the :meth:`~pandas.melt` method is the R equivalent: ) pd.melt(cheese, id_vars=["first", "last"]) - cheese.set_index(["first", "last"]).stack(v3=True) # alternative way + cheese.set_index(["first", "last"]).stack(future_stack=True) # alternative way For more details and examples see :ref:`the reshaping documentation `. diff --git a/doc/source/user_guide/10min.rst b/doc/source/user_guide/10min.rst index d0aa0dcc5bce7..51168f74c2657 100644 --- a/doc/source/user_guide/10min.rst +++ b/doc/source/user_guide/10min.rst @@ -579,7 +579,7 @@ columns: .. ipython:: python - stacked = df2.stack(v3=True) + stacked = df2.stack(future_stack=True) stacked With a "stacked" DataFrame or Series (having a :class:`MultiIndex` as the diff --git a/doc/source/user_guide/cookbook.rst b/doc/source/user_guide/cookbook.rst index a34d6ebe15e54..66ee571d6b5a5 100644 --- a/doc/source/user_guide/cookbook.rst +++ b/doc/source/user_guide/cookbook.rst @@ -311,7 +311,7 @@ The :ref:`multindexing ` docs. df.columns = pd.MultiIndex.from_tuples([tuple(c.split("_")) for c in df.columns]) df # Now stack & Reset - df = df.stack(0, v3=True).reset_index(1) + df = df.stack(0, future_stack=True).reset_index(1) df # And fix the labels (Notice the label 'level_1' got added automatically) df.columns = ["Sample", "All_X", "All_Y"] @@ -688,7 +688,7 @@ The :ref:`Pivot ` docs. aggfunc="sum", margins=True, ) - table.stack("City", v3=True) + table.stack("City", future_stack=True) `Frequency table like plyr in R `__ diff --git a/doc/source/user_guide/groupby.rst b/doc/source/user_guide/groupby.rst index 39e2f726d4f8c..75c816f66d5e4 100644 --- a/doc/source/user_guide/groupby.rst +++ b/doc/source/user_guide/groupby.rst @@ -1713,4 +1713,4 @@ column index name will be used as the name of the inserted column: result - result.stack(v3=True) + result.stack(future_stack=True) diff --git a/doc/source/user_guide/reshaping.rst b/doc/source/user_guide/reshaping.rst index 1463a1284d645..1e73b7672782e 100644 --- a/doc/source/user_guide/reshaping.rst +++ b/doc/source/user_guide/reshaping.rst @@ -127,7 +127,7 @@ stacked level becomes the new lowest level in a :class:`MultiIndex` on the colum .. ipython:: python - stacked = df2.stack(v3=True) + stacked = df2.stack(future_stack=True) stacked With a "stacked" :class:`DataFrame` or :class:`Series` (having a :class:`MultiIndex` as the @@ -163,7 +163,7 @@ will result in a **sorted** copy of the original :class:`DataFrame` or :class:`S index = pd.MultiIndex.from_product([[2, 1], ["a", "b"]]) df = pd.DataFrame(np.random.randn(4), index=index, columns=["A"]) df - all(df.unstack().stack(v3=True) == df.sort_index()) + all(df.unstack().stack(future_stack=True) == df.sort_index()) The above code will raise a ``TypeError`` if the call to :meth:`~DataFrame.sort_index` is removed. @@ -191,16 +191,16 @@ processed individually. df = pd.DataFrame(np.random.randn(4, 4), columns=columns) df - df.stack(level=["animal", "hair_length"], v3=True) + df.stack(level=["animal", "hair_length"], future_stack=True) The list of levels can contain either level names or level numbers (but not a mixture of the two). .. ipython:: python - # df.stack(level=['animal', 'hair_length'], v3=True) + # df.stack(level=['animal', 'hair_length'], future_stack=True) # from above is equivalent to: - df.stack(level=[1, 2], v3=True) + df.stack(level=[1, 2], future_stack=True) Missing data ~~~~~~~~~~~~ @@ -233,8 +233,8 @@ which level in the columns to stack: .. ipython:: python - df2.stack("exp", v3=True) - df2.stack("animal", v3=True) + df2.stack("exp", future_stack=True) + df2.stack("animal", future_stack=True) Unstacking can result in missing values if subgroups do not have the same set of labels. By default, missing values will be replaced with the default @@ -345,12 +345,12 @@ some very expressive and fast data manipulations. .. ipython:: python df - df.stack(v3=True).mean(1).unstack() + df.stack(future_stack=True).mean(1).unstack() # same result, another way df.T.groupby(level=1).mean() - df.stack(v3=True).groupby(level=1).mean() + df.stack(future_stack=True).groupby(level=1).mean() df.mean().unstack(0) @@ -460,7 +460,7 @@ as having a multi-level index: .. ipython:: python - table.stack(v3=True) + table.stack(future_stack=True) .. _reshaping.crosstabulations: diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst index 15da71d5b605c..2cc288177bd03 100644 --- a/doc/source/whatsnew/v2.1.0.rst +++ b/doc/source/whatsnew/v2.1.0.rst @@ -133,14 +133,14 @@ to ``na_action=None``, like for all the other array types. New implementation of :meth:`DataFrame.stack` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -pandas has reimplemented :meth:`DataFrame.stack`. To use the new implementation, pass the argument ``v3=True``. This will become the only option in pandas 3.0. +pandas has reimplemented :meth:`DataFrame.stack`. To use the new implementation, pass the argument ``future_stack=True``. This will become the only option in pandas 3.0. The previous implementation had two main behavioral downsides. 1. The previous implementation would unnecessarily introduce NA values into the result. The user could have NA values automatically removed by passing ``dropna=True`` (the default), but doing this could also remove NA values from the result that existed in the input. See the examples below. 2. The previous implementation with ``sort=True`` (the default) would sometimes sort part of the resulting index, and sometimes not. If the input's columns are *not* a :class:`MultiIndex`, then the resulting index would never be sorted. If the columns are a :class:`MultiIndex`, then in most cases the level(s) in the resulting index that come from stacking the column level(s) would be sorted. In rare cases such level(s) would be sorted in a non-standard order, depending on how the columns were created. -The new implementation (``v3=True``) will no longer unnecessarily introduce NA values when stacking multiple levels and will never sort. As such, the arguments ``dropna`` and ``sort`` are not utilized and must remain unspecified when using ``v3=True``. These arguments will be removed in the next major release. +The new implementation (``future_stack=True``) will no longer unnecessarily introduce NA values when stacking multiple levels and will never sort. As such, the arguments ``dropna`` and ``sort`` are not utilized and must remain unspecified when using ``future_stack=True``. These arguments will be removed in the next major release. .. ipython:: python @@ -148,13 +148,13 @@ The new implementation (``v3=True``) will no longer unnecessarily introduce NA v df = pd.DataFrame([[0, 2], [1, 3]], index=["z", "y"], columns=columns) df -In the previous version (``v3=False``), the default of ``dropna=True`` would remove unnecessarily introduced NA values but still coerce the dtype to ``float64`` in the process. In the new version, no NAs are introduced and so there is no coercion of the dtype. +In the previous version (``future_stack=False``), the default of ``dropna=True`` would remove unnecessarily introduced NA values but still coerce the dtype to ``float64`` in the process. In the new version, no NAs are introduced and so there is no coercion of the dtype. .. ipython:: python :okwarning: - df.stack([0, 1], v3=False, dropna=True) - df.stack([0, 1], v3=True) + df.stack([0, 1], future_stack=False, dropna=True) + df.stack([0, 1], future_stack=True) If the input contains NA values, the previous version would drop those as well with ``dropna=True`` or introduce new NA values with ``dropna=False``. The new version persists all values from the input. @@ -163,9 +163,9 @@ If the input contains NA values, the previous version would drop those as well w df = pd.DataFrame([[0, 2], [np.nan, np.nan]], columns=columns) df - df.stack([0, 1], v3=False, dropna=True) - df.stack([0, 1], v3=False, dropna=False) - df.stack([0, 1], v3=True) + df.stack([0, 1], future_stack=False, dropna=True) + df.stack([0, 1], future_stack=False, dropna=False) + df.stack([0, 1], future_stack=True) .. _whatsnew_210.enhancements.other: diff --git a/pandas/core/frame.py b/pandas/core/frame.py index dca1d4dc406bb..b0bfe1e69b31a 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -9174,7 +9174,7 @@ def stack( level: IndexLabel = -1, dropna: bool | lib.NoDefault = lib.no_default, sort: bool | lib.NoDefault = lib.no_default, - v3: bool = False, + future_stack: bool = False, ): """ Stack the prescribed level(s) from columns to index. @@ -9203,7 +9203,7 @@ def stack( section. sort : bool, default True Whether to sort the levels of the resulting MultiIndex. - v3: bool, default False + future_stack: bool, default False Whether to use the new implementation that will replace the current implementation in pandas 3.0. When True, dropna and sort have no impact on the result and must remain unspecified. See :ref:`pandas 2.1.0 Release @@ -9247,7 +9247,7 @@ def stack( weight height cat 0 1 dog 2 3 - >>> df_single_level_cols.stack(v3=True) + >>> df_single_level_cols.stack(future_stack=True) cat weight 0 height 1 dog weight 2 @@ -9269,7 +9269,7 @@ def stack( kg pounds cat 1 2 dog 2 4 - >>> df_multi_level_cols1.stack(v3=True) + >>> df_multi_level_cols1.stack(future_stack=True) weight cat kg 1 pounds 2 @@ -9294,7 +9294,7 @@ def stack( kg m cat 1.0 2.0 dog 3.0 4.0 - >>> df_multi_level_cols2.stack(v3=True) + >>> df_multi_level_cols2.stack(future_stack=True) weight height cat kg 1.0 NaN m NaN 2.0 @@ -9305,13 +9305,13 @@ def stack( The first parameter controls which level or levels are stacked: - >>> df_multi_level_cols2.stack(0, v3=True) + >>> df_multi_level_cols2.stack(0, future_stack=True) kg m cat height NaN 2.0 weight 1.0 NaN dog height NaN 4.0 weight 3.0 NaN - >>> df_multi_level_cols2.stack([0, 1], v3=True) + >>> df_multi_level_cols2.stack([0, 1], future_stack=True) cat height m 2.0 weight kg 1.0 dog height m 4.0 @@ -9345,7 +9345,7 @@ def stack( dog kg 2.0 NaN m NaN 3.0 """ - if not v3: + if not future_stack: from pandas.core.reshape.reshape import ( stack, stack_multiple, @@ -9365,14 +9365,16 @@ def stack( if dropna is not lib.no_default: raise ValueError( - "Cannot specify dropna with v3=True, this argument will be " - "removed in a future version of pandas" + "dropna must be unspecified with future_stack=True as the new " + "implementation does not introduce rows of NA values. This " + "argument will be removed in a future version of pandas." ) if sort is not lib.no_default: raise ValueError( - "Cannot specify sort with v3=True, this argument will be " - "removed in a future version of pandas" + "Cannot specify sort with future_stack=True, this argument will be " + "removed in a future version of pandas. Sort the result using " + ".sort_index instead." ) if ( diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py index 21c3c233e6030..d0b0f636617cd 100644 --- a/pandas/core/groupby/generic.py +++ b/pandas/core/groupby/generic.py @@ -416,7 +416,7 @@ def _wrap_applied_output( res_df = self._reindex_output(res_df) # if self.observed is False, # keep all-NaN rows created while re-indexing - res_ser = res_df.stack(v3=True) + res_ser = res_df.stack(future_stack=True) res_ser.name = self.obj.name return res_ser elif isinstance(values[0], (Series, DataFrame)): diff --git a/pandas/core/resample.py b/pandas/core/resample.py index c76e0bda495e4..9b8d1c870091d 100644 --- a/pandas/core/resample.py +++ b/pandas/core/resample.py @@ -1497,7 +1497,7 @@ def size(self): # If the result is a non-empty DataFrame we stack to get a Series # GH 46826 if isinstance(result, ABCDataFrame) and not result.empty: - result = result.stack(v3=True) + result = result.stack(future_stack=True) if not len(self.ax): from pandas import Series diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index d520a188cca6e..444c4a6bd8ce5 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -441,7 +441,7 @@ def _all_key(key): if len(cols) > 0: row_margin = data[cols + values].groupby(cols, observed=observed).agg(aggfunc) - row_margin = row_margin.stack(v3=True) + row_margin = row_margin.stack(future_stack=True) # slight hack new_order = [len(cols)] + list(range(len(cols))) diff --git a/pandas/core/reshape/reshape.py b/pandas/core/reshape/reshape.py index 34d867df0412b..fc8d827cd31bb 100644 --- a/pandas/core/reshape/reshape.py +++ b/pandas/core/reshape/reshape.py @@ -503,7 +503,7 @@ def unstack(obj: Series | DataFrame, level, fill_value=None, sort: bool = True): if isinstance(obj.index, MultiIndex): return _unstack_frame(obj, level, fill_value=fill_value, sort=sort) else: - return obj.T.stack(v3=True) + return obj.T.stack(future_stack=True) elif not isinstance(obj.index, MultiIndex): # GH 36113 # Give nicer error messages when unstack a Series whose @@ -615,7 +615,7 @@ def stack_factorize(index): levels=new_levels, codes=new_codes, names=new_names, verify_integrity=False ) else: - levels, (ilab, clab) = zip(*map(factorize, (frame.index, frame.columns))) + levels, (ilab, clab) = zip(*map(stack_factorize, (frame.index, frame.columns))) codes = ilab.repeat(K), np.tile(clab, N).ravel() new_index = MultiIndex( levels=levels, @@ -887,7 +887,7 @@ def stack_v3(frame: DataFrame, level: list[int]) -> Series | DataFrame: raise ValueError("Columns with duplicate values are not supported in stack") # If we need to drop `level` from columns, it needs to be in descending order - drop_levnums = sorted(level)[::-1] + drop_levnums = sorted(level, reverse=True) stack_cols = frame.columns._drop_level_numbers( [k for k in range(frame.columns.nlevels) if k not in level][::-1] ) diff --git a/pandas/tests/extension/base/reshaping.py b/pandas/tests/extension/base/reshaping.py index 5a6c20af35f5f..7282249304ee2 100644 --- a/pandas/tests/extension/base/reshaping.py +++ b/pandas/tests/extension/base/reshaping.py @@ -252,12 +252,12 @@ def test_merge_on_extension_array_duplicates(self, data): ), ], ) - @pytest.mark.parametrize("v3", [True, False]) - def test_stack(self, data, columns, v3): + @pytest.mark.parametrize("future_stack", [True, False]) + def test_stack(self, data, columns, future_stack): df = pd.DataFrame({"A": data[:5], "B": data[:5]}) df.columns = columns - result = df.stack(v3=v3) - expected = df.astype(object).stack(v3=v3) + result = df.stack(future_stack=future_stack) + expected = df.astype(object).stack(future_stack=future_stack) # we need a second astype(object), in case the constructor inferred # object -> specialized, as is done for period. expected = expected.astype(object) diff --git a/pandas/tests/extension/json/test_json.py b/pandas/tests/extension/json/test_json.py index 94a1f6f07eedc..1b5ce38194ba8 100644 --- a/pandas/tests/extension/json/test_json.py +++ b/pandas/tests/extension/json/test_json.py @@ -199,7 +199,7 @@ class TestReshaping(BaseJSON, base.BaseReshapingTests): @pytest.mark.xfail(reason="Different definitions of NA") def test_stack(self): """ - The test does .astype(object).stack(v3=True). If we happen to have + The test does .astype(object).stack(future_stack=True). If we happen to have any missing values in `data`, then we'll end up with different rows since we consider `{}` NA, but `.astype(object)` doesn't. """ diff --git a/pandas/tests/extension/test_sparse.py b/pandas/tests/extension/test_sparse.py index a7f0b0c8b7522..9fc1df891562b 100644 --- a/pandas/tests/extension/test_sparse.py +++ b/pandas/tests/extension/test_sparse.py @@ -156,9 +156,9 @@ def test_concat_mixed_dtypes(self, data): ), ], ) - @pytest.mark.parametrize("v3", [True, False]) - def test_stack(self, data, columns, v3): - super().test_stack(data, columns, v3) + @pytest.mark.parametrize("future_stack", [True, False]) + def test_stack(self, data, columns, future_stack): + super().test_stack(data, columns, future_stack) def test_concat_columns(self, data, na_value): self._check_unsupported(data) diff --git a/pandas/tests/frame/methods/test_reset_index.py b/pandas/tests/frame/methods/test_reset_index.py index f844b49f22e26..bdff73b304935 100644 --- a/pandas/tests/frame/methods/test_reset_index.py +++ b/pandas/tests/frame/methods/test_reset_index.py @@ -112,7 +112,7 @@ def test_reset_index_with_intervals(self): tm.assert_frame_equal(result2, original) def test_reset_index(self, float_frame): - stacked = float_frame.stack(v3=True)[::2] + stacked = float_frame.stack(future_stack=True)[::2] stacked = DataFrame({"foo": stacked, "bar": stacked}) names = ["first", "second"] @@ -749,7 +749,7 @@ def test_reset_index_rename(float_frame): def test_reset_index_rename_multiindex(float_frame): # GH 6878 - stacked_df = float_frame.stack(v3=True)[::2] + stacked_df = float_frame.stack(future_stack=True)[::2] stacked_df = DataFrame({"foo": stacked_df, "bar": stacked_df}) names = ["first", "second"] @@ -763,7 +763,7 @@ def test_reset_index_rename_multiindex(float_frame): def test_errorreset_index_rename(float_frame): # GH 6878 - stacked_df = float_frame.stack(v3=True)[::2] + stacked_df = float_frame.stack(future_stack=True)[::2] stacked_df = DataFrame({"first": stacked_df, "second": stacked_df}) with pytest.raises( diff --git a/pandas/tests/frame/test_stack_unstack.py b/pandas/tests/frame/test_stack_unstack.py index 130e8fb98be4e..f686b0097cc88 100644 --- a/pandas/tests/frame/test_stack_unstack.py +++ b/pandas/tests/frame/test_stack_unstack.py @@ -24,16 +24,16 @@ @pytest.fixture(params=[True, False]) -def v3(request): +def future_stack(request): return request.param class TestDataFrameReshape: - def test_stack_unstack(self, float_frame, v3): + def test_stack_unstack(self, float_frame, future_stack): df = float_frame.copy() df[:] = np.arange(np.prod(df.shape)).reshape(df.shape) - stacked = df.stack(v3=v3) + stacked = df.stack(future_stack=future_stack) stacked_df = DataFrame({"foo": stacked, "bar": stacked}) unstacked = stacked.unstack() @@ -47,26 +47,26 @@ def test_stack_unstack(self, float_frame, v3): tm.assert_frame_equal(unstacked_cols.T, df) tm.assert_frame_equal(unstacked_cols_df["bar"].T, df) - def test_stack_mixed_level(self, v3): + def test_stack_mixed_level(self, future_stack): # GH 18310 levels = [range(3), [3, "a", "b"], [1, 2]] # flat columns: df = DataFrame(1, index=levels[0], columns=levels[1]) - result = df.stack(v3=v3) + result = df.stack(future_stack=future_stack) expected = Series(1, index=MultiIndex.from_product(levels[:2])) tm.assert_series_equal(result, expected) # MultiIndex columns: df = DataFrame(1, index=levels[0], columns=MultiIndex.from_product(levels[1:])) - result = df.stack(1, v3=v3) + result = df.stack(1, future_stack=future_stack) expected = DataFrame( 1, index=MultiIndex.from_product([levels[0], levels[2]]), columns=levels[1] ) tm.assert_frame_equal(result, expected) # as above, but used labels in level are actually of homogeneous type - result = df[["a", "b"]].stack(1, v3=v3) + result = df[["a", "b"]].stack(1, future_stack=future_stack) expected = expected[["a", "b"]] tm.assert_frame_equal(result, expected) @@ -82,7 +82,7 @@ def test_unstack_not_consolidated(self, using_array_manager): expected = df.unstack() tm.assert_series_equal(res, expected) - def test_unstack_fill(self, v3): + def test_unstack_fill(self, future_stack): # GH #9746: fill_value keyword argument for Series # and DataFrame unstack @@ -115,7 +115,7 @@ def test_unstack_fill(self, v3): result = Series([0, 0, 2], index=unstacked.index, name=key) tm.assert_series_equal(result, expected) - stacked = unstacked.stack(["x", "y"], v3=v3) + stacked = unstacked.stack(["x", "y"], future_stack=future_stack) stacked.index = stacked.index.reorder_levels(df.index.names) # Workaround for GH #17886 (unnecessarily casts to float): stacked = stacked.astype(np.int64) @@ -388,17 +388,21 @@ def unstack_and_compare(df, column_name): s = df1["A"] unstack_and_compare(s, "index") - def test_stack_ints(self, v3): + def test_stack_ints(self, future_stack): columns = MultiIndex.from_tuples(list(itertools.product(range(3), repeat=3))) df = DataFrame(np.random.randn(30, 27), columns=columns) tm.assert_frame_equal( - df.stack(level=[1, 2], v3=v3), - df.stack(level=1, v3=v3).stack(level=1, v3=v3), + df.stack(level=[1, 2], future_stack=future_stack), + df.stack(level=1, future_stack=future_stack).stack( + level=1, future_stack=future_stack + ), ) tm.assert_frame_equal( - df.stack(level=[-2, -1], v3=v3), - df.stack(level=1, v3=v3).stack(level=1, v3=v3), + df.stack(level=[-2, -1], future_stack=future_stack), + df.stack(level=1, future_stack=future_stack).stack( + level=1, future_stack=future_stack + ), ) df_named = df.copy() @@ -406,11 +410,13 @@ def test_stack_ints(self, v3): assert return_value is None tm.assert_frame_equal( - df_named.stack(level=[1, 2], v3=v3), - df_named.stack(level=1, v3=v3).stack(level=1, v3=v3), + df_named.stack(level=[1, 2], future_stack=future_stack), + df_named.stack(level=1, future_stack=future_stack).stack( + level=1, future_stack=future_stack + ), ) - def test_stack_mixed_levels(self, v3): + def test_stack_mixed_levels(self, future_stack): columns = MultiIndex.from_tuples( [ ("A", "cat", "long"), @@ -422,8 +428,12 @@ def test_stack_mixed_levels(self, v3): ) df = DataFrame(np.random.randn(4, 4), columns=columns) - animal_hair_stacked = df.stack(level=["animal", "hair_length"], v3=v3) - exp_hair_stacked = df.stack(level=["exp", "hair_length"], v3=v3) + animal_hair_stacked = df.stack( + level=["animal", "hair_length"], future_stack=future_stack + ) + exp_hair_stacked = df.stack( + level=["exp", "hair_length"], future_stack=future_stack + ) # GH #8584: Need to check that stacking works when a number # is passed that is both a level name and in the range of @@ -431,12 +441,12 @@ def test_stack_mixed_levels(self, v3): df2 = df.copy() df2.columns.names = ["exp", "animal", 1] tm.assert_frame_equal( - df2.stack(level=["animal", 1], v3=v3), + df2.stack(level=["animal", 1], future_stack=future_stack), animal_hair_stacked, check_names=False, ) tm.assert_frame_equal( - df2.stack(level=["exp", 1], v3=v3), + df2.stack(level=["exp", 1], future_stack=future_stack), exp_hair_stacked, check_names=False, ) @@ -448,19 +458,19 @@ def test_stack_mixed_levels(self, v3): "a mixture of the two" ) with pytest.raises(ValueError, match=msg): - df2.stack(level=["animal", 0], v3=v3) + df2.stack(level=["animal", 0], future_stack=future_stack) # GH #8584: Having 0 in the level names could raise a # strange error about lexsort depth df3 = df.copy() df3.columns.names = ["exp", "animal", 0] tm.assert_frame_equal( - df3.stack(level=["animal", 0], v3=v3), + df3.stack(level=["animal", 0], future_stack=future_stack), animal_hair_stacked, check_names=False, ) - def test_stack_int_level_names(self, v3): + def test_stack_int_level_names(self, future_stack): columns = MultiIndex.from_tuples( [ ("A", "cat", "long"), @@ -472,24 +482,30 @@ def test_stack_int_level_names(self, v3): ) df = DataFrame(np.random.randn(4, 4), columns=columns) - exp_animal_stacked = df.stack(level=["exp", "animal"], v3=v3) - animal_hair_stacked = df.stack(level=["animal", "hair_length"], v3=v3) - exp_hair_stacked = df.stack(level=["exp", "hair_length"], v3=v3) + exp_animal_stacked = df.stack( + level=["exp", "animal"], future_stack=future_stack + ) + animal_hair_stacked = df.stack( + level=["animal", "hair_length"], future_stack=future_stack + ) + exp_hair_stacked = df.stack( + level=["exp", "hair_length"], future_stack=future_stack + ) df2 = df.copy() df2.columns.names = [0, 1, 2] tm.assert_frame_equal( - df2.stack(level=[1, 2], v3=v3), + df2.stack(level=[1, 2], future_stack=future_stack), animal_hair_stacked, check_names=False, ) tm.assert_frame_equal( - df2.stack(level=[0, 1], v3=v3), + df2.stack(level=[0, 1], future_stack=future_stack), exp_animal_stacked, check_names=False, ) tm.assert_frame_equal( - df2.stack(level=[0, 2], v3=v3), + df2.stack(level=[0, 2], future_stack=future_stack), exp_hair_stacked, check_names=False, ) @@ -498,17 +514,17 @@ def test_stack_int_level_names(self, v3): df3 = df.copy() df3.columns.names = [2, 0, 1] tm.assert_frame_equal( - df3.stack(level=[0, 1], v3=v3), + df3.stack(level=[0, 1], future_stack=future_stack), animal_hair_stacked, check_names=False, ) tm.assert_frame_equal( - df3.stack(level=[2, 0], v3=v3), + df3.stack(level=[2, 0], future_stack=future_stack), exp_animal_stacked, check_names=False, ) tm.assert_frame_equal( - df3.stack(level=[2, 1], v3=v3), + df3.stack(level=[2, 1], future_stack=future_stack), exp_hair_stacked, check_names=False, ) @@ -527,7 +543,7 @@ def test_unstack_bool(self): ) tm.assert_frame_equal(rs, xp) - def test_unstack_level_binding(self, v3): + def test_unstack_level_binding(self, future_stack): # GH9856 mi = MultiIndex( levels=[["foo", "bar"], ["one", "two"], ["a", "b"]], @@ -535,7 +551,7 @@ def test_unstack_level_binding(self, v3): names=["first", "second", "third"], ) s = Series(0, index=mi) - result = s.unstack([1, 2]).stack(0, v3=v3) + result = s.unstack([1, 2]).stack(0, future_stack=future_stack) expected_mi = MultiIndex( levels=[["foo", "bar"], ["one", "two"]], @@ -654,7 +670,7 @@ def test_unstack_dtypes_mixed_date(self, c, d): assert left.shape == (3, 2) tm.assert_frame_equal(left, right) - def test_unstack_non_unique_index_names(self, v3): + def test_unstack_non_unique_index_names(self, future_stack): idx = MultiIndex.from_tuples([("a", "b"), ("c", "d")], names=["c1", "c1"]) df = DataFrame([1, 2], index=idx) msg = "The name c1 occurs multiple times, use a level number" @@ -662,7 +678,7 @@ def test_unstack_non_unique_index_names(self, v3): df.unstack("c1") with pytest.raises(ValueError, match=msg): - df.T.stack("c1", v3=v3) + df.T.stack("c1", future_stack=future_stack) def test_unstack_unused_levels(self): # GH 17845: unused codes in index make unstack() cast int to float @@ -1018,11 +1034,11 @@ def test_unstack_nan_index5(self): key = r["1st"], (col, r["2nd"], r["3rd"]) assert r[col] == left.loc[key] - def test_stack_datetime_column_multiIndex(self, v3): + def test_stack_datetime_column_multiIndex(self, future_stack): # GH 8039 t = datetime(2014, 1, 1) df = DataFrame([1, 2, 3, 4], columns=MultiIndex.from_tuples([(t, "A", "B")])) - result = df.stack(v3=v3) + result = df.stack(future_stack=future_stack) eidx = MultiIndex.from_product([(0, 1, 2, 3), ("B",)]) ecols = MultiIndex.from_tuples([(t, "A")]) @@ -1056,9 +1072,9 @@ def test_stack_datetime_column_multiIndex(self, v3): ], ) @pytest.mark.parametrize("level", (-1, 0, 1, [0, 1], [1, 0])) - def test_stack_partial_multiIndex(self, multiindex_columns, level, v3): + def test_stack_partial_multiIndex(self, multiindex_columns, level, future_stack): # GH 8844 - dropna = False if not v3 else lib.no_default + dropna = False if not future_stack else lib.no_default full_multiindex = MultiIndex.from_tuples( [("B", "x"), ("B", "z"), ("A", "y"), ("C", "x"), ("C", "u")], names=["Upper", "Lower"], @@ -1068,13 +1084,13 @@ def test_stack_partial_multiIndex(self, multiindex_columns, level, v3): np.arange(3 * len(multiindex)).reshape(3, len(multiindex)), columns=multiindex, ) - result = df.stack(level=level, dropna=dropna, v3=v3) + result = df.stack(level=level, dropna=dropna, future_stack=future_stack) - if isinstance(level, int) and not v3: + if isinstance(level, int) and not future_stack: # Stacking a single level should not make any all-NaN rows, # so df.stack(level=level, dropna=False) should be the same # as df.stack(level=level, dropna=True). - expected = df.stack(level=level, dropna=True, v3=v3) + expected = df.stack(level=level, dropna=True, future_stack=future_stack) if isinstance(expected, Series): tm.assert_series_equal(result, expected) else: @@ -1083,21 +1099,21 @@ def test_stack_partial_multiIndex(self, multiindex_columns, level, v3): df.columns = MultiIndex.from_tuples( df.columns.to_numpy(), names=df.columns.names ) - expected = df.stack(level=level, dropna=dropna, v3=v3) + expected = df.stack(level=level, dropna=dropna, future_stack=future_stack) if isinstance(expected, Series): tm.assert_series_equal(result, expected) else: tm.assert_frame_equal(result, expected) - def test_stack_full_multiIndex(self, v3): + def test_stack_full_multiIndex(self, future_stack): # GH 8844 full_multiindex = MultiIndex.from_tuples( [("B", "x"), ("B", "z"), ("A", "y"), ("C", "x"), ("C", "u")], names=["Upper", "Lower"], ) df = DataFrame(np.arange(6).reshape(2, 3), columns=full_multiindex[[0, 1, 3]]) - dropna = False if not v3 else lib.no_default - result = df.stack(dropna=dropna, v3=v3) + dropna = False if not future_stack else lib.no_default + result = df.stack(dropna=dropna, future_stack=future_stack) expected = DataFrame( [[0, 2], [1, np.nan], [3, 5], [4, np.nan]], index=MultiIndex( @@ -1111,11 +1127,11 @@ def test_stack_full_multiIndex(self, v3): tm.assert_frame_equal(result, expected) @pytest.mark.parametrize("ordered", [False, True]) - def test_stack_preserve_categorical_dtype(self, ordered, v3): + def test_stack_preserve_categorical_dtype(self, ordered, future_stack): # GH13854 cidx = pd.CategoricalIndex(list("yxz"), categories=list("xyz"), ordered=ordered) df = DataFrame([[10, 11, 12]], columns=cidx) - result = df.stack(v3=v3) + result = df.stack(future_stack=future_stack) # `MultiIndex.from_product` preserves categorical dtype - # it's tested elsewhere. @@ -1132,28 +1148,30 @@ def test_stack_preserve_categorical_dtype(self, ordered, v3): (list("zyx"), [14, 15, 12, 13, 10, 11]), ], ) - def test_stack_multi_preserve_categorical_dtype(self, ordered, labels, data, v3): + def test_stack_multi_preserve_categorical_dtype( + self, ordered, labels, data, future_stack + ): # GH-36991 cidx = pd.CategoricalIndex(labels, categories=sorted(labels), ordered=ordered) cidx2 = pd.CategoricalIndex(["u", "v"], ordered=ordered) midx = MultiIndex.from_product([cidx, cidx2]) df = DataFrame([sorted(data)], columns=midx) - result = df.stack([0, 1], v3=v3) + result = df.stack([0, 1], future_stack=future_stack) - labels = labels if v3 else sorted(labels) + labels = labels if future_stack else sorted(labels) s_cidx = pd.CategoricalIndex(labels, ordered=ordered) - expected_data = sorted(data) if v3 else data + expected_data = sorted(data) if future_stack else data expected = Series( expected_data, index=MultiIndex.from_product([[0], s_cidx, cidx2]) ) tm.assert_series_equal(result, expected) - def test_stack_preserve_categorical_dtype_values(self, v3): + def test_stack_preserve_categorical_dtype_values(self, future_stack): # GH-23077 cat = pd.Categorical(["a", "a", "b", "c"]) df = DataFrame({"A": cat, "B": cat}) - result = df.stack(v3=v3) + result = df.stack(future_stack=future_stack) index = MultiIndex.from_product([[0, 1, 2, 3], ["A", "B"]]) expected = Series( pd.Categorical(["a", "a", "a", "a", "b", "b", "c", "c"]), index=index @@ -1168,10 +1186,10 @@ def test_stack_preserve_categorical_dtype_values(self, v3): ([0, 1, 2, 3], MultiIndex.from_product([[1, 2], ["a", "b"]])), ], ) - def test_stack_multi_columns_non_unique_index(self, index, columns, v3): + def test_stack_multi_columns_non_unique_index(self, index, columns, future_stack): # GH-28301 df = DataFrame(index=index, columns=columns).fillna(1) - stacked = df.stack(v3=v3) + stacked = df.stack(future_stack=future_stack) new_index = MultiIndex.from_tuples(stacked.index.to_numpy()) expected = DataFrame( stacked.to_numpy(), index=new_index, columns=stacked.columns @@ -1189,7 +1207,7 @@ def test_stack_multi_columns_non_unique_index(self, index, columns, v3): ], ) def test_stack_multi_columns_mixed_extension_types( - self, vals1, vals2, dtype1, dtype2, expected_dtype, v3 + self, vals1, vals2, dtype1, dtype2, expected_dtype, future_stack ): # GH45740 df = DataFrame( @@ -1198,8 +1216,10 @@ def test_stack_multi_columns_mixed_extension_types( ("A", 2): Series(vals2, dtype=dtype2), } ) - result = df.stack(v3=v3) - expected = df.astype(object).stack(v3=v3).astype(expected_dtype) + result = df.stack(future_stack=future_stack) + expected = ( + df.astype(object).stack(future_stack=future_stack).astype(expected_dtype) + ) tm.assert_frame_equal(result, expected) @pytest.mark.parametrize("level", [0, 1]) @@ -1329,11 +1349,11 @@ def test_unstack_timezone_aware_values(): tm.assert_frame_equal(result, expected) -def test_stack_timezone_aware_values(v3): +def test_stack_timezone_aware_values(future_stack): # GH 19420 ts = date_range(freq="D", start="20180101", end="20180103", tz="America/New_York") df = DataFrame({"A": ts}, index=["a", "b", "c"]) - result = df.stack(v3=v3) + result = df.stack(future_stack=future_stack) expected = Series( ts, index=MultiIndex(levels=[["a", "b", "c"], ["A"]], codes=[[0, 1, 2], [0, 0, 0]]), @@ -1342,31 +1362,33 @@ def test_stack_timezone_aware_values(v3): @pytest.mark.parametrize("dropna", [True, False, lib.no_default]) -def test_stack_empty_frame(dropna, v3): +def test_stack_empty_frame(dropna, future_stack): # GH 36113 levels = [np.array([], dtype=np.int64), np.array([], dtype=np.int64)] expected = Series(dtype=np.float64, index=MultiIndex(levels=levels, codes=[[], []])) - if v3 and dropna is not lib.no_default: - with pytest.raises(ValueError, match="Cannot specify dropna"): - DataFrame(dtype=np.float64).stack(dropna=dropna, v3=v3) + if future_stack and dropna is not lib.no_default: + with pytest.raises(ValueError, match="dropna must be unspecified"): + DataFrame(dtype=np.float64).stack(dropna=dropna, future_stack=future_stack) else: - result = DataFrame(dtype=np.float64).stack(dropna=dropna, v3=v3) + result = DataFrame(dtype=np.float64).stack( + dropna=dropna, future_stack=future_stack + ) tm.assert_series_equal(result, expected) @pytest.mark.parametrize("dropna", [True, False, lib.no_default]) @pytest.mark.parametrize("fill_value", [None, 0]) -def test_stack_unstack_empty_frame(dropna, fill_value, v3): +def test_stack_unstack_empty_frame(dropna, fill_value, future_stack): # GH 36113 - if v3 and dropna is not lib.no_default: - with pytest.raises(ValueError, match="Cannot specify dropna"): - DataFrame(dtype=np.int64).stack(dropna=dropna, v3=v3).unstack( - fill_value=fill_value - ) + if future_stack and dropna is not lib.no_default: + with pytest.raises(ValueError, match="dropna must be unspecified"): + DataFrame(dtype=np.int64).stack( + dropna=dropna, future_stack=future_stack + ).unstack(fill_value=fill_value) else: result = ( DataFrame(dtype=np.int64) - .stack(dropna=dropna, v3=v3) + .stack(dropna=dropna, future_stack=future_stack) .unstack(fill_value=fill_value) ) expected = DataFrame(dtype=np.int64) @@ -1411,11 +1433,11 @@ def test_unstacking_multi_index_df(): tm.assert_frame_equal(result, expected) -def test_stack_positional_level_duplicate_column_names(v3): +def test_stack_positional_level_duplicate_column_names(future_stack): # https://github.com/pandas-dev/pandas/issues/36353 columns = MultiIndex.from_product([("x", "y"), ("y", "z")], names=["a", "a"]) df = DataFrame([[1, 1, 1, 1]], columns=columns) - result = df.stack(0, v3=v3) + result = df.stack(0, future_stack=future_stack) new_columns = Index(["y", "z"], name="a") new_index = MultiIndex.from_tuples([(0, "x"), (0, "y")], names=[None, "a"]) @@ -1446,7 +1468,7 @@ def test_unstack_non_slice_like_blocks(using_array_manager): tm.assert_frame_equal(res, expected) -def test_stack_sort_false(v3): +def test_stack_sort_false(future_stack): # GH 15105 data = [[1, 2, 3.0, 4.0], [2, 3, 4.0, 5.0], [3, 4, np.nan, np.nan]] df = DataFrame( @@ -1455,9 +1477,9 @@ def test_stack_sort_false(v3): levels=[["B", "A"], ["x", "y"]], codes=[[0, 0, 1, 1], [0, 1, 0, 1]] ), ) - kwargs = {} if v3 else {"sort": False} - result = df.stack(level=0, v3=v3, **kwargs) - if v3: + kwargs = {} if future_stack else {"sort": False} + result = df.stack(level=0, future_stack=future_stack, **kwargs) + if future_stack: expected = DataFrame( { "x": [1.0, 3.0, 2.0, 4.0, 3.0, np.nan], @@ -1479,17 +1501,17 @@ def test_stack_sort_false(v3): data, columns=MultiIndex.from_arrays([["B", "B", "A", "A"], ["x", "y", "x", "y"]]), ) - kwargs = {} if v3 else {"sort": False} - result = df.stack(level=0, v3=v3, **kwargs) + kwargs = {} if future_stack else {"sort": False} + result = df.stack(level=0, future_stack=future_stack, **kwargs) tm.assert_frame_equal(result, expected) -def test_stack_sort_false_multi_level(v3): +def test_stack_sort_false_multi_level(future_stack): # GH 15105 idx = MultiIndex.from_tuples([("weight", "kg"), ("height", "m")]) df = DataFrame([[1.0, 2.0], [3.0, 4.0]], index=["cat", "dog"], columns=idx) - kwargs = {} if v3 else {"sort": False} - result = df.stack([0, 1], v3=v3, **kwargs) + kwargs = {} if future_stack else {"sort": False} + result = df.stack([0, 1], future_stack=future_stack, **kwargs) expected_index = MultiIndex.from_tuples( [ ("cat", "weight", "kg"), @@ -1570,13 +1592,13 @@ def test_unstack_multiple_no_empty_columns(self): expected = unstacked.dropna(axis=1, how="all") tm.assert_frame_equal(unstacked, expected) - def test_stack(self, multiindex_year_month_day_dataframe_random_data, v3): + def test_stack(self, multiindex_year_month_day_dataframe_random_data, future_stack): ymd = multiindex_year_month_day_dataframe_random_data # regular roundtrip unstacked = ymd.unstack() - restacked = unstacked.stack(v3=v3) - if v3: + restacked = unstacked.stack(future_stack=future_stack) + if future_stack: # NA values in unstacked persist to restacked in version 3 restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked, ymd) @@ -1584,32 +1606,32 @@ def test_stack(self, multiindex_year_month_day_dataframe_random_data, v3): unlexsorted = ymd.sort_index(level=2) unstacked = unlexsorted.unstack(2) - restacked = unstacked.stack(v3=v3) - if v3: + restacked = unstacked.stack(future_stack=future_stack) + if future_stack: # NA values in unstacked persist to restacked in version 3 restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked.sort_index(level=0), ymd) unlexsorted = unlexsorted[::-1] unstacked = unlexsorted.unstack(1) - restacked = unstacked.stack(v3=v3).swaplevel(1, 2) - if v3: + restacked = unstacked.stack(future_stack=future_stack).swaplevel(1, 2) + if future_stack: # NA values in unstacked persist to restacked in version 3 restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked.sort_index(level=0), ymd) unlexsorted = unlexsorted.swaplevel(0, 1) unstacked = unlexsorted.unstack(0).swaplevel(0, 1, axis=1) - restacked = unstacked.stack(0, v3=v3).swaplevel(1, 2) - if v3: + restacked = unstacked.stack(0, future_stack=future_stack).swaplevel(1, 2) + if future_stack: # NA values in unstacked persist to restacked in version 3 restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked.sort_index(level=0), ymd) # columns unsorted unstacked = ymd.unstack() - restacked = unstacked.stack(v3=v3) - if v3: + restacked = unstacked.stack(future_stack=future_stack) + if future_stack: # NA values in unstacked persist to restacked in version 3 restacked = restacked.dropna(how="all") tm.assert_frame_equal(restacked, ymd) @@ -1617,31 +1639,33 @@ def test_stack(self, multiindex_year_month_day_dataframe_random_data, v3): # more than 2 levels in the columns unstacked = ymd.unstack(1).unstack(1) - result = unstacked.stack(1, v3=v3) + result = unstacked.stack(1, future_stack=future_stack) expected = ymd.unstack() tm.assert_frame_equal(result, expected) - result = unstacked.stack(2, v3=v3) + result = unstacked.stack(2, future_stack=future_stack) expected = ymd.unstack(1) tm.assert_frame_equal(result, expected) - result = unstacked.stack(0, v3=v3) - expected = ymd.stack(v3=v3).unstack(1).unstack(1) + result = unstacked.stack(0, future_stack=future_stack) + expected = ymd.stack(future_stack=future_stack).unstack(1).unstack(1) tm.assert_frame_equal(result, expected) # not all levels present in each echelon unstacked = ymd.unstack(2).loc[:, ::3] - stacked = unstacked.stack(v3=v3).stack(v3=v3) - ymd_stacked = ymd.stack(v3=v3) - if v3: + stacked = unstacked.stack(future_stack=future_stack).stack( + future_stack=future_stack + ) + ymd_stacked = ymd.stack(future_stack=future_stack) + if future_stack: # NA values in unstacked persist to restacked in version 3 stacked = stacked.dropna(how="all") ymd_stacked = ymd_stacked.dropna(how="all") tm.assert_series_equal(stacked, ymd_stacked.reindex(stacked.index)) # stack with negative number - result = ymd.unstack(0).stack(-2, v3=v3) - expected = ymd.unstack(0).stack(0, v3=v3) + result = ymd.unstack(0).stack(-2, future_stack=future_stack) + expected = ymd.unstack(0).stack(0, future_stack=future_stack) tm.assert_equal(result, expected) @pytest.mark.parametrize( @@ -1669,26 +1693,26 @@ def test_stack(self, multiindex_year_month_day_dataframe_random_data, v3): ], ], ) - def test_stack_duplicate_index(self, idx, columns, exp_idx, v3): + def test_stack_duplicate_index(self, idx, columns, exp_idx, future_stack): # GH10417 df = DataFrame( np.arange(12).reshape(4, 3), index=idx, columns=columns, ) - if v3: + if future_stack: msg = "Columns with duplicate values are not supported in stack" with pytest.raises(ValueError, match=msg): - df.stack(v3=v3) + df.stack(future_stack=future_stack) else: - result = df.stack(v3=v3) + result = df.stack(future_stack=future_stack) expected = Series(np.arange(12), index=exp_idx) tm.assert_series_equal(result, expected) assert result.index.is_unique is False li, ri = result.index, expected.index tm.assert_index_equal(li, ri) - def test_unstack_odd_failure(self, v3): + def test_unstack_odd_failure(self, future_stack): data = """day,time,smoker,sum,len Fri,Dinner,No,8.25,3. Fri,Dinner,Yes,27.03,9 @@ -1707,26 +1731,26 @@ def test_unstack_odd_failure(self, v3): # it works, #2100 result = df.unstack(2) - recons = result.stack(v3=v3) - if v3: + recons = result.stack(future_stack=future_stack) + if future_stack: # NA values in unstacked persist to restacked in version 3 recons = recons.dropna(how="all") tm.assert_frame_equal(recons, df) - def test_stack_mixed_dtype(self, multiindex_dataframe_random_data, v3): + def test_stack_mixed_dtype(self, multiindex_dataframe_random_data, future_stack): frame = multiindex_dataframe_random_data df = frame.T df["foo", "four"] = "foo" df = df.sort_index(level=1, axis=1) - stacked = df.stack(v3=v3) - result = df["foo"].stack(v3=v3).sort_index() + stacked = df.stack(future_stack=future_stack) + result = df["foo"].stack(future_stack=future_stack).sort_index() tm.assert_series_equal(stacked["foo"], result, check_names=False) assert result.name is None assert stacked["bar"].dtype == np.float_ - def test_unstack_bug(self, v3): + def test_unstack_bug(self, future_stack): df = DataFrame( { "state": ["naive", "naive", "naive", "active", "active", "active"], @@ -1740,22 +1764,24 @@ def test_unstack_bug(self, v3): result = df.groupby(["state", "exp", "barcode", "v"]).apply(len) unstacked = result.unstack() - restacked = unstacked.stack(v3=v3) + restacked = unstacked.stack(future_stack=future_stack) tm.assert_series_equal(restacked, result.reindex(restacked.index).astype(float)) - def test_stack_unstack_preserve_names(self, multiindex_dataframe_random_data, v3): + def test_stack_unstack_preserve_names( + self, multiindex_dataframe_random_data, future_stack + ): frame = multiindex_dataframe_random_data unstacked = frame.unstack() assert unstacked.index.name == "first" assert unstacked.columns.names == ["exp", "second"] - restacked = unstacked.stack(v3=v3) + restacked = unstacked.stack(future_stack=future_stack) assert restacked.index.names == frame.index.names @pytest.mark.parametrize("method", ["stack", "unstack"]) def test_stack_unstack_wrong_level_name( - self, method, multiindex_dataframe_random_data, v3 + self, method, multiindex_dataframe_random_data, future_stack ): # GH 18303 - wrong level name should raise frame = multiindex_dataframe_random_data @@ -1763,7 +1789,7 @@ def test_stack_unstack_wrong_level_name( # A DataFrame with flat axes: df = frame.loc["foo"] - kwargs = {"v3": v3} if method == "stack" else {} + kwargs = {"future_stack": future_stack} if method == "stack" else {} with pytest.raises(KeyError, match="does not match index name"): getattr(df, method)("mistake", **kwargs) @@ -1780,20 +1806,20 @@ def test_unstack_level_name(self, multiindex_dataframe_random_data): expected = frame.unstack(level=1) tm.assert_frame_equal(result, expected) - def test_stack_level_name(self, multiindex_dataframe_random_data, v3): + def test_stack_level_name(self, multiindex_dataframe_random_data, future_stack): frame = multiindex_dataframe_random_data unstacked = frame.unstack("second") - result = unstacked.stack("exp", v3=v3) - expected = frame.unstack().stack(0, v3=v3) + result = unstacked.stack("exp", future_stack=future_stack) + expected = frame.unstack().stack(0, future_stack=future_stack) tm.assert_frame_equal(result, expected) - result = frame.stack("exp", v3=v3) - expected = frame.stack(v3=v3) + result = frame.stack("exp", future_stack=future_stack) + expected = frame.stack(future_stack=future_stack) tm.assert_series_equal(result, expected) def test_stack_unstack_multiple( - self, multiindex_year_month_day_dataframe_random_data, v3 + self, multiindex_year_month_day_dataframe_random_data, future_stack ): ymd = multiindex_year_month_day_dataframe_random_data @@ -1807,8 +1833,8 @@ def test_stack_unstack_multiple( s_unstacked = s.unstack(["year", "month"]) tm.assert_frame_equal(s_unstacked, expected["A"]) - restacked = unstacked.stack(["year", "month"], v3=v3) - if v3: + restacked = unstacked.stack(["year", "month"], future_stack=future_stack) + if future_stack: # NA values in unstacked persist to restacked in version 3 restacked = restacked.dropna(how="all") restacked = restacked.swaplevel(0, 1).swaplevel(1, 2) @@ -1827,7 +1853,7 @@ def test_stack_unstack_multiple( tm.assert_frame_equal(unstacked, expected.loc[:, unstacked.columns]) def test_stack_names_and_numbers( - self, multiindex_year_month_day_dataframe_random_data, v3 + self, multiindex_year_month_day_dataframe_random_data, future_stack ): ymd = multiindex_year_month_day_dataframe_random_data @@ -1835,10 +1861,10 @@ def test_stack_names_and_numbers( # Can't use mixture of names and numbers to stack with pytest.raises(ValueError, match="level should contain"): - unstacked.stack([0, "month"], v3=v3) + unstacked.stack([0, "month"], future_stack=future_stack) def test_stack_multiple_out_of_bounds( - self, multiindex_year_month_day_dataframe_random_data, v3 + self, multiindex_year_month_day_dataframe_random_data, future_stack ): # nlevels == 3 ymd = multiindex_year_month_day_dataframe_random_data @@ -1846,9 +1872,9 @@ def test_stack_multiple_out_of_bounds( unstacked = ymd.unstack(["year", "month"]) with pytest.raises(IndexError, match="Too many levels"): - unstacked.stack([2, 3], v3=v3) + unstacked.stack([2, 3], future_stack=future_stack) with pytest.raises(IndexError, match="not a valid level number"): - unstacked.stack([-4, -3], v3=v3) + unstacked.stack([-4, -3], future_stack=future_stack) def test_unstack_period_series(self): # GH4342 @@ -1966,7 +1992,7 @@ def test_unstack_period_frame(self): tm.assert_frame_equal(result3, expected) - def test_stack_multiple_bug(self, v3): + def test_stack_multiple_bug(self, future_stack): # bug when some uniques are not present in the data GH#3170 id_col = ([1] * 3) + ([2] * 3) name = (["a"] * 3) + (["b"] * 3) @@ -1981,28 +2007,33 @@ def test_stack_multiple_bug(self, v3): with pytest.raises(TypeError, match=msg): unst.resample("W-THU").mean() down = unst.resample("W-THU").mean(numeric_only=True) - rs = down.stack("ID", v3=v3) - xp = unst.loc[:, ["VAR1"]].resample("W-THU").mean().stack("ID", v3=v3) + rs = down.stack("ID", future_stack=future_stack) + xp = ( + unst.loc[:, ["VAR1"]] + .resample("W-THU") + .mean() + .stack("ID", future_stack=future_stack) + ) xp.columns.name = "Params" tm.assert_frame_equal(rs, xp) - def test_stack_dropna(self, v3): + def test_stack_dropna(self, future_stack): # GH#3997 df = DataFrame({"A": ["a1", "a2"], "B": ["b1", "b2"], "C": [1, 1]}) df = df.set_index(["A", "B"]) - dropna = False if not v3 else lib.no_default - stacked = df.unstack().stack(dropna=dropna, v3=v3) + dropna = False if not future_stack else lib.no_default + stacked = df.unstack().stack(dropna=dropna, future_stack=future_stack) assert len(stacked) > len(stacked.dropna()) - if v3: - with pytest.raises(ValueError, match="Cannot specify dropna"): - df.unstack().stack(dropna=True, v3=v3) + if future_stack: + with pytest.raises(ValueError, match="dropna must be unspecified"): + df.unstack().stack(dropna=True, future_stack=future_stack) else: - stacked = df.unstack().stack(dropna=True, v3=v3) + stacked = df.unstack().stack(dropna=True, future_stack=future_stack) tm.assert_frame_equal(stacked, stacked.dropna()) - def test_unstack_multiple_hierarchical(self, v3): + def test_unstack_multiple_hierarchical(self, future_stack): df = DataFrame( index=[ [0, 0, 0, 0, 1, 1, 1, 1], @@ -2039,7 +2070,7 @@ def test_unstack_sparse_keyspace(self): # it works! is sufficient idf.unstack("E") - def test_unstack_unobserved_keys(self, v3): + def test_unstack_unobserved_keys(self, future_stack): # related to GH#2278 refactoring levels = [[0, 1], [0, 1, 2, 3]] codes = [[0, 0, 1, 1], [0, 2, 0, 2]] @@ -2051,7 +2082,7 @@ def test_unstack_unobserved_keys(self, v3): result = df.unstack() assert len(result.columns) == 4 - recons = result.stack(v3=v3) + recons = result.stack(future_stack=future_stack) tm.assert_frame_equal(recons, df) @pytest.mark.slow @@ -2085,13 +2116,15 @@ def __init__(self, *args, **kwargs) -> None: ) @pytest.mark.parametrize("stack_lev", range(2)) @pytest.mark.parametrize("sort", [True, False]) - def test_stack_order_with_unsorted_levels(self, levels, stack_lev, sort, v3): + def test_stack_order_with_unsorted_levels( + self, levels, stack_lev, sort, future_stack + ): # GH#16323 # deep check for 1-row case columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]]) df = DataFrame(columns=columns, data=[range(4)]) - kwargs = {} if v3 else {"sort": sort} - df_stacked = df.stack(stack_lev, v3=v3, **kwargs) + kwargs = {} if future_stack else {"sort": sort} + df_stacked = df.stack(stack_lev, future_stack=future_stack, **kwargs) for row in df.index: for col in df.columns: expected = df.loc[row, col] @@ -2100,7 +2133,7 @@ def test_stack_order_with_unsorted_levels(self, levels, stack_lev, sort, v3): result = df_stacked.loc[result_row, result_col] assert result == expected - def test_stack_order_with_unsorted_levels_multi_row(self, v3): + def test_stack_order_with_unsorted_levels_multi_row(self, future_stack): # GH#16323 # check multi-row case @@ -2112,19 +2145,20 @@ def test_stack_order_with_unsorted_levels_multi_row(self, v3): columns=mi, index=range(5), data=np.arange(5 * len(mi)).reshape(5, -1) ) assert all( - df.loc[row, col] == df.stack(0, v3=v3).loc[(row, col[0]), col[1]] + df.loc[row, col] + == df.stack(0, future_stack=future_stack).loc[(row, col[0]), col[1]] for row in df.index for col in df.columns ) - def test_stack_order_with_unsorted_levels_multi_row_2(self, v3): + def test_stack_order_with_unsorted_levels_multi_row_2(self, future_stack): # GH#53636 levels = ((0, 1), (1, 0)) stack_lev = 1 columns = MultiIndex(levels=levels, codes=[[0, 0, 1, 1], [0, 1, 0, 1]]) df = DataFrame(columns=columns, data=[range(4)], index=[1, 0, 2, 3]) - kwargs = {} if v3 else {"sort": True} - result = df.stack(stack_lev, v3=v3, **kwargs) + kwargs = {} if future_stack else {"sort": True} + result = df.stack(stack_lev, future_stack=future_stack, **kwargs) expected_index = MultiIndex( levels=[[0, 1, 2, 3], [0, 1]], codes=[[1, 1, 0, 0, 2, 2, 3, 3], [1, 0, 1, 0, 1, 0, 1, 0]], @@ -2138,7 +2172,7 @@ def test_stack_order_with_unsorted_levels_multi_row_2(self, v3): ) tm.assert_frame_equal(result, expected) - def test_stack_unstack_unordered_multiindex(self, v3): + def test_stack_unstack_unordered_multiindex(self, future_stack): # GH# 18265 values = np.arange(5) data = np.vstack( @@ -2153,7 +2187,9 @@ def test_stack_unstack_unordered_multiindex(self, v3): multi_level_df = pd.concat(second_level_dict, axis=1) multi_level_df.columns.names = ["second", "first"] df = multi_level_df.reindex(sorted(multi_level_df.columns), axis=1) - result = df.stack(["first", "second"], v3=v3).unstack(["first", "second"]) + result = df.stack(["first", "second"], future_stack=future_stack).unstack( + ["first", "second"] + ) expected = DataFrame( [["a0", "b0"], ["a1", "b1"], ["a2", "b2"], ["a3", "b3"], ["a4", "b4"]], index=[0, 1, 2, 3, 4], @@ -2176,7 +2212,7 @@ def test_unstack_preserve_types( assert unstacked["E", 1].dtype == np.object_ assert unstacked["F", 1].dtype == np.float64 - def test_unstack_group_index_overflow(self, v3): + def test_unstack_group_index_overflow(self, future_stack): codes = np.tile(np.arange(500), 2) level = np.arange(500) @@ -2190,7 +2226,7 @@ def test_unstack_group_index_overflow(self, v3): assert result.shape == (500, 2) # test roundtrip - stacked = result.stack(v3=v3) + stacked = result.stack(future_stack=future_stack) tm.assert_series_equal(s, stacked.reindex(s.index)) # put it at beginning @@ -2269,7 +2305,7 @@ def test_unstack_with_level_has_nan(self): tm.assert_index_equal(result, expected) - def test_stack_nan_in_multiindex_columns(self, v3): + def test_stack_nan_in_multiindex_columns(self, future_stack): # GH#39481 df = DataFrame( np.zeros([1, 5]), @@ -2283,8 +2319,8 @@ def test_stack_nan_in_multiindex_columns(self, v3): ], ), ) - result = df.stack(2, v3=v3) - if v3: + result = df.stack(2, future_stack=future_stack) + if future_stack: index = MultiIndex(levels=[[0], [0.0, 1.0]], codes=[[0, 0, 0], [-1, 0, 1]]) columns = MultiIndex(levels=[[0], [2, 3]], codes=[[0, 0, 0], [-1, 0, 1]]) else: @@ -2297,7 +2333,7 @@ def test_stack_nan_in_multiindex_columns(self, v3): ) tm.assert_frame_equal(result, expected) - def test_multi_level_stack_categorical(self, v3): + def test_multi_level_stack_categorical(self, future_stack): # GH 15239 midx = MultiIndex.from_arrays( [ @@ -2307,8 +2343,8 @@ def test_multi_level_stack_categorical(self, v3): ] ) df = DataFrame(np.arange(8).reshape(2, 4), columns=midx) - result = df.stack([1, 2], v3=v3) - if v3: + result = df.stack([1, 2], future_stack=future_stack) + if future_stack: expected = DataFrame( [ [0, np.nan], @@ -2352,7 +2388,7 @@ def test_multi_level_stack_categorical(self, v3): ) tm.assert_frame_equal(result, expected) - def test_stack_nan_level(self, v3): + def test_stack_nan_level(self, future_stack): # GH 9406 df_nan = DataFrame( np.arange(4).reshape(2, 2), @@ -2362,8 +2398,8 @@ def test_stack_nan_level(self, v3): index=Index([0, 1], name="Num"), dtype=np.float64, ) - result = df_nan.stack(v3=v3) - if v3: + result = df_nan.stack(future_stack=future_stack) + if future_stack: index = MultiIndex( levels=[[0, 1], [np.nan, "b"]], codes=[[0, 0, 1, 1], [0, 1, 0, 1]], @@ -2395,7 +2431,7 @@ def test_unstack_categorical_columns(self): expected.columns = MultiIndex.from_tuples([("cat", 0), ("cat", 1)]) tm.assert_frame_equal(result, expected) - def test_stack_unsorted(self, v3): + def test_stack_unsorted(self, future_stack): # GH 16925 PAE = ["ITA", "FRA"] VAR = ["A1", "A2"] @@ -2409,11 +2445,15 @@ def test_stack_unsorted(self, v3): DF.columns = DF.columns.droplevel(0) DF.loc[:, ("A0", "NET")] = 9999 - result = DF.stack(["VAR", "TYP"], v3=v3).sort_index() - expected = DF.sort_index(axis=1).stack(["VAR", "TYP"], v3=v3).sort_index() + result = DF.stack(["VAR", "TYP"], future_stack=future_stack).sort_index() + expected = ( + DF.sort_index(axis=1) + .stack(["VAR", "TYP"], future_stack=future_stack) + .sort_index() + ) tm.assert_series_equal(result, expected) - def test_stack_nullable_dtype(self, v3): + def test_stack_nullable_dtype(self, future_stack): # GH#43561 columns = MultiIndex.from_product( [["54511", "54515"], ["r", "t_mean"]], names=["station", "element"] @@ -2423,14 +2463,18 @@ def test_stack_nullable_dtype(self, v3): arr = np.array([[50, 226, 10, 215], [10, 215, 9, 220], [305, 232, 111, 220]]) df = DataFrame(arr, columns=columns, index=index, dtype=pd.Int64Dtype()) - result = df.stack("station", v3=v3) + result = df.stack("station", future_stack=future_stack) - expected = df.astype(np.int64).stack("station", v3=v3).astype(pd.Int64Dtype()) + expected = ( + df.astype(np.int64) + .stack("station", future_stack=future_stack) + .astype(pd.Int64Dtype()) + ) tm.assert_frame_equal(result, expected) # non-homogeneous case df[df.columns[0]] = df[df.columns[0]].astype(pd.Float64Dtype()) - result = df.stack("station", v3=v3) + result = df.stack("station", future_stack=future_stack) expected = DataFrame( { diff --git a/pandas/tests/frame/test_subclass.py b/pandas/tests/frame/test_subclass.py index a7c8373933feb..11a21a9733a08 100644 --- a/pandas/tests/frame/test_subclass.py +++ b/pandas/tests/frame/test_subclass.py @@ -216,7 +216,7 @@ def test_subclass_stack(self): columns=["X", "Y", "Z"], ) - res = df.stack(v3=True) + res = df.stack(future_stack=True) exp = tm.SubclassedSeries( [1, 2, 3, 4, 5, 6, 7, 8, 9], index=[list("aaabbbccc"), list("XYZXYZXYZ")] ) @@ -253,10 +253,10 @@ def test_subclass_stack_multi(self): columns=Index(["W", "X"], name="www"), ) - res = df.stack(v3=True) + res = df.stack(future_stack=True) tm.assert_frame_equal(res, exp) - res = df.stack("yyy", v3=True) + res = df.stack("yyy", future_stack=True) tm.assert_frame_equal(res, exp) exp = tm.SubclassedDataFrame( @@ -277,7 +277,7 @@ def test_subclass_stack_multi(self): columns=Index(["y", "z"], name="yyy"), ) - res = df.stack("www", v3=True) + res = df.stack("www", future_stack=True) tm.assert_frame_equal(res, exp) def test_subclass_stack_multi_mixed(self): @@ -315,10 +315,10 @@ def test_subclass_stack_multi_mixed(self): columns=Index(["W", "X"], name="www"), ) - res = df.stack(v3=True) + res = df.stack(future_stack=True) tm.assert_frame_equal(res, exp) - res = df.stack("yyy", v3=True) + res = df.stack("yyy", future_stack=True) tm.assert_frame_equal(res, exp) exp = tm.SubclassedDataFrame( @@ -339,7 +339,7 @@ def test_subclass_stack_multi_mixed(self): columns=Index(["y", "z"], name="yyy"), ) - res = df.stack("www", v3=True) + res = df.stack("www", future_stack=True) tm.assert_frame_equal(res, exp) def test_subclass_unstack(self): diff --git a/pandas/tests/groupby/aggregate/test_cython.py b/pandas/tests/groupby/aggregate/test_cython.py index 2bb4f1fdf1dec..a9cdbe5907cee 100644 --- a/pandas/tests/groupby/aggregate/test_cython.py +++ b/pandas/tests/groupby/aggregate/test_cython.py @@ -68,7 +68,7 @@ def test_cythonized_aggers(op_name): expd = {} for (cat1, cat2), group in grouped: expd.setdefault(cat1, {})[cat2] = op(group["C"]) - exp = DataFrame(expd).T.stack(v3=True) + exp = DataFrame(expd).T.stack(future_stack=True) exp.index.names = ["A", "B"] exp.name = "C" diff --git a/pandas/tests/groupby/test_categorical.py b/pandas/tests/groupby/test_categorical.py index 49ebffb6a8a51..b33a846684f13 100644 --- a/pandas/tests/groupby/test_categorical.py +++ b/pandas/tests/groupby/test_categorical.py @@ -237,9 +237,13 @@ def f(x): # GH 10460 expc = Categorical.from_codes(np.arange(4).repeat(8), levels, ordered=True) exp = CategoricalIndex(expc) - tm.assert_index_equal((desc_result.stack(v3=True).index.get_level_values(0)), exp) + tm.assert_index_equal( + (desc_result.stack(future_stack=True).index.get_level_values(0)), exp + ) exp = Index(["count", "mean", "std", "min", "25%", "50%", "75%", "max"] * 4) - tm.assert_index_equal((desc_result.stack(v3=True).index.get_level_values(1)), exp) + tm.assert_index_equal( + (desc_result.stack(future_stack=True).index.get_level_values(1)), exp + ) def test_level_get_group(observed): @@ -673,9 +677,13 @@ def test_datetime(): # GH 10460 expc = Categorical.from_codes(np.arange(4).repeat(8), levels, ordered=True) exp = CategoricalIndex(expc) - tm.assert_index_equal((desc_result.stack(v3=True).index.get_level_values(0)), exp) + tm.assert_index_equal( + (desc_result.stack(future_stack=True).index.get_level_values(0)), exp + ) exp = Index(["count", "mean", "std", "min", "25%", "50%", "75%", "max"] * 4) - tm.assert_index_equal((desc_result.stack(v3=True).index.get_level_values(1)), exp) + tm.assert_index_equal( + (desc_result.stack(future_stack=True).index.get_level_values(1)), exp + ) def test_categorical_index(): @@ -713,8 +721,10 @@ def test_describe_categorical_columns(): df = DataFrame(np.random.randn(20, 4), columns=cats) result = df.groupby([1, 2, 3, 4] * 5).describe() - tm.assert_index_equal(result.stack(v3=True).columns, cats) - tm.assert_categorical_equal(result.stack(v3=True).columns.values, cats.values) + tm.assert_index_equal(result.stack(future_stack=True).columns, cats) + tm.assert_categorical_equal( + result.stack(future_stack=True).columns.values, cats.values + ) def test_unstack_categorical(): diff --git a/pandas/tests/groupby/test_function.py b/pandas/tests/groupby/test_function.py index e14044bc22e57..84b417f9ed5b4 100644 --- a/pandas/tests/groupby/test_function.py +++ b/pandas/tests/groupby/test_function.py @@ -1114,7 +1114,7 @@ def test_series_describe_single(): ts = tm.makeTimeSeries() grouped = ts.groupby(lambda x: x.month) result = grouped.apply(lambda x: x.describe()) - expected = grouped.describe().stack(v3=True) + expected = grouped.describe().stack(future_stack=True) tm.assert_series_equal(result, expected) diff --git a/pandas/tests/indexes/datetimes/test_partial_slicing.py b/pandas/tests/indexes/datetimes/test_partial_slicing.py index fe90cee5a1922..c74d5d9c1d3a6 100644 --- a/pandas/tests/indexes/datetimes/test_partial_slicing.py +++ b/pandas/tests/indexes/datetimes/test_partial_slicing.py @@ -359,7 +359,7 @@ def test_partial_slicing_with_multiindex_series(self): # partial slice on a series mi ser = DataFrame( np.random.rand(1000, 1000), index=date_range("2000-1-1", periods=1000) - ).stack(v3=True) + ).stack(future_stack=True) s2 = ser[:-1].copy() expected = s2["2000-1-4"] diff --git a/pandas/tests/indexes/multi/test_indexing.py b/pandas/tests/indexes/multi/test_indexing.py index 50bcbaf1b3702..78b2c493ec116 100644 --- a/pandas/tests/indexes/multi/test_indexing.py +++ b/pandas/tests/indexes/multi/test_indexing.py @@ -37,12 +37,12 @@ def test_slice_locs_partial(self, idx): def test_slice_locs(self): df = tm.makeTimeDataFrame() - stacked = df.stack(v3=True) + stacked = df.stack(future_stack=True) idx = stacked.index slob = slice(*idx.slice_locs(df.index[5], df.index[15])) sliced = stacked[slob] - expected = df[5:16].stack(v3=True) + expected = df[5:16].stack(future_stack=True) tm.assert_almost_equal(sliced.values, expected.values) slob = slice( @@ -52,19 +52,19 @@ def test_slice_locs(self): ) ) sliced = stacked[slob] - expected = df[6:15].stack(v3=True) + expected = df[6:15].stack(future_stack=True) tm.assert_almost_equal(sliced.values, expected.values) def test_slice_locs_with_type_mismatch(self): df = tm.makeTimeDataFrame() - stacked = df.stack(v3=True) + stacked = df.stack(future_stack=True) idx = stacked.index with pytest.raises(TypeError, match="^Level type mismatch"): idx.slice_locs((1, 3)) with pytest.raises(TypeError, match="^Level type mismatch"): idx.slice_locs(df.index[5] + timedelta(seconds=30), (5, 2)) df = tm.makeCustomDataframe(5, 5) - stacked = df.stack(v3=True) + stacked = df.stack(future_stack=True) idx = stacked.index with pytest.raises(TypeError, match="^Level type mismatch"): idx.slice_locs(timedelta(seconds=30)) diff --git a/pandas/tests/indexes/multi/test_integrity.py b/pandas/tests/indexes/multi/test_integrity.py index e15435a0f6068..45dd484eff4c6 100644 --- a/pandas/tests/indexes/multi/test_integrity.py +++ b/pandas/tests/indexes/multi/test_integrity.py @@ -235,7 +235,10 @@ def test_rangeindex_fallback_coercion_bug(): # GH 12893 df1 = pd.DataFrame(np.arange(100).reshape((10, 10))) df2 = pd.DataFrame(np.arange(100).reshape((10, 10))) - df = pd.concat({"df1": df1.stack(v3=True), "df2": df2.stack(v3=True)}, axis=1) + df = pd.concat( + {"df1": df1.stack(future_stack=True), "df2": df2.stack(future_stack=True)}, + axis=1, + ) df.index.names = ["fizz", "buzz"] str(df) diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index 08ace46836ffa..dd4fded7bce96 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1873,7 +1873,7 @@ def test_frame_int_overflow(self): ], ) def test_json_multiindex(self, dataframe, expected): - series = dataframe.stack(v3=True) + series = dataframe.stack(future_stack=True) result = series.to_json(orient="index") assert result == expected @@ -1912,7 +1912,7 @@ def test_to_json_multiindex_escape(self): True, index=pd.date_range("2017-01-20", "2017-01-23"), columns=["foo", "bar"], - ).stack(v3=True) + ).stack(future_stack=True) result = df.to_json() expected = ( "{\"(Timestamp('2017-01-20 00:00:00'), 'foo')\":true," diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py index fe49748791baf..40c8ac7a3cd0d 100644 --- a/pandas/tests/io/pytables/test_append.py +++ b/pandas/tests/io/pytables/test_append.py @@ -131,7 +131,7 @@ def test_append_series(setup_path): mi["C"] = "foo" mi.loc[3:5, "C"] = "bar" mi.set_index(["C", "B"], inplace=True) - s = mi.stack(v3=True) + s = mi.stack(future_stack=True) s.index = s.index.droplevel(2) store.append("mi", s) tm.assert_series_equal(store["mi"], s, check_index_type=True) diff --git a/pandas/tests/series/methods/test_reset_index.py b/pandas/tests/series/methods/test_reset_index.py index 91ad8de76766b..120261b933a38 100644 --- a/pandas/tests/series/methods/test_reset_index.py +++ b/pandas/tests/series/methods/test_reset_index.py @@ -34,7 +34,7 @@ def test_reset_index_dti_round_trip(self): def test_reset_index(self): df = tm.makeDataFrame()[:5] - ser = df.stack(v3=True) + ser = df.stack(future_stack=True) ser.index.names = ["hash", "category"] ser.name = "value" diff --git a/pandas/tests/series/methods/test_unstack.py b/pandas/tests/series/methods/test_unstack.py index e79df357abdde..9160d33dd6055 100644 --- a/pandas/tests/series/methods/test_unstack.py +++ b/pandas/tests/series/methods/test_unstack.py @@ -133,7 +133,9 @@ def test_unstack_mixed_type_name_in_multiindex( def test_unstack_multi_index_categorical_values(): - mi = tm.makeTimeDataFrame().stack(v3=True).index.rename(["major", "minor"]) + mi = ( + tm.makeTimeDataFrame().stack(future_stack=True).index.rename(["major", "minor"]) + ) ser = Series(["foo"] * len(mi), index=mi, name="category", dtype="category") result = ser.unstack() From f08540c27132468c9d3c7ff87654e790d4a644d9 Mon Sep 17 00:00:00 2001 From: Richard Shadrach Date: Tue, 1 Aug 2023 16:18:50 -0400 Subject: [PATCH 7/8] Fixup docstring --- pandas/core/frame.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index f4d92cde7968b..968e7f815a264 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -9304,15 +9304,15 @@ def stack( >>> df_multi_level_cols2.stack(0, future_stack=True) kg m - cat height NaN 2.0 - weight 1.0 NaN - dog height NaN 4.0 - weight 3.0 NaN + cat weight 1.0 NaN + height NaN 2.0 + dog weight 3.0 NaN + height NaN 4.0 >>> df_multi_level_cols2.stack([0, 1], future_stack=True) - cat height m 2.0 - weight kg 1.0 - dog height m 4.0 - weight kg 3.0 + cat weight kg 1.0 + height m 2.0 + dog weight kg 3.0 + height m 4.0 dtype: float64 **Dropping missing values** From 35c3f5a54407b58dd3d6602c75a053c750819101 Mon Sep 17 00:00:00 2001 From: richard Date: Tue, 1 Aug 2023 21:35:58 -0400 Subject: [PATCH 8/8] Docstring fixup --- pandas/core/frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 968e7f815a264..3b2fe1699e996 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -9200,7 +9200,7 @@ def stack( section. sort : bool, default True Whether to sort the levels of the resulting MultiIndex. - future_stack: bool, default False + future_stack : bool, default False Whether to use the new implementation that will replace the current implementation in pandas 3.0. When True, dropna and sort have no impact on the result and must remain unspecified. See :ref:`pandas 2.1.0 Release