From 85d1a304393d1158ac2fc70a59956849a451efe6 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Fri, 16 Oct 2020 22:04:15 -0700 Subject: [PATCH 01/12] CLN: Weighted rolling window methods --- pandas/core/window/rolling.py | 79 +++++++++++++++-------------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 1fcc47931e882..1c8796e6fab9a 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -999,27 +999,24 @@ def _constructor(self): def validate(self): super().validate() - window = self.window - if isinstance(window, BaseIndexer): + if isinstance(self.window, BaseIndexer): raise NotImplementedError( "BaseIndexer subclasses not implemented with win_types." ) - elif isinstance(window, (list, tuple, np.ndarray)): + elif isinstance(self.window, (list, tuple, np.ndarray)): pass - elif is_integer(window): - if window <= 0: + elif is_integer(self.window): + if self.window <= 0: raise ValueError("window must be > 0 ") - import_optional_dependency( - "scipy", extra="Scipy is required to generate window weight." + sig = import_optional_dependency( + "scipy.signal", extra="Scipy is required to generate window weight." ) - import scipy.signal as sig - if not isinstance(self.win_type, str): raise ValueError(f"Invalid win_type {self.win_type}") if getattr(sig, self.win_type, None) is None: raise ValueError(f"Invalid win_type {self.win_type}") else: - raise ValueError(f"Invalid window {window}") + raise ValueError(f"Invalid window {self.window}") def _get_win_type(self, kwargs: Dict[str, Any]) -> Union[str, Tuple]: """ @@ -1034,36 +1031,27 @@ def _get_win_type(self, kwargs: Dict[str, Any]) -> Union[str, Tuple]: ------- win_type : str, or tuple """ - # the below may pop from kwargs - def _validate_win_type(win_type, kwargs): - arg_map = { - "kaiser": ["beta"], - "gaussian": ["std"], - "general_gaussian": ["power", "width"], - "slepian": ["width"], - "exponential": ["tau"], - } - - if win_type in arg_map: - win_args = _pop_args(win_type, arg_map[win_type], kwargs) - if win_type == "exponential": - # exponential window requires the first arg (center) - # to be set to None (necessary for symmetric window) - win_args.insert(0, None) - - return tuple([win_type] + win_args) - - return win_type - - def _pop_args(win_type, arg_names, kwargs): - all_args = [] - for n in arg_names: - if n not in kwargs: - raise ValueError(f"{win_type} window requires {n}") - all_args.append(kwargs.pop(n)) - return all_args - - return _validate_win_type(self.win_type, kwargs) + arg_map = { + "kaiser": ["beta"], + "gaussian": ["std"], + "general_gaussian": ["power", "width"], + "slepian": ["width"], + "exponential": ["tau"], + } + window_arguments = arg_map.get(self.win_type) + if window_arguments is not None: + extracted_arguments = [] + for argument in window_arguments: + if argument not in kwargs: + raise ValueError(f"{self.win_type} window requires {argument}") + extracted_arguments.append(kwargs.pop(argument)) + if self.win_type == "exponential": + # exponential window requires the first arg (center) + # to be set to None (necessary for symmetric window) + extracted_arguments.insert(0, None) + + return tuple([self.win_type] + extracted_arguments) + return self.win_type def _center_window(self, result: np.ndarray, offset: int) -> np.ndarray: """ @@ -1094,14 +1082,15 @@ def _get_window_weights( window : ndarray the window, weights """ - window = self.window - if isinstance(window, (list, tuple, np.ndarray)): - return com.asarray_tuplesafe(window).astype(float) - elif is_integer(window): + if isinstance(self.window, (list, tuple, np.ndarray)): + return com.asarray_tuplesafe(self.window).astype(float, copy=False) + elif is_integer(self.window): import scipy.signal as sig # GH #15662. `False` makes symmetric window, rather than periodic. - return sig.get_window(win_type, window, False).astype(float) + return sig.get_window(win_type, self.window, False).astype( + float, copy=False + ) def _apply( self, From 2ead3d597b3863851b6647728ff6a5640a72b1b6 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sat, 17 Oct 2020 13:18:42 -0700 Subject: [PATCH 02/12] typing --- pandas/core/window/rolling.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 1c8796e6fab9a..e5f988d7bf04d 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -1018,7 +1018,7 @@ def validate(self): else: raise ValueError(f"Invalid window {self.window}") - def _get_win_type(self, kwargs: Dict[str, Any]) -> Union[str, Tuple]: + def _get_win_type(self, kwargs: Dict[str, Any]) -> Optional[Union[str, Tuple]]: """ Extract arguments for the window type, provide validation for it and return the validated window type. @@ -1031,17 +1031,16 @@ def _get_win_type(self, kwargs: Dict[str, Any]) -> Union[str, Tuple]: ------- win_type : str, or tuple """ - arg_map = { + arg_map: Dict[str, List[str]] = { "kaiser": ["beta"], "gaussian": ["std"], "general_gaussian": ["power", "width"], "slepian": ["width"], "exponential": ["tau"], } - window_arguments = arg_map.get(self.win_type) - if window_arguments is not None: + if self.win_type in arg_map: extracted_arguments = [] - for argument in window_arguments: + for argument in arg_map[self.win_type]: if argument not in kwargs: raise ValueError(f"{self.win_type} window requires {argument}") extracted_arguments.append(kwargs.pop(argument)) From bc698d12c71298563f27fc9f2338b0473ac22f8a Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Oct 2020 17:32:55 -0700 Subject: [PATCH 03/12] Support all scipy windows with arguments --- doc/source/user_guide/computation.rst | 4 + doc/source/whatsnew/v1.2.0.rst | 1 + pandas/core/window/rolling.py | 84 ++++--------------- .../window/moments/test_moments_rolling.py | 4 +- 4 files changed, 21 insertions(+), 72 deletions(-) diff --git a/doc/source/user_guide/computation.rst b/doc/source/user_guide/computation.rst index b24020848b363..e03ec8a7918bc 100644 --- a/doc/source/user_guide/computation.rst +++ b/doc/source/user_guide/computation.rst @@ -451,6 +451,10 @@ The list of recognized types are the `scipy.signal window functions * ``slepian`` (needs width) * ``exponential`` (needs tau). +.. versionadded:: 1.2.0 + +All Scipy window types, concurrent with your installed version, are recognized ``win_types``. + .. ipython:: python ser = pd.Series(np.random.randn(10), index=pd.date_range("1/1/2000", periods=10)) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index d8961f5fdb959..c826525e260e4 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -221,6 +221,7 @@ Other enhancements - :meth:`Rolling.var()` and :meth:`Rolling.std()` use Kahan summation and Welfords Method to avoid numerical issues (:issue:`37051`) - :meth:`DataFrame.plot` now recognizes ``xlabel`` and ``ylabel`` arguments for plots of type ``scatter`` and ``hexbin`` (:issue:`37001`) - :class:`DataFrame` now supports ``divmod`` operation (:issue:`37165`) +- :class:`Window` now supports all Scipy supported ``win_type``s with flexible keyword argument support (:issue:`34556`) .. _whatsnew_120.api_breaking.python: diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index e5f988d7bf04d..1759227df2214 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -872,7 +872,8 @@ class Window(BaseWindow): To learn more about the offsets & frequency strings, please see `this link `__. - The recognized win_types are: + If ``win_type=None``, all points are evenly weighted; otherwise, + the recognized ``win_type``s are: * ``boxcar`` * ``triang`` @@ -890,13 +891,16 @@ class Window(BaseWindow): * ``slepian`` (needs parameter: width) * ``exponential`` (needs parameter: tau), center is set to None. - If ``win_type=None`` all points are evenly weighted. To learn more about - different window types see `scipy.signal window functions - `__. - Certain window types require additional parameters to be passed. Please see the third example below on how to add the additional parameters. + To learn more about different window types see `scipy.signal window functions + `__. + + .. versionadded:: 1.2.0 + + All Scipy window types, concurrent with your installed version, are recognized ``win_types``. + Examples -------- >>> df = pd.DataFrame({'B': [0, 1, 2, np.nan, 4]}) @@ -1003,8 +1007,6 @@ def validate(self): raise NotImplementedError( "BaseIndexer subclasses not implemented with win_types." ) - elif isinstance(self.window, (list, tuple, np.ndarray)): - pass elif is_integer(self.window): if self.window <= 0: raise ValueError("window must be > 0 ") @@ -1018,40 +1020,6 @@ def validate(self): else: raise ValueError(f"Invalid window {self.window}") - def _get_win_type(self, kwargs: Dict[str, Any]) -> Optional[Union[str, Tuple]]: - """ - Extract arguments for the window type, provide validation for it - and return the validated window type. - - Parameters - ---------- - kwargs : dict - - Returns - ------- - win_type : str, or tuple - """ - arg_map: Dict[str, List[str]] = { - "kaiser": ["beta"], - "gaussian": ["std"], - "general_gaussian": ["power", "width"], - "slepian": ["width"], - "exponential": ["tau"], - } - if self.win_type in arg_map: - extracted_arguments = [] - for argument in arg_map[self.win_type]: - if argument not in kwargs: - raise ValueError(f"{self.win_type} window requires {argument}") - extracted_arguments.append(kwargs.pop(argument)) - if self.win_type == "exponential": - # exponential window requires the first arg (center) - # to be set to None (necessary for symmetric window) - extracted_arguments.insert(0, None) - - return tuple([self.win_type] + extracted_arguments) - return self.win_type - def _center_window(self, result: np.ndarray, offset: int) -> np.ndarray: """ Center the result in the window for weighted rolling aggregations. @@ -1065,32 +1033,6 @@ def _center_window(self, result: np.ndarray, offset: int) -> np.ndarray: result = np.copy(result[tuple(lead_indexer)]) return result - def _get_window_weights( - self, win_type: Optional[Union[str, Tuple]] = None - ) -> np.ndarray: - """ - Get the window, weights. - - Parameters - ---------- - win_type : str, or tuple - type of window to create - - Returns - ------- - window : ndarray - the window, weights - """ - if isinstance(self.window, (list, tuple, np.ndarray)): - return com.asarray_tuplesafe(self.window).astype(float, copy=False) - elif is_integer(self.window): - import scipy.signal as sig - - # GH #15662. `False` makes symmetric window, rather than periodic. - return sig.get_window(win_type, self.window, False).astype( - float, copy=False - ) - def _apply( self, func: Callable[[np.ndarray, int, int], np.ndarray], @@ -1111,14 +1053,16 @@ def _apply( whether to cache a numba compiled function. Only available for numba enabled methods (so far only apply) **kwargs - additional arguments for rolling function and window function + additional arguments for scipy windows if necessary Returns ------- y : type of input """ - win_type = self._get_win_type(kwargs) - window = self._get_window_weights(win_type=win_type) + signal = import_optional_dependency( + "scipy.signal", extra="Scipy is required to generate window weight." + ) + window = getattr(signal, self.win_type)(self.window, **kwargs) offset = (len(window) - 1) // 2 if self.center else 0 def homogeneous_func(values: np.ndarray): diff --git a/pandas/tests/window/moments/test_moments_rolling.py b/pandas/tests/window/moments/test_moments_rolling.py index 488306d0585c5..28e0c9872df30 100644 --- a/pandas/tests/window/moments/test_moments_rolling.py +++ b/pandas/tests/window/moments/test_moments_rolling.py @@ -431,7 +431,7 @@ def test_cmov_window_special(win_types_special): kwds = { "kaiser": {"beta": 1.0}, "gaussian": {"std": 1.0}, - "general_gaussian": {"power": 2.0, "width": 2.0}, + "general_gaussian": {"p": 2.0, "sig": 2.0}, "exponential": {"tau": 10}, } @@ -503,7 +503,7 @@ def test_cmov_window_special_linear_range(win_types_special): kwds = { "kaiser": {"beta": 1.0}, "gaussian": {"std": 1.0}, - "general_gaussian": {"power": 2.0, "width": 2.0}, + "general_gaussian": {"p": 2.0, "sig": 2.0}, "slepian": {"width": 0.5}, "exponential": {"tau": 10}, } From 66bdb26af8cdb51802e7d019b20b00fea31241c6 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Oct 2020 17:34:15 -0700 Subject: [PATCH 04/12] clarify whatsnew --- doc/source/whatsnew/v1.2.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index c826525e260e4..16dc9a2104e6a 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -221,7 +221,7 @@ Other enhancements - :meth:`Rolling.var()` and :meth:`Rolling.std()` use Kahan summation and Welfords Method to avoid numerical issues (:issue:`37051`) - :meth:`DataFrame.plot` now recognizes ``xlabel`` and ``ylabel`` arguments for plots of type ``scatter`` and ``hexbin`` (:issue:`37001`) - :class:`DataFrame` now supports ``divmod`` operation (:issue:`37165`) -- :class:`Window` now supports all Scipy supported ``win_type``s with flexible keyword argument support (:issue:`34556`) +- :class:`Window` now supports all Scipy window types in ``win_type`` with flexible keyword argument support (:issue:`34556`) .. _whatsnew_120.api_breaking.python: From 5214f1b39d399a399a783b29526139ad84f275ee Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Oct 2020 17:36:37 -0700 Subject: [PATCH 05/12] flake8 --- pandas/core/window/rolling.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 1759227df2214..03b304590d8df 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -899,7 +899,8 @@ class Window(BaseWindow): .. versionadded:: 1.2.0 - All Scipy window types, concurrent with your installed version, are recognized ``win_types``. + All Scipy window types, concurrent with your installed version, + are recognized ``win_types``. Examples -------- From 3ed2bc77c614b6d9b6784a6d0a615515bcd5cbc7 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Oct 2020 17:41:11 -0700 Subject: [PATCH 06/12] clarify documentation --- pandas/core/window/rolling.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 03b304590d8df..107c9c6dbdcbf 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -891,8 +891,10 @@ class Window(BaseWindow): * ``slepian`` (needs parameter: width) * ``exponential`` (needs parameter: tau), center is set to None. - Certain window types require additional parameters to be passed. Please see - the third example below on how to add the additional parameters. + Certain window types require additional parameters to be passed + in the aggregation function. The additional parameters must match + the keywords specified in the Scipy window type method signature. + Please see the third example below on how to add the additional parameters. To learn more about different window types see `scipy.signal window functions `__. From 24273176393f629c68c22a92f5648a283a1dd4d3 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Oct 2020 18:16:32 -0700 Subject: [PATCH 07/12] Add supported windows as of scipy 1.2.0 --- pandas/core/window/rolling.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 107c9c6dbdcbf..37a7dbafb4a15 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -902,7 +902,8 @@ class Window(BaseWindow): .. versionadded:: 1.2.0 All Scipy window types, concurrent with your installed version, - are recognized ``win_types``. + are recognized ``win_types``, now supporting + ``flattop``, ``dpss``, ``chebwinn``, and ``turkey`` as of Scipy version 1.2.0. Examples -------- From 377d2259f296baee3268ebd794141a83f3359008 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 18 Oct 2020 18:28:55 -0700 Subject: [PATCH 08/12] Fix test and mypy error --- pandas/core/window/rolling.py | 1 + pandas/tests/window/test_window.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 37a7dbafb4a15..a9b271482a081 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -1066,6 +1066,7 @@ def _apply( signal = import_optional_dependency( "scipy.signal", extra="Scipy is required to generate window weight." ) + assert self.win_type is not None # for mypy window = getattr(signal, self.win_type)(self.window, **kwargs) offset = (len(window) - 1) // 2 if self.center else 0 diff --git a/pandas/tests/window/test_window.py b/pandas/tests/window/test_window.py index a3fff3122f80a..b9e16f7f1410a 100644 --- a/pandas/tests/window/test_window.py +++ b/pandas/tests/window/test_window.py @@ -50,7 +50,7 @@ def test_constructor_with_win_type(which, win_types): @pytest.mark.parametrize("method", ["sum", "mean"]) def test_numpy_compat(method): # see gh-12811 - w = Window(Series([2, 4, 6]), window=[0, 2]) + w = Window(Series([2, 4, 6]), window=0) msg = "numpy operations are not valid with window objects" From c0563701930a4a1a55d2a30cffd3a46f0b3e8c41 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 19 Oct 2020 16:29:20 -0700 Subject: [PATCH 09/12] Use valid window in test --- pandas/tests/window/test_window.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/window/test_window.py b/pandas/tests/window/test_window.py index b9e16f7f1410a..d32464d28d3b4 100644 --- a/pandas/tests/window/test_window.py +++ b/pandas/tests/window/test_window.py @@ -50,7 +50,7 @@ def test_constructor_with_win_type(which, win_types): @pytest.mark.parametrize("method", ["sum", "mean"]) def test_numpy_compat(method): # see gh-12811 - w = Window(Series([2, 4, 6]), window=0) + w = Window(Series([2, 4, 6]), window=2) msg = "numpy operations are not valid with window objects" From b2be37368f355064197f0caab6505cd0e16d311f Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Mon, 19 Oct 2020 16:36:00 -0700 Subject: [PATCH 10/12] Simplify documentation --- pandas/core/window/rolling.py | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index a9b271482a081..3a06a46066a63 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -872,39 +872,15 @@ class Window(BaseWindow): To learn more about the offsets & frequency strings, please see `this link `__. - If ``win_type=None``, all points are evenly weighted; otherwise, - the recognized ``win_type``s are: - - * ``boxcar`` - * ``triang`` - * ``blackman`` - * ``hamming`` - * ``bartlett`` - * ``parzen`` - * ``bohman`` - * ``blackmanharris`` - * ``nuttall`` - * ``barthann`` - * ``kaiser`` (needs parameter: beta) - * ``gaussian`` (needs parameter: std) - * ``general_gaussian`` (needs parameters: power, width) - * ``slepian`` (needs parameter: width) - * ``exponential`` (needs parameter: tau), center is set to None. - - Certain window types require additional parameters to be passed + If ``win_type=None``, all points are evenly weighted; otherwise, ``win_type`` + can accept a string of any `scipy.signal window function + `__. + + Certain Scipy window types require additional parameters to be passed in the aggregation function. The additional parameters must match the keywords specified in the Scipy window type method signature. Please see the third example below on how to add the additional parameters. - To learn more about different window types see `scipy.signal window functions - `__. - - .. versionadded:: 1.2.0 - - All Scipy window types, concurrent with your installed version, - are recognized ``win_types``, now supporting - ``flattop``, ``dpss``, ``chebwinn``, and ``turkey`` as of Scipy version 1.2.0. - Examples -------- >>> df = pd.DataFrame({'B': [0, 1, 2, np.nan, 4]}) From 135283ca226ae2190b686592593b75f86141e952 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Tue, 20 Oct 2020 16:54:55 -0700 Subject: [PATCH 11/12] Change test to use public constructor --- pandas/tests/window/test_window.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/tests/window/test_window.py b/pandas/tests/window/test_window.py index d32464d28d3b4..e0179a9264760 100644 --- a/pandas/tests/window/test_window.py +++ b/pandas/tests/window/test_window.py @@ -6,7 +6,6 @@ import pandas as pd from pandas import Series -from pandas.core.window import Window @td.skip_if_no_scipy @@ -50,7 +49,7 @@ def test_constructor_with_win_type(which, win_types): @pytest.mark.parametrize("method", ["sum", "mean"]) def test_numpy_compat(method): # see gh-12811 - w = Window(Series([2, 4, 6]), window=2) + w = Series([2, 4, 6]).rolling(window=2) msg = "numpy operations are not valid with window objects" From 5e4c886e0b44c7979c41dc9d4cd0183b676291e9 Mon Sep 17 00:00:00 2001 From: Matt Roeschke Date: Sun, 25 Oct 2020 12:57:53 -0700 Subject: [PATCH 12/12] Add test for invalid scipy arg --- pandas/tests/window/test_window.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pandas/tests/window/test_window.py b/pandas/tests/window/test_window.py index e0179a9264760..eab62b3383283 100644 --- a/pandas/tests/window/test_window.py +++ b/pandas/tests/window/test_window.py @@ -74,3 +74,11 @@ def test_agg_function_support(arg): with pytest.raises(AttributeError, match=msg): roll.agg({"A": arg}) + + +@td.skip_if_no_scipy +def test_invalid_scipy_arg(): + # This error is raised by scipy + msg = r"boxcar\(\) got an unexpected" + with pytest.raises(TypeError, match=msg): + Series(range(3)).rolling(1, win_type="boxcar").mean(foo="bar")