From 7e461a18d9f6928132afec6f48ce968b3e989ba6 Mon Sep 17 00:00:00 2001 From: Kaiqi Dong Date: Mon, 3 Dec 2018 17:43:52 +0100 Subject: [PATCH 01/32] remove \n from docstring --- pandas/core/arrays/datetimes.py | 26 +++++++++++++------------- pandas/core/arrays/timedeltas.py | 16 ++++++++-------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index cfe3afcf3730a..b3df505d56d78 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -82,7 +82,7 @@ def f(self): return result f.__name__ = name - f.__doc__ = docstring + f.__doc__ = "\n{}\n".format(docstring) return property(f) @@ -1072,19 +1072,19 @@ def date(self): return tslib.ints_to_pydatetime(timestamps, box="date") - year = _field_accessor('year', 'Y', "\n The year of the datetime\n") + year = _field_accessor('year', 'Y', "The year of the datetime") month = _field_accessor('month', 'M', - "\n The month as January=1, December=12 \n") - day = _field_accessor('day', 'D', "\nThe days of the datetime\n") - hour = _field_accessor('hour', 'h', "\nThe hours of the datetime\n") - minute = _field_accessor('minute', 'm', "\nThe minutes of the datetime\n") - second = _field_accessor('second', 's', "\nThe seconds of the datetime\n") + "The month as January=1, December=12") + day = _field_accessor('day', 'D', "The days of the datetime") + hour = _field_accessor('hour', 'h', "The hours of the datetime") + minute = _field_accessor('minute', 'm', "The minutes of the datetime") + second = _field_accessor('second', 's', "The seconds of the datetime") microsecond = _field_accessor('microsecond', 'us', - "\nThe microseconds of the datetime\n") + "The microseconds of the datetime") nanosecond = _field_accessor('nanosecond', 'ns', - "\nThe nanoseconds of the datetime\n") + "The nanoseconds of the datetime") weekofyear = _field_accessor('weekofyear', 'woy', - "\nThe week ordinal of the year\n") + "The week ordinal of the year") week = weekofyear _dayofweek_doc = """ The day of the week with Monday=0, Sunday=6. @@ -1129,12 +1129,12 @@ def date(self): "The name of day in a week (ex: Friday)\n\n.. deprecated:: 0.23.0") dayofyear = _field_accessor('dayofyear', 'doy', - "\nThe ordinal day of the year\n") - quarter = _field_accessor('quarter', 'q', "\nThe quarter of the date\n") + "The ordinal day of the year") + quarter = _field_accessor('quarter', 'q', "The quarter of the date") days_in_month = _field_accessor( 'days_in_month', 'dim', - "\nThe number of days in the month\n") + "The number of days in the month") daysinmonth = days_in_month _is_month_doc = """ Indicates whether the date is the {first_or_last} day of the month. diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 830283d31a929..4afc9f5483c2a 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -59,7 +59,7 @@ def f(self): return result f.__name__ = name - f.__doc__ = docstring + f.__doc__ = "\n{}\n".format(docstring) return property(f) @@ -684,16 +684,16 @@ def to_pytimedelta(self): return tslibs.ints_to_pytimedelta(self.asi8) days = _field_accessor("days", "days", - "\nNumber of days for each element.\n") + "Number of days for each element.") seconds = _field_accessor("seconds", "seconds", - "\nNumber of seconds (>= 0 and less than 1 day) " - "for each element.\n") + "Number of seconds (>= 0 and less than 1 day) " + "for each element.") microseconds = _field_accessor("microseconds", "microseconds", - "\nNumber of microseconds (>= 0 and less " - "than 1 second) for each element.\n") + "Number of microseconds (>= 0 and less " + "than 1 second) for each element.") nanoseconds = _field_accessor("nanoseconds", "nanoseconds", - "\nNumber of nanoseconds (>= 0 and less " - "than 1 microsecond) for each element.\n") + "Number of nanoseconds (>= 0 and less " + "than 1 microsecond) for each element.") @property def components(self): From dea38f24c0067ae3fe9484b837c9649714213bba Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 14 Jan 2020 21:26:31 +0100 Subject: [PATCH 02/32] fix issue 17038 --- pandas/core/reshape/pivot.py | 4 +++- pandas/tests/reshape/test_pivot.py | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index b443ba142369c..9743d90f4dd04 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -117,7 +117,9 @@ def pivot_table( agged[v] = maybe_downcast_to_dtype(agged[v], data[v].dtype) table = agged - if table.index.nlevels > 1: + + # GH 17038, this check should only happen if index is specified + if table.index.nlevels > 1 and index: # Related GH #17123 # If index_names are integers, determine whether the integers refer # to the level position or name. diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index 743fc50c87e96..46a05123c9fdd 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -896,12 +896,6 @@ def _check_output( totals = table.loc[("All", ""), value_col] assert totals == self.data[value_col].mean() - # no rows - rtable = self.data.pivot_table( - columns=["AA", "BB"], margins=True, aggfunc=np.mean - ) - assert isinstance(rtable, Series) - table = self.data.pivot_table(index=["AA", "BB"], margins=True, aggfunc="mean") for item in ["DD", "EE", "FF"]: totals = table.loc[("All", ""), item] @@ -972,6 +966,20 @@ def test_pivot_integer_columns(self): tm.assert_frame_equal(table, table2, check_names=False) + @pytest.mark.parametrize("cols", [(1, 2), ("a", "b"), (1, "b"), ("a", 1)]) + def test_pivot_table_multiindex_only(self, cols): + # GH 17038 + df2 = DataFrame({cols[0]: [1, 2, 3], cols[1]: [1, 2, 3], "v": [4, 5, 6]}) + + result = df2.pivot_table(values="v", columns=cols) + expected = DataFrame( + [[4, 5, 6]], + columns=MultiIndex.from_tuples([(1, 1), (2, 2), (3, 3)], names=cols), + index=Index(["v"]), + ) + + tm.assert_frame_equal(result, expected) + def test_pivot_no_level_overlap(self): # GH #1181 From cd9e7ac3f31ffaf95cd628863df911dea9fa1248 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 14 Jan 2020 21:29:43 +0100 Subject: [PATCH 03/32] revert change --- pandas/core/reshape/pivot.py | 3 +-- pandas/tests/reshape/test_pivot.py | 20 ++++++-------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 9743d90f4dd04..a7cdbb0da7a4e 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -118,8 +118,7 @@ def pivot_table( table = agged - # GH 17038, this check should only happen if index is specified - if table.index.nlevels > 1 and index: + if table.index.nlevels > 1: # Related GH #17123 # If index_names are integers, determine whether the integers refer # to the level position or name. diff --git a/pandas/tests/reshape/test_pivot.py b/pandas/tests/reshape/test_pivot.py index 46a05123c9fdd..743fc50c87e96 100644 --- a/pandas/tests/reshape/test_pivot.py +++ b/pandas/tests/reshape/test_pivot.py @@ -896,6 +896,12 @@ def _check_output( totals = table.loc[("All", ""), value_col] assert totals == self.data[value_col].mean() + # no rows + rtable = self.data.pivot_table( + columns=["AA", "BB"], margins=True, aggfunc=np.mean + ) + assert isinstance(rtable, Series) + table = self.data.pivot_table(index=["AA", "BB"], margins=True, aggfunc="mean") for item in ["DD", "EE", "FF"]: totals = table.loc[("All", ""), item] @@ -966,20 +972,6 @@ def test_pivot_integer_columns(self): tm.assert_frame_equal(table, table2, check_names=False) - @pytest.mark.parametrize("cols", [(1, 2), ("a", "b"), (1, "b"), ("a", 1)]) - def test_pivot_table_multiindex_only(self, cols): - # GH 17038 - df2 = DataFrame({cols[0]: [1, 2, 3], cols[1]: [1, 2, 3], "v": [4, 5, 6]}) - - result = df2.pivot_table(values="v", columns=cols) - expected = DataFrame( - [[4, 5, 6]], - columns=MultiIndex.from_tuples([(1, 1), (2, 2), (3, 3)], names=cols), - index=Index(["v"]), - ) - - tm.assert_frame_equal(result, expected) - def test_pivot_no_level_overlap(self): # GH #1181 From e5e912be0f596943067a7df812442764d311a086 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 14 Jan 2020 21:30:16 +0100 Subject: [PATCH 04/32] revert change --- pandas/core/reshape/pivot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index a7cdbb0da7a4e..b443ba142369c 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -117,7 +117,6 @@ def pivot_table( agged[v] = maybe_downcast_to_dtype(agged[v], data[v].dtype) table = agged - if table.index.nlevels > 1: # Related GH #17123 # If index_names are integers, determine whether the integers refer From 1cc1225a579584caa85b0abe08a8ec96d03c215f Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 23 Feb 2020 11:30:44 +0100 Subject: [PATCH 05/32] Add signature for pivot --- pandas/core/reshape/pivot.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index b04e4e1ac4d48..8f83791b10da7 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Tuple, Union, Optional import numpy as np @@ -7,6 +7,7 @@ from pandas.core.dtypes.cast import maybe_downcast_to_dtype from pandas.core.dtypes.common import is_integer_dtype, is_list_like, is_scalar from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries +from pandas._typing import Label import pandas.core.common as com from pandas.core.frame import _shared_docs @@ -422,13 +423,18 @@ def _convert_by(by): @Substitution("\ndata : DataFrame") @Appender(_shared_docs["pivot"], indents=1) -def pivot(data: "DataFrame", index=None, columns=None, values=None) -> "DataFrame": +def pivot( + data: "DataFrame", + index: Optional[Union[Label, List[Label]]] = None, + columns: Optional[Union[Label, List[Label]]] = None, + values: Optional[Union[Label, List[Label]]] = None, +) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") columns = columns if is_list_like(columns) else [columns] if values is None: - cols: List[str] = [] + cols: List[Label] = [] if index is None: pass elif is_list_like(index): From 046cdc9c01571077d3fdc1c6c52549924b450d57 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 23 Feb 2020 11:32:11 +0100 Subject: [PATCH 06/32] sorting --- pandas/core/reshape/pivot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 8f83791b10da7..cf2764e9fd50a 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,13 +1,13 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Tuple, Union, Optional +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union import numpy as np +from pandas._typing import Label from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.cast import maybe_downcast_to_dtype from pandas.core.dtypes.common import is_integer_dtype, is_list_like, is_scalar from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries -from pandas._typing import Label import pandas.core.common as com from pandas.core.frame import _shared_docs From 2807b63aede6d0fb0333e7a695c17e77bc46627a Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 23 Feb 2020 14:18:43 +0100 Subject: [PATCH 07/32] try fix annotation --- pandas/core/reshape/pivot.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index cf2764e9fd50a..2a823beca50b4 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -425,8 +425,8 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Label, List[Label]]] = None, - columns: Optional[Union[Label, List[Label]]] = None, + index: Optional[Union[Label, Collection[Label]]] = None, + columns: Union[Label, List[Label]] = None, values: Optional[Union[Label, List[Label]]] = None, ) -> "DataFrame": if columns is None: @@ -446,16 +446,17 @@ def pivot( append = index is None indexed = data.set_index(cols, append=append) else: + idx_list: List[Series] = [] if index is None: - index = [Series(data.index, name=data.index.name)] + idx_list = [Series(data.index, name=data.index.name)] elif is_list_like(index): - index = [data[idx] for idx in index] + idx_list = [data[idx] for idx in index] else: - index = [data[index]] + idx_list = [data[index]] data_columns = [data[col] for col in columns] - index.extend(data_columns) - index = MultiIndex.from_arrays(index) + idx_list.extend(data_columns) + index = MultiIndex.from_arrays(idx_list) if is_list_like(values) and not isinstance(values, tuple): # Exclude tuple because it is seen as a single column name From 2fdd875cfe568b0ee3cdd021bc0c998bc5725af2 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 20:05:55 +0100 Subject: [PATCH 08/32] fixup --- pandas/core/reshape/pivot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 2a823beca50b4..d8200888f7ae0 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -425,7 +425,7 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Label, Collection[Label]]] = None, + index: Optional[Union[Label, List[Label]]] = None, columns: Union[Label, List[Label]] = None, values: Optional[Union[Label, List[Label]]] = None, ) -> "DataFrame": @@ -446,7 +446,6 @@ def pivot( append = index is None indexed = data.set_index(cols, append=append) else: - idx_list: List[Series] = [] if index is None: idx_list = [Series(data.index, name=data.index.name)] elif is_list_like(index): From 545323739711b138c8eb5a4f094a2adfd6ebdbc7 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 20:39:01 +0100 Subject: [PATCH 09/32] fixup --- pandas/core/reshape/pivot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index d8200888f7ae0..49cab48ff63c2 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -425,9 +425,9 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Label, List[Label]]] = None, - columns: Union[Label, List[Label]] = None, - values: Optional[Union[Label, List[Label]]] = None, + index: Optional[Union[Label, List[Optional[Label]]]] = None, + columns: Optional[Union[Label, List[Optional[Label]]]] = None, + values: Optional[Union[Label, List[Optional[Label]]]] = None, ) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") From 18e85bbb533fe4bd03a237a88d641c6db263447b Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 21:27:49 +0100 Subject: [PATCH 10/32] fixup --- pandas/core/reshape/pivot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 49cab48ff63c2..ced1dff983e08 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union, Sequence import numpy as np @@ -425,9 +425,9 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Label, List[Optional[Label]]]] = None, - columns: Optional[Union[Label, List[Optional[Label]]]] = None, - values: Optional[Union[Label, List[Optional[Label]]]] = None, + index: Optional[Union[Label, Sequence[Optional[Label]]]] = None, + columns: Optional[Union[Label, Sequence[Optional[Label]]]] = None, + values: Optional[Union[Label, Sequence[Optional[Label]]]] = None, ) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") From f61dcacb1bb5069830fc8da2bceec9384e71b782 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 21:28:07 +0100 Subject: [PATCH 11/32] fixup --- pandas/core/reshape/pivot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index ced1dff983e08..d2543d935787f 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union, Sequence +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union import numpy as np From 5e3ac3f955073fff54211af43b7af909e2d0aa15 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 22:11:48 +0100 Subject: [PATCH 12/32] fixup --- pandas/core/reshape/pivot.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index d2543d935787f..a87f88519793d 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union, Iterable import numpy as np @@ -434,7 +434,7 @@ def pivot( columns = columns if is_list_like(columns) else [columns] if values is None: - cols: List[Label] = [] + cols: Iterable[Label] = [] if index is None: pass elif is_list_like(index): @@ -446,6 +446,7 @@ def pivot( append = index is None indexed = data.set_index(cols, append=append) else: + idx_list: Iterable[Series] if index is None: idx_list = [Series(data.index, name=data.index.name)] elif is_list_like(index): @@ -455,12 +456,12 @@ def pivot( data_columns = [data[col] for col in columns] idx_list.extend(data_columns) - index = MultiIndex.from_arrays(idx_list) + mi_index = MultiIndex.from_arrays(idx_list) if is_list_like(values) and not isinstance(values, tuple): # Exclude tuple because it is seen as a single column name indexed = data._constructor( - data[values].values, index=index, columns=values + data[values].values, index=mi_index, columns=values ) else: indexed = data._constructor_sliced(data[values].values, index=index) From ffda679e4f1278e1e3dcc0b574c8da70b25901f5 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 22:12:13 +0100 Subject: [PATCH 13/32] fixup --- pandas/core/reshape/pivot.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index a87f88519793d..fa06c4b739995 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,4 +1,14 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union, Iterable +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + Union, +) import numpy as np From 6b81abb648524e475b2cef16f3bf811fe6869cd0 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 22:55:22 +0100 Subject: [PATCH 14/32] fixup --- pandas/core/reshape/pivot.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index fa06c4b739995..0f92471ecf148 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -2,7 +2,6 @@ TYPE_CHECKING, Callable, Dict, - Iterable, List, Optional, Sequence, @@ -12,7 +11,7 @@ import numpy as np -from pandas._typing import Label +from pandas._typing import Axis from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.cast import maybe_downcast_to_dtype @@ -435,16 +434,16 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Label, Sequence[Optional[Label]]]] = None, - columns: Optional[Union[Label, Sequence[Optional[Label]]]] = None, - values: Optional[Union[Label, Sequence[Optional[Label]]]] = None, + index: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, + columns: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, + values: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, ) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") columns = columns if is_list_like(columns) else [columns] if values is None: - cols: Iterable[Label] = [] + cols: List[Axis] = [] if index is None: pass elif is_list_like(index): @@ -456,7 +455,6 @@ def pivot( append = index is None indexed = data.set_index(cols, append=append) else: - idx_list: Iterable[Series] if index is None: idx_list = [Series(data.index, name=data.index.name)] elif is_list_like(index): From c01f221065626220e7ce1ab1169c483a322bacd2 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 23:22:38 +0100 Subject: [PATCH 15/32] fixup --- pandas/core/reshape/pivot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 0f92471ecf148..3589b5f687cc1 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -443,7 +443,7 @@ def pivot( columns = columns if is_list_like(columns) else [columns] if values is None: - cols: List[Axis] = [] + cols: List[Axis, Sequence[Optional[Axis]]] = [] if index is None: pass elif is_list_like(index): @@ -472,7 +472,7 @@ def pivot( data[values].values, index=mi_index, columns=values ) else: - indexed = data._constructor_sliced(data[values].values, index=index) + indexed = data._constructor_sliced(data[values].values, index=mi_index) return indexed.unstack(columns) From 1552f4b5399ce837627416a769808577500dd2b8 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 24 Feb 2020 23:22:53 +0100 Subject: [PATCH 16/32] fixup --- pandas/core/reshape/pivot.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 3589b5f687cc1..0110499a2eac0 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,13 +1,4 @@ -from typing import ( - TYPE_CHECKING, - Callable, - Dict, - List, - Optional, - Sequence, - Tuple, - Union, -) +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union import numpy as np From 29b06058733020e31529f6d9d64ee755123c4f4d Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 25 Feb 2020 20:29:37 +0100 Subject: [PATCH 17/32] fixup --- pandas/core/reshape/pivot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 0110499a2eac0..e5cc9212d3c8d 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -2,7 +2,7 @@ import numpy as np -from pandas._typing import Axis +from pandas._typing import Axes from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.cast import maybe_downcast_to_dtype @@ -425,9 +425,9 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, - columns: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, - values: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, + index: Optional[Union[Axes, Sequence[Optional[Axes]]]] = None, + columns: Optional[Union[Axes, Sequence[Optional[Axes]]]] = None, + values: Optional[Union[Axes, Sequence[Optional[Axes]]]] = None, ) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") From 4d2d6d3a461491aacfb117a65dad0ff49d7fdcdb Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 25 Feb 2020 20:38:01 +0100 Subject: [PATCH 18/32] fixup --- pandas/core/reshape/pivot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index e5cc9212d3c8d..d5481a0cf123f 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -434,7 +434,7 @@ def pivot( columns = columns if is_list_like(columns) else [columns] if values is None: - cols: List[Axis, Sequence[Optional[Axis]]] = [] + cols: List[Axes, Sequence[Optional[Axes]]] = [] if index is None: pass elif is_list_like(index): From f7fb25ad11bcf48078103493f251e295cc36431c Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Tue, 25 Feb 2020 20:43:55 +0100 Subject: [PATCH 19/32] fixup --- pandas/core/reshape/pivot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index d5481a0cf123f..c504e79ff66a7 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -434,7 +434,7 @@ def pivot( columns = columns if is_list_like(columns) else [columns] if values is None: - cols: List[Axes, Sequence[Optional[Axes]]] = [] + cols: List[Optional[Axes]] = [] if index is None: pass elif is_list_like(index): From d44b3df0f98972811b44f59fddd6807ea66b8da6 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Fri, 3 Apr 2020 21:54:54 +0200 Subject: [PATCH 20/32] improve annotation --- pandas/core/reshape/pivot.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 3d6786a240b07..19ba0577d9837 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -2,7 +2,7 @@ import numpy as np -from pandas._typing import Axes +from pandas._typing import Axis from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.cast import maybe_downcast_to_dtype @@ -427,16 +427,16 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Axes, Sequence[Optional[Axes]]]] = None, - columns: Optional[Union[Axes, Sequence[Optional[Axes]]]] = None, - values: Optional[Union[Axes, Sequence[Optional[Axes]]]] = None, + index: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, + columns: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, + values: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, ) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") columns = columns if is_list_like(columns) else [columns] if values is None: - cols: List[Optional[Axes]] = [] + cols: List[Optional[Axis]] = [] if index is None: pass elif is_list_like(index): From 3c3c7a405c02b4bfa61ee7def920854395c070f9 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Fri, 3 Apr 2020 22:42:08 +0200 Subject: [PATCH 21/32] try out --- pandas/core/reshape/pivot.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 19ba0577d9837..0f53d0306ef2c 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union, Iterable import numpy as np @@ -427,16 +427,15 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, - columns: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, - values: Optional[Union[Axis, Sequence[Optional[Axis]]]] = None, + index: Optional[Union[Axis, List]] = None, + columns: Optional[Union[Axis, List]] = None, + values: Optional[Union[Axis, List]] = None, ) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") columns = columns if is_list_like(columns) else [columns] if values is None: - cols: List[Optional[Axis]] = [] if index is None: pass elif is_list_like(index): From cb6473d083115ca16343074bc7c1acf6eb377a39 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Fri, 3 Apr 2020 23:14:55 +0200 Subject: [PATCH 22/32] fixup --- pandas/core/reshape/pivot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 0f53d0306ef2c..f02f5f57a7d82 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,8 +1,8 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union, Iterable +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union import numpy as np -from pandas._typing import Axis +from pandas._typing import Label from pandas.util._decorators import Appender, Substitution from pandas.core.dtypes.cast import maybe_downcast_to_dtype @@ -427,9 +427,9 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Axis, List]] = None, - columns: Optional[Union[Axis, List]] = None, - values: Optional[Union[Axis, List]] = None, + index: Optional[Union[Label, Sequence[Optional[Label]]]] = None, + columns: Optional[Union[Label, Sequence[Optional[Label]]]] = None, + values: Optional[Union[Label, Sequence[Optional[Label]]]] = None, ) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") @@ -437,7 +437,7 @@ def pivot( if values is None: if index is None: - pass + cols = [] elif is_list_like(index): cols = list(index) else: From 76d319ac1f34fae62c10e9ec20db942209fbd855 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Wed, 8 Apr 2020 20:36:57 +0200 Subject: [PATCH 23/32] add type ignore --- pandas/core/reshape/pivot.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 71948709ef8f5..df786615eb9b8 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -436,13 +436,14 @@ def pivot( columns = columns if is_list_like(columns) else [columns] if values is None: + cols: List[Optional[Sequence]] = [] if index is None: - cols = [] + pass elif is_list_like(index): - cols = list(index) + cols = list(index) # type: ignore else: - cols = [index] - cols.extend(columns) + cols = [index] # type: ignore + cols.extend(columns) # type: ignore append = index is None indexed = data.set_index(cols, append=append) @@ -450,18 +451,18 @@ def pivot( if index is None: idx_list = [Series(data.index, name=data.index.name)] elif is_list_like(index): - idx_list = [data[idx] for idx in index] + idx_list = [data[idx] for idx in index] # type: ignore else: idx_list = [data[index]] - data_columns = [data[col] for col in columns] + data_columns = [data[col] for col in columns] # type: ignore idx_list.extend(data_columns) mi_index = MultiIndex.from_arrays(idx_list) if is_list_like(values) and not isinstance(values, tuple): # Exclude tuple because it is seen as a single column name indexed = data._constructor( - data[values]._values, index=mi_index, columns=values + data[values]._values, index=mi_index, columns=values # type: ignore ) else: indexed = data._constructor_sliced(data[values]._values, index=mi_index) From 45637fcfa43f7a11b32bf5bfb53a984cbe20f6d9 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Wed, 8 Apr 2020 20:38:37 +0200 Subject: [PATCH 24/32] add comment --- pandas/core/reshape/pivot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index df786615eb9b8..e49dd4c9bb041 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -440,6 +440,7 @@ def pivot( if index is None: pass elif is_list_like(index): + # Remove type ignore once mypy-5206 is implemented, same for below cols = list(index) # type: ignore else: cols = [index] # type: ignore From f063a243e293af504a31aadfd919734d70367012 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Sun, 17 May 2020 20:27:16 +0200 Subject: [PATCH 25/32] use cast --- pandas/core/reshape/pivot.py | 43 +++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 38510a0533c7a..6c9535c451cab 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -1,4 +1,14 @@ -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, + Union, + cast, +) import numpy as np @@ -427,24 +437,29 @@ def _convert_by(by): @Appender(_shared_docs["pivot"], indents=1) def pivot( data: "DataFrame", - index: Optional[Union[Label, Sequence[Optional[Label]]]] = None, - columns: Optional[Union[Label, Sequence[Optional[Label]]]] = None, - values: Optional[Union[Label, Sequence[Optional[Label]]]] = None, + index: Optional[Union[Label, Sequence[Label]]] = None, + columns: Optional[Union[Label, Sequence[Label]]] = None, + values: Optional[Union[Label, Sequence[Label]]] = None, ) -> "DataFrame": if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") - columns = columns if is_list_like(columns) else [columns] + + if is_list_like(columns): + columns = cast(Sequence[Label], columns) + columns = columns + else: + columns = [columns] if values is None: - cols: List[Optional[Sequence]] = [] + cols: List[Label] = [] if index is None: pass elif is_list_like(index): - # Remove type ignore once mypy-5206 is implemented, same for below - cols = list(index) # type: ignore + index = cast(Sequence[Label], index) + cols = list(index) else: - cols = [index] # type: ignore - cols.extend(columns) # type: ignore + cols = [index] + cols.extend(columns) append = index is None indexed = data.set_index(cols, append=append) @@ -452,18 +467,20 @@ def pivot( if index is None: idx_list = [Series(data.index, name=data.index.name)] elif is_list_like(index): - idx_list = [data[idx] for idx in index] # type: ignore + index = cast(Sequence[Label], index) + idx_list = [data[idx] for idx in index] else: idx_list = [data[index]] - data_columns = [data[col] for col in columns] # type: ignore + data_columns = [data[col] for col in columns] idx_list.extend(data_columns) mi_index = MultiIndex.from_arrays(idx_list) if is_list_like(values) and not isinstance(values, tuple): # Exclude tuple because it is seen as a single column name + values = cast(Sequence[Label], values) indexed = data._constructor( - data[values]._values, index=mi_index, columns=values # type: ignore + data[values]._values, index=mi_index, columns=values ) else: indexed = data._constructor_sliced(data[values]._values, index=mi_index) From 1ac9e0f1ae5d835bd18a37e821dccbe1498c9bc9 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 18 May 2020 12:42:31 +0200 Subject: [PATCH 26/32] use convert_list_like --- pandas/core/reshape/pivot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 6c9535c451cab..90b9a77d1a2bc 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -455,8 +455,7 @@ def pivot( if index is None: pass elif is_list_like(index): - index = cast(Sequence[Label], index) - cols = list(index) + cols = com.convert_to_list_like(index) else: cols = [index] cols.extend(columns) From c17dd8069e3db02937080308a3a9119b31b69659 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 18 May 2020 15:55:54 +0200 Subject: [PATCH 27/32] commit uncommited change --- pandas/core/reshape/pivot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 90b9a77d1a2bc..8cf491a908c8d 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -445,8 +445,7 @@ def pivot( raise TypeError("pivot() missing 1 required argument: 'columns'") if is_list_like(columns): - columns = cast(Sequence[Label], columns) - columns = columns + columns = com.convert_to_list_like(columns) else: columns = [columns] From 16798b1055e3de2993a4f080518a669904680eaa Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 18 May 2020 17:06:07 +0200 Subject: [PATCH 28/32] fixup --- pandas/core/reshape/pivot.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 8cf491a908c8d..6f7631a5d3a6d 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -444,19 +444,14 @@ def pivot( if columns is None: raise TypeError("pivot() missing 1 required argument: 'columns'") - if is_list_like(columns): - columns = com.convert_to_list_like(columns) - else: - columns = [columns] + columns = com.convert_to_list_like(columns) if values is None: cols: List[Label] = [] if index is None: pass - elif is_list_like(index): - cols = com.convert_to_list_like(index) else: - cols = [index] + cols = com.convert_to_list_like(index) cols.extend(columns) append = index is None From 63643aab1b7b95a87967a9ee568334a62220388f Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 18 May 2020 17:08:15 +0200 Subject: [PATCH 29/32] get rid of cast --- pandas/core/reshape/pivot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 6f7631a5d3a6d..f807fa2ea0d66 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -460,7 +460,7 @@ def pivot( if index is None: idx_list = [Series(data.index, name=data.index.name)] elif is_list_like(index): - index = cast(Sequence[Label], index) + index = com.convert_to_list_like(index) idx_list = [data[idx] for idx in index] else: idx_list = [data[index]] From 7b1076146ac7f062f62c3acedd6fd46578e1f30b Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 18 May 2020 17:21:52 +0200 Subject: [PATCH 30/32] simplify --- pandas/core/reshape/pivot.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index f807fa2ea0d66..25bf96127516d 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -447,10 +447,7 @@ def pivot( columns = com.convert_to_list_like(columns) if values is None: - cols: List[Label] = [] - if index is None: - pass - else: + if index is not None: cols = com.convert_to_list_like(index) cols.extend(columns) @@ -459,11 +456,9 @@ def pivot( else: if index is None: idx_list = [Series(data.index, name=data.index.name)] - elif is_list_like(index): + else: index = com.convert_to_list_like(index) idx_list = [data[idx] for idx in index] - else: - idx_list = [data[index]] data_columns = [data[col] for col in columns] idx_list.extend(data_columns) From 9870d0f484dce6e41485e8669aa64668bd79183d Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 18 May 2020 17:36:00 +0200 Subject: [PATCH 31/32] fix failed test --- pandas/core/reshape/pivot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 25bf96127516d..4ffe2664ddc40 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -449,6 +449,8 @@ def pivot( if values is None: if index is not None: cols = com.convert_to_list_like(index) + else: + cols = [] cols.extend(columns) append = index is None From 4e05ec0f6ca1753221ac1d65ab47f5d131d7e681 Mon Sep 17 00:00:00 2001 From: Kaiqi Date: Mon, 18 May 2020 20:57:47 +0200 Subject: [PATCH 32/32] less diff --- pandas/core/indexes/multi.py | 2 +- pandas/core/reshape/pivot.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index f1e1ebcaca1c4..dfb329b3ddbdf 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -387,7 +387,7 @@ def _verify_integrity( return new_codes @classmethod - def from_arrays(cls, arrays, sortorder=None, names=lib.no_default): + def from_arrays(cls, arrays, sortorder=None, names=lib.no_default) -> "MultiIndex": """ Convert arrays to MultiIndex. diff --git a/pandas/core/reshape/pivot.py b/pandas/core/reshape/pivot.py index 4ffe2664ddc40..ea5916eff3afa 100644 --- a/pandas/core/reshape/pivot.py +++ b/pandas/core/reshape/pivot.py @@ -457,23 +457,23 @@ def pivot( indexed = data.set_index(cols, append=append) else: if index is None: - idx_list = [Series(data.index, name=data.index.name)] + index = [Series(data.index, name=data.index.name)] else: index = com.convert_to_list_like(index) - idx_list = [data[idx] for idx in index] + index = [data[idx] for idx in index] data_columns = [data[col] for col in columns] - idx_list.extend(data_columns) - mi_index = MultiIndex.from_arrays(idx_list) + index.extend(data_columns) + index = MultiIndex.from_arrays(index) if is_list_like(values) and not isinstance(values, tuple): # Exclude tuple because it is seen as a single column name values = cast(Sequence[Label], values) indexed = data._constructor( - data[values]._values, index=mi_index, columns=values + data[values]._values, index=index, columns=values ) else: - indexed = data._constructor_sliced(data[values]._values, index=mi_index) + indexed = data._constructor_sliced(data[values]._values, index=index) return indexed.unstack(columns)