Merge remote-tracking branch 'upstream/master' into series_rolling_count_ignores_min_periods

Author: fujiaxiang

doc/source/whatsnew/v1.1.0.rst

@@ -156,7 +156,7 @@ Reshaping
 - Bug in :meth:`DataFrame.pivot_table` when ``margin`` is ``True`` and only ``column`` is defined (:issue:`31016`)
 - Fix incorrect error message in :meth:`DataFrame.pivot` when ``columns`` is set to ``None``. (:issue:`30924`)
 - Bug in :func:`crosstab` when inputs are two Series and have tuple names, the output will keep dummy MultiIndex as columns. (:issue:`18321`)
-
+- Bug in :func:`concat` where the resulting indices are not copied when ``copy=True`` (:issue:`29879`)
 
 Sparse
 ^^^^^^

pandas/core/indexes/api.py

@@ -63,7 +63,7 @@
 
 
 def get_objs_combined_axis(
-    objs, intersect: bool = False, axis=0, sort: bool = True
+    objs, intersect: bool = False, axis=0, sort: bool = True, copy: bool = False
 ) -> Index:
     """
     Extract combined index: return intersection or union (depending on the
@@ -81,13 +81,15 @@ def get_objs_combined_axis(
         The axis to extract indexes from.
     sort : bool, default True
         Whether the result index should come out sorted or not.
+    copy : bool, default False
+        If True, return a copy of the combined index.
 
     Returns
     -------
     Index
     """
     obs_idxes = [obj._get_axis(axis) for obj in objs]
-    return _get_combined_index(obs_idxes, intersect=intersect, sort=sort)
+    return _get_combined_index(obs_idxes, intersect=intersect, sort=sort, copy=copy)
 
 
 def _get_distinct_objs(objs: List[Index]) -> List[Index]:
@@ -105,7 +107,10 @@ def _get_distinct_objs(objs: List[Index]) -> List[Index]:
 
 
 def _get_combined_index(
-    indexes: List[Index], intersect: bool = False, sort: bool = False
+    indexes: List[Index],
+    intersect: bool = False,
+    sort: bool = False,
+    copy: bool = False,
 ) -> Index:
     """
     Return the union or intersection of indexes.
@@ -119,6 +124,8 @@ def _get_combined_index(
         calculate the union.
     sort : bool, default False
         Whether the result index should come out sorted or not.
+    copy : bool, default False
+        If True, return a copy of the combined index.
 
     Returns
     -------
@@ -143,6 +150,11 @@ def _get_combined_index(
             index = index.sort_values()
         except TypeError:
             pass
+
+    # GH 29879
+    if copy:
+        index = index.copy()
+
     return index
 
 

pandas/core/reshape/concat.py

@@ -517,7 +517,11 @@ def _get_new_axes(self) -> List[Index]:
     def _get_comb_axis(self, i: int) -> Index:
         data_axis = self.objs[0]._get_block_manager_axis(i)
         return get_objs_combined_axis(
-            self.objs, axis=data_axis, intersect=self.intersect, sort=self.sort
+            self.objs,
+            axis=data_axis,
+            intersect=self.intersect,
+            sort=self.sort,
+            copy=self.copy,
         )
 
     def _get_concat_axis(self) -> Index:

pandas/tests/io/data/html/computer_sales_page.html

@@ -14,7 +14,7 @@
 from pandas.errors import ParserError
 import pandas.util._test_decorators as td
 
-from pandas import DataFrame, Index, MultiIndex, Series, Timestamp, date_range, read_csv
+from pandas import DataFrame, MultiIndex, Series, Timestamp, date_range, read_csv
 import pandas._testing as tm
 
 from pandas.io.common import file_path_to_url
@@ -373,32 +373,6 @@ def test_python_docs_table(self):
         zz = [df.iloc[0, 0][0:4] for df in dfs]
         assert sorted(zz) == sorted(["Repo", "What"])
 
-    @pytest.mark.slow
-    def test_thousands_macau_stats(self, datapath):
-        all_non_nan_table_index = -2
-        macau_data = datapath("io", "data", "html", "macau.html")
-        dfs = self.read_html(macau_data, index_col=0, attrs={"class": "style1"})
-        df = dfs[all_non_nan_table_index]
-
-        assert not any(s.isna().any() for _, s in df.items())
-
-    @pytest.mark.slow
-    def test_thousands_macau_index_col(self, datapath, request):
-        # https://github.com/pandas-dev/pandas/issues/29622
-        # This tests fails for bs4 >= 4.8.0 - so handle xfail accordingly
-        if self.read_html.keywords.get("flavor") == "bs4" and td.safe_import(
-            "bs4", "4.8.0"
-        ):
-            reason = "fails for bs4 version >= 4.8.0"
-            request.node.add_marker(pytest.mark.xfail(reason=reason))
-
-        all_non_nan_table_index = -2
-        macau_data = datapath("io", "data", "html", "macau.html")
-        dfs = self.read_html(macau_data, index_col=0, header=0)
-        df = dfs[all_non_nan_table_index]
-
-        assert not any(s.isna().any() for _, s in df.items())
-
     def test_empty_tables(self):
         """
         Make sure that read_html ignores empty tables.
@@ -571,23 +545,6 @@ def test_parse_header_of_non_string_column(self):
 
         tm.assert_frame_equal(result, expected)
 
-    def test_nyse_wsj_commas_table(self, datapath):
-        data = datapath("io", "data", "html", "nyse_wsj.html")
-        df = self.read_html(data, index_col=0, header=0, attrs={"class": "mdcTable"})[0]
-
-        expected = Index(
-            [
-                "Issue(Roll over for charts and headlines)",
-                "Volume",
-                "Price",
-                "Chg",
-                "% Chg",
-            ]
-        )
-        nrows = 100
-        assert df.shape[0] == nrows
-        tm.assert_index_equal(df.columns, expected)
-
     @pytest.mark.slow
     def test_banklist_header(self, datapath):
         from pandas.io.html import _remove_whitespace
@@ -894,24 +851,23 @@ def test_parse_dates_combine(self):
         newdf = DataFrame({"datetime": raw_dates})
         tm.assert_frame_equal(newdf, res[0])
 
-    def test_computer_sales_page(self, datapath):
-        data = datapath("io", "data", "html", "computer_sales_page.html")
-        msg = (
-            r"Passed header=\[0,1\] are too many "
-            r"rows for this multi_index of columns"
-        )
-        with pytest.raises(ParserError, match=msg):
-            self.read_html(data, header=[0, 1])
-
-        data = datapath("io", "data", "html", "computer_sales_page.html")
-        assert self.read_html(data, header=[1, 2])
-
     def test_wikipedia_states_table(self, datapath):
         data = datapath("io", "data", "html", "wikipedia_states.html")
         assert os.path.isfile(data), f"{repr(data)} is not a file"
         assert os.path.getsize(data), f"{repr(data)} is an empty file"
         result = self.read_html(data, "Arizona", header=1)[0]
+        assert result.shape == (60, 12)
+        assert "Unnamed" in result.columns[-1]
         assert result["sq mi"].dtype == np.dtype("float64")
+        assert np.allclose(result.loc[0, "sq mi"], 665384.04)
+
+    def test_wikipedia_states_multiindex(self, datapath):
+        data = datapath("io", "data", "html", "wikipedia_states.html")
+        result = self.read_html(data, "Arizona", index_col=0)[0]
+        assert result.shape == (60, 11)
+        assert "Unnamed" in result.columns[-1][1]
+        assert result.columns.nlevels == 2
+        assert np.allclose(result.loc["Alaska", ("Total area[2]", "sq mi")], 665384.04)
 
     def test_parser_error_on_empty_header_row(self):
         msg = (

pandas/tests/io/data/html/macau.html

@@ -2750,3 +2750,17 @@ def test_concat_sparse():
     )
     result = pd.concat([a, a], axis=1)
     tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("test_series", [True, False])
+def test_concat_copy_index(test_series, axis):
+    # GH 29879
+    if test_series:
+        ser = Series([1, 2])
+        comb = concat([ser, ser], axis=axis, copy=True)
+        assert comb.index is not ser.index
+    else:
+        df = DataFrame([[1, 2], [3, 4]], columns=["a", "b"])
+        comb = concat([df, df], axis=axis, copy=True)
+        assert comb.index is not df.index
+        assert comb.columns is not df.columns