COMPAT: Pandas 0.23 duplicate names in MI

Pandas 0.23 is disallowing duplicate names in MultiIndexes. This
adjusts a test that relied on that behavior, and `groupby().nunique` which
produced it as a by-product.

Closes dask#3039

xref pandas-dev/pandas#18882

Author: TomAugspurger

dask/dataframe/groupby.py

@@ -289,8 +289,18 @@ def _nunique_df_chunk(df, *index, **kwargs):
     return grouped
 
 
+def _drop_duplicates_rename(df):
+    # Avoid duplicate index labels in a groupby().apply() context
+    # https://github.com/dask/dask/issues/3039
+    # https://github.com/pandas-dev/pandas/pull/18882
+    names = [None] * df.index.nlevels
+    result = df.drop_duplicates().rename_axis(names)
+    return result
+
+
 def _nunique_df_combine(df, levels):
-    result = df.groupby(level=levels, sort=False).apply(pd.DataFrame.drop_duplicates)
+    result = df.groupby(level=levels,
+                        sort=False).apply(_drop_duplicates_rename)
 
     if isinstance(levels, list):
         result.index = pd.MultiIndex.from_arrays([

dask/dataframe/tests/test_groupby.py

@@ -160,6 +160,9 @@ def test_groupby_on_index(get):
     def func(df):
         return df.assign(b=df.b - df.b.mean())
 
+    def func2(df):
+        return df[['b']] - df[['b']].mean()
+
     with dask.set_options(get=get):
         with pytest.warns(None):
             assert_eq(ddf.groupby('a').apply(func),
@@ -168,8 +171,8 @@ def func(df):
             assert_eq(ddf.groupby('a').apply(func).set_index('a'),
                       pdf.groupby('a').apply(func).set_index('a'))
 
-            assert_eq(pdf2.groupby(pdf2.index).apply(func),
-                      ddf2.groupby(ddf2.index).apply(func))
+            assert_eq(pdf2.groupby(pdf2.index).apply(func2),
+                      ddf2.groupby(ddf2.index).apply(func2))
 
 
 def test_groupby_multilevel_getitem():