Add has_duplicates property for Index and MultiIndex (#946)

Author: charlesdong1991

databricks/koalas/indexes.py

@@ -127,6 +127,30 @@ def spark_type(self):
         """ Returns the data type as defined by Spark, as a Spark DataType object."""
         return self.to_series().spark_type
 
+    @property
+    def has_duplicates(self) -> bool:
+        """
+        If index has duplicates, return True, otherwise False.
+
+        Examples
+        --------
+        >>> kdf = ks.DataFrame({'a': [1, 2, 3]}, index=list('aac'))
+        >>> kdf.index.has_duplicates
+        True
+
+        >>> kdf = ks.DataFrame({'a': [1, 2, 3]}, index=[list('abc'), list('def')])
+        >>> kdf.index.has_duplicates
+        False
+
+        >>> kdf = ks.DataFrame({'a': [1, 2, 3]}, index=[list('aac'), list('eef')])
+        >>> kdf.index.has_duplicates
+        True
+        """
+        df = self._kdf._sdf.select(self._scol)
+        col = df.columns[0]
+
+        return df.select(F.count(col) != F.countDistinct(col)).first()[0]
+
     @property
     def name(self) -> Union[str, Tuple[str, ...]]:
         """Return name of the Index."""

databricks/koalas/missing/indexes.py

@@ -31,7 +31,6 @@ class _MissingPandasLikeIndex(object):
 
     # Properties
     T = unsupported_property('T')
-    has_duplicates = unsupported_property('has_duplicates')
     nbytes = unsupported_property('nbytes')
     nlevels = unsupported_property('nlevels')
     shape = unsupported_property('shape')
@@ -129,7 +128,6 @@ class _MissingPandasLikeMultiIndex(object):
     # Properties
     T = unsupported_property('T')
     codes = unsupported_property('codes')
-    has_duplicates = unsupported_property('has_duplicates')
     is_all_dates = unsupported_property('is_all_dates')
     levels = unsupported_property('levels')
     levshape = unsupported_property('levshape')

databricks/koalas/tests/test_indexes.py

@@ -239,6 +239,28 @@ def test_missing(self):
                                         "property.*Index.*{}.*is deprecated".format(name)):
                 getattr(kdf.set_index(['a', 'b']).index, name)
 
+    def test_index_has_duplicates(self):
+        indexes = [("a", "b", "c"), ("a", "a", "c"), (1, 3, 3), (1, 2, 3)]
+        names = [None, 'ks', 'ks', None]
+        has_dup = [False, True, True, False]
+
+        for idx, name, expected in zip(indexes, names, has_dup):
+            pdf = pd.DataFrame({"a": [1, 2, 3]}, index=pd.Index(idx, name=name))
+            kdf = ks.from_pandas(pdf)
+
+            self.assertEqual(kdf.index.has_duplicates, expected)
+
+    def test_multiindex_has_duplicates(self):
+        indexes = [[list("abc"), list("edf")], [list("aac"), list("edf")],
+                   [list("aac"), list("eef")], [[1, 4, 4], [4, 6, 6]]]
+        has_dup = [False, False, True, True]
+
+        for idx, expected in zip(indexes, has_dup):
+            pdf = pd.DataFrame({"a": [1, 2, 3]}, index=idx)
+            kdf = ks.from_pandas(pdf)
+
+            self.assertEqual(kdf.index.has_duplicates, expected)
+
     def test_multi_index_not_supported(self):
         kdf = ks.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6], 'c': [7, 8, 9]})