ENH: add data hashing routines

xref dask/dask#1807

Author: jreback

doc/source/whatsnew/v0.19.2.txt

@@ -16,6 +16,13 @@ Highlights include:
     :backlinks: none
 
 
+.. _whatsnew_0192.enhancements:
+
+Enhancements
+~~~~~~~~~~~~
+- Routines to provide a hash for pandas objects (:issue:``)
+
+
 .. _whatsnew_0192.performance:
 
 Performance Improvements

pandas/core/base.py

@@ -16,6 +16,7 @@
 from pandas.util.decorators import (Appender, cache_readonly,
                                     deprecate_kwarg, Substitution)
 from pandas.core.common import AbstractMethodError
+from pandas.tools.hashing import hash_pandas_object
 from pandas.formats.printing import pprint_thing
 
 _shared_docs = dict()
@@ -795,7 +796,50 @@ def __unicode__(self):
         return "%s(%s, dtype='%s')" % (type(self).__name__, prepr, self.dtype)
 
 
-class IndexOpsMixin(object):
+class HashableMixin(object):
+    """ provide methods for hashable pandas objects """
+
+    def hash(self, index=True):
+        """
+        Return a data hash of the Series/DataFrame
+
+        Parameters
+        ----------
+        index : boolean, default True
+            include the index in the hash (if Series/DataFrame)
+
+        Returns
+        -------
+        1d uint64 numpy array of hash values, same length as the
+        object
+
+        Examples
+        --------
+        >>> pd.Index([1, 2, 3]).hash()
+        array([6238072747940578789, 15839785061582574730,
+               2185194620014831856], dtype=uint64)
+
+        >>>  pd.Series([1, 2, 3]).hash()
+        array([267474170112184751, 16863939785269199747,
+               3948624847917518682], dtype=uint64)
+
+        >>> pd.Series([1, 2, 3]).hash(index=False)
+        array([6238072747940578789, 15839785061582574730,
+               2185194620014831856], dtype=uint64)
+
+        >>> pd.DataFrame({'A': [1, 2, 3]}).hash()
+        array([267474170112184751, 16863939785269199747,
+               3948624847917518682], dtype=uint64)
+
+        >>> pd.DataFrame({'A': [1, 2, 3], 'B': ['foo', 'bar', 'baz']}).hash()
+        array([10443389771536978168, 14762841401800935363,
+               9751253963311919054], dtype=uint64)
+
+        """
+        return hash_pandas_object(self, index=index)
+
+
+class IndexOpsMixin(HashableMixin):
     """ common ops mixin to support a unified inteface / docs for Series /
     Index
     """

pandas/core/frame.py

@@ -201,7 +201,7 @@
 # DataFrame class
 
 
-class DataFrame(NDFrame):
+class DataFrame(NDFrame, base.HashableMixin):
     """ Two-dimensional size-mutable, potentially heterogeneous tabular data
     structure with labeled axes (rows and columns). Arithmetic operations
     align on both row and column labels. Can be thought of as a dict-like

pandas/tools/hashing.py

@@ -0,0 +1,82 @@
+"""
+data hash pandas / numpy objects
+"""
+
+import numpy as np
+from pandas.types.generic import ABCIndexClass, ABCSeries, ABCDataFrame
+from pandas.types.common import is_categorical_dtype
+
+
+def hash_pandas_object(obj, index=True):
+    """
+    Return a data hash of the Index/Series/DataFrame
+
+    Parameters
+    ----------
+    index : boolean, default True
+    include the index in the hash (if Series/DataFrame)
+
+    Returns
+    -------
+    1d uint64 numpy array of hash values, same length as the
+    object
+    """
+
+    def adder(h, hashed_to_add):
+        h = np.multiply(h, np.uint(3), h)
+        return np.add(h, hashed_to_add, h)
+
+    if isinstance(obj, ABCIndexClass):
+        h = hash_array(obj.values).astype('uint64')
+    elif isinstance(obj, ABCSeries):
+        h = hash_array(obj.values).astype('uint64')
+        if index:
+            h = adder(h, hash_pandas_object(obj.index))
+    elif isinstance(obj, ABCDataFrame):
+        cols = obj.iteritems()
+        first_series = next(cols)[1]
+        h = hash_array(first_series.values).astype('uint64')
+        for _, col in cols:
+            h = adder(h, hash_array(col.values))
+        if index:
+            h = adder(h, hash_pandas_object(obj.index))
+    else:
+        raise TypeError("Unexpected type %s" % type(obj))
+    return h
+
+
+def hash_array(vals):
+    """Given a 1d array, return an array of deterministic integers."""
+    # work with cagegoricals as ints. (This check is above the complex
+    # check so that we don't ask numpy if categorical is a subdtype of
+    # complex, as it will choke.
+    if is_categorical_dtype(vals.dtype):
+        vals = vals.codes
+
+    # we'll be working with everything as 64-bit values, so handle this
+    # 128-bit value early
+    if np.issubdtype(vals.dtype, np.complex128):
+        return hash_array(vals.real) + 23 * hash_array(vals.imag)
+
+    # MAIN LOGIC:
+
+    # First, turn whatever array this is into unsigned 64-bit ints, if we can
+    # manage it.
+    if vals.dtype == np.bool:
+        vals = vals.astype('u8')
+
+    if (np.issubdtype(vals.dtype, np.datetime64) or
+       np.issubdtype(vals.dtype, np.timedelta64) or
+       np.issubdtype(vals.dtype, np.number)) and vals.dtype.itemsize <= 8:
+
+        vals = vals.view('u{}'.format(vals.dtype.itemsize)).astype('u8')
+    else:
+        vals = np.array([hash(x) for x in vals], dtype=np.uint64)
+
+    # Then, redistribute these 64-bit ints within the space of 64-bit ints
+    vals ^= vals >> 30
+    vals *= np.uint64(0xbf58476d1ce4e5b9)
+    vals ^= vals >> 27
+    vals *= np.uint64(0x94d049bb133111eb)
+    vals ^= vals >> 31
+    return vals

pandas/tools/tests/test_hashing.py

@@ -0,0 +1,75 @@
+import numpy as np
+import pandas as pd
+
+from pandas import DataFrame, Series, Index
+from pandas.tools.hashing import hash_array, hash_pandas_object
+import pandas.util.testing as tm
+
+
+class TestHashing(tm.TestCase):
+
+    _multiprocess_can_split_ = True
+
+    def setUp(self):
+        self.df = DataFrame(
+            {'i32': np.array([1, 2, 3] * 3, dtype='int32'),
+             'f32': np.array([None, 2.5, 3.5] * 3, dtype='float32'),
+             'cat': Series(['a', 'b', 'c'] * 3).astype('category'),
+             'obj': Series(['d', 'e', 'f'] * 3),
+             'bool': np.array([True, False, True] * 3),
+             'dt': Series(pd.date_range('20130101', periods=9)),
+             'dt_tz': Series(pd.date_range('20130101', periods=9,
+                                           tz='US/Eastern')),
+             'td': Series(pd.timedelta_range('2000', periods=9))})
+
+    def test_hash_array(self):
+        for name, s in self.df.iteritems():
+            a = s.values
+            tm.assert_numpy_array_equal(hash_array(a), hash_array(a))
+
+    def check_equal(self, obj):
+        a = hash_pandas_object(obj)
+        b = hash_pandas_object(obj)
+        tm.assert_numpy_array_equal(a, b)
+
+        a = obj.hash()
+        b = obj.hash()
+        tm.assert_numpy_array_equal(a, b)
+
+        a = obj.hash(index=False)
+        b = obj.hash(index=False)
+        tm.assert_numpy_array_equal(a, b)
+
+        # check that we are not hashing the same if
+        # we include the index
+        if not isinstance(obj, Index):
+            a = obj.hash(index=True)
+            b = obj.hash(index=False)
+            self.assertFalse((a == b).all())
+
+    def test_hash_pandas_object(self):
+
+        for obj in [Series([1, 2, 3]),
+                    Series([1.0, 1.5, 3.2]),
+                    Series([1.0, 1.5, 3.2], index=[1.5, 1.1, 3.3]),
+                    Series(['a', 'b', 'c']),
+                    Series([True, False, True]),
+                    Index([1, 2, 3]),
+                    Index([True, False, True]),
+                    DataFrame({'x': ['a', 'b', 'c'], 'y': [1, 2, 3]}),
+                    tm.makeMissingDataframe(),
+                    tm.makeMixedDataFrame(),
+                    tm.makeTimeDataFrame(),
+                    tm.makeTimeSeries(),
+                    tm.makeTimedeltaIndex(),
+                    Series([1, 2, 3], index=pd.MultiIndex.from_tuples(
+                        [('a', 1), ('a', 2), ('b', 1)]))]:
+            self.check_equal(obj)
+
+    def test_errors(self):
+
+        for obj in [pd.Timestamp('20130101'), tm.makePanel()]:
+            def f():
+                hash_pandas_object(f)
+
+            self.assertRaises(TypeError, f)