add in siphash

Author: jreback

pandas/core/strings.py

@@ -154,7 +154,7 @@ def _na_map(f, arr, na_result=np.nan, dtype=object):
     return _map(f, arr, na_mask=True, na_value=na_result, dtype=dtype)
 
 
-def _map(f, arr, na_mask=False, na_value=np.nan, dtype=object):
+def _map(f, arr, na_mask=False, na_value=np.nan, dtype=object, convert=True):
     if not len(arr):
         return np.ndarray(0, dtype=dtype)
 
@@ -165,8 +165,8 @@ def _map(f, arr, na_mask=False, na_value=np.nan, dtype=object):
     if na_mask:
         mask = isnull(arr)
         try:
-            convert = not all(mask)
-            result = lib.map_infer_mask(arr, f, mask.view(np.uint8), convert)
+            _convert = convert and not all(mask)
+            result = lib.map_infer_mask(arr, f, mask.view(np.uint8), _convert)
         except (TypeError, AttributeError):
 
             def g(x):

pandas/src/hash.pyx

@@ -0,0 +1,131 @@
+# cython: profile=False
+# Translated from the reference implementation at https://github.com/veorq/SipHash
+
+import cython
+cimport numpy as cnp
+import numpy as np
+from numpy cimport ndarray, uint8_t, uint32_t, uint64_t
+
+DEF cROUNDS = 2
+DEF dROUNDS = 4
+
+
+def hash_object_array(ndarray[object] arr, object key):
+    """
+    Parameters
+    ----------
+    arr : 1-d object ndarray of objects
+
+    Returns
+    -------
+    1-d uint64 ndarray of hashes
+
+    """
+    cdef:
+        Py_ssize_t i, n
+        ndarray[uint64_t] result
+        bytes data, k
+
+    k = <bytes>key.encode('utf8')
+    n = len(arr)
+    result = np.empty(n, dtype=np.uint64)
+    for i in range(n):
+        data = <bytes>(arr[i].encode('utf8'))
+        result[i] = low_level_siphash(data, len(data), k)
+    return result
+
+cdef inline uint64_t _rotl(uint64_t x, uint64_t b) nogil:
+    return (x << b) | (x >> (64 - b))
+
+cdef inline void u32to8_le(uint8_t* p, uint32_t v) nogil:
+    p[0] = <uint8_t>(v)
+    p[1] = <uint8_t>(v >> 8)
+    p[2] = <uint8_t>(v >> 16)
+    p[3] = <uint8_t>(v >> 24)
+
+cdef inline void u64to8_le(uint8_t* p, uint64_t v) nogil:
+    u32to8_le(p, <uint32_t>v)
+    u32to8_le(p + 4, <uint32_t>(v >> 32))
+
+cdef inline uint64_t u8to64_le(uint8_t* p) nogil:
+    return (<uint64_t>p[0] |
+            <uint64_t>p[1] <<  8 |
+            <uint64_t>p[2] << 16 |
+            <uint64_t>p[3] << 24 |
+            <uint64_t>p[4] << 32 |
+            <uint64_t>p[5] << 40 |
+            <uint64_t>p[6] << 48 |
+            <uint64_t>p[7] << 56)
+
+cdef inline void _sipround(uint64_t* v0, uint64_t* v1,
+                           uint64_t* v2, uint64_t* v3) nogil:
+    v0[0] += v1[0]
+    v1[0] = _rotl(v1[0], 13)
+    v1[0] ^= v0[0]
+    v0[0] = _rotl(v0[0], 32)
+    v2[0] += v3[0]
+    v3[0] = _rotl(v3[0], 16)
+    v3[0] ^= v2[0]
+    v0[0] += v3[0]
+    v3[0] = _rotl(v3[0], 21)
+    v3[0] ^= v0[0]
+    v2[0] += v1[0]
+    v1[0] = _rotl(v1[0], 17)
+    v1[0] ^= v2[0]
+    v2[0] = _rotl(v2[0], 32)
+
+cpdef uint64_t siphash(bytes data, bytes key) except? 0:
+    if len(key) != 16:
+        raise ValueError(
+            'key should be a 16-byte bytestring, got {!r} (len {})'.format(
+                key, len(key)))
+    return low_level_siphash(data, len(data), key)
+
+@cython.cdivision(True)
+cdef uint64_t low_level_siphash(uint8_t* data, size_t datalen,
+                                uint8_t* key) nogil:
+    cdef uint64_t v0 = 0x736f6d6570736575ULL
+    cdef uint64_t v1 = 0x646f72616e646f6dULL
+    cdef uint64_t v2 = 0x6c7967656e657261ULL
+    cdef uint64_t v3 = 0x7465646279746573ULL
+    cdef uint64_t b
+    cdef uint64_t k0 = u8to64_le(key)
+    cdef uint64_t k1 = u8to64_le(key + 8)
+    cdef uint64_t m
+    cdef int i
+    cdef uint8_t* end = data + datalen - (datalen % sizeof(uint64_t))
+    cdef int left = datalen & 7
+    cdef int left_byte
+
+    b = (<uint64_t>datalen) << 56
+    v3 ^= k1
+    v2 ^= k0
+    v1 ^= k1
+    v0 ^= k0
+
+    while (data != end):
+        m = u8to64_le(data)
+        v3 ^= m
+        for i in range(cROUNDS):
+            _sipround(&v0, &v1, &v2, &v3)
+        v0 ^= m
+
+        data += sizeof(uint64_t)
+
+    for i in range(left-1, -1, -1):
+        b |= (<uint64_t>data[i]) << (i * 8)
+
+    v3 ^= b
+
+    for i in range(cROUNDS):
+        _sipround(&v0, &v1, &v2, &v3)
+
+    v0 ^= b
+    v2 ^= 0xff
+
+    for i in range(dROUNDS):
+        _sipround(&v0, &v1, &v2, &v3)
+
+    b = v0 ^ v1 ^ v2 ^ v3
+
+    return b

pandas/tools/hashing.py

@@ -2,9 +2,10 @@
 data hash pandas / numpy objects
 """
 
-from hashlib import md5
 import numpy as np
 from pandas import Series
+from pandas import _hash
+from pandas.lib import infer_dtype
 from pandas.types.generic import ABCIndexClass, ABCSeries, ABCDataFrame
 from pandas.types.common import is_categorical_dtype
 
@@ -61,10 +62,11 @@ def hash_array(vals):
         return hash_array(vals.real) + 23 * hash_array(vals.imag)
 
     # MAIN LOGIC:
+    inferred = infer_dtype(vals)
 
     # First, turn whatever array this is into unsigned 64-bit ints, if we can
     # manage it.
-    if vals.dtype == np.bool:
+    if inferred == 'boolean':
         vals = vals.astype('u8')
 
     if (np.issubdtype(vals.dtype, np.datetime64) or
@@ -76,10 +78,7 @@ def hash_array(vals):
 
         # we want to stringify
         # then apply a consistent hashing scheme
-        def f(v):
-            return int(md5(v).hexdigest(), 16) % (10 ** 8)
-        vals = Series(vals).astype(str).str.encode('utf8').values
-        vals = np.array([f(v) for v in vals], dtype='uint64')
+        vals = _hash.hash_object_array(vals, 'foo')
 
     # Then, redistribute these 64-bit ints within the space of 64-bit ints
     vals ^= vals >> 30

setup.py

@@ -331,6 +331,7 @@ class CheckSDist(sdist_class):
                  'pandas/src/period.pyx',
                  'pandas/src/sparse.pyx',
                  'pandas/src/testing.pyx',
+                 'pandas/src/hash.pyx',
                  'pandas/io/sas/saslib.pyx']
 
     def initialize_options(self):
@@ -501,10 +502,12 @@ def pxd(name):
             'sources': ['pandas/src/parser/tokenizer.c',
                         'pandas/src/parser/io.c']},
     _sparse={'pyxfile': 'src/sparse',
-             'depends': ([srcpath('sparse', suffix='.pyx')]
-                         + _pxi_dep['_sparse'])},
+             'depends': ([srcpath('sparse', suffix='.pyx')] +
+                         _pxi_dep['_sparse'])},
     _testing={'pyxfile': 'src/testing',
               'depends': [srcpath('testing', suffix='.pyx')]},
+    _hash={'pyxfile': 'src/hash',
+           'depends': [srcpath('hash', suffix='.pyx')]},
 )
 
 ext_data["io.sas.saslib"] = {'pyxfile': 'io/sas/saslib'}