refactor utility functions

Author: alimanfoo

numcodecs/astype.py

@@ -4,7 +4,7 @@
 import numpy as np
 
 from .abc import Codec
-from .compat import buffer_copy, ndarray_from_buffer
+from .compat import memory_copy, ensure_ndarray_from_memory
 
 
 class AsType(Codec):
@@ -50,7 +50,7 @@ def __init__(self, encode_dtype, decode_dtype):
     def encode(self, buf):
 
         # view input data as 1D array
-        arr = ndarray_from_buffer(buf, self.decode_dtype)
+        arr = ensure_ndarray_from_memory(buf).view(self.decode_dtype)
 
         # convert and copy
         enc = arr.astype(self.encode_dtype)
@@ -60,13 +60,13 @@ def encode(self, buf):
     def decode(self, buf, out=None):
 
         # view encoded data as 1D array
-        enc = ndarray_from_buffer(buf, self.encode_dtype)
+        enc = ensure_ndarray_from_memory(buf).view(self.encode_dtype)
 
         # convert and copy
         dec = enc.astype(self.decode_dtype)
 
         # handle output
-        out = buffer_copy(dec, out)
+        out = memory_copy(dec, out)
 
         return out
 

numcodecs/bz2.py

@@ -1,14 +1,10 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 import bz2 as _bz2
-import array
-
-
-import numpy as np
 
 
 from numcodecs.abc import Codec
-from numcodecs.compat import buffer_copy, ensure_memoryview
+from numcodecs.compat import memory_copy, ensure_memoryview
 
 
 class BZ2(Codec):
@@ -46,4 +42,4 @@ def decode(self, buf, out=None):
         # handle destination - Python standard library bz2 module does not
         # support direct decompression into buffer, so we have to copy into
         # out if given
-        return buffer_copy(dec, out)
+        return memory_copy(dec, out)

numcodecs/categorize.py

@@ -3,7 +3,7 @@
 
 
 from .abc import Codec
-from .compat import ndarray_from_buffer, buffer_copy, ensure_text
+from .compat import ensure_ndarray_from_memory, memory_copy, ensure_text
 
 
 import numpy as np
@@ -53,7 +53,10 @@ def __init__(self, labels, dtype, astype='u1'):
     def encode(self, buf):
 
         # view input as ndarray
-        arr = ndarray_from_buffer(buf, self.dtype)
+        if self.dtype == object:
+            arr = np.asanyarray(buf, dtype=object).reshape(-1, order='A')
+        else:
+            arr = ensure_ndarray_from_memory(buf).view(self.dtype)
 
         # setup output array
         enc = np.zeros_like(arr, dtype=self.astype)
@@ -67,7 +70,7 @@ def encode(self, buf):
     def decode(self, buf, out=None):
 
         # view encoded data as ndarray
-        enc = ndarray_from_buffer(buf, self.astype)
+        enc = ensure_ndarray_from_memory(buf).view(self.astype)
 
         # setup output
         if isinstance(out, np.ndarray):
@@ -84,7 +87,7 @@ def decode(self, buf, out=None):
 
         # handle output
         if copy_needed:
-            dec = buffer_copy(dec, out)
+            dec = memory_copy(dec, out)
 
         return dec
 

numcodecs/checksum32.py

@@ -7,30 +7,28 @@
 
 
 from .abc import Codec
-from .compat import ndarray_from_buffer, buffer_copy
+from .compat import ensure_ndarray_from_memory, memory_copy
 
 
 class Checksum32(Codec):
 
     checksum = None
 
     def encode(self, buf):
-        if isinstance(buf, np.ndarray) and buf.dtype == object:
-            raise ValueError('cannot encode object array')
-        arr = ndarray_from_buffer(buf, dtype='u1')
+        arr = ensure_ndarray_from_memory(buf).view('u1')
         checksum = self.checksum(arr) & 0xffffffff
         enc = np.empty(arr.nbytes + 4, dtype='u1')
         enc[:4].view('<u4')[0] = checksum
         enc[4:] = arr
         return enc
 
     def decode(self, buf, out=None):
-        arr = ndarray_from_buffer(buf, dtype='u1')
+        arr = ensure_ndarray_from_memory(buf).view('u1')
         expect = arr[:4].view('<u4')[0]
         checksum = self.checksum(arr[4:]) & 0xffffffff
         if expect != checksum:
             raise RuntimeError('checksum failed')
-        return buffer_copy(arr[4:], out)
+        return memory_copy(arr[4:], out)
 
 
 class CRC32(Checksum32):

numcodecs/compat.py

@@ -25,68 +25,31 @@
     from functools import reduce
 
 
-def buffer_copy(buf, out=None):
+def memory_copy(buf, out=None):
     """Copy the contents of the memory buffer from `buf` to `out`."""
 
     if out is None:
         # no-op
         return buf
 
-    # handle ndarray destination
-    if isinstance(out, np.ndarray):
+    # obtain ndarrays, casting to the same data type
+    buf = ensure_ndarray_from_memory(buf).view('u1')
+    out = ensure_ndarray_from_memory(out).view('u1')
 
-        # view source as destination dtype
-        if isinstance(buf, np.ndarray):
-            buf = buf.view(dtype=out.dtype).reshape(-1, order='A')
-        else:
-            buf = np.frombuffer(buf, dtype=out.dtype)
-
-        # ensure shapes are compatible
-        if buf.shape != out.shape:
-            if out.flags.f_contiguous:
-                order = 'F'
-            else:
-                order = 'C'
-            buf = buf.reshape(out.shape, order=order)
-
-        # copy via numpy
-        np.copyto(out, buf)
-
-    # handle generic buffer destination
-    else:
-
-        # obtain memoryview of destination
-        dest = memoryview(out)
-
-        # ensure source is 1D
-        if isinstance(buf, np.ndarray):
-            buf = buf.reshape(-1, order='A')
-            # try to match itemsize
-            dtype = 'u%s' % dest.itemsize
-            buf = buf.view(dtype=dtype)
-
-        # try to copy via memoryview
-        dest[:] = buf
+    # copy memory
+    np.copyto(out, buf)
 
     return out
 
 
-def ndarray_from_buffer(buf, dtype):
-    if isinstance(buf, np.ndarray):
-        arr = buf.reshape(-1, order='A').view(dtype)
-    else:
-        arr = np.frombuffer(buf, dtype=dtype)
-    return arr
-
-
 def ensure_text(l, encoding='utf-8'):
     if isinstance(l, text_type):
         return l
     else:  # pragma: py3 no cover
         return text_type(l, encoding=encoding)
 
 
-def ensure_ndarray_from_memory(o):
+def ensure_ndarray_from_memory(o, flatten=True):
     """Convenience function to obtain a numpy ndarray using memory exposed by object `o`,
     ensuring that no memory copies are made, and that `o` is not an object array.
 
@@ -96,6 +59,8 @@ def ensure_ndarray_from_memory(o):
         Any object exposing a memory buffer. On Python 3 this must be an object exposing
         the new-style buffer interface. On Python 2 this can also be an object exposing
         the old-style buffer interface.
+    flatten : bool, optional
+        If True, flatten any multi-dimensional inputs into a one-dimensional memoryview.
 
     Returns
     -------
@@ -106,7 +71,7 @@ def ensure_ndarray_from_memory(o):
     if not isinstance(o, np.ndarray):
 
         # first try to obtain a memoryview or buffer, needed to ensure that we don't
-        # accidentally copy memory when going via np.array()
+        # subsequently copy memory when going via np.array()
 
         if PY2:  # pragma: py3 no cover
             # accept objects exposing either old-style or new-style buffer interface
@@ -120,13 +85,18 @@ def ensure_ndarray_from_memory(o):
 
         # N.B., this is not documented, but np.array() will accept an object exposing
         # a buffer interface, and will take a view of the memory rather than making a
-        # copy, preserving type information
+        # copy, preserving type information where present
         o = np.array(o, copy=False)
 
     # check for object arrays
     if o.dtype == object:
         raise ValueError('object arrays are not supported')
 
+    if flatten:
+
+        # flatten the array to 1 dimension
+        o = o.reshape(-1, order='A')
+
     return o
 
 
@@ -149,17 +119,12 @@ def ensure_memoryview(o, flatten=True):
     """
 
     # go via numpy, for convenience
-    o = ensure_ndarray_from_memory(o)
+    o = ensure_ndarray_from_memory(o, flatten=flatten)
 
     # check for datetime or timedelta ndarray, cannot take a memoryview of those
     if o.dtype.kind in 'Mm':
         o = o.view(np.int64)
 
-    if flatten:
-
-        # flatten the array
-        o = o.reshape(-1, order='A')
-
     # expose as memoryview
     o = memoryview(o)
 
@@ -203,9 +168,6 @@ def ensure_buffer(o):
         # go via numpy, for convenience
         o = ensure_ndarray_from_memory(o)
 
-        # N.B., no need to flatten multi-dimensional arrays, as the old-style buffer
-        # interface just exposes the flat memory
-
         # expose as buffer
         o = buffer(o)
 

numcodecs/delta.py

@@ -6,7 +6,7 @@
 
 
 from .abc import Codec
-from .compat import ndarray_from_buffer, buffer_copy
+from .compat import ensure_ndarray_from_memory, memory_copy
 
 
 class Delta(Codec):
@@ -57,7 +57,7 @@ def __init__(self, dtype, astype=None):
     def encode(self, buf):
 
         # view input data as 1D array
-        arr = ndarray_from_buffer(buf, self.dtype)
+        arr = ensure_ndarray_from_memory(buf).reshape(-1, order='A').view(self.dtype)
 
         # setup encoded output
         enc = np.empty_like(arr, dtype=self.astype)
@@ -73,7 +73,7 @@ def encode(self, buf):
     def decode(self, buf, out=None):
 
         # view encoded data as 1D array
-        enc = ndarray_from_buffer(buf, self.astype)
+        enc = ensure_ndarray_from_memory(buf).reshape(-1, order='A').view(self.astype)
 
         # setup decoded output
         if isinstance(out, np.ndarray):
@@ -89,7 +89,7 @@ def decode(self, buf, out=None):
 
         # handle output
         if copy_needed:
-            out = buffer_copy(dec, out)
+            out = memory_copy(dec, out)
 
         return out
 

numcodecs/fixedscaleoffset.py

@@ -6,7 +6,7 @@
 
 
 from .abc import Codec
-from .compat import ndarray_from_buffer, buffer_copy
+from .compat import ensure_ndarray_from_memory, memory_copy
 
 
 class FixedScaleOffset(Codec):
@@ -88,7 +88,7 @@ def __init__(self, offset, scale, dtype, astype=None):
     def encode(self, buf):
 
         # interpret buffer as 1D array
-        arr = ndarray_from_buffer(buf, self.dtype)
+        arr = ensure_ndarray_from_memory(buf).view(self.dtype)
 
         # compute scale offset
         enc = (arr - self.offset) * self.scale
@@ -104,7 +104,7 @@ def encode(self, buf):
     def decode(self, buf, out=None):
 
         # interpret buffer as 1D array
-        enc = ndarray_from_buffer(buf, self.astype)
+        enc = ensure_ndarray_from_memory(buf).view(self.astype)
 
         # decode scale offset
         dec = (enc / self.scale) + self.offset
@@ -113,7 +113,7 @@ def decode(self, buf, out=None):
         dec = dec.astype(self.dtype, copy=False)
 
         # handle output
-        return buffer_copy(dec, out)
+        return memory_copy(dec, out)
 
     def get_config(self):
         # override to handle encoding dtypes

numcodecs/gzip.py

@@ -5,7 +5,7 @@
 
 
 from .abc import Codec
-from .compat import buffer_copy, ensure_memoryview
+from .compat import memory_copy, ensure_memoryview
 
 
 class GZip(Codec):
@@ -52,4 +52,4 @@ def decode(self, buf, out=None):
         # handle destination - Python standard library zlib module does not
         # support direct decompression into buffer, so we have to copy into
         # out if given
-        return buffer_copy(decompressed, out)
+        return memory_copy(decompressed, out)

numcodecs/lzma.py

@@ -14,9 +14,8 @@
 
 if _lzma:
 
-    import numpy as np
     from .abc import Codec
-    from .compat import buffer_copy, ensure_memoryview
+    from .compat import memory_copy, ensure_memoryview
 
     # noinspection PyShadowingBuiltins
     class LZMA(Codec):
@@ -64,7 +63,7 @@ def decode(self, buf, out=None):
             dec = _lzma.decompress(buf, format=self.format, filters=self.filters)
 
             # handle destination
-            return buffer_copy(dec, out)
+            return memory_copy(dec, out)
 
         def __repr__(self):
             r = '%s(format=%r, check=%r, preset=%r, filters=%r)' % \