Switch Buffers to memoryviews & remove extra copies/allocations

docs/release.rst

@@ -37,6 +37,13 @@ Fixes
 ~~~~~
 * Remove redundant ``id`` from codec metadata serialization in Zarr3 codecs.
   By :user:`Norman Rzepka <normanrz>`, :issue:`685`
+* Preallocate output buffers and resize directly as needed.
+  By :user:`John Kirkham <jakirkham>`, :issue:`656`
+
+Maintenance
+~~~~~~~~~~~
+* Replace internal ``Buffer`` usage with ``memoryview``\ s.
+  By :user:`John Kirkham <jakirkham>`, :issue:`656`
 
 .. _release_0.15.0:
 

numcodecs/blosc.pyx

@@ -9,12 +9,11 @@ import os
 from deprecated import deprecated
 
 
-from cpython.buffer cimport PyBUF_ANY_CONTIGUOUS, PyBUF_WRITEABLE
-from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING
+from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize
+from cpython.memoryview cimport PyMemoryView_GET_BUFFER
 
+from .compat_ext cimport PyBytes_RESIZE, ensure_continguous_memoryview
 
-from .compat_ext cimport Buffer
-from .compat_ext import Buffer
 from .compat import ensure_contiguous_ndarray
 from .abc import Codec
 
@@ -154,17 +153,16 @@ def _cbuffer_sizes(source):
 
     """
     cdef:
-        Buffer buffer
+        memoryview source_mv
+        const Py_buffer* source_pb
         size_t nbytes, cbytes, blocksize
 
-    # obtain buffer
-    buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
 
     # determine buffer size
-    blosc_cbuffer_sizes(buffer.ptr, &nbytes, &cbytes, &blocksize)
-
-    # release buffers
-    buffer.release()
+    blosc_cbuffer_sizes(source_pb.buf, &nbytes, &cbytes, &blocksize)
 
     return nbytes, cbytes, blocksize
 
@@ -173,16 +171,15 @@ cbuffer_sizes = deprecated(_cbuffer_sizes)
 def cbuffer_complib(source):
     """Return the name of the compression library used to compress `source`."""
     cdef:
-        Buffer buffer
+        memoryview source_mv
+        const Py_buffer* source_pb
 
-    # obtain buffer
-    buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
 
     # determine buffer size
-    complib = blosc_cbuffer_complib(buffer.ptr)
-
-    # release buffers
-    buffer.release()
+    complib = blosc_cbuffer_complib(source_pb.buf)
 
     complib = complib.decode('ascii')
 
@@ -202,18 +199,17 @@ def _cbuffer_metainfo(source):
 
     """
     cdef:
-        Buffer buffer
+        memoryview source_mv
+        const Py_buffer* source_pb
         size_t typesize
         int flags
 
-    # obtain buffer
-    buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
 
     # determine buffer size
-    blosc_cbuffer_metainfo(buffer.ptr, &typesize, &flags)
-
-    # release buffers
-    buffer.release()
+    blosc_cbuffer_metainfo(source_pb.buf, &typesize, &flags)
 
     # decompose flags
     if flags & BLOSC_DOSHUFFLE:
@@ -263,28 +259,34 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE,
     """
 
     cdef:
-        char *source_ptr
-        char *dest_ptr
-        Buffer source_buffer
+        memoryview source_mv
+        const Py_buffer* source_pb
+        const char* source_ptr
         size_t nbytes, itemsize
         int cbytes
         bytes dest
+        char* dest_ptr
 
     # check valid cname early
     cname_str = cname.decode('ascii')
     if cname_str not in list_compressors():
         _err_bad_cname(cname_str)
 
-    # setup source buffer
-    source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
-    source_ptr = source_buffer.ptr
-    nbytes = source_buffer.nbytes
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
+
+    # extract metadata
+    source_ptr = <const char*>source_pb.buf
+    nbytes = source_pb.len
+
+    # validate typesize
     if isinstance(typesize, int):
         if typesize < 1:
             raise ValueError(f"Cannot use typesize {typesize} less than 1.")
         itemsize = typesize
     else:
-        itemsize = source_buffer.itemsize
+        itemsize = source_pb.itemsize
 
     # determine shuffle
     if shuffle == AUTOSHUFFLE:
@@ -333,16 +335,14 @@ def compress(source, char* cname, int clevel, int shuffle=SHUFFLE,
                                             cname, blocksize, 1)
 
     finally:
-
-        # release buffers
-        source_buffer.release()
+        pass
 
     # check compression was successful
     if cbytes <= 0:
         raise RuntimeError('error during blosc compression: %d' % cbytes)
 
     # resize after compression
-    dest = dest[:cbytes]
+    PyBytes_RESIZE(dest, cbytes)
 
     return dest
 
@@ -366,30 +366,36 @@ def decompress(source, dest=None):
     """
     cdef:
         int ret
-        char *source_ptr
-        char *dest_ptr
-        Buffer source_buffer
-        Buffer dest_buffer = None
+        memoryview source_mv
+        const Py_buffer* source_pb
+        const char* source_ptr
+        memoryview dest_mv
+        Py_buffer* dest_pb
+        char* dest_ptr
         size_t nbytes, cbytes, blocksize
 
-    # setup source buffer
-    source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
-    source_ptr = source_buffer.ptr
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
+
+    # get source pointer
+    source_ptr = <const char*>source_pb.buf
 
     # determine buffer size
     blosc_cbuffer_sizes(source_ptr, &nbytes, &cbytes, &blocksize)
 
     # setup destination buffer
     if dest is None:
         # allocate memory
-        dest = PyBytes_FromStringAndSize(NULL, nbytes)
-        dest_ptr = PyBytes_AS_STRING(dest)
-        dest_nbytes = nbytes
+        dest_1d = dest = PyBytes_FromStringAndSize(NULL, nbytes)
     else:
-        arr = ensure_contiguous_ndarray(dest)
-        dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
-        dest_ptr = dest_buffer.ptr
-        dest_nbytes = dest_buffer.nbytes
+        dest_1d = ensure_contiguous_ndarray(dest)
+
+    # obtain dest memoryview
+    dest_mv = memoryview(dest_1d)
+    dest_pb = PyMemoryView_GET_BUFFER(dest_mv)
+    dest_ptr = <char*>dest_pb.buf
+    dest_nbytes = dest_pb.len
 
     try:
 
@@ -408,11 +414,7 @@ def decompress(source, dest=None):
                 ret = blosc_decompress_ctx(source_ptr, dest_ptr, nbytes, 1)
 
     finally:
-
-        # release buffers
-        source_buffer.release()
-        if dest_buffer is not None:
-            dest_buffer.release()
+        pass
 
     # handle errors
     if ret <= 0:
@@ -449,14 +451,20 @@ def _decompress_partial(source, start, nitems, dest=None):
         int encoding_size
         int nitems_bytes
         int start_bytes
-        char *source_ptr
-        char *dest_ptr
-        Buffer source_buffer
-        Buffer dest_buffer = None
+        memoryview source_mv
+        const Py_buffer* source_pb
+        const char* source_ptr
+        memoryview dest_mv
+        Py_buffer* dest_pb
+        char* dest_ptr
+        size_t dest_nbytes
 
-    # setup source buffer
-    source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS)
-    source_ptr = source_buffer.ptr
+    # obtain source memoryview
+    source_mv = ensure_continguous_memoryview(source)
+    source_pb = PyMemoryView_GET_BUFFER(source_mv)
+
+    # setup source pointer
+    source_ptr = <const char*>source_pb.buf
 
     # get encoding size from source buffer header
     encoding_size = source[3]
@@ -467,26 +475,25 @@ def _decompress_partial(source, start, nitems, dest=None):
 
     # setup destination buffer
     if dest is None:
-        dest = PyBytes_FromStringAndSize(NULL, nitems_bytes)
-        dest_ptr = PyBytes_AS_STRING(dest)
-        dest_nbytes = nitems_bytes
+        # allocate memory
+        dest_1d = dest = PyBytes_FromStringAndSize(NULL, nitems_bytes)
     else:
-        arr = ensure_contiguous_ndarray(dest)
-        dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE)
-        dest_ptr = dest_buffer.ptr
-        dest_nbytes = dest_buffer.nbytes
+        dest_1d = ensure_contiguous_ndarray(dest)
+
+    # obtain dest memoryview
+    dest_mv = memoryview(dest_1d)
+    dest_pb = PyMemoryView_GET_BUFFER(dest_mv)
+    dest_ptr = <char*>dest_pb.buf
+    dest_nbytes = dest_pb.len
 
     # try decompression
     try:
         if dest_nbytes < nitems_bytes:
             raise ValueError('destination buffer too small; expected at least %s, '
                              'got %s' % (nitems_bytes, dest_nbytes))
         ret = blosc_getitem(source_ptr, start, nitems, dest_ptr)
-
     finally:
-        source_buffer.release()
-        if dest_buffer is not None:
-            dest_buffer.release()
+        pass
 
     # ret refers to the number of bytes returned from blosc_getitem.
     if ret <= 0:

numcodecs/compat_ext.pxd

@@ -1,12 +1,11 @@
 # cython: language_level=3
 
 
-cdef class Buffer:
-    cdef:
-        char *ptr
-        Py_buffer buffer
-        size_t nbytes
-        size_t itemsize
-        bint acquired
-
-    cpdef release(self)
+cdef extern from *:
+    """
+    #define PyBytes_RESIZE(b, n) _PyBytes_Resize(&b, n)
+    """
+    int PyBytes_RESIZE(object b, Py_ssize_t n) except -1
+
+
+cpdef memoryview ensure_continguous_memoryview(obj)

numcodecs/compat_ext.pyx

@@ -3,26 +3,17 @@
 # cython: linetrace=False
 # cython: binding=False
 # cython: language_level=3
-from cpython.buffer cimport PyObject_GetBuffer, PyBuffer_Release
 
+from cpython.buffer cimport PyBuffer_IsContiguous
+from cpython.memoryview cimport PyMemoryView_GET_BUFFER
 
-from .compat import ensure_contiguous_ndarray
 
-
-cdef class Buffer:
-    """Convenience class for buffer interface."""
-
-    def __cinit__(self, obj, int flags):
-        PyObject_GetBuffer(obj, &(self.buffer), flags)
-        self.acquired = True
-        self.ptr = <char *> self.buffer.buf
-        self.itemsize = self.buffer.itemsize
-        self.nbytes = self.buffer.len
-
-    cpdef release(self):
-        if self.acquired:
-            PyBuffer_Release(&(self.buffer))
-            self.acquired = False
-
-    def __dealloc__(self):
-        self.release()
+cpdef memoryview ensure_continguous_memoryview(obj):
+    cdef memoryview mv
+    if type(obj) is memoryview:
+        mv = <memoryview>obj
+    else:
+        mv = memoryview(obj)
+    if not PyBuffer_IsContiguous(PyMemoryView_GET_BUFFER(mv), b'A'):
+        raise BufferError("Expected contiguous memory")
+    return mv