Skip to content

Return ndarrays from encode and (where possible) decode #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 29, 2018
Merged

Return ndarrays from encode and (where possible) decode #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
97fc7c9
Ensure `encode` returns a `ndarray`
jakirkham Nov 29, 2018
f1ed57b
Return `ndarray`s from `decode` where possible
jakirkham Nov 29, 2018
90505c9
Regenerate C files from Cython files
jakirkham Nov 29, 2018
97c6b18
release notes [ci skip]
alimanfoo Nov 29, 2018
2012553
Merge branch 'master' into ret_ndarray_encode_decode
alimanfoo Nov 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/release.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ Release notes
code readability and maintainability. By :user:`John Kirkham <jakirkham>` and
:user:`Alistair Miles <alimanfoo>`; :issue:`119`, :issue:`121`, :issue:`128`.

* Return values from encode() and decode() methods are now returned as numpy
arrays for consistency across codecs. By :user:`John Kirkham <jakirkham>`,
:issue:`136`.

* Improvements to handling of errors in the :class:`numcodecs.blosc.Blosc` and
:class:`numcodecs.lz4.LZ4` codecs when the maximum allowed size of an input
buffer is exceeded. By :user:`Jerome Kelleher <jeromekelleher>`, :issue:`80`,
Expand Down
300 changes: 184 additions & 116 deletions numcodecs/blosc.c
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 5 additions & 3 deletions numcodecs/blosc.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING

from .compat_ext cimport Buffer
from .compat_ext import Buffer
from .compat import PY2, text_type, ensure_contiguous_ndarray
from .compat import PY2, text_type, ensure_ndarray, ensure_contiguous_ndarray
from .abc import Codec


Expand Down Expand Up @@ -488,11 +488,13 @@ class Blosc(Codec):

def encode(self, buf):
buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
return compress(buf, self._cname_bytes, self.clevel, self.shuffle, self.blocksize)
out = compress(buf, self._cname_bytes, self.clevel, self.shuffle, self.blocksize)
return ensure_ndarray(out)

def decode(self, buf, out=None):
buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
return decompress(buf, out)
out = decompress(buf, out)
return ensure_ndarray(out)

def __repr__(self):
r = '%s(cname=%r, clevel=%r, shuffle=%s, blocksize=%s)' % \
Expand Down
4 changes: 2 additions & 2 deletions numcodecs/bz2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@


from numcodecs.abc import Codec
from numcodecs.compat import ndarray_copy, ensure_contiguous_ndarray
from numcodecs.compat import ndarray_copy, ensure_ndarray, ensure_contiguous_ndarray


class BZ2(Codec):
Expand All @@ -28,7 +28,7 @@ def encode(self, buf):
buf = ensure_contiguous_ndarray(buf)

# do compression
return _bz2.compress(buf, self.level)
return ensure_ndarray(_bz2.compress(buf, self.level))

# noinspection PyMethodMayBeStatic
def decode(self, buf, out=None):
Expand Down
222 changes: 137 additions & 85 deletions numcodecs/lz4.c
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions numcodecs/lz4.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING

from .compat_ext cimport Buffer
from .compat_ext import Buffer
from .compat import PY2, ensure_contiguous_ndarray
from .compat import PY2, ensure_ndarray, ensure_contiguous_ndarray
from .abc import Codec


Expand Down Expand Up @@ -219,11 +219,11 @@ class LZ4(Codec):

def encode(self, buf):
buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
return compress(buf, self.acceleration)
return ensure_ndarray(compress(buf, self.acceleration))

def decode(self, buf, out=None):
buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
return decompress(buf, out)
return ensure_ndarray(decompress(buf, out))

def __repr__(self):
r = '%s(acceleration=%r)' % \
Expand Down
6 changes: 3 additions & 3 deletions numcodecs/lzma.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
if _lzma:

from .abc import Codec
from .compat import ndarray_copy, ensure_contiguous_ndarray
from .compat import ndarray_copy, ensure_ndarray, ensure_contiguous_ndarray

# noinspection PyShadowingBuiltins
class LZMA(Codec):
Expand Down Expand Up @@ -51,8 +51,8 @@ def encode(self, buf):
buf = ensure_contiguous_ndarray(buf)

# do compression
return _lzma.compress(buf, format=self.format, check=self.check,
preset=self.preset, filters=self.filters)
return ensure_ndarray(_lzma.compress(buf, format=self.format, check=self.check,
preset=self.preset, filters=self.filters))

def decode(self, buf, out=None):

Expand Down
107 changes: 67 additions & 40 deletions numcodecs/vlen.c
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions numcodecs/vlen.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import numpy as np
from .abc import Codec
from .compat_ext cimport Buffer
from .compat_ext import Buffer
from .compat import ensure_contiguous_ndarray
from .compat import ensure_ndarray, ensure_contiguous_ndarray
from cpython cimport (PyBytes_GET_SIZE, PyBytes_AS_STRING, PyBytes_Check,
PyBytes_FromStringAndSize, PyUnicode_AsUTF8String)
from cpython.buffer cimport PyBUF_ANY_CONTIGUOUS
Expand Down Expand Up @@ -130,7 +130,7 @@ class VLenUTF8(Codec):
memcpy(data, encv, l)
data += l

return out
return ensure_ndarray(out)

@cython.wraparound(False)
@cython.boundscheck(False)
Expand Down
4 changes: 2 additions & 2 deletions numcodecs/zlib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@


from .abc import Codec
from .compat import ndarray_copy, ensure_contiguous_ndarray
from .compat import ndarray_copy, ensure_ndarray, ensure_contiguous_ndarray


class Zlib(Codec):
Expand All @@ -28,7 +28,7 @@ def encode(self, buf):
buf = ensure_contiguous_ndarray(buf)

# do compression
return _zlib.compress(buf, self.level)
return ensure_ndarray(_zlib.compress(buf, self.level))

# noinspection PyMethodMayBeStatic
def decode(self, buf, out=None):
Expand Down
Loading