Starter property-based test suite

Author: Zac-HD

.gitignore

@@ -1,6 +1,9 @@
 *.py[cod]
 __pycache__
 
+# example caches from Hypothesis
+.hypothesis/
+
 # temp files from docs build
 doc/auto_gallery
 doc/example.nc

.travis.yml

@@ -49,6 +49,8 @@ matrix:
     env: CONDA_ENV=py36-netcdftime-dev
   - python: 3.5
     env: CONDA_ENV=docs
+  - python: 3.6
+    env: CONDA_ENV=py36-hypothesis
   allow_failures:
   - python: 3.6
     env:
@@ -108,6 +110,8 @@ script:
   - if [[ "$CONDA_ENV" == "docs" ]]; then
       conda install -c conda-forge sphinx_rtd_theme;
       sphinx-build -n -b html -d _build/doctrees doc _build/html;
+    elif [[ "$CONDA_ENV" == "py36-hypothesis" ]]; then
+      pytest properties ;
     else
       py.test xarray --cov=xarray --cov-config ci/.coveragerc --cov-report term-missing --verbose $EXTRA_FLAGS;
     fi

ci/requirements-py36-hypothesis.yml

@@ -0,0 +1,27 @@
+name: test_env
+channels:
+  - conda-forge
+dependencies:
+  - python=3.6
+  - dask
+  - distributed
+  - h5py
+  - h5netcdf
+  - matplotlib
+  - netcdf4
+  - pytest
+  - flake8
+  - numpy
+  - pandas
+  - scipy
+  - seaborn
+  - toolz
+  - rasterio
+  - bottleneck
+  - zarr
+  - pip:
+    - coveralls
+    - pytest-cov
+    - pydap
+    - lxml
+    - hypothesis

properties/README.md

@@ -0,0 +1,22 @@
+# Property-based tests using Hypothesis
+
+This directory contains property-based tests using a library
+called [Hypothesis](https://github.com/HypothesisWorks/hypothesis-python).
+
+The property tests for Xarray are a work in progress - more are always welcome.
+They are stored in a separate directory because they tend to run more examples
+and thus take longer, and so that local development can run a test suite
+without needing to `pip install hypothesis`.
+
+## Hang on, "property-based" tests?
+
+Instead of making assertions about operations on a particular piece of
+data, you use Hypothesis to describe a *kind* of data, then make assertions
+that should hold for *any* example of this kind.
+
+For example: "given a 2d ndarray of dtype uint8 `arr`,
+`xr.DataArray(arr).plot.imshow()` never raises an exception".
+
+Hypothesis will then try many random examples, and report a minimised
+failing input for each error it finds.
+[See the docs for more info.](https://hypothesis.readthedocs.io/en/master/)

properties/test_encode_decode.py

@@ -0,0 +1,46 @@
+"""
+Property-based tests for encoding/decoding methods.
+
+These ones pass, just as you'd hope!
+
+"""
+from __future__ import absolute_import, division, print_function
+
+from hypothesis import given, settings
+import hypothesis.strategies as st
+import hypothesis.extra.numpy as npst
+
+import xarray as xr
+
+# Run for a while - arrays are a bigger search space than usual
+settings.deadline = None
+
+
+an_array = npst.arrays(
+    dtype=st.one_of(
+        npst.unsigned_integer_dtypes(),
+        npst.integer_dtypes(),
+        npst.floating_dtypes(),
+    ),
+    shape=npst.array_shapes(max_side=3),  # max_side specified for performance
+)
+
+
+@given(st.data(), an_array)
+def test_CFMask_coder_roundtrip(data, arr):
+    names = data.draw(st.lists(st.text(), min_size=arr.ndim,
+                               max_size=arr.ndim, unique=True).map(tuple))
+    original = xr.Variable(names, arr)
+    coder = xr.coding.variables.CFMaskCoder()
+    roundtripped = coder.decode(coder.encode(original))
+    xr.testing.assert_identical(original, roundtripped)
+
+
+@given(st.data(), an_array)
+def test_CFScaleOffset_coder_roundtrip(data, arr):
+    names = data.draw(st.lists(st.text(), min_size=arr.ndim,
+                               max_size=arr.ndim, unique=True).map(tuple))
+    original = xr.Variable(names, arr)
+    coder = xr.coding.variables.CFScaleOffsetCoder()
+    roundtripped = coder.decode(coder.encode(original))
+    xr.testing.assert_identical(original, roundtripped)

properties/test_plotting.py

@@ -0,0 +1,53 @@
+"""
+Property-based tests for plotting methods.
+
+Note that Hypothesis can find a 2x2 array that causes an internal error
+for *every plot method tested*.  Most are inside matplotlib.  Good luck.
+
+"""
+from __future__ import absolute_import, division, print_function
+
+import pytest
+
+from hypothesis import given, settings
+import hypothesis.strategies as st
+import hypothesis.extra.numpy as npst
+
+import xarray as xr
+
+# Run for a while - arrays are a bigger search space than usual
+settings.deadline = None
+
+
+two_dimensional_array = npst.arrays(
+    dtype=st.one_of(
+        npst.unsigned_integer_dtypes(),
+        npst.integer_dtypes(),
+        npst.floating_dtypes(),
+    ),
+    shape=npst.array_shapes(min_dims=2, max_dims=2, min_side=2, max_side=4),
+)
+
+
+@pytest.mark.xfail
+@given(two_dimensional_array)
+def test_imshow(arr):
+    xr.DataArray(arr).plot.imshow()
+
+
+@pytest.mark.xfail
+@given(two_dimensional_array)
+def test_pcolormesh(arr):
+    xr.DataArray(arr).plot.pcolormesh()
+
+
+@pytest.mark.xfail
+@given(two_dimensional_array)
+def test_contour(arr):
+    xr.DataArray(arr).plot.contour()
+
+
+@pytest.mark.xfail
+@given(two_dimensional_array)
+def test_contourf(arr):
+    xr.DataArray(arr).plot.contourf()