Merge branch 'master' into dt_accessor

Author: Daniel Rothenberg

conftest.py

@@ -5,5 +5,3 @@ def pytest_addoption(parser):
     """Add command-line flags for pytest."""
     parser.addoption("--run-flaky", action="store_true",
                      help="runs flaky tests")
-    parser.addoption("--skip-slow", action="store_true",
-                     help="skips slow tests")

doc/api.rst

@@ -80,6 +80,7 @@ Dataset contents
    Dataset.merge
    Dataset.rename
    Dataset.swap_dims
+   Dataset.expand_dims
    Dataset.drop
    Dataset.set_coords
    Dataset.reset_coords
@@ -223,6 +224,7 @@ DataArray contents
    DataArray.pipe
    DataArray.rename
    DataArray.swap_dims
+   DataArray.expand_dims
    DataArray.drop
    DataArray.reset_coords
    DataArray.copy
@@ -422,6 +424,7 @@ Dataset methods
    Dataset.from_dict
    Dataset.close
    Dataset.compute
+   Dataset.persist
    Dataset.load
    Dataset.chunk
    Dataset.filter_by_attrs
@@ -447,6 +450,7 @@ DataArray methods
    DataArray.from_cdms2
    DataArray.from_dict
    DataArray.compute
+   DataArray.persist
    DataArray.load
    DataArray.chunk
 

doc/dask.rst

@@ -144,12 +144,23 @@ Explicit conversion by wrapping a DataArray with ``np.asarray`` also works:
             [  1.337e+00,  -1.531e+00, ...,   8.726e-01,  -1.538e+00],
             ...
 
-With the current version of dask, there is no automatic alignment of chunks when
-performing operations between dask arrays with different chunk sizes. If your
-computation involves multiple dask arrays with different chunks, you may need to
-explicitly rechunk each array to ensure compatibility. With xarray, both
-converting data to a dask arrays and converting the chunk sizes of dask arrays
-is done with the :py:meth:`~xarray.Dataset.chunk` method:
+Alternatively you can load the data into memory but keep the arrays as
+dask arrays using the `~xarray.Dataset.persist` method:
+
+.. ipython::
+
+   ds = ds.persist()
+
+This is particularly useful when using a distributed cluster because the data
+will be loaded into distributed memory across your machines and be much faster
+to use than reading repeatedly from disk.  Warning that on a single machine
+this operation will try to load all of your data into memory.  You should make
+sure that your dataset is not larger than available memory.
+
+For performance you may wish to consider chunk sizes.  The correct choice of
+chunk size depends both on your data and on the operations you want to perform.
+With xarray, both converting data to a dask arrays and converting the chunk
+sizes of dask arrays is done with the :py:meth:`~xarray.Dataset.chunk` method:
 
 .. ipython:: python
     :suppress:
@@ -226,6 +237,7 @@ larger chunksizes.
     import os
     os.remove('example-data.nc')
 
+
 Optimization Tips
 -----------------
 

doc/faq.rst

@@ -140,18 +140,24 @@ If you are using xarray and would like to cite it in academic publication, we
 would certainly appreciate it. We recommend two citations.
 
   1. At a minimum, we recommend citing the xarray overview journal article,
-     submitted to the Journal of Open Research Software.
+     published in the Journal of Open Research Software.
 
-     - Hoyer, S., Hamman, J. (In revision). Xarray: N-D labeled arrays and
-       datasets in Python. Journal of Open Research Software.
+     - Hoyer, S. & Hamman, J., (2017). xarray: N-D labeled Arrays and
+       Datasets in Python. Journal of Open Research Software. 5(1), p.10.
+       DOI: http://doi.org/10.5334/jors.148
 
        Here’s an example of a BibTeX entry::
 
            @article{hoyer2017xarray,
-             title   = {xarray: {N-D} labeled arrays and datasets in {Python}},
-             author  = {Hoyer, S. and J. Hamman},
-             journal = {In revision, J. Open Res. Software},
-             year    = {2017}
+             title     = {xarray: {N-D} labeled arrays and datasets in {Python}},
+             author    = {Hoyer, S. and J. Hamman},
+             journal   = {Journal of Open Research Software},
+             volume    = {5},
+             number    = {1},
+             year      = {2017},
+             publisher = {Ubiquity Press},
+             doi       = {10.5334/jors.148},
+             url       = {http://doi.org/10.5334/jors.148}
            }
 
   2. You may also want to cite a specific version of the xarray package. We

doc/index.rst

@@ -57,11 +57,15 @@ Documentation
 See also
 --------
 
+- Stephan Hoyer and Joe Hamman's `Journal of Open Research Software paper`_ describing the xarray project.
+- The `UW eScience Institute's Geohackweek`_ tutorial on xarray for geospatial data scientists.
 - Stephan Hoyer's `SciPy2015 talk`_ introducing xarray to a general audience.
 - Stephan Hoyer's `2015 Unidata Users Workshop talk`_ and `tutorial`_ (`with answers`_) introducing
   xarray to users familiar with netCDF.
 - `Nicolas Fauchereau's tutorial`_ on xarray for netCDF users.
 
+.. _Journal of Open Research Software paper: http://doi.org/10.5334/jors.148
+.. _UW eScience Institute's Geohackweek : https://geohackweek.github.io/nDarrays/
 .. _SciPy2015 talk: https://www.youtube.com/watch?v=X0pAhJgySxk
 .. _2015 Unidata Users Workshop talk: https://www.youtube.com/watch?v=J9ypQOnt5l8
 .. _tutorial: https://github.com/Unidata/unidata-users-workshop/blob/master/notebooks/xray-tutorial.ipynb

doc/reshaping.rst

@@ -27,6 +27,28 @@ on a :py:class:`~xarray.Dataset`, use :py:meth:`~xarray.DataArray.transpose` or
     ds.transpose('y', 'z', 'x')
     ds.T
 
+Expand and squeeze dimensions
+-----------------------------
+
+To expand a :py:class:`~xarray.DataArray` or all
+variables on a :py:class:`~xarray.Dataset` along a new dimension,
+use :py:meth:`~xarray.DataArray.expand_dims`
+
+.. ipython:: python
+
+    expanded  = ds.expand_dims('w')
+    expanded
+
+This method attaches a new dimension with size 1 to all data variables.
+
+To remove such a size-1 dimension from the :py:class:`~xarray.DataArray`
+or :py:class:`~xarray.Dataset`,
+use :py:meth:`~xarray.DataArray.squeeze`
+
+.. ipython:: python
+
+    expanded.squeeze('w')
+
 Converting between datasets and arrays
 --------------------------------------
 

doc/whats-new.rst

@@ -22,17 +22,17 @@ v0.9.3 (unreleased)
 Enhancements
 ~~~~~~~~~~~~
 
-- Add ``.dt`` accessor to DataArrays for computing datetime-like properties
-  for the values they contain, similar to ``pandas.Series`` (:issue:`358`).
-  By `Daniel Rothenberg <https://github.com/darothen>`_.
-
 - Add ``.persist()`` method to Datasets and DataArrays to enable persisting
   data in distributed memory (:issue:`1344`).
   By `Matthew Rocklin <https://github.com/mrocklin>`_.
 
 - New :py:meth:`~xarray.DataArray.expand_dims` method for ``DataArray`` and
   ``Dataset`` (:issue:`1326`).
   By `Keisuke Fujii <https://github.com/fujiisoup>`_.
+  
+- Add ``.dt`` accessor to DataArrays for computing datetime-like properties
+  for the values they contain, similar to ``pandas.Series`` (:issue:`358`).
+  By `Daniel Rothenberg <https://github.com/darothen>`_.
 
 Bug fixes
 ~~~~~~~~~

xarray/backends/api.py

@@ -318,12 +318,8 @@ def maybe_decode_store(store, lock=False):
     return maybe_decode_store(store)
 
 
-def open_dataarray(filename_or_obj, group=None, decode_cf=True,
-                   mask_and_scale=True, decode_times=True,
-                   concat_characters=True, decode_coords=True, engine=None,
-                   chunks=None, lock=None, cache=None, drop_variables=None):
-    """
-    Opens an DataArray from a netCDF file containing a single data variable.
+def open_dataarray(*args, **kwargs):
+    """Open an DataArray from a netCDF file containing a single data variable.
 
     This is designed to read netCDF files with only one data variable. If
     multiple variables are present then a ValueError is raised.
@@ -353,6 +349,10 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True,
     decode_times : bool, optional
         If True, decode times encoded in the standard NetCDF datetime format
         into datetime objects. Otherwise, leave them encoded as numbers.
+    autoclose : bool, optional
+        If True, automatically close files to avoid OS Error of too many files
+        being open.  However, this option doesn't work with streams, e.g.,
+        BytesIO.
     concat_characters : bool, optional
         If True, concatenate along the last dimension of character arrays to
         form string arrays. Dimensions will only be concatenated over (and
@@ -400,10 +400,7 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True,
     --------
     open_dataset
     """
-    dataset = open_dataset(filename_or_obj, group, decode_cf,
-                           mask_and_scale, decode_times,
-                           concat_characters, decode_coords, engine,
-                           chunks, lock, cache, drop_variables)
+    dataset = open_dataset(*args, **kwargs)
 
     if len(dataset.data_vars) != 1:
         raise ValueError('Given file dataset contains more than one data '
@@ -536,7 +533,7 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT,
                     'h5netcdf': backends.H5NetCDFStore}
 
 
-def to_netcdf(dataset, path=None, mode='w', format=None, group=None,
+def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None,
               engine=None, writer=None, encoding=None, unlimited_dims=None):
     """This function creates an appropriate datastore for writing a dataset to
     disk as a netCDF file
@@ -547,18 +544,19 @@ def to_netcdf(dataset, path=None, mode='w', format=None, group=None,
     """
     if encoding is None:
         encoding = {}
-    if path is None:
-        path = BytesIO()
+    if path_or_file is None:
         if engine is None:
             engine = 'scipy'
-        elif engine is not None:
+        elif engine != 'scipy':
             raise ValueError('invalid engine for creating bytes with '
                              'to_netcdf: %r. Only the default engine '
                              "or engine='scipy' is supported" % engine)
-    else:
+    elif isinstance(path_or_file, basestring):
         if engine is None:
-            engine = _get_default_engine(path)
-        path = _normalize_path(path)
+            engine = _get_default_engine(path_or_file)
+        path_or_file = _normalize_path(path_or_file)
+    else:  # file-like object
+        engine = 'scipy'
 
     # validate Dataset keys, DataArray names, and attr keys/values
     _validate_dataset_names(dataset)
@@ -575,17 +573,18 @@ def to_netcdf(dataset, path=None, mode='w', format=None, group=None,
     # if a writer is provided, store asynchronously
     sync = writer is None
 
-    store = store_cls(path, mode, format, group, writer)
+    target = path_or_file if path_or_file is not None else BytesIO()
+    store = store_cls(target, mode, format, group, writer)
 
     if unlimited_dims is None:
         unlimited_dims = dataset.encoding.get('unlimited_dims', None)
     try:
         dataset.dump_to_store(store, sync=sync, encoding=encoding,
                               unlimited_dims=unlimited_dims)
-        if isinstance(path, BytesIO):
-            return path.getvalue()
+        if path_or_file is None:
+            return target.getvalue()
     finally:
-        if sync:
+        if sync and isinstance(path_or_file, basestring):
             store.close()
 
     if not sync:

xarray/backends/netcdf3.py

@@ -6,7 +6,7 @@
 import numpy as np
 
 from .. import conventions, Variable
-from ..core import ops
+from ..core import duck_array_ops
 from ..core.pycompat import basestring, unicode_type, OrderedDict
 
 
@@ -45,7 +45,7 @@ def coerce_nc3_dtype(arr):
         if ((('int' in dtype or 'U' in dtype) and
                 not (cast_arr == arr).all()) or
                 ('float' in dtype and
-                    not ops.allclose_or_equiv(cast_arr, arr))):
+                    not duck_array_ops.allclose_or_equiv(cast_arr, arr))):
             raise ValueError('could not safely cast array from dtype %s to %s'
                              % (dtype, new_dtype))
         arr = cast_arr

xarray/conventions.py

@@ -11,7 +11,7 @@
 from collections import defaultdict
 from pandas.tslib import OutOfBoundsDatetime
 
-from .core import indexing, ops, utils
+from .core import duck_array_ops, indexing, ops, utils
 from .core.formatting import format_timestamp, first_n_items, last_item
 from .core.variable import as_variable, Variable
 from .core.pycompat import iteritems, OrderedDict, PY3, basestring
@@ -632,7 +632,7 @@ def maybe_encode_dtype(var, name=None):
                                   'point data as an integer dtype without '
                                   'any _FillValue to use for NaNs' % name,
                                   RuntimeWarning, stacklevel=3)
-                data = ops.around(data)[...]
+                data = duck_array_ops.around(data)[...]
             if dtype == 'S1' and data.dtype != 'S1':
                 data = string_to_char(np.asarray(data, 'S'))
                 dims = dims + ('string%s' % data.shape[-1],)