Fixes for Dataset.to_netcdf (#1368)

Fixes GH1320 (writing to file-like objects)
Fixed GH1321 (explicitly setting ``engine='scipy'``)

Author: shoyer

doc/whats-new.rst

@@ -26,9 +26,20 @@ Enhancements
   data in distributed memory (:issue:`1344`).
   By `Matthew Rocklin <https://github.com/mrocklin>`_.
 
+- New :py:meth:`~xarray.DataArray.expand_dims` method for ``DataArray`` and
+  ``Dataset`` (:issue:`1326`).
+  By `Keisuke Fujii <https://github.com/fujiisoup>`_.
+
 Bug fixes
 ~~~~~~~~~
 
+- Fixed writing to file-like objects with :py:meth:`~xarray.Dataset.to_netcdf`
+  (:issue:`1320`).
+  `Stephan Hoyer <https://github.com/shoyer>`_.
+- Fixed explicitly setting ``engine='scipy'`` with ``to_netcdf`` when not
+  providing a path (:issue:`1321`).
+  `Stephan Hoyer <https://github.com/shoyer>`_.
+
 - Fixed open_dataarray does not pass properly its parameters to open_dataset
   (:issue:`1359`).
   `Stephan Hoyer <https://github.com/shoyer>`_.
@@ -47,8 +58,6 @@ The minor release includes bug-fixes and backwards compatible enhancements.
 
 Enhancements
 ~~~~~~~~~~~~
-- `expand_dims` on DataArray is newly supported (:issue:`1326`)
-  By `Keisuke Fujii <https://github.com/fujiisoup>`_.
 
 - ``rolling`` on Dataset is now supported (:issue:`859`).
 

xarray/backends/api.py

@@ -533,7 +533,7 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT,
                     'h5netcdf': backends.H5NetCDFStore}
 
 
-def to_netcdf(dataset, path=None, mode='w', format=None, group=None,
+def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None,
               engine=None, writer=None, encoding=None, unlimited_dims=None):
     """This function creates an appropriate datastore for writing a dataset to
     disk as a netCDF file
@@ -544,18 +544,19 @@ def to_netcdf(dataset, path=None, mode='w', format=None, group=None,
     """
     if encoding is None:
         encoding = {}
-    if path is None:
-        path = BytesIO()
+    if path_or_file is None:
         if engine is None:
             engine = 'scipy'
-        elif engine is not None:
+        elif engine != 'scipy':
             raise ValueError('invalid engine for creating bytes with '
                              'to_netcdf: %r. Only the default engine '
                              "or engine='scipy' is supported" % engine)
-    else:
+    elif isinstance(path_or_file, basestring):
         if engine is None:
-            engine = _get_default_engine(path)
-        path = _normalize_path(path)
+            engine = _get_default_engine(path_or_file)
+        path_or_file = _normalize_path(path_or_file)
+    else:  # file-like object
+        engine = 'scipy'
 
     # validate Dataset keys, DataArray names, and attr keys/values
     _validate_dataset_names(dataset)
@@ -572,17 +573,18 @@ def to_netcdf(dataset, path=None, mode='w', format=None, group=None,
     # if a writer is provided, store asynchronously
     sync = writer is None
 
-    store = store_cls(path, mode, format, group, writer)
+    target = path_or_file if path_or_file is not None else BytesIO()
+    store = store_cls(target, mode, format, group, writer)
 
     if unlimited_dims is None:
         unlimited_dims = dataset.encoding.get('unlimited_dims', None)
     try:
         dataset.dump_to_store(store, sync=sync, encoding=encoding,
                               unlimited_dims=unlimited_dims)
-        if isinstance(path, BytesIO):
-            return path.getvalue()
+        if path_or_file is None:
+            return target.getvalue()
     finally:
-        if sync:
+        if sync and isinstance(path_or_file, basestring):
             store.close()
 
     if not sync:

xarray/core/dataset.py

@@ -921,6 +921,7 @@ def dump_to_store(self, store, encoder=None, sync=True, encoding=None,
         store.store(variables, attrs, check_encoding,
                     unlimited_dims=unlimited_dims)
         if sync:
+            print('syncing')
             store.sync()
 
     def to_netcdf(self, path=None, mode='w', format=None, group=None,
@@ -929,11 +930,12 @@ def to_netcdf(self, path=None, mode='w', format=None, group=None,
 
         Parameters
         ----------
-        path : str, optional
-            Path to which to save this dataset. If no path is provided, this
-            function returns the resulting netCDF file as a bytes object; in
-            this case, we need to use scipy.io.netcdf, which does not support
-            netCDF version 4 (the default format becomes NETCDF3_64BIT).
+        path : str or file-like object, optional
+            Path to which to save this dataset. File-like objects are only
+            supported by the scipy engine. If no path is provided, this
+            function returns the resulting netCDF file as bytes; in this case,
+            we need to use scipy, which does not support netCDF version 4 (the
+            default format becomes NETCDF3_64BIT).
         mode : {'w', 'a'}, optional
             Write ('w') or append ('a') mode. If mode='w', any existing file at
             this location will be overwritten.

xarray/tests/test_backends.py

@@ -814,6 +814,10 @@ def roundtrip(self, data, save_kwargs={}, open_kwargs={},
                           autoclose=self.autoclose, **open_kwargs) as ds:
             yield ds
 
+    def test_to_netcdf_explicit_engine(self):
+        # regression test for GH1321
+        Dataset({'foo': 42}).to_netcdf(engine='scipy')
+
     @pytest.mark.skipif(PY2, reason='cannot pickle BytesIO on Python 2')
     def test_bytesio_pickle(self):
         data = Dataset({'foo': ('x', [1, 2, 3])})
@@ -828,7 +832,33 @@ class ScipyInMemoryDataTestAutocloseTrue(ScipyInMemoryDataTest):
 
 
 @requires_scipy
-class ScipyOnDiskDataTest(CFEncodedDataTest, Only32BitTypes, TestCase):
+class ScipyFileObjectTest(CFEncodedDataTest, Only32BitTypes, TestCase):
+    @contextlib.contextmanager
+    def create_store(self):
+        fobj = BytesIO()
+        yield backends.ScipyDataStore(fobj, 'w')
+
+    @contextlib.contextmanager
+    def roundtrip(self, data, save_kwargs={}, open_kwargs={},
+                  allow_cleanup_failure=False):
+        with create_tmp_file() as tmp_file:
+            with open(tmp_file, 'wb') as f:
+                data.to_netcdf(f, **save_kwargs)
+            with open(tmp_file, 'rb') as f:
+                with open_dataset(f, engine='scipy', **open_kwargs) as ds:
+                    yield ds
+
+    @pytest.mark.skip(reason='cannot pickle file objects')
+    def test_pickle(self):
+        pass
+
+    @pytest.mark.skip(reason='cannot pickle file objects')
+    def test_pickle_dataarray(self):
+        pass
+
+
+@requires_scipy
+class ScipyFilePathTest(CFEncodedDataTest, Only32BitTypes, TestCase):
     @contextlib.contextmanager
     def create_store(self):
         with create_tmp_file() as tmp_file:
@@ -868,7 +898,7 @@ def test_netcdf3_endianness(self):
             self.assertTrue(var.dtype.isnative)
 
 
-class ScipyOnDiskDataTestAutocloseTrue(ScipyOnDiskDataTest):
+class ScipyFilePathTestAutocloseTrue(ScipyFilePathTest):
     autoclose = True