From cb55c45a3e5c4fdccfc0d090d10c9a35e3d92e60 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Mon, 21 Aug 2017 19:47:16 +0200 Subject: [PATCH 01/23] Add pathlib support --- xarray/backends/api.py | 12 ++++++++++++ xarray/tests/test_backends.py | 37 ++++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 616471ca7aa..2484878c808 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -6,6 +6,8 @@ from glob import glob from io import BytesIO from numbers import Number +from pathlib import Path +from types import GeneratorType import numpy as np @@ -253,6 +255,9 @@ def maybe_decode_store(store, lock=False): return ds2 + if isinstance(filename_or_obj, Path): + filename_or_obj = str(filename_or_obj) + if isinstance(filename_or_obj, backends.AbstractDataStore): store = filename_or_obj elif isinstance(filename_or_obj, basestring): @@ -494,8 +499,15 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, auto_combine open_dataset """ + # handle output of pathlib.Path.glob() + if isinstance(paths, GeneratorType): + paths = list(paths) + if isinstance(paths[0], Path): + paths = sorted(str(p) for p in paths) + if isinstance(paths, basestring): paths = sorted(glob(paths)) + if not paths: raise IOError('no files to open') diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index f5bd615cfe2..bc907afd8d3 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -6,6 +6,7 @@ import contextlib import itertools import os.path +from pathlib import Path import pickle import shutil import tempfile @@ -49,6 +50,11 @@ def open_example_dataset(name, *args, **kwargs): *args, **kwargs) +def open_example_dataset_pathlib(name, *args, **kwargs): + return open_dataset(Path(os.path.dirname(__file__)) / 'data' / name, + *args, **kwargs) + + def create_masked_and_scaled_data(): x = np.array([np.nan, np.nan, 10, 10.1, 10.2]) encoding = {'_FillValue': -1, 'add_offset': 10, @@ -315,6 +321,15 @@ def test_roundtrip_example_1_netcdf(self): # a dtype attribute. self.assertDatasetEqual(expected, actual) + def test_roundtrip_example_1_netcdf_pathlib(self): + expected = open_example_dataset_pathlib('example_1.nc') + with self.roundtrip(expected) as actual: + # we allow the attributes to differ since that + # will depend on the encoding used. For example, + # without CF encoding 'actual' will end up with + # a dtype attribute. + self.assertDatasetEqual(expected, actual) + def test_roundtrip_coordinates(self): original = Dataset({'foo': ('x', [0, 1])}, {'x': [2, 3], 'y': ('a', [42]), 'z': ('x', [4, 5])}) @@ -569,6 +584,20 @@ def create_tmp_file(suffix='.nc', allow_cleanup_failure=False): raise +@contextlib.contextmanager +def create_tmp_file_pathlib(suffix='.nc', allow_cleanup_failure=False): + temp_dir = tempfile.mkdtemp() + path = Path(os.path.join(temp_dir, 'temp-%s%s' % (next(_counter), suffix))) + try: + yield path + finally: + try: + shutil.rmtree(temp_dir) + except OSError: + if not allow_cleanup_failure: + raise + + @contextlib.contextmanager def create_tmp_files(nfiles, suffix='.nc', allow_cleanup_failure=False): with ExitStack() as stack: @@ -1273,10 +1302,12 @@ def test_dataset_caching(self): actual.foo.values # no caching assert not actual.foo.variable._in_memory - def test_open_mfdataset(self): + @pytest.mark.parametrize("_create_tmp_file", [create_tmp_file, + create_tmp_file_pathlib]) + def test_open_mfdataset(self, _create_tmp_file): original = Dataset({'foo': ('x', np.random.randn(10))}) - with create_tmp_file() as tmp1: - with create_tmp_file() as tmp2: + with _create_tmp_file() as tmp1: + with _create_tmp_file() as tmp2: original.isel(x=slice(5)).to_netcdf(tmp1) original.isel(x=slice(5, 10)).to_netcdf(tmp2) with open_mfdataset([tmp1, tmp2], From f9922d674d9061db7637c1ddd0e31004dc1a593d Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Mon, 21 Aug 2017 20:04:01 +0200 Subject: [PATCH 02/23] Loop over tmpfile functions --- xarray/tests/test_backends.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index bc907afd8d3..04bfd2aece4 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1302,24 +1302,24 @@ def test_dataset_caching(self): actual.foo.values # no caching assert not actual.foo.variable._in_memory - @pytest.mark.parametrize("_create_tmp_file", [create_tmp_file, - create_tmp_file_pathlib]) - def test_open_mfdataset(self, _create_tmp_file): + def test_open_mfdataset(self): original = Dataset({'foo': ('x', np.random.randn(10))}) - with _create_tmp_file() as tmp1: - with _create_tmp_file() as tmp2: - original.isel(x=slice(5)).to_netcdf(tmp1) - original.isel(x=slice(5, 10)).to_netcdf(tmp2) - with open_mfdataset([tmp1, tmp2], - autoclose=self.autoclose) as actual: - self.assertIsInstance(actual.foo.variable.data, da.Array) - self.assertEqual(actual.foo.variable.data.chunks, - ((5, 5),)) - self.assertDatasetAllClose(original, actual) - with open_mfdataset([tmp1, tmp2], chunks={'x': 3}, - autoclose=self.autoclose) as actual: - self.assertEqual(actual.foo.variable.data.chunks, - ((3, 2, 3, 2),)) + for _create_tmp_file in [create_tmp_file, create_tmp_file_pathlib]: + with _create_tmp_file() as tmp1: + with _create_tmp_file() as tmp2: + original.isel(x=slice(5)).to_netcdf(tmp1) + original.isel(x=slice(5, 10)).to_netcdf(tmp2) + with open_mfdataset([tmp1, tmp2], + autoclose=self.autoclose) as actual: + self.assertIsInstance(actual.foo.variable.data, + da.Array) + self.assertEqual(actual.foo.variable.data.chunks, + ((5, 5),)) + self.assertDatasetAllClose(original, actual) + with open_mfdataset([tmp1, tmp2], chunks={'x': 3}, + autoclose=self.autoclose) as actual: + self.assertEqual(actual.foo.variable.data.chunks, + ((3, 2, 3, 2),)) with self.assertRaisesRegexp(IOError, 'no files to open'): open_mfdataset('foo-bar-baz-*.nc', autoclose=self.autoclose) From 02023edc7178ea9cb1fd64169afa659b82dfd67c Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 23 Aug 2017 11:01:49 -0500 Subject: [PATCH 03/23] Added show_commit_url to asv.conf (#1515) * Added show_commit_url to asv.conf This should setup the proper links from the published output to the commit on Github. FYI the benchmarks should be running stably now, and posted to http://pandas.pydata.org/speed/xarray. http://pandas.pydata.org/speed/xarray/regressions.xml has an RSS feed to the regressions. * Update asv.conf.json --- asv_bench/asv.conf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json index ecbcd250310..a2878a7bf50 100644 --- a/asv_bench/asv.conf.json +++ b/asv_bench/asv.conf.json @@ -36,7 +36,7 @@ "install_timeout": 600, // the base URL to show a commit for the project. - // "show_commit_url": "http://github.com/owner/project/commit/", + "show_commit_url": "https://github.com/pydata/xarray/commit/", // The Pythons you'd like to test against. If not provided, defaults // to the current version of Python used to run `asv`. From 4276bb81bffb294516b38ddac8a4d87b7ad56888 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Fri, 25 Aug 2017 14:24:37 +0900 Subject: [PATCH 04/23] Small documentation fixes (#1516) * Clarify in docs that inferring DataArray dimensions is deprecated * Fix DataArray docstring * Clarify DataArray coords documentation --- doc/data-structures.rst | 8 +++++--- xarray/core/dataarray.py | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/doc/data-structures.rst b/doc/data-structures.rst index 48c8e2d9874..95f755680df 100644 --- a/doc/data-structures.rst +++ b/doc/data-structures.rst @@ -46,9 +46,11 @@ The :py:class:`~xarray.DataArray` constructor takes: - ``data``: a multi-dimensional array of values (e.g., a numpy ndarray, :py:class:`~pandas.Series`, :py:class:`~pandas.DataFrame` or :py:class:`~pandas.Panel`) -- ``coords``: a list or dictionary of coordinates -- ``dims``: a list of dimension names. If omitted, dimension names are - taken from ``coords`` if possible. +- ``coords``: a list or dictionary of coordinates. If a list, it should be a + list of tuples where the first element is the dimension name and the second + element is the corresponding coordinate array_like object. +- ``dims``: a list of dimension names. If omitted and ``coords`` is a list of + tuples, dimension names are taken from ``coords``. - ``attrs``: a dictionary of attributes to add to the instance - ``name``: a string that names the instance diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 8700446295c..6b4d28e1006 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -175,9 +175,11 @@ def __init__(self, data, coords=None, dims=None, name=None, coords : sequence or dict of array_like objects, optional Coordinates (tick labels) to use for indexing along each dimension. If dict-like, should be a mapping from dimension names to the - corresponding coordinates. + corresponding coordinates. If sequence-like, should be a sequence + of tuples where the first element is the dimension name and the + second element is the corresponding coordinate array_like object. dims : str or sequence of str, optional - Name(s) of the the data dimension(s). Must be either a string (only + Name(s) of the data dimension(s). Must be either a string (only for 1D data) or a sequence of strings with length equal to the number of dimensions. If this argument is omitted, dimension names are taken from ``coords`` (if possible) and otherwise default to From 812a48381a0d7a05cc3427033ad734dc5c6c054e Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 16:16:28 +0200 Subject: [PATCH 05/23] Condense pathlib handling for open_mf_dataset --- xarray/backends/api.py | 12 +++---- xarray/tests/test_backends.py | 59 +++++++++-------------------------- 2 files changed, 19 insertions(+), 52 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 2484878c808..8cf55fcf914 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -499,14 +499,10 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, auto_combine open_dataset """ - # handle output of pathlib.Path.glob() - if isinstance(paths, GeneratorType): - paths = list(paths) - if isinstance(paths[0], Path): - paths = sorted(str(p) for p in paths) - if isinstance(paths, basestring): paths = sorted(glob(paths)) + else: + paths = [str(p) if isinstance(p, Path) else p for p in paths] if not paths: raise IOError('no files to open') @@ -613,7 +609,9 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, mode : {'w', 'a'}, optional Write ('w') or append ('a') mode. If mode='w', any existing file at these locations will be overwritten. - format : {'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', 'NETCDF3_CLASSIC'}, optional + format : {'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', 'NETCDF3_CLASSIC'}, + optional + File format for the resulting netCDF file: * NETCDF4: Data is stored in an HDF5 file, using netCDF4 API diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 04bfd2aece4..f5bd615cfe2 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -6,7 +6,6 @@ import contextlib import itertools import os.path -from pathlib import Path import pickle import shutil import tempfile @@ -50,11 +49,6 @@ def open_example_dataset(name, *args, **kwargs): *args, **kwargs) -def open_example_dataset_pathlib(name, *args, **kwargs): - return open_dataset(Path(os.path.dirname(__file__)) / 'data' / name, - *args, **kwargs) - - def create_masked_and_scaled_data(): x = np.array([np.nan, np.nan, 10, 10.1, 10.2]) encoding = {'_FillValue': -1, 'add_offset': 10, @@ -321,15 +315,6 @@ def test_roundtrip_example_1_netcdf(self): # a dtype attribute. self.assertDatasetEqual(expected, actual) - def test_roundtrip_example_1_netcdf_pathlib(self): - expected = open_example_dataset_pathlib('example_1.nc') - with self.roundtrip(expected) as actual: - # we allow the attributes to differ since that - # will depend on the encoding used. For example, - # without CF encoding 'actual' will end up with - # a dtype attribute. - self.assertDatasetEqual(expected, actual) - def test_roundtrip_coordinates(self): original = Dataset({'foo': ('x', [0, 1])}, {'x': [2, 3], 'y': ('a', [42]), 'z': ('x', [4, 5])}) @@ -584,20 +569,6 @@ def create_tmp_file(suffix='.nc', allow_cleanup_failure=False): raise -@contextlib.contextmanager -def create_tmp_file_pathlib(suffix='.nc', allow_cleanup_failure=False): - temp_dir = tempfile.mkdtemp() - path = Path(os.path.join(temp_dir, 'temp-%s%s' % (next(_counter), suffix))) - try: - yield path - finally: - try: - shutil.rmtree(temp_dir) - except OSError: - if not allow_cleanup_failure: - raise - - @contextlib.contextmanager def create_tmp_files(nfiles, suffix='.nc', allow_cleanup_failure=False): with ExitStack() as stack: @@ -1304,22 +1275,20 @@ def test_dataset_caching(self): def test_open_mfdataset(self): original = Dataset({'foo': ('x', np.random.randn(10))}) - for _create_tmp_file in [create_tmp_file, create_tmp_file_pathlib]: - with _create_tmp_file() as tmp1: - with _create_tmp_file() as tmp2: - original.isel(x=slice(5)).to_netcdf(tmp1) - original.isel(x=slice(5, 10)).to_netcdf(tmp2) - with open_mfdataset([tmp1, tmp2], - autoclose=self.autoclose) as actual: - self.assertIsInstance(actual.foo.variable.data, - da.Array) - self.assertEqual(actual.foo.variable.data.chunks, - ((5, 5),)) - self.assertDatasetAllClose(original, actual) - with open_mfdataset([tmp1, tmp2], chunks={'x': 3}, - autoclose=self.autoclose) as actual: - self.assertEqual(actual.foo.variable.data.chunks, - ((3, 2, 3, 2),)) + with create_tmp_file() as tmp1: + with create_tmp_file() as tmp2: + original.isel(x=slice(5)).to_netcdf(tmp1) + original.isel(x=slice(5, 10)).to_netcdf(tmp2) + with open_mfdataset([tmp1, tmp2], + autoclose=self.autoclose) as actual: + self.assertIsInstance(actual.foo.variable.data, da.Array) + self.assertEqual(actual.foo.variable.data.chunks, + ((5, 5),)) + self.assertDatasetAllClose(original, actual) + with open_mfdataset([tmp1, tmp2], chunks={'x': 3}, + autoclose=self.autoclose) as actual: + self.assertEqual(actual.foo.variable.data.chunks, + ((3, 2, 3, 2),)) with self.assertRaisesRegexp(IOError, 'no files to open'): open_mfdataset('foo-bar-baz-*.nc', autoclose=self.autoclose) From 47be4b7480d3d4380304b7a31e23fc8adad3de53 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 17:01:14 +0200 Subject: [PATCH 06/23] Add and test pathlib support for backends --- xarray/backends/api.py | 15 ++++++++------- xarray/core/dataarray.py | 8 +++++--- xarray/core/dataset.py | 5 +++-- xarray/tests/test_backends.py | 26 ++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 8cf55fcf914..22c724b16f6 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -7,7 +7,6 @@ from io import BytesIO from numbers import Number from pathlib import Path -from types import GeneratorType import numpy as np @@ -141,8 +140,8 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, Parameters ---------- - filename_or_obj : str, file or xarray.backends.*DataStore - Strings are interpreted as a path to a netCDF file or an OpenDAP URL + filename_or_obj : str, Path, file or xarray.backends.*DataStore + Strings and Path objects are interpreted as a path to a netCDF file oran OpenDAP URL and opened with python-netCDF4, unless the filename ends with .gz, in which case the file is gunzipped and opened with scipy.io.netcdf (only netCDF3 supported). File-like objects are opened with scipy.io.netcdf @@ -442,7 +441,7 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, ---------- paths : str or sequence Either a string glob in the form "path/to/my/files/*.nc" or an explicit - list of files to open. + list of files to open. Paths can be given as strings or as pathlib Paths. chunks : int or dict, optional Dictionary with keys given by dimension names and values given by chunk sizes. In general, these should divide the dimensions of each dataset. @@ -540,6 +539,8 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, The ``writer`` argument is only for the private use of save_mfdataset. """ + if isinstance(path_or_file, Path): + path_or_file = str(path_or_file) if encoding is None: encoding = {} if path_or_file is None: @@ -604,13 +605,13 @@ def save_mfdataset(datasets, paths, mode='w', format=None, groups=None, ---------- datasets : list of xarray.Dataset List of datasets to save. - paths : list of str + paths : list of str or list of Paths List of paths to which to save each corresponding dataset. mode : {'w', 'a'}, optional Write ('w') or append ('a') mode. If mode='w', any existing file at these locations will be overwritten. - format : {'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', 'NETCDF3_CLASSIC'}, - optional + format : {'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', + 'NETCDF3_CLASSIC'}, optional File format for the resulting netCDF file: diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 6b4d28e1006..0bde9a4adf3 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1286,7 +1286,7 @@ def to_netcdf(self, *args, **kwargs): Parameters ---------- - path : str, optional + path : str or Path, optional Path to which to save this dataset. If no path is provided, this function returns the resulting netCDF file as a bytes object; in this case, we need to use scipy.io.netcdf, which does not support @@ -1294,7 +1294,8 @@ def to_netcdf(self, *args, **kwargs): mode : {'w', 'a'}, optional Write ('w') or append ('a') mode. If mode='w', any existing file at this location will be overwritten. - format : {'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', 'NETCDF3_CLASSIC'}, optional + format : {'NETCDF4', 'NETCDF4_CLASSIC', 'NETCDF3_64BIT', + 'NETCDF3_CLASSIC'}, optional File format for the resulting netCDF file: * NETCDF4: Data is stored in an HDF5 file, using netCDF4 API @@ -1324,7 +1325,8 @@ def to_netcdf(self, *args, **kwargs): encoding : dict, optional Nested dictionary with variable names as keys and dictionaries of variable specific encodings as values, e.g., - ``{'my_variable': {'dtype': 'int16', 'scale_factor': 0.1, 'zlib': True}, ...}`` + ``{'my_variable': {'dtype': 'int16', 'scale_factor': 0.1, + 'zlib': True}, ...}`` Notes ----- diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index bffdbf10724..aa49d8a73b0 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -924,7 +924,7 @@ def to_netcdf(self, path=None, mode='w', format=None, group=None, Parameters ---------- - path : str or file-like object, optional + path : str, Path or file-like object, optional Path to which to save this dataset. File-like objects are only supported by the scipy engine. If no path is provided, this function returns the resulting netCDF file as bytes; in this case, @@ -963,7 +963,8 @@ def to_netcdf(self, path=None, mode='w', format=None, group=None, encoding : dict, optional Nested dictionary with variable names as keys and dictionaries of variable specific encodings as values, e.g., - ``{'my_variable': {'dtype': 'int16', 'scale_factor': 0.1, 'zlib': True}, ...}`` + ``{'my_variable': {'dtype': 'int16', 'scale_factor': 0.1, + 'zlib': True}, ...}`` unlimited_dims : sequence of str, optional Dimension(s) that should be serialized as unlimited dimensions. By default, no dimensions are treated as unlimited dimensions. diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index f5bd615cfe2..a2d33528626 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -6,6 +6,7 @@ import contextlib import itertools import os.path +from pathlib import Path import pickle import shutil import tempfile @@ -1293,6 +1294,18 @@ def test_open_mfdataset(self): with self.assertRaisesRegexp(IOError, 'no files to open'): open_mfdataset('foo-bar-baz-*.nc', autoclose=self.autoclose) + def test_open_mfdataset_pathlib(self): + original = Dataset({'foo': ('x', np.random.randn(10))}) + with create_tmp_file() as tmp1: + with create_tmp_file() as tmp2: + tmp1 = Path(tmp1) + tmp2 = Path(tmp2) + original.isel(x=slice(5)).to_netcdf(tmp1) + original.isel(x=slice(5, 10)).to_netcdf(tmp2) + with open_mfdataset([tmp1, tmp2], + autoclose=self.autoclose) as actual: + self.assertDatasetAllClose(original, actual) + def test_attrs_mfdataset(self): original = Dataset({'foo': ('x', np.random.randn(10))}) with create_tmp_file() as tmp1: @@ -1343,6 +1356,19 @@ def test_save_mfdataset_invalid(self): with self.assertRaisesRegexp(ValueError, 'same length'): save_mfdataset([ds, ds], ['only one path']) + def test_save_mfdataset_pathlib_roundtrip(self): + original = Dataset({'foo': ('x', np.random.randn(10))}) + datasets = [original.isel(x=slice(5)), + original.isel(x=slice(5, 10))] + with create_tmp_file() as tmp1: + with create_tmp_file() as tmp2: + tmp1 = Path(tmp1) + tmp2 = Path(tmp2) + save_mfdataset(datasets, [tmp1, tmp2]) + with open_mfdataset([tmp1, tmp2], + autoclose=self.autoclose) as actual: + self.assertDatasetIdentical(actual, original) + def test_open_and_do_math(self): original = Dataset({'foo': ('x', np.random.randn(10))}) with create_tmp_file() as tmp: From aac0760e84d8d8e0ea1037882ec9f961394dccf7 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 17:07:00 +0200 Subject: [PATCH 07/23] Add pathlib2 for python < 3 --- ci/requirements-py27-cdat+pynio.yml | 1 + ci/requirements-py27-min.yml | 1 + ci/requirements-py27-windows.yml | 1 + setup.py | 3 +++ 4 files changed, 6 insertions(+) diff --git a/ci/requirements-py27-cdat+pynio.yml b/ci/requirements-py27-cdat+pynio.yml index 113714cbfd6..0258c8c9672 100644 --- a/ci/requirements-py27-cdat+pynio.yml +++ b/ci/requirements-py27-cdat+pynio.yml @@ -13,6 +13,7 @@ dependencies: - netcdf4 - numpy - pandas + - pathlib2 - pynio - pytest - scipy diff --git a/ci/requirements-py27-min.yml b/ci/requirements-py27-min.yml index 7499157dbe9..2be15416dd8 100644 --- a/ci/requirements-py27-min.yml +++ b/ci/requirements-py27-min.yml @@ -1,6 +1,7 @@ name: test_env dependencies: - python=2.7 + - pathlib2 - pytest - numpy==1.9.3 - pandas==0.15.0 diff --git a/ci/requirements-py27-windows.yml b/ci/requirements-py27-windows.yml index cfd3d4262cc..e953b5ffdcb 100644 --- a/ci/requirements-py27-windows.yml +++ b/ci/requirements-py27-windows.yml @@ -9,6 +9,7 @@ dependencies: - h5netcdf - matplotlib - netcdf4 + - pathlib2 - pytest - numpy - pandas diff --git a/setup.py b/setup.py index 7f68adb8aa2..1b0abfe0951 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,9 @@ INSTALL_REQUIRES = ['numpy >= 1.7', 'pandas >= 0.15.0'] TESTS_REQUIRE = ['pytest >= 2.7.1'] +if sys.version_info < (3, 0): + INSTALL_REQUIRES.append('pathlib2') + TESTS_REQUIRE.append('pathlib2') DESCRIPTION = "N-D labeled arrays and datasets in Python" LONG_DESCRIPTION = """ From 3ca8c9eff03dfddfaf99881965438ed1400e58d5 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 17:19:57 +0200 Subject: [PATCH 08/23] Use pathlib backport if available. This follows who argues for sticking to pathlib2. --- xarray/backends/api.py | 6 +++++- xarray/tests/test_backends.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 22c724b16f6..00d1262c3bc 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -6,7 +6,11 @@ from glob import glob from io import BytesIO from numbers import Number -from pathlib import Path +try: + from pathlib2 import Path +except ImportError: + from pathlib import Path + import numpy as np diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a2d33528626..3eb9db779d7 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -6,7 +6,10 @@ import contextlib import itertools import os.path -from pathlib import Path +try: + from pathlib2 import Path +except ImportError: + from pathlib import Path import pickle import shutil import tempfile From aae32a8981d1ee1a576fe54ccdf00678cfe66933 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 17:47:05 +0200 Subject: [PATCH 09/23] Use pathlib w DataArray.to_netcdf --- xarray/backends/api.py | 25 +++++++++++++------------ xarray/tests/test_backends.py | 10 ++++++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 00d1262c3bc..6a2f6be6f81 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -145,11 +145,11 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, Parameters ---------- filename_or_obj : str, Path, file or xarray.backends.*DataStore - Strings and Path objects are interpreted as a path to a netCDF file oran OpenDAP URL - and opened with python-netCDF4, unless the filename ends with .gz, in - which case the file is gunzipped and opened with scipy.io.netcdf (only - netCDF3 supported). File-like objects are opened with scipy.io.netcdf - (only netCDF3 supported). + Strings and Path objects are interpreted as a path to a netCDF file + oran OpenDAP URL and opened with python-netCDF4, unless the filename + ends with .gz, in which case the file is gunzipped and opened with + scipy.io.netcdf (only netCDF3 supported). File-like objects are opened + with scipy.io.netcdf (only netCDF3 supported). group : str, optional Path to the netCDF4 group in the given file to open (only works for netCDF4 files). @@ -325,12 +325,12 @@ def open_dataarray(*args, **kwargs): Parameters ---------- - filename_or_obj : str, file or xarray.backends.*DataStore - Strings are interpreted as a path to a netCDF file or an OpenDAP URL - and opened with python-netCDF4, unless the filename ends with .gz, in - which case the file is gunzipped and opened with scipy.io.netcdf (only - netCDF3 supported). File-like objects are opened with scipy.io.netcdf - (only netCDF3 supported). + filename_or_obj : str, Path, file or xarray.backends.*DataStore + Strings and Paths are interpreted as a path to a netCDF file or an + OpenDAP URL and opened with python-netCDF4, unless the filename ends + with .gz, in which case the file is gunzipped and opened with + scipy.io.netcdf (only netCDF3 supported). File-like objects are opened + with scipy.io.netcdf (only netCDF3 supported). group : str, optional Path to the netCDF4 group in the given file to open (only works for netCDF4 files). @@ -445,7 +445,8 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, ---------- paths : str or sequence Either a string glob in the form "path/to/my/files/*.nc" or an explicit - list of files to open. Paths can be given as strings or as pathlib Paths. + list of files to open. Paths can be given as strings or as pathlib + Paths. chunks : int or dict, optional Dictionary with keys given by dimension names and values given by chunk sizes. In general, these should divide the dimensions of each dataset. diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 3eb9db779d7..f77956ecc6e 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1963,3 +1963,13 @@ def test_open_dataarray_options(self): expected = data.drop('y') with open_dataarray(tmp, drop_variables=['y']) as loaded: self.assertDataArrayIdentical(expected, loaded) + + def test_dataarray_to_netcdf_no_name_pathlib(self): + original_da = DataArray(np.arange(12).reshape((3, 4))) + + with create_tmp_file() as tmp: + tmp = Path(tmp) + original_da.to_netcdf(tmp) + + with open_dataarray(tmp) as loaded_da: + self.assertDataArrayIdentical(original_da, loaded_da) From 2cc69f47fa918b3987bd44d70ce4afc866045111 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 18:46:06 +0200 Subject: [PATCH 10/23] Handle case of completely missing pathlib --- xarray/backends/api.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index 6a2f6be6f81..b28c7d9cfc2 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -7,9 +7,13 @@ from io import BytesIO from numbers import Number try: - from pathlib2 import Path -except ImportError: - from pathlib import Path + try: + from pathlib import Path + except ImportError as e: + from pathlib2 import Path + path_type = (Path, ) +except ImportError as e: + path_type = () import numpy as np @@ -258,7 +262,7 @@ def maybe_decode_store(store, lock=False): return ds2 - if isinstance(filename_or_obj, Path): + if isinstance(filename_or_obj, path_type): filename_or_obj = str(filename_or_obj) if isinstance(filename_or_obj, backends.AbstractDataStore): @@ -506,7 +510,7 @@ def open_mfdataset(paths, chunks=None, concat_dim=_CONCAT_DIM_DEFAULT, if isinstance(paths, basestring): paths = sorted(glob(paths)) else: - paths = [str(p) if isinstance(p, Path) else p for p in paths] + paths = [str(p) if isinstance(p, path_type) else p for p in paths] if not paths: raise IOError('no files to open') @@ -544,7 +548,7 @@ def to_netcdf(dataset, path_or_file=None, mode='w', format=None, group=None, The ``writer`` argument is only for the private use of save_mfdataset. """ - if isinstance(path_or_file, Path): + if isinstance(path_or_file, path_type): path_or_file = str(path_or_file) if encoding is None: encoding = {} From 3033433587dabbdf223e5605bca019b3ebde94b8 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 18:46:27 +0200 Subject: [PATCH 11/23] Remove pathlib requirement --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index 1b0abfe0951..7f68adb8aa2 100644 --- a/setup.py +++ b/setup.py @@ -37,9 +37,6 @@ INSTALL_REQUIRES = ['numpy >= 1.7', 'pandas >= 0.15.0'] TESTS_REQUIRE = ['pytest >= 2.7.1'] -if sys.version_info < (3, 0): - INSTALL_REQUIRES.append('pathlib2') - TESTS_REQUIRE.append('pathlib2') DESCRIPTION = "N-D labeled arrays and datasets in Python" LONG_DESCRIPTION = """ From c8722db27deb6b67842f8e053871ca62683209d7 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 18:46:49 +0200 Subject: [PATCH 12/23] Drop pathlib from minimal test env --- ci/requirements-py27-min.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/requirements-py27-min.yml b/ci/requirements-py27-min.yml index 2be15416dd8..7499157dbe9 100644 --- a/ci/requirements-py27-min.yml +++ b/ci/requirements-py27-min.yml @@ -1,7 +1,6 @@ name: test_env dependencies: - python=2.7 - - pathlib2 - pytest - numpy==1.9.3 - pandas==0.15.0 From aeed776ea6d4dc227b550afcee6eb611ed6cfcd3 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 19:09:39 +0200 Subject: [PATCH 13/23] Add what's-new entry on pathlib support --- doc/whats-new.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 7d06a49d486..88bc32e1c76 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -21,9 +21,38 @@ v0.9.7 (unreleased) Enhancements ~~~~~~~~~~~~ +- Support for `pathlib.Path` objects added to + :py:func:`~xarray.open_dataset`, :py:func:`~xarray.open_mfdataset`, + :py:func:`~xarray.to_netcdf`, and :py:func:`~xarray.save_mfdataset`: + + .. ipython:: + :verbatim: + In [1]: import xarray as xr + + In [2]: from pathlib import Path # In Python 2, use pathlib2! + + In [3]: data_dir = Path("data/") + + In [4]: one_file = data_dir / "dta_for_month_01.nc" + + In [5]: print(xr.open_dataset(one_file)) + Out[5]: + + [...] + + In [6]: all_files = data_dir.glob("dta_for_month_*.nc") + + In [7]: print(xr.open_mfdataset(all_files)) + Out[7]: + + [...] + + By `Willi Rath `_. + - More attributes available in :py:attr:`~xarray.Dataset.attrs` dictionary when raster files are opened with :py:func:`~xarray.open_rasterio`. By `Greg Brener `_ + - Support for NetCDF files using an ``_Unsigned`` attribute to indicate that a a signed integer data type should be interpreted as unsigned bytes (:issue:`1444`). @@ -55,6 +84,8 @@ Enhancements (:issue:`576`). By `Stephan Hoyer `_. + + Bug fixes ~~~~~~~~~ From 137dff21bea20ff56297d29873eba75afb2b049b Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 19:14:50 +0200 Subject: [PATCH 14/23] Prefer stdlib pathlib --- xarray/tests/test_backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index f77956ecc6e..a2ee3fdebc9 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -7,9 +7,9 @@ import itertools import os.path try: - from pathlib2 import Path -except ImportError: from pathlib import Path +except ImportError: + from pathlib2 import Path import pickle import shutil import tempfile From b55b0136730f714cc07ad657c51f7202018dede5 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 19:34:48 +0200 Subject: [PATCH 15/23] Suppress ImportError's for pathlib --- xarray/tests/test_backends.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a2ee3fdebc9..08aecdb2dbd 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -6,10 +6,6 @@ import contextlib import itertools import os.path -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path import pickle import shutil import tempfile @@ -44,6 +40,12 @@ except ImportError: pass +with suppress(ImportError): + try: + from pathlib import Path + except ImportError: + from pathlib2 import Path + ON_WINDOWS = sys.platform == 'win32' From 422615fd960006ed2af9c585da7a4c348dd3cae3 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Fri, 25 Aug 2017 20:05:26 +0200 Subject: [PATCH 16/23] Acutally get suppress function --- xarray/tests/test_backends.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 08aecdb2dbd..ee6926640bb 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -22,7 +22,8 @@ from xarray.backends.common import robust_getitem from xarray.backends.netCDF4_ import _extract_nc4_variable_encoding from xarray.core import indexing -from xarray.core.pycompat import iteritems, PY2, ExitStack, basestring +from xarray.core.pycompat import (iteritems, PY2, ExitStack, basestring, + suppress) from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap, requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf, From 8c9ee31450b371349b2a1f35cc81f3a7bff47f73 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Sat, 26 Aug 2017 21:24:49 +0200 Subject: [PATCH 17/23] Add decorator for tests requiring pathlib(2) --- xarray/tests/__init__.py | 14 ++++++++++++++ xarray/tests/test_backends.py | 23 +++++++++++++++-------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 424484b438a..7afad6ffe92 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -83,6 +83,17 @@ except ImportError: has_rasterio = False +try: + import pathlib + has_pathlib = True +except ImportError: + try: + import pathlib2 + has_pathlib = True + except ImportError: + has_pathlib = False + + # slighly simpler construction that the full functions. # Generally `pytest.importorskip('package')` inline is even easier requires_matplotlib = pytest.mark.skipif( @@ -105,6 +116,9 @@ not has_bottleneck, reason='requires bottleneck') requires_rasterio = pytest.mark.skipif( not has_rasterio, reason='requires rasterio') +requires_pathlib = pytest.mark.skipif( + not has_pathlib, reason='requires pathlib / pathlib2' +) try: diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index ee6926640bb..33f1d33881b 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -22,13 +22,13 @@ from xarray.backends.common import robust_getitem from xarray.backends.netCDF4_ import _extract_nc4_variable_encoding from xarray.core import indexing -from xarray.core.pycompat import (iteritems, PY2, ExitStack, basestring, - suppress) +from xarray.core.pycompat import (iteritems, PY2, ExitStack, basestring) from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap, requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf, - requires_pynio, has_netCDF4, has_scipy, assert_allclose, - flaky, network, requires_rasterio, assert_identical) + requires_pynio, requires_pathlib, has_netCDF4, has_scipy, + assert_allclose, flaky, network, requires_rasterio, + assert_identical) from .test_dataset import create_test_data try: @@ -41,11 +41,13 @@ except ImportError: pass -with suppress(ImportError): +try: + from pathlib import Path +except ImportError: try: from pathlib import Path except ImportError: - from pathlib2 import Path + pass ON_WINDOWS = sys.platform == 'win32' @@ -309,7 +311,8 @@ def test_roundtrip_timedelta_data(self): self.assertDatasetIdentical(expected, actual) def test_roundtrip_float64_data(self): - expected = Dataset({'x': ('y', np.array([1.0, 2.0, np.pi], dtype='float64'))}) + expected = Dataset({'x': ('y', np.array([1.0, 2.0, np.pi], + dtype='float64'))}) with self.roundtrip(expected) as actual: self.assertDatasetIdentical(expected, actual) @@ -745,7 +748,8 @@ def test_mask_and_scale(self): v.scale_factor = 0.1 v[:] = np.array([-1, -1, 0, 1, 2]) - # first make sure netCDF4 reads the masked and scaled data correctly + # first make sure netCDF4 reads the masked and scaled data + # correctly with nc4.Dataset(tmp_file, mode='r') as nc: expected = np.ma.array([-1, -1, 10, 10.1, 10.2], mask=[True, True, False, False, False]) @@ -1300,6 +1304,7 @@ def test_open_mfdataset(self): with self.assertRaisesRegexp(IOError, 'no files to open'): open_mfdataset('foo-bar-baz-*.nc', autoclose=self.autoclose) + @requires_pathlib def test_open_mfdataset_pathlib(self): original = Dataset({'foo': ('x', np.random.randn(10))}) with create_tmp_file() as tmp1: @@ -1362,6 +1367,7 @@ def test_save_mfdataset_invalid(self): with self.assertRaisesRegexp(ValueError, 'same length'): save_mfdataset([ds, ds], ['only one path']) + @requires_pathlib def test_save_mfdataset_pathlib_roundtrip(self): original = Dataset({'foo': ('x', np.random.randn(10))}) datasets = [original.isel(x=slice(5)), @@ -1967,6 +1973,7 @@ def test_open_dataarray_options(self): with open_dataarray(tmp, drop_variables=['y']) as loaded: self.assertDataArrayIdentical(expected, loaded) + @requires_pathlib def test_dataarray_to_netcdf_no_name_pathlib(self): original_da = DataArray(np.arange(12).reshape((3, 4))) From f3dbf4b0bcf86ab834631ce625e7ed41a88eee9e Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Sat, 26 Aug 2017 21:29:02 +0200 Subject: [PATCH 18/23] Move path_type to central submodule --- xarray/backends/api.py | 10 +--------- xarray/core/pycompat.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index b28c7d9cfc2..cdc296194c0 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -6,14 +6,6 @@ from glob import glob from io import BytesIO from numbers import Number -try: - try: - from pathlib import Path - except ImportError as e: - from pathlib2 import Path - path_type = (Path, ) -except ImportError as e: - path_type = () import numpy as np @@ -23,7 +15,7 @@ from ..core import indexing from ..core.combine import auto_combine from ..core.utils import close_on_error, is_remote_uri -from ..core.pycompat import basestring +from ..core.pycompat import basestring, path_type DATAARRAY_NAME = '__xarray_dataarray_name__' DATAARRAY_VARIABLE = '__xarray_dataarray_variable__' diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 894608ef22d..7eaa0ccc450 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -59,6 +59,16 @@ def itervalues(d): except ImportError: # pragma: no cover dask_array_type = () +try: + try: + from pathlib import Path + except ImportError as e: + from pathlib2 import Path + path_type = (Path, ) +except ImportError as e: + path_type = () + + try: from contextlib import suppress except ImportError: @@ -188,7 +198,7 @@ def __exit__(self, *exc_details): # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements frame_exc = sys.exc_info()[1] - + def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: From efdc88374969e2afb5af0739f811413505587918 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Sat, 26 Aug 2017 21:29:29 +0200 Subject: [PATCH 19/23] Remove unnecessary parens --- xarray/tests/test_backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 33f1d33881b..e6a2c041b81 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -22,7 +22,7 @@ from xarray.backends.common import robust_getitem from xarray.backends.netCDF4_ import _extract_nc4_variable_encoding from xarray.core import indexing -from xarray.core.pycompat import (iteritems, PY2, ExitStack, basestring) +from xarray.core.pycompat import iteritems, PY2, ExitStack, basestring from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap, requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf, From 999d21d5b6fcd73501fda0883154998db517fc75 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Sat, 26 Aug 2017 21:32:15 +0200 Subject: [PATCH 20/23] Revert "Added show_commit_url to asv.conf (#1515)" This reverts commit 02023edc7178ea9cb1fd64169afa659b82dfd67c. --- asv_bench/asv.conf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json index a2878a7bf50..ecbcd250310 100644 --- a/asv_bench/asv.conf.json +++ b/asv_bench/asv.conf.json @@ -36,7 +36,7 @@ "install_timeout": 600, // the base URL to show a commit for the project. - "show_commit_url": "https://github.com/pydata/xarray/commit/", + // "show_commit_url": "http://github.com/owner/project/commit/", // The Pythons you'd like to test against. If not provided, defaults // to the current version of Python used to run `asv`. From 04216f1e0fa4e8abe95178bf1fe14199649725d5 Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Sat, 26 Aug 2017 21:32:33 +0200 Subject: [PATCH 21/23] Revert "Small documentation fixes (#1516)" This reverts commit 4276bb81bffb294516b38ddac8a4d87b7ad56888. --- doc/data-structures.rst | 8 +++----- xarray/core/dataarray.py | 6 ++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/doc/data-structures.rst b/doc/data-structures.rst index 95f755680df..48c8e2d9874 100644 --- a/doc/data-structures.rst +++ b/doc/data-structures.rst @@ -46,11 +46,9 @@ The :py:class:`~xarray.DataArray` constructor takes: - ``data``: a multi-dimensional array of values (e.g., a numpy ndarray, :py:class:`~pandas.Series`, :py:class:`~pandas.DataFrame` or :py:class:`~pandas.Panel`) -- ``coords``: a list or dictionary of coordinates. If a list, it should be a - list of tuples where the first element is the dimension name and the second - element is the corresponding coordinate array_like object. -- ``dims``: a list of dimension names. If omitted and ``coords`` is a list of - tuples, dimension names are taken from ``coords``. +- ``coords``: a list or dictionary of coordinates +- ``dims``: a list of dimension names. If omitted, dimension names are + taken from ``coords`` if possible. - ``attrs``: a dictionary of attributes to add to the instance - ``name``: a string that names the instance diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 0bde9a4adf3..1f664ab34be 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -175,11 +175,9 @@ def __init__(self, data, coords=None, dims=None, name=None, coords : sequence or dict of array_like objects, optional Coordinates (tick labels) to use for indexing along each dimension. If dict-like, should be a mapping from dimension names to the - corresponding coordinates. If sequence-like, should be a sequence - of tuples where the first element is the dimension name and the - second element is the corresponding coordinate array_like object. + corresponding coordinates. dims : str or sequence of str, optional - Name(s) of the data dimension(s). Must be either a string (only + Name(s) of the the data dimension(s). Must be either a string (only for 1D data) or a sequence of strings with length equal to the number of dimensions. If this argument is omitted, dimension names are taken from ``coords`` (if possible) and otherwise default to From ce156a8dd7008f80e2ae83405bda62c163550e3a Mon Sep 17 00:00:00 2001 From: Willi Rath Date: Sat, 26 Aug 2017 21:44:39 +0200 Subject: [PATCH 22/23] Fix typo in docstring and fallback-module name --- xarray/backends/api.py | 2 +- xarray/tests/test_backends.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/api.py b/xarray/backends/api.py index cdc296194c0..15f393813fe 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -142,7 +142,7 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True, ---------- filename_or_obj : str, Path, file or xarray.backends.*DataStore Strings and Path objects are interpreted as a path to a netCDF file - oran OpenDAP URL and opened with python-netCDF4, unless the filename + or an OpenDAP URL and opened with python-netCDF4, unless the filename ends with .gz, in which case the file is gunzipped and opened with scipy.io.netcdf (only netCDF3 supported). File-like objects are opened with scipy.io.netcdf (only netCDF3 supported). diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index e6a2c041b81..dd4495a2260 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -45,7 +45,7 @@ from pathlib import Path except ImportError: try: - from pathlib import Path + from pathlib2 import Path except ImportError: pass From b22a3893c8f3329f03800fe72a902ac610745550 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 31 Aug 2017 08:44:54 -0700 Subject: [PATCH 23/23] Tweak what's new for pathlib support --- doc/whats-new.rst | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 88bc32e1c76..94d2e750b4a 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -23,30 +23,23 @@ Enhancements - Support for `pathlib.Path` objects added to :py:func:`~xarray.open_dataset`, :py:func:`~xarray.open_mfdataset`, - :py:func:`~xarray.to_netcdf`, and :py:func:`~xarray.save_mfdataset`: + :py:func:`~xarray.to_netcdf`, and :py:func:`~xarray.save_mfdataset` + (:issue:`799`): .. ipython:: :verbatim: - In [1]: import xarray as xr - In [2]: from pathlib import Path # In Python 2, use pathlib2! + In [2]: from pathlib import Path # In Python 2, use pathlib2! In [3]: data_dir = Path("data/") In [4]: one_file = data_dir / "dta_for_month_01.nc" - In [5]: print(xr.open_dataset(one_file)) + In [5]: xr.open_dataset(one_file) Out[5]: [...] - In [6]: all_files = data_dir.glob("dta_for_month_*.nc") - - In [7]: print(xr.open_mfdataset(all_files)) - Out[7]: - - [...] - By `Willi Rath `_. - More attributes available in :py:attr:`~xarray.Dataset.attrs` dictionary when