From d7e3be977e21fbd7c394e02957d90d6add020cef Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Sat, 15 May 2021 09:51:42 +0200 Subject: [PATCH 01/14] fix normalize_path in pynio_.py --- xarray/backends/pynio_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index bb57e0bea81..c0feefee591 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -112,13 +112,13 @@ def open_dataset( mode="r", lock=None, ): + filename_or_obj = _normalize_path(filename_or_obj) store = NioDataStore( filename_or_obj, mode=mode, lock=lock, ) - filename_or_obj = _normalize_path(filename_or_obj) store_entrypoint = StoreBackendEntrypoint() with close_on_error(store): ds = store_entrypoint.open_dataset( From 79d8237ef8e586bd6edd3a5d7d2fa07ebae3ff38 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Sat, 15 May 2021 14:44:49 +0200 Subject: [PATCH 02/14] draft: refactor to improve not found engine error --- xarray/backends/cfgrib_.py | 6 ++++-- xarray/backends/h5netcdf_.py | 6 ++++-- xarray/backends/netCDF4_.py | 6 ++++-- xarray/backends/plugins.py | 28 +++++++++++++++++++++++++--- xarray/backends/pseudonetcdf_.py | 6 ++++-- xarray/backends/pydap_.py | 6 ++++-- xarray/backends/pynio_.py | 6 ++++-- xarray/backends/scipy_.py | 6 ++++-- xarray/backends/store.py | 3 +++ xarray/backends/zarr.py | 7 +++++++ 10 files changed, 63 insertions(+), 17 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 24a075aa811..775f74ac8c4 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -147,6 +147,8 @@ def open_dataset( ) return ds + def installed(self) -> bool: + return has_cfgrib -if has_cfgrib: - BACKEND_ENTRYPOINTS["cfgrib"] = CfgribfBackendEntrypoint + +BACKEND_ENTRYPOINTS["cfgrib"] = CfgribfBackendEntrypoint diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 84e89f80dae..a4f6fedb64d 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -392,6 +392,8 @@ def open_dataset( ) return ds + def installed(self) -> bool: + return has_h5netcdf -if has_h5netcdf: - BACKEND_ENTRYPOINTS["h5netcdf"] = H5netcdfBackendEntrypoint + +BACKEND_ENTRYPOINTS["h5netcdf"] = H5netcdfBackendEntrypoint diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index a60c940c3c4..0126c9737c7 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -570,6 +570,8 @@ def open_dataset( ) return ds + def installed(self) -> bool: + return has_netcdf4 -if has_netcdf4: - BACKEND_ENTRYPOINTS["netcdf4"] = NetCDF4BackendEntrypoint + +BACKEND_ENTRYPOINTS["netcdf4"] = NetCDF4BackendEntrypoint diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 23e83b0021e..875424fa69a 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -96,7 +96,9 @@ def build_engines(pkg_entrypoints): @functools.lru_cache(maxsize=1) def list_engines(): pkg_entrypoints = pkg_resources.iter_entry_points("xarray.backends") - return build_engines(pkg_entrypoints) + engines = build_engines(pkg_entrypoints) + engines = {key: engines[key] for key in engines if engines[key].installed()} + return engines def guess_engine(store_spec): @@ -104,12 +106,32 @@ def guess_engine(store_spec): for engine, backend in engines.items(): try: - if backend.guess_can_open and backend.guess_can_open(store_spec): + if backend.guess_can_open(store_spec): return engine except Exception: warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) - raise ValueError("cannot guess the engine, try passing one explicitly") + compatible_engines = [] + for engine, backend in BACKEND_ENTRYPOINTS.items(): + try: + if backend.guess_can_open(store_spec): + compatible_engines.append(engine) + except Exception: + warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) + + if len(compatible_engines): + error_message = ( + f"Xarray cannot find a matching installed engine for this file in the installed engines" + f"{engines}. Try to pass one explicitly or consider installing one of the " + f"following engine which reports a match: {compatible_engines}." + ) + else: + error_message = ( + "Xarray cannot detect the proper engine to open this file. " + "Check if it is installed and try to pass it explicitly." + ) + + raise ValueError(error_message) def get_backend(engine): diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index c97c7141bd1..a4a47022cb0 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -152,6 +152,8 @@ def open_dataset( ) return ds + def installed(self) -> bool: + return has_pseudonetcdf -if has_pseudonetcdf: - BACKEND_ENTRYPOINTS["pseudonetcdf"] = PseudoNetCDFBackendEntrypoint + +BACKEND_ENTRYPOINTS["pseudonetcdf"] = PseudoNetCDFBackendEntrypoint diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index 148f32cf982..c7949c5ec16 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -142,6 +142,8 @@ def open_dataset( ) return ds + def installed(self) -> bool: + return has_pydap -if has_pydap: - BACKEND_ENTRYPOINTS["pydap"] = PydapBackendEntrypoint + +BACKEND_ENTRYPOINTS["pydap"] = PydapBackendEntrypoint diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index c0feefee591..6e8c07ddde3 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -133,6 +133,8 @@ def open_dataset( ) return ds + def installed(self) -> bool: + return has_pynio -if has_pynio: - BACKEND_ENTRYPOINTS["pynio"] = PynioBackendEntrypoint + +BACKEND_ENTRYPOINTS["pynio"] = PynioBackendEntrypoint diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index c27716ea44d..64e95e5bdfd 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -282,6 +282,8 @@ def open_dataset( ) return ds + def installed(self) -> bool: + return has_scipy -if has_scipy: - BACKEND_ENTRYPOINTS["scipy"] = ScipyBackendEntrypoint + +BACKEND_ENTRYPOINTS["scipy"] = ScipyBackendEntrypoint diff --git a/xarray/backends/store.py b/xarray/backends/store.py index 860a0254b64..f506e068f1a 100644 --- a/xarray/backends/store.py +++ b/xarray/backends/store.py @@ -41,5 +41,8 @@ def open_dataset( return ds + def installed(self) -> bool: + return True + BACKEND_ENTRYPOINTS["store"] = StoreBackendEntrypoint diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index a4086eacece..447df6549b2 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -704,6 +704,13 @@ def open_zarr( class ZarrBackendEntrypoint(BackendEntrypoint): + def guess_can_open(self, filename_or_obj): + try: + _, ext = os.path.splitext(filename_or_obj) + except TypeError: + return False + return ext in {".zarr"} + def open_dataset( self, filename_or_obj, From 9f67bc37342cc66a7bb85e98369b83e457957d1f Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 17 May 2021 14:44:20 +0200 Subject: [PATCH 03/14] fix backend registration and tests --- xarray/backends/cfgrib_.py | 3 ++- xarray/backends/h5netcdf_.py | 3 ++- xarray/backends/netCDF4_.py | 9 ++------- xarray/backends/plugins.py | 6 ++++-- xarray/backends/pseudonetcdf_.py | 3 ++- xarray/backends/pydap_.py | 3 ++- xarray/backends/pynio_.py | 3 ++- xarray/backends/scipy_.py | 4 ++-- xarray/backends/store.py | 3 ++- xarray/backends/zarr.py | 7 +++++-- xarray/tests/test_backends.py | 4 ++-- 11 files changed, 27 insertions(+), 21 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 775f74ac8c4..c1b30f81db9 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -147,7 +147,8 @@ def open_dataset( ) return ds - def installed(self) -> bool: + @staticmethod + def installed() -> bool: return has_cfgrib diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index a4f6fedb64d..849fdb9791c 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -392,7 +392,8 @@ def open_dataset( ) return ds - def installed(self) -> bool: + @staticmethod + def installed() -> bool: return has_h5netcdf diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 0126c9737c7..abf7b16bbd5 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -348,12 +348,6 @@ def open( if isinstance(filename, pathlib.Path): filename = os.fspath(filename) - if not isinstance(filename, str): - raise ValueError( - "can only read bytes or file-like objects " - "with engine='scipy' or 'h5netcdf'" - ) - if format is None: format = "NETCDF4" @@ -570,7 +564,8 @@ def open_dataset( ) return ds - def installed(self) -> bool: + @staticmethod + def installed() -> bool: return has_netcdf4 diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 875424fa69a..0caca23bfd3 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -81,7 +81,10 @@ def sort_backends(backend_entrypoints): def build_engines(pkg_entrypoints): - backend_entrypoints = BACKEND_ENTRYPOINTS.copy() + backend_entrypoints = {} + for backend_name, backend in BACKEND_ENTRYPOINTS.items(): + if backend.installed(): + backend_entrypoints[backend_name] = backend pkg_entrypoints = remove_duplicates(pkg_entrypoints) external_backend_entrypoints = backends_dict_from_pkg(pkg_entrypoints) backend_entrypoints.update(external_backend_entrypoints) @@ -97,7 +100,6 @@ def build_engines(pkg_entrypoints): def list_engines(): pkg_entrypoints = pkg_resources.iter_entry_points("xarray.backends") engines = build_engines(pkg_entrypoints) - engines = {key: engines[key] for key in engines if engines[key].installed()} return engines diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index a4a47022cb0..8218186dc8e 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -152,7 +152,8 @@ def open_dataset( ) return ds - def installed(self) -> bool: + @staticmethod + def installed() -> bool: return has_pseudonetcdf diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index c7949c5ec16..b857d97a727 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -142,7 +142,8 @@ def open_dataset( ) return ds - def installed(self) -> bool: + @staticmethod + def installed() -> bool: return has_pydap diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 6e8c07ddde3..d0dc073bc5c 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -133,7 +133,8 @@ def open_dataset( ) return ds - def installed(self) -> bool: + @staticmethod + def installed() -> bool: return has_pynio diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index 64e95e5bdfd..fa789456aad 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -282,8 +282,8 @@ def open_dataset( ) return ds - def installed(self) -> bool: - return has_scipy + @staticmethod + def installed() -> bool: return has_scipy BACKEND_ENTRYPOINTS["scipy"] = ScipyBackendEntrypoint diff --git a/xarray/backends/store.py b/xarray/backends/store.py index f506e068f1a..a7f11f1db69 100644 --- a/xarray/backends/store.py +++ b/xarray/backends/store.py @@ -41,7 +41,8 @@ def open_dataset( return ds - def installed(self) -> bool: + @staticmethod + def installed() -> bool: return True diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 447df6549b2..2b2db20bcc6 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -756,6 +756,9 @@ def open_dataset( ) return ds + @staticmethod + def installed() -> bool: + return has_zarr -if has_zarr: - BACKEND_ENTRYPOINTS["zarr"] = ZarrBackendEntrypoint + +BACKEND_ENTRYPOINTS["zarr"] = ZarrBackendEntrypoint diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 95531ec5062..2f0c3ef002c 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2764,7 +2764,7 @@ def test_open_badbytes(self): with pytest.raises(ValueError, match=r"HDF5 as bytes"): with open_dataset(b"\211HDF\r\n\032\n", engine="h5netcdf"): pass - with pytest.raises(ValueError, match=r"cannot guess the engine"): + with pytest.raises(ValueError, match=r"Xarray cannot detect the proper engine"): with open_dataset(b"garbage"): pass with pytest.raises(ValueError, match=r"can only read bytes"): @@ -2816,7 +2816,7 @@ def test_open_fileobj(self): # `raises_regex`?). Ref https://github.com/pydata/xarray/pull/5191 with open(tmp_file, "rb") as f: f.seek(8) - with pytest.raises(ValueError, match="cannot guess the engine"): + with pytest.raises(ValueError, match="Xarray cannot detect the proper engine"): open_dataset(f) From e528603600ef087dc01685143fb21991af0daa4b Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 19 May 2021 13:43:43 +0200 Subject: [PATCH 04/14] fix not workng jet --- xarray/backends/plugins.py | 45 +++++++++++++++----------------------- xarray/backends/scipy_.py | 3 ++- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 2b5543cf664..ff9a4b541b0 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -109,46 +109,37 @@ def guess_engine(store_spec): except Exception: warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) + compatible = [] + for engine, backend in BACKEND_ENTRYPOINTS.items(): + try: + if backend.guess_can_open(store_spec): + compatible.append(engine) + except Exception: + warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) + installed = [k for k in engines if k != "store"] if installed: - raise ValueError( + error_msg = ( "did not find a match in any of xarray's currently installed IO " f"backends {installed}. Consider explicitly selecting one of the " - "installed backends via the ``engine`` parameter to " - "xarray.open_dataset(), or installing additional IO dependencies:\n" + "installed engines via the ``engine`` parameter, or installing " + "additional IO dependencies:\n" "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" "http://xarray.pydata.org/en/stable/user-guide/io.html" ) else: - raise ValueError( + error_msg = ( "xarray is unable to open this file because it has no currently " - "installed IO backends. Xarray's read/write support requires " - "installing optional dependencies:\n" + "installed IO engines. Xarray's read/write support requires " + "optional dependencies:\n" "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" - "http://xarray.pydata.org/en/stable/user-guide/io.html" + "http://xarray.pydata.org/en/stable/user-guide/io" ) - compatible_engines = [] - for engine, backend in BACKEND_ENTRYPOINTS.items(): - try: - if backend.guess_can_open(store_spec): - compatible_engines.append(engine) - except Exception: - warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) - - if len(compatible_engines): - error_message = ( - f"Xarray cannot find a matching installed engine for this file in the installed engines" - f"{engines}. Try to pass one explicitly or consider installing one of the " - f"following engine which reports a match: {compatible_engines}." - ) - else: - error_message = ( - "Xarray cannot detect the proper engine to open this file. " - "Check if it is installed and try to pass it explicitly." - ) + if compatible: + error_msg = error_msg + "\nThe following engines reports a match with the input file: {compatible}." - raise ValueError(error_message) + raise ValueError(error_msg) def get_backend(engine): diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index 856c80736bd..a4edc4aaa16 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -290,7 +290,8 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: return has_scipy + def installed() -> bool: + return has_scipy BACKEND_ENTRYPOINTS["scipy"] = ScipyBackendEntrypoint From e1c5a5e22855e1a050b1950bfa3eabbaf346176c Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 19 May 2021 14:02:18 +0200 Subject: [PATCH 05/14] fix error call guess_can_open --- xarray/backends/plugins.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index ff9a4b541b0..6ec31456536 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -127,6 +127,10 @@ def guess_engine(store_spec): "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" "http://xarray.pydata.org/en/stable/user-guide/io.html" ) + if compatible: + error_msg = \ + error_msg + \ + "\nThe following engines reports a match with the input file: {compatible}." else: error_msg = ( "xarray is unable to open this file because it has no currently " @@ -135,9 +139,11 @@ def guess_engine(store_spec): "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" "http://xarray.pydata.org/en/stable/user-guide/io" ) - - if compatible: - error_msg = error_msg + "\nThe following engines reports a match with the input file: {compatible}." + if compatible: + error_msg = \ + error_msg + \ + "\nConsider to install the dependencies of following engines that reports "\ + "a match with the input file: {compatible}." raise ValueError(error_msg) From f2abf6b9987dbef74e245100b06c2e6439a8f8b0 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 19 May 2021 14:52:44 +0200 Subject: [PATCH 06/14] fix --- xarray/backends/netCDF4_.py | 6 ++++++ xarray/backends/plugins.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index ef3152833ab..8a960f83461 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -349,6 +349,12 @@ def open( if isinstance(filename, pathlib.Path): filename = os.fspath(filename) + if not isinstance(filename, str): + raise ValueError( + "can only read bytes or file-like objects " + "with engine='scipy' or 'h5netcdf'" + ) + if format is None: format = "NETCDF4" diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 6ec31456536..f248d63e681 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -110,8 +110,9 @@ def guess_engine(store_spec): warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) compatible = [] - for engine, backend in BACKEND_ENTRYPOINTS.items(): + for engine, backend_cls in BACKEND_ENTRYPOINTS.items(): try: + backend = backend_cls() if backend.guess_can_open(store_spec): compatible.append(engine) except Exception: From 733a7dd7eca7d55a1e875fb2b01edd8d4f769729 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 31 May 2021 18:15:09 +0200 Subject: [PATCH 07/14] fix --- xarray/backends/plugins.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index f248d63e681..a79afb53245 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -131,7 +131,7 @@ def guess_engine(store_spec): if compatible: error_msg = \ error_msg + \ - "\nThe following engines reports a match with the input file: {compatible}." + f"\nThe following engines reports a match with the input file: {compatible}." else: error_msg = ( "xarray is unable to open this file because it has no currently " @@ -143,8 +143,8 @@ def guess_engine(store_spec): if compatible: error_msg = \ error_msg + \ - "\nConsider to install the dependencies of following engines that reports "\ - "a match with the input file: {compatible}." + f"\nConsider to install one of the following engines that reports "\ + f"a match with the input file: {compatible}." raise ValueError(error_msg) From be8ead74a3d677154ccc9ae3f47964d3806647c6 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 31 May 2021 18:15:32 +0200 Subject: [PATCH 08/14] add tests --- xarray/tests/test_plugins.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index b35971e185b..f485b891067 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -177,3 +177,14 @@ def test_no_matching_engine_found(): def test_no_engines_installed(): with pytest.raises(ValueError, match="no currently installed IO backends."): plugins.guess_engine("not-valid") + + +@mock.patch( + "xarray.backends.plugins.list_engines", + mock.MagicMock(return_value={"dummy": DummyBackendEntrypointArgs()}), +) +def test_no_matching_engine_found(): + with pytest.raises( + ValueError, match="match in any of xarray's currently installed IO" + ): + plugins.guess_engine("not-valid") From 3e028ab548dd14e1a915d4089405357243c8e8e0 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 31 May 2021 19:29:59 +0200 Subject: [PATCH 09/14] update message error engine not fuond --- xarray/backends/plugins.py | 53 ++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index a79afb53245..1157dc423d2 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -120,31 +120,40 @@ def guess_engine(store_spec): installed = [k for k in engines if k != "store"] if installed: - error_msg = ( - "did not find a match in any of xarray's currently installed IO " - f"backends {installed}. Consider explicitly selecting one of the " - "installed engines via the ``engine`` parameter, or installing " - "additional IO dependencies:\n" - "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" - "http://xarray.pydata.org/en/stable/user-guide/io.html" - ) + if not compatible: + error_msg = ( + "did not find a match in any of xarray's currently installed IO " + f"backends {installed}. Consider explicitly selecting one of the " + "installed engines via the ``engine`` parameter, or installing " + "additional IO dependencies, see:\n" + "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" + "http://xarray.pydata.org/en/stable/user-guide/io.html" + ) if compatible: - error_msg = \ - error_msg + \ - f"\nThe following engines reports a match with the input file: {compatible}." + error_msg = ( + "did not find a match in any of xarray's currently installed IO " + f"backends {installed}. Consider explicitly selecting one of the " + "installed engines via the ``engine`` parameter or to install one " + f"of the following engines that report a match with the input file: " + f"{compatible}" + ) else: - error_msg = ( - "xarray is unable to open this file because it has no currently " - "installed IO engines. Xarray's read/write support requires " - "optional dependencies:\n" - "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" - "http://xarray.pydata.org/en/stable/user-guide/io" - ) + if not compatible: + error_msg = ( + "xarray is unable to open this file because it has no currently " + "installed IO engines. Xarray's read/write support requires " + "optional IO dependencies, see:\n" + "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" + "http://xarray.pydata.org/en/stable/user-guide/io" + ) if compatible: - error_msg = \ - error_msg + \ - f"\nConsider to install one of the following engines that reports "\ - f"a match with the input file: {compatible}." + error_msg = ( + "xarray is unable to open this file because it has no currently " + "installed IO engines. Xarray's read/write support requires " + "optional dependencies." + f"\nConsider to install one of the following engines that report " + f"a match with the input file: {compatible}" + ) raise ValueError(error_msg) From f32ace9b1f518cd7f205a36c9b8ddfc8869dba89 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Mon, 31 May 2021 19:34:51 +0200 Subject: [PATCH 10/14] update tests --- xarray/tests/test_plugins.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index f485b891067..5e0552d4cfc 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -175,16 +175,28 @@ def test_no_matching_engine_found(): mock.MagicMock(return_value={}), ) def test_no_engines_installed(): - with pytest.raises(ValueError, match="no currently installed IO backends."): + with pytest.raises( + ValueError, match=r"xarray is unable to open.*IO dependencies, see:" + ): plugins.guess_engine("not-valid") + with pytest.raises( + ValueError, match=r"xarray is unable to open.*engines that report a match" + ): + plugins.guess_engine("foo.nc") + @mock.patch( "xarray.backends.plugins.list_engines", mock.MagicMock(return_value={"dummy": DummyBackendEntrypointArgs()}), ) -def test_no_matching_engine_found(): +def test_engines_installed(): with pytest.raises( - ValueError, match="match in any of xarray's currently installed IO" + ValueError, match=r"did not find a match in any.*IO dependencies, see:" ): plugins.guess_engine("not-valid") + + with pytest.raises( + ValueError, match=r"did not find a match in any.*engines that report a match" + ): + plugins.guess_engine("foo.nc") From 682ce0e112ba39cc2671f1d1569375fa4a1e66a9 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 9 Jun 2021 14:59:48 +0200 Subject: [PATCH 11/14] fix if else --- xarray/backends/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 1157dc423d2..b8fc662366b 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -129,7 +129,7 @@ def guess_engine(store_spec): "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" "http://xarray.pydata.org/en/stable/user-guide/io.html" ) - if compatible: + else: error_msg = ( "did not find a match in any of xarray's currently installed IO " f"backends {installed}. Consider explicitly selecting one of the " @@ -146,7 +146,7 @@ def guess_engine(store_spec): "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" "http://xarray.pydata.org/en/stable/user-guide/io" ) - if compatible: + else: error_msg = ( "xarray is unable to open this file because it has no currently " "installed IO engines. Xarray's read/write support requires " From defe2b60da2edb46716e71b7a7cbe2b7c6135973 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 9 Jun 2021 17:09:37 +0200 Subject: [PATCH 12/14] fix message error and tests --- xarray/backends/cfgrib_.py | 2 +- xarray/backends/h5netcdf_.py | 2 +- xarray/backends/netCDF4_.py | 2 +- xarray/backends/plugins.py | 30 ++++++++++-------------------- xarray/backends/pseudonetcdf_.py | 2 +- xarray/backends/pydap_.py | 2 +- xarray/backends/pynio_.py | 2 +- xarray/backends/scipy_.py | 2 +- xarray/backends/store.py | 2 +- xarray/backends/zarr.py | 2 +- xarray/tests/test_plugins.py | 31 ++++++------------------------- 11 files changed, 25 insertions(+), 54 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index 36cc5a31446..f7758ec4f70 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -147,7 +147,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return has_cfgrib diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index f01fe591879..618971648da 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -394,7 +394,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return has_h5netcdf diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 8a960f83461..96a5ccae69e 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -573,7 +573,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return has_netcdf4 diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index b8fc662366b..08b9e2c01b9 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -83,7 +83,7 @@ def sort_backends(backend_entrypoints): def build_engines(pkg_entrypoints): backend_entrypoints = {} for backend_name, backend in BACKEND_ENTRYPOINTS.items(): - if backend.installed(): + if backend.are_dependencies_installed(): backend_entrypoints[backend_name] = backend pkg_entrypoints = remove_duplicates(pkg_entrypoints) external_backend_entrypoints = backends_dict_from_pkg(pkg_entrypoints) @@ -119,8 +119,8 @@ def guess_engine(store_spec): warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) installed = [k for k in engines if k != "store"] - if installed: - if not compatible: + if not compatible: + if installed: error_msg = ( "did not find a match in any of xarray's currently installed IO " f"backends {installed}. Consider explicitly selecting one of the " @@ -130,15 +130,6 @@ def guess_engine(store_spec): "http://xarray.pydata.org/en/stable/user-guide/io.html" ) else: - error_msg = ( - "did not find a match in any of xarray's currently installed IO " - f"backends {installed}. Consider explicitly selecting one of the " - "installed engines via the ``engine`` parameter or to install one " - f"of the following engines that report a match with the input file: " - f"{compatible}" - ) - else: - if not compatible: error_msg = ( "xarray is unable to open this file because it has no currently " "installed IO engines. Xarray's read/write support requires " @@ -146,14 +137,13 @@ def guess_engine(store_spec): "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" "http://xarray.pydata.org/en/stable/user-guide/io" ) - else: - error_msg = ( - "xarray is unable to open this file because it has no currently " - "installed IO engines. Xarray's read/write support requires " - "optional dependencies." - f"\nConsider to install one of the following engines that report " - f"a match with the input file: {compatible}" - ) + else: + error_msg = ( + "found the following matches with the input file in xarray's IO " + f"backends: {compatible}. But their dependencies may not be installed, see:\n" + "http://xarray.pydata.org/en/stable/user-guide/io.html \n" + "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html" + ) raise ValueError(error_msg) diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index 8218186dc8e..a6f9c88b7b4 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -153,7 +153,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return has_pseudonetcdf diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index d3a7f78facb..eaebb1dfc06 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -154,7 +154,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return has_pydap diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index d0dc073bc5c..80ef0bca8e9 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -134,7 +134,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return has_pynio diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index a4edc4aaa16..858692d3896 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -290,7 +290,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return has_scipy diff --git a/xarray/backends/store.py b/xarray/backends/store.py index a7f11f1db69..9c0c4245b8b 100644 --- a/xarray/backends/store.py +++ b/xarray/backends/store.py @@ -42,7 +42,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return True diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 8ecbb228f40..1df97a0533c 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -764,7 +764,7 @@ def open_dataset( return ds @staticmethod - def installed() -> bool: + def are_dependencies_installed() -> bool: return has_zarr diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py index 5e0552d4cfc..b7a5f9405d1 100644 --- a/xarray/tests/test_plugins.py +++ b/xarray/tests/test_plugins.py @@ -164,39 +164,20 @@ def test_build_engines_sorted(): mock.MagicMock(return_value={"dummy": DummyBackendEntrypointArgs()}), ) def test_no_matching_engine_found(): - with pytest.raises( - ValueError, match="match in any of xarray's currently installed IO" - ): + with pytest.raises(ValueError, match=r"did not find a match in any"): plugins.guess_engine("not-valid") - -@mock.patch( - "xarray.backends.plugins.list_engines", - mock.MagicMock(return_value={}), -) -def test_no_engines_installed(): - with pytest.raises( - ValueError, match=r"xarray is unable to open.*IO dependencies, see:" - ): - plugins.guess_engine("not-valid") - - with pytest.raises( - ValueError, match=r"xarray is unable to open.*engines that report a match" - ): + with pytest.raises(ValueError, match=r"found the following matches with the input"): plugins.guess_engine("foo.nc") @mock.patch( "xarray.backends.plugins.list_engines", - mock.MagicMock(return_value={"dummy": DummyBackendEntrypointArgs()}), + mock.MagicMock(return_value={}), ) -def test_engines_installed(): - with pytest.raises( - ValueError, match=r"did not find a match in any.*IO dependencies, see:" - ): +def test_engines_not_installed(): + with pytest.raises(ValueError, match=r"xarray is unable to open"): plugins.guess_engine("not-valid") - with pytest.raises( - ValueError, match=r"did not find a match in any.*engines that report a match" - ): + with pytest.raises(ValueError, match=r"found the following matches with the input"): plugins.guess_engine("foo.nc") From 36a18df026e72974c07ae2b164d0ac341d19f19a Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 9 Jun 2021 17:12:42 +0200 Subject: [PATCH 13/14] revert changes in error messages --- xarray/backends/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 08b9e2c01b9..bff4dfcb10b 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -132,8 +132,8 @@ def guess_engine(store_spec): else: error_msg = ( "xarray is unable to open this file because it has no currently " - "installed IO engines. Xarray's read/write support requires " - "optional IO dependencies, see:\n" + "installed IO backends. Xarray's read/write support requires " + "installing optional IO dependencies, see:\n" "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" "http://xarray.pydata.org/en/stable/user-guide/io" ) From f640ff69681a7b0ffd0489954c746310acae672d Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 11 Jun 2021 14:24:28 +0200 Subject: [PATCH 14/14] revert changes in error messages --- xarray/backends/cfgrib_.py | 6 ++---- xarray/backends/h5netcdf_.py | 6 ++---- xarray/backends/netCDF4_.py | 6 ++---- xarray/backends/plugins.py | 16 ++++++++-------- xarray/backends/pseudonetcdf_.py | 5 +---- xarray/backends/pydap_.py | 6 ++---- xarray/backends/pynio_.py | 6 ++---- xarray/backends/scipy_.py | 6 ++---- xarray/backends/store.py | 6 ++---- xarray/backends/zarr.py | 6 ++---- 10 files changed, 25 insertions(+), 44 deletions(-) diff --git a/xarray/backends/cfgrib_.py b/xarray/backends/cfgrib_.py index f7758ec4f70..e7aeaaba83a 100644 --- a/xarray/backends/cfgrib_.py +++ b/xarray/backends/cfgrib_.py @@ -94,6 +94,8 @@ def get_encoding(self): class CfgribfBackendEntrypoint(BackendEntrypoint): + available = has_cfgrib + def guess_can_open(self, filename_or_obj): try: _, ext = os.path.splitext(filename_or_obj) @@ -146,9 +148,5 @@ def open_dataset( ) return ds - @staticmethod - def are_dependencies_installed() -> bool: - return has_cfgrib - BACKEND_ENTRYPOINTS["cfgrib"] = CfgribfBackendEntrypoint diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 618971648da..3a49928ec65 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -337,6 +337,8 @@ def close(self, **kwargs): class H5netcdfBackendEntrypoint(BackendEntrypoint): + available = has_h5netcdf + def guess_can_open(self, filename_or_obj): magic_number = try_read_magic_number_from_file_or_path(filename_or_obj) if magic_number is not None: @@ -393,9 +395,5 @@ def open_dataset( ) return ds - @staticmethod - def are_dependencies_installed() -> bool: - return has_h5netcdf - BACKEND_ENTRYPOINTS["h5netcdf"] = H5netcdfBackendEntrypoint diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index 96a5ccae69e..769c96c99ce 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -512,6 +512,8 @@ def close(self, **kwargs): class NetCDF4BackendEntrypoint(BackendEntrypoint): + available = has_netcdf4 + def guess_can_open(self, filename_or_obj): if isinstance(filename_or_obj, str) and is_remote_uri(filename_or_obj): return True @@ -572,9 +574,5 @@ def open_dataset( ) return ds - @staticmethod - def are_dependencies_installed() -> bool: - return has_netcdf4 - BACKEND_ENTRYPOINTS["netcdf4"] = NetCDF4BackendEntrypoint diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index bff4dfcb10b..08c1bec8325 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -83,7 +83,7 @@ def sort_backends(backend_entrypoints): def build_engines(pkg_entrypoints): backend_entrypoints = {} for backend_name, backend in BACKEND_ENTRYPOINTS.items(): - if backend.are_dependencies_installed(): + if backend.available: backend_entrypoints[backend_name] = backend pkg_entrypoints = remove_duplicates(pkg_entrypoints) external_backend_entrypoints = backends_dict_from_pkg(pkg_entrypoints) @@ -109,21 +109,21 @@ def guess_engine(store_spec): except Exception: warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) - compatible = [] + compatible_engines = [] for engine, backend_cls in BACKEND_ENTRYPOINTS.items(): try: backend = backend_cls() if backend.guess_can_open(store_spec): - compatible.append(engine) + compatible_engines.append(engine) except Exception: warnings.warn(f"{engine!r} fails while guessing", RuntimeWarning) - installed = [k for k in engines if k != "store"] - if not compatible: - if installed: + installed_engines = [k for k in engines if k != "store"] + if not compatible_engines: + if installed_engines: error_msg = ( "did not find a match in any of xarray's currently installed IO " - f"backends {installed}. Consider explicitly selecting one of the " + f"backends {installed_engines}. Consider explicitly selecting one of the " "installed engines via the ``engine`` parameter, or installing " "additional IO dependencies, see:\n" "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html\n" @@ -140,7 +140,7 @@ def guess_engine(store_spec): else: error_msg = ( "found the following matches with the input file in xarray's IO " - f"backends: {compatible}. But their dependencies may not be installed, see:\n" + f"backends: {compatible_engines}. But their dependencies may not be installed, see:\n" "http://xarray.pydata.org/en/stable/user-guide/io.html \n" "http://xarray.pydata.org/en/stable/getting-started-guide/installing.html" ) diff --git a/xarray/backends/pseudonetcdf_.py b/xarray/backends/pseudonetcdf_.py index a6f9c88b7b4..da178926dbe 100644 --- a/xarray/backends/pseudonetcdf_.py +++ b/xarray/backends/pseudonetcdf_.py @@ -102,6 +102,7 @@ def close(self): class PseudoNetCDFBackendEntrypoint(BackendEntrypoint): + available = has_pseudonetcdf # *args and **kwargs are not allowed in open_backend_dataset_ kwargs, # unless the open_dataset_parameters are explicity defined like this: @@ -152,9 +153,5 @@ def open_dataset( ) return ds - @staticmethod - def are_dependencies_installed() -> bool: - return has_pseudonetcdf - BACKEND_ENTRYPOINTS["pseudonetcdf"] = PseudoNetCDFBackendEntrypoint diff --git a/xarray/backends/pydap_.py b/xarray/backends/pydap_.py index eaebb1dfc06..bc479f9a71d 100644 --- a/xarray/backends/pydap_.py +++ b/xarray/backends/pydap_.py @@ -110,6 +110,8 @@ def get_dimensions(self): class PydapBackendEntrypoint(BackendEntrypoint): + available = has_pydap + def guess_can_open(self, filename_or_obj): return isinstance(filename_or_obj, str) and is_remote_uri(filename_or_obj) @@ -153,9 +155,5 @@ def open_dataset( ) return ds - @staticmethod - def are_dependencies_installed() -> bool: - return has_pydap - BACKEND_ENTRYPOINTS["pydap"] = PydapBackendEntrypoint diff --git a/xarray/backends/pynio_.py b/xarray/backends/pynio_.py index 80ef0bca8e9..4e912f3e1ef 100644 --- a/xarray/backends/pynio_.py +++ b/xarray/backends/pynio_.py @@ -99,6 +99,8 @@ def close(self): class PynioBackendEntrypoint(BackendEntrypoint): + available = has_pynio + def open_dataset( self, filename_or_obj, @@ -133,9 +135,5 @@ def open_dataset( ) return ds - @staticmethod - def are_dependencies_installed() -> bool: - return has_pynio - BACKEND_ENTRYPOINTS["pynio"] = PynioBackendEntrypoint diff --git a/xarray/backends/scipy_.py b/xarray/backends/scipy_.py index 858692d3896..4c1ce1ef09d 100644 --- a/xarray/backends/scipy_.py +++ b/xarray/backends/scipy_.py @@ -238,6 +238,8 @@ def close(self): class ScipyBackendEntrypoint(BackendEntrypoint): + available = has_scipy + def guess_can_open(self, filename_or_obj): magic_number = try_read_magic_number_from_file_or_path(filename_or_obj) @@ -289,9 +291,5 @@ def open_dataset( ) return ds - @staticmethod - def are_dependencies_installed() -> bool: - return has_scipy - BACKEND_ENTRYPOINTS["scipy"] = ScipyBackendEntrypoint diff --git a/xarray/backends/store.py b/xarray/backends/store.py index 9c0c4245b8b..b774d2bce95 100644 --- a/xarray/backends/store.py +++ b/xarray/backends/store.py @@ -4,6 +4,8 @@ class StoreBackendEntrypoint(BackendEntrypoint): + available = True + def guess_can_open(self, filename_or_obj): return isinstance(filename_or_obj, AbstractDataStore) @@ -41,9 +43,5 @@ def open_dataset( return ds - @staticmethod - def are_dependencies_installed() -> bool: - return True - BACKEND_ENTRYPOINTS["store"] = StoreBackendEntrypoint diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 1df97a0533c..5b228f77d24 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -703,6 +703,8 @@ def open_zarr( class ZarrBackendEntrypoint(BackendEntrypoint): + available = has_zarr + def guess_can_open(self, filename_or_obj): try: _, ext = os.path.splitext(filename_or_obj) @@ -763,9 +765,5 @@ def open_dataset( ) return ds - @staticmethod - def are_dependencies_installed() -> bool: - return has_zarr - BACKEND_ENTRYPOINTS["zarr"] = ZarrBackendEntrypoint