diff --git a/CHANGELOG.md b/CHANGELOG.md index 91b397d03..97f4a9b31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - `__geo_interface__` for items ([#885](https://github.com/stac-utils/pystac/pull/885)) - Optional `strategy` parameter to `catalog.add_items()` ([#967](https://github.com/stac-utils/pystac/pull/967)) - `start_datetime` and `end_datetime` arguments to the `Item` constructor ([#918](https://github.com/stac-utils/pystac/pull/918)) +- `RetryStacIO` ([#986](https://github.com/stac-utils/pystac/pull/986)) ### Removed diff --git a/README.md b/README.md index 57ef096e4..ac6e2fc75 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,15 @@ optional `orjson` requirements: pip install pystac[orjson] ``` -### Install from source: +If you would like to use a custom `RetryStacIO` class for automatically retrying +network requests when reading with PySTAC, you'll need +[`urllib3`](https://urllib3.readthedocs.io/en/stable/): + +```shell +pip install pystac[urllib3] +``` + +### Install from source ```shell git clone https://github.com/stac-utils/pystac.git diff --git a/docs/conf.py b/docs/conf.py index de96d5461..12c1b2d28 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -232,6 +232,7 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "dateutil": ("https://dateutil.readthedocs.io/en/stable", None), + "urllib3": ("https://urllib3.readthedocs.io/en/stable", None), } # -- Substutition variables diff --git a/docs/installation.rst b/docs/installation.rst index 3c2b76a07..616d095dc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -66,6 +66,18 @@ additional functionality: pip install pystac[orjson] +* ``urllib3`` + + Installs the additional `urllib3 `__ dependency. + For now, this is only used in :py:class:`pystac.stac_io.RetryStacIO`, but it + may be used more extensively in the future. + + To install: + + .. code-block:: bash + + pip install pystac[urllib3] + Versions ======== diff --git a/pystac/catalog.py b/pystac/catalog.py index 0cdc48256..40174966f 100644 --- a/pystac/catalog.py +++ b/pystac/catalog.py @@ -320,7 +320,8 @@ def add_items( items: Iterable[Item], strategy: Optional[HrefLayoutStrategy] = None, ) -> None: - """Adds links to multiple :class:`~pystac.Item`s. + """Adds links to multiple :class:`Items `. + This method will set each item's parent to this object, and their root to this Catalog's root. diff --git a/pystac/stac_io.py b/pystac/stac_io.py index 18650e027..2bac4ec96 100644 --- a/pystac/stac_io.py +++ b/pystac/stac_io.py @@ -23,6 +23,14 @@ except ImportError: orjson = None # type: ignore[assignment] +# Is urllib3 available? +try: + import urllib3 # noqa +except ImportError: + HAS_URLLIB3 = False +else: + HAS_URLLIB3 = True + if TYPE_CHECKING: from pystac.catalog import Catalog from pystac.stac_object import STACObject @@ -281,9 +289,8 @@ def read_text_from_href(self, href: str) -> str: href : The URI of the file to open. """ - parsed = safe_urlparse(href) href_contents: str - if parsed.scheme != "": + if _is_url(href): try: req = Request(href, headers=self.headers) with urlopen(req) as f: @@ -373,3 +380,62 @@ def _report_duplicate_object_names( else: result[key] = value return result + + +def _is_url(href: str) -> bool: + parsed = safe_urlparse(href) + return parsed.scheme != "" + + +if HAS_URLLIB3: + from typing import cast + + from urllib3 import PoolManager + from urllib3.util import Retry + + class RetryStacIO(DefaultStacIO): + """A customized StacIO that retries requests, using + :py:class:`urllib3.util.retry.Retry`. + + The headers are passed to :py:class:`DefaultStacIO`. If retry is not + provided, a default retry is used. + + To use this class, you'll need to install PySTAC with urllib3: + + .. code-block:: shell + + pip install pystac[urllib3] + + """ + + retry: Retry + """The :py:class:`urllib3.util.retry.Retry` to use with all reading network + requests.""" + + def __init__( + self, + headers: Optional[Dict[str, str]] = None, + retry: Optional[Retry] = None, + ): + super().__init__(headers) + self.retry = retry or Retry() + + def read_text_from_href(self, href: str) -> str: + """Reads file as a UTF-8 string, with retry support. + + Args: + href : The URI of the file to open. + """ + if _is_url(href): + # TODO provide a pooled StacIO to enable more efficient network + # access (probably named `PooledStacIO`). + http = PoolManager() + try: + response = http.request( + "GET", href, retries=self.retry # type: ignore + ) + return cast(str, response.data.decode("utf-8")) + except HTTPError as e: + raise Exception("Could not read uri {}".format(href)) from e + else: + return super().read_text_from_href(href) diff --git a/pystac/utils.py b/pystac/utils.py index 196dcd5c1..d30cd7a0d 100644 --- a/pystac/utils.py +++ b/pystac/utils.py @@ -308,9 +308,9 @@ def datetime_to_str(dt: datetime, timespec: str = "auto") -> str: Args: dt : The datetime to convert. timespec: An optional argument that specifies the number of additional - terms of the time to include. Valid options are 'auto', 'hours', - 'minutes', 'seconds', 'milliseconds' and 'microseconds'. The default value - is 'auto'. + terms of the time to include. Valid options are 'auto', 'hours', + 'minutes', 'seconds', 'milliseconds' and 'microseconds'. The default value + is 'auto'. Returns: str: The ISO8601 (RFC 3339) formatted string representing the datetime. diff --git a/requirements-test.txt b/requirements-test.txt index 0ea559c64..27ebebffe 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,23 +1,20 @@ -mypy==1.0.0 -flake8==6.0.0 black==23.1.0 -pytest==7.2.1 -pytest-cov==4.0.0 -pytest-mock==3.10.0 codespell==2.2.2 -isort==5.12.0 - -jsonschema==4.17.3 coverage==7.1.0 doc8==0.11.2 -jinja2<4.0 +flake8==6.0.0 html5lib==1.1 - +isort==5.12.0 +jinja2<4.0 +jsonschema==4.17.3 +mypy==1.0.0 +orjson==3.8.6 +pre-commit==3.0.4 +pytest-cov==4.0.0 +pytest-mock==3.10.0 +pytest-vcr==1.0.2 +pytest==7.2.1 types-html5lib==1.1.11.11 -types-python-dateutil==2.8.19.7 types-orjson==3.6.2 - -pre-commit==3.0.4 - -# optional dependencies -orjson==3.8.6 +types-python-dateutil==2.8.19.7 +types-urllib3==1.26.25.5 diff --git a/setup.py b/setup.py index b1e680a0f..215181c73 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,11 @@ py_modules=[splitext(basename(path))[0] for path in glob("pystac/*.py")], python_requires=">=3.8", install_requires=["python-dateutil>=2.7.0"], - extras_require={"validation": ["jsonschema>=4.0.1"], "orjson": ["orjson>=3.5"]}, + extras_require={ + "validation": ["jsonschema>=4.0.1"], + "orjson": ["orjson>=3.5"], + "urllib3": ["urllib3>=1.26"], + }, license="Apache Software License 2.0", license_files=["LICENSE"], zip_safe=False, diff --git a/tests/cassettes/test_retry_stac_io.yaml b/tests/cassettes/test_retry_stac_io.yaml new file mode 100644 index 000000000..277c15431 --- /dev/null +++ b/tests/cassettes/test_retry_stac_io.yaml @@ -0,0 +1,139 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: https://planetarycomputer.microsoft.com/api/stac/v1 + response: + body: + string: '{"type":"Catalog","id":"microsoft-pc","title":"Microsoft Planetary + Computer STAC API","description":"Searchable spatiotemporal metadata describing + Earth science datasets hosted by the Microsoft Planetary Computer","stac_version":"1.0.0","conformsTo":["http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core","http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson","http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30","http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/filter","https://api.stacspec.org/v1.0.0-rc.1/collections","https://api.stacspec.org/v1.0.0-rc.1/core","https://api.stacspec.org/v1.0.0-rc.1/item-search","https://api.stacspec.org/v1.0.0-rc.1/item-search#fields","https://api.stacspec.org/v1.0.0-rc.1/item-search#filter","https://api.stacspec.org/v1.0.0-rc.1/item-search#filter:basic-cql","https://api.stacspec.org/v1.0.0-rc.1/item-search#filter:cql-json","https://api.stacspec.org/v1.0.0-rc.1/item-search#filter:cql-text","https://api.stacspec.org/v1.0.0-rc.1/item-search#query","https://api.stacspec.org/v1.0.0-rc.1/item-search#sort","https://api.stacspec.org/v1.0.0-rc.1/ogcapi-features"],"links":[{"rel":"self","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/"},{"rel":"root","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/"},{"rel":"data","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections"},{"rel":"conformance","type":"application/json","title":"STAC/WFS3 + conformance classes implemented by this server","href":"https://planetarycomputer.microsoft.com/api/stac/v1/conformance"},{"rel":"search","type":"application/geo+json","title":"STAC + search","href":"https://planetarycomputer.microsoft.com/api/stac/v1/search","method":"GET"},{"rel":"search","type":"application/geo+json","title":"STAC + search","href":"https://planetarycomputer.microsoft.com/api/stac/v1/search","method":"POST"},{"rel":"child","type":"application/json","title":"Daymet + Annual Puerto Rico","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-annual-pr"},{"rel":"child","type":"application/json","title":"Daymet + Daily Hawaii","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-daily-hi"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Seamless DEMs","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-seamless"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Digital Surface Model","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-dsm"},{"rel":"child","type":"application/json","title":"Forest + Inventory and Analysis","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/fia"},{"rel":"child","type":"application/json","title":"ESA + WorldCover 2020","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-worldcover"},{"rel":"child","type":"application/json","title":"Sentinel + 1 Radiometrically Terrain Corrected (RTC)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-1-rtc"},{"rel":"child","type":"application/json","title":"gridMET","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/gridmet"},{"rel":"child","type":"application/json","title":"Daymet + Annual North America","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-annual-na"},{"rel":"child","type":"application/json","title":"Daymet + Monthly North America","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-monthly-na"},{"rel":"child","type":"application/json","title":"Daymet + Annual Hawaii","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-annual-hi"},{"rel":"child","type":"application/json","title":"Daymet + Monthly Hawaii","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-monthly-hi"},{"rel":"child","type":"application/json","title":"Daymet + Monthly Puerto Rico","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-monthly-pr"},{"rel":"child","type":"application/json","title":"gNATSGO + Soil Database - Tables","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/gnatsgo-tables"},{"rel":"child","type":"application/json","title":"HGB: + Harmonized Global Biomass for 2010","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/hgb"},{"rel":"child","type":"application/json","title":"Copernicus + DEM GLO-30","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/cop-dem-glo-30"},{"rel":"child","type":"application/json","title":"Copernicus + DEM GLO-90","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/cop-dem-glo-90"},{"rel":"child","type":"application/json","title":"GOES-R + Cloud & Moisture Imagery","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/goes-cmi"},{"rel":"child","type":"application/json","title":"TerraClimate","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/terraclimate"},{"rel":"child","type":"application/json","title":"Earth + Exchange Global Daily Downscaled Projections (NEX-GDDP-CMIP6)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/nasa-nex-gddp-cmip6"},{"rel":"child","type":"application/json","title":"GPM + IMERG","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/gpm-imerg-hhr"},{"rel":"child","type":"application/json","title":"10m + Annual Land Use Land Cover (9-class)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/io-lulc-9-class"},{"rel":"child","type":"application/json","title":"gNATSGO + Soil Database - Rasters","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/gnatsgo-rasters"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Height above Ground","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-hag"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Intensity","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-intensity"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Point Source","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-pointsourceid"},{"rel":"child","type":"application/json","title":"MTBS: + Monitoring Trends in Burn Severity","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/mtbs"},{"rel":"child","type":"application/json","title":"Landsat + 8 Collection 2 Level-2","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/landsat-8-c2-l2"},{"rel":"child","type":"application/json","title":"C-CAP + Regional Land Cover and Change","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-c-cap"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Point Cloud","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-copc"},{"rel":"child","type":"application/json","title":"MODIS + Burned Area Monthly","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-64A1-061"},{"rel":"child","type":"application/json","title":"ALOS + Forest/Non-Forest Annual Mosaic","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/alos-fnf-mosaic"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Returns","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-returns"},{"rel":"child","type":"application/json","title":"MoBI: + Map of Biodiversity Importance","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/mobi"},{"rel":"child","type":"application/json","title":"Landsat + Collection 2 Level-2","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/landsat-c2-l2"},{"rel":"child","type":"application/json","title":"ERA5 + - PDS","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/era5-pds"},{"rel":"child","type":"application/json","title":"NAIP: + National Agriculture Imagery Program","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/naip"},{"rel":"child","type":"application/json","title":"Chloris + Biomass","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/chloris-biomass"},{"rel":"child","type":"application/json","title":"HydroForecast + - Kwando & Upper Zambezi Rivers","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/kaza-hydroforecast"},{"rel":"child","type":"application/json","title":"Planet-NICFI + Basemaps (Analytic)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/planet-nicfi-analytic"},{"rel":"child","type":"application/json","title":"MODIS + Gross Primary Productivity 8-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-17A2H-061"},{"rel":"child","type":"application/json","title":"MODIS + Land Surface Temperature/Emissivity 8-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-11A2-061"},{"rel":"child","type":"application/json","title":"Daymet + Daily Puerto Rico","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-daily-pr"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Digital Terrain Model (Native)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-dtm-native"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Classification","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-classification"},{"rel":"child","type":"application/json","title":"USGS + 3DEP Lidar Digital Terrain Model","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/3dep-lidar-dtm"},{"rel":"child","type":"application/json","title":"USGS + Gap Land Cover","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/gap"},{"rel":"child","type":"application/json","title":"MODIS + Gross Primary Productivity 8-Day Gap-Filled","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-17A2HGF-061"},{"rel":"child","type":"application/json","title":"Planet-NICFI + Basemaps (Visual)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/planet-nicfi-visual"},{"rel":"child","type":"application/json","title":"Global + Biodiversity Information Facility (GBIF)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/gbif"},{"rel":"child","type":"application/json","title":"MODIS + Net Primary Production Yearly Gap-Filled","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-17A3HGF-061"},{"rel":"child","type":"application/json","title":"MODIS + Surface Reflectance 8-Day (500m)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-09A1-061"},{"rel":"child","type":"application/json","title":"ALOS + World 3D-30m","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/alos-dem"},{"rel":"child","type":"application/json","title":"ALOS + PALSAR Annual Mosaic","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/alos-palsar-mosaic"},{"rel":"child","type":"application/json","title":"Deltares + Global Water Availability","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/deltares-water-availability"},{"rel":"child","type":"application/json","title":"MODIS + Net Evapotranspiration Yearly Gap-Filled","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-16A3GF-061"},{"rel":"child","type":"application/json","title":"MODIS + Land Surface Temperature/3-Band Emissivity 8-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-21A2-061"},{"rel":"child","type":"application/json","title":"US + Census","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/us-census"},{"rel":"child","type":"application/json","title":"JRC + Global Surface Water","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/jrc-gsw"},{"rel":"child","type":"application/json","title":"Deltares + Global Flood Maps","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/deltares-floods"},{"rel":"child","type":"application/json","title":"MODIS + Nadir BRDF-Adjusted Reflectance (NBAR) Daily","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-43A4-061"},{"rel":"child","type":"application/json","title":"MODIS + Surface Reflectance 8-Day (250m)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-09Q1-061"},{"rel":"child","type":"application/json","title":"MODIS + Thermal Anomalies/Fire Daily","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-14A1-061"},{"rel":"child","type":"application/json","title":"HREA: + High Resolution Electricity Access","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/hrea"},{"rel":"child","type":"application/json","title":"MODIS + Vegetation Indices 16-Day (250m)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-13Q1-061"},{"rel":"child","type":"application/json","title":"MODIS + Thermal Anomalies/Fire 8-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-14A2-061"},{"rel":"child","type":"application/json","title":"Sentinel-2 + Level-2A","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a"},{"rel":"child","type":"application/json","title":"MODIS + Leaf Area Index/FPAR 8-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-15A2H-061"},{"rel":"child","type":"application/json","title":"MODIS + Land Surface Temperature/Emissivity Daily","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-11A1-061"},{"rel":"child","type":"application/json","title":"MODIS + Leaf Area Index/FPAR 4-Day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-15A3H-061"},{"rel":"child","type":"application/json","title":"MODIS + Snow Cover 8-day","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-10A2-061"},{"rel":"child","type":"application/json","title":"MODIS + Snow Cover Daily","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-10A1-061"},{"rel":"child","type":"application/json","title":"MODIS + Vegetation Indices 16-Day (500m)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/modis-13A1-061"},{"rel":"child","type":"application/json","title":"Daymet + Daily North America","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/daymet-daily-na"},{"rel":"child","type":"application/json","title":"Land + Cover of Canada","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/nrcan-landcover"},{"rel":"child","type":"application/json","title":"ECMWF + Open Data (real-time)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/ecmwf-forecast"},{"rel":"child","type":"application/json","title":"NOAA + MRMS QPE 24-Hour Pass 2","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-mrms-qpe-24h-pass2"},{"rel":"child","type":"application/json","title":"Sentinel + 1 Level-1 Ground Range Detected (GRD)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-1-grd"},{"rel":"child","type":"application/json","title":"NASADEM + HGT v001","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/nasadem"},{"rel":"child","type":"application/json","title":"Esri + 10-Meter Land Cover (10-class)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/io-lulc"},{"rel":"child","type":"application/json","title":"Landsat + Collection 2 Level-1","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/landsat-c2-l1"},{"rel":"child","type":"application/json","title":"Denver + Regional Council of Governments Land Use Land Cover","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/drcog-lulc"},{"rel":"child","type":"application/json","title":"Chesapeake + Land Cover (7-class)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/chesapeake-lc-7"},{"rel":"child","type":"application/json","title":"Chesapeake + Land Cover (13-class)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/chesapeake-lc-13"},{"rel":"child","type":"application/json","title":"Chesapeake + Land Use","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/chesapeake-lu"},{"rel":"child","type":"application/json","title":"NOAA + MRMS QPE 1-Hour Pass 1","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-mrms-qpe-1h-pass1"},{"rel":"child","type":"application/json","title":"NOAA + MRMS QPE 1-Hour Pass 2","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-mrms-qpe-1h-pass2"},{"rel":"child","type":"application/json","title":"Monthly + NOAA U.S. Climate Gridded Dataset (NClimGrid)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-nclimgrid-monthly"},{"rel":"child","type":"application/json","title":"GOES-R + Lightning Detection","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/goes-glm"},{"rel":"child","type":"application/json","title":"USDA + Cropland Data Layers (CDLs)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/usda-cdl"},{"rel":"child","type":"application/json","title":"Urban + Innovation Eclipse Sensor Data","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/eclipse"},{"rel":"child","type":"application/json","title":"ESA + Climate Change Initiative Land Cover Maps (Cloud Optimized GeoTIFF)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-cci-lc"},{"rel":"child","type":"application/json","title":"ESA + Climate Change Initiative Land Cover Maps (NetCDF)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-cci-lc-netcdf"},{"rel":"child","type":"application/json","title":"FWS + National Wetlands Inventory","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/fws-nwi"},{"rel":"child","type":"application/json","title":"USGS + LCMAP CONUS Collection 1.3","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/usgs-lcmap-conus-v13"},{"rel":"child","type":"application/json","title":"USGS + LCMAP Hawaii Collection 1.0","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/usgs-lcmap-hawaii-v10"},{"rel":"child","type":"application/json","title":"NOAA + US Tabular Climate Normals","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-climate-normals-tabular"},{"rel":"child","type":"application/json","title":"NOAA + US Gridded Climate Normals (NetCDF)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-climate-normals-netcdf"},{"rel":"child","type":"application/json","title":"NOAA + US Gridded Climate Normals (Cloud-Optimized GeoTIFF)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/noaa-climate-normals-gridded"},{"rel":"child","type":"application/json","title":"ASTER + L1T","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/aster-l1t"},{"rel":"child","type":"application/json","title":"CIL + Global Downscaled Projections for Climate Impacts Research (CC-BY-SA-4.0)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/cil-gdpcir-cc-by-sa"},{"rel":"child","type":"application/json","title":"CIL + Global Downscaled Projections for Climate Impacts Research (CC-BY-4.0)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/cil-gdpcir-cc-by"},{"rel":"child","type":"application/json","title":"CIL + Global Downscaled Projections for Climate Impacts Research (CC0-1.0)","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/cil-gdpcir-cc0"},{"rel":"child","type":"application/json","title":"Microsoft + Building Footprints","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/ms-buildings"},{"rel":"service-desc","type":"application/vnd.oai.openapi+json;version=3.0","title":"OpenAPI + service description","href":"https://planetarycomputer.microsoft.com/api/stac/v1/openapi.json"},{"rel":"service-doc","type":"text/html","title":"OpenAPI + service documentation","href":"https://planetarycomputer.microsoft.com/api/stac/v1/docs"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Origin: + - '*' + Content-Length: + - '20440' + Content-Type: + - application/json + Date: + - Tue, 14 Feb 2023 21:19:45 GMT + Strict-Transport-Security: + - max-age=15724800; includeSubDomains + X-Azure-Ref: + - 08frrYwAAAACZRrmBI+hZT56du4hepCBFV1NURURHRTAxMDkAOTI3YWJmYTYtMTlmNi00YWYxLWEwOWQtYzk1OWQ5YTFlNjQ0 + X-Cache: + - CONFIG_NOCACHE + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_retry_stac_io_404.yaml b/tests/cassettes/test_retry_stac_io_404.yaml new file mode 100644 index 000000000..0381d57b2 --- /dev/null +++ b/tests/cassettes/test_retry_stac_io_404.yaml @@ -0,0 +1,33 @@ +interactions: +- request: + body: null + headers: {} + method: GET + uri: https://planetarycomputer.microsoft.com/api/stac/v1/collections/not-a-collection-id + response: + body: + string: '{"code":"NotFoundError","description":"No collection with id ''not-a-collection-id'' + found!"}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Origin: + - '*' + Connection: + - keep-alive + Content-Length: + - '91' + Content-Type: + - application/json + Date: + - Tue, 14 Feb 2023 21:19:47 GMT + Strict-Transport-Security: + - max-age=15724800; includeSubDomains + X-Cache: + - CONFIG_NOCACHE + x-azure-ref: + - 20230214T211946Z-4y4d7db50t7213q871yh9z53gs000000018g0000000081rq + status: + code: 404 + message: Not Found +version: 1 diff --git a/tests/test_stac_io.py b/tests/test_stac_io.py index 5d086f7ee..76f4afb6c 100644 --- a/tests/test_stac_io.py +++ b/tests/test_stac_io.py @@ -3,6 +3,8 @@ import tempfile import unittest +import pytest + import pystac from pystac.stac_io import DefaultStacIO, DuplicateKeyReportingMixin, StacIO from tests.utils import TestCases @@ -114,3 +116,29 @@ def test_headers_stac_io(self, urlopen_mock: unittest.mock.MagicMock) -> None: request_obj = urlopen_mock.call_args[0][0] self.assertEqual(request_obj.headers, stac_io.headers) + + +@pytest.mark.vcr() +def test_retry_stac_io() -> None: + # This doesn't test any retry behavior, but it does make sure that we can + # still read objects. + _ = pytest.importorskip("urllib3") + from pystac.stac_io import RetryStacIO + + stac_io = RetryStacIO() + _ = stac_io.read_stac_object("https://planetarycomputer.microsoft.com/api/stac/v1") + + +@pytest.mark.vcr() +def test_retry_stac_io_404() -> None: + # This doesn't test any retry behavior, but it does make sure that we can + # error when an object doesn't exist. + _ = pytest.importorskip("urllib3") + from pystac.stac_io import RetryStacIO + + stac_io = RetryStacIO() + with pytest.raises(Exception): + _ = stac_io.read_stac_object( + "https://planetarycomputer.microsoft.com" + "/api/stac/v1/collections/not-a-collection-id" + )