make test_psm3.py robust to API overuse errors

azure-pipelines.yml

@@ -19,14 +19,16 @@ jobs:
       Python37:
         python.version: '3.7'
 
+
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '$(python.version)'
 
   - script: |
-      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines
+      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures
       pip install -e .
+      export NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
     displayName: 'Test with pytest'
 
@@ -74,6 +76,7 @@ jobs:
     displayName: 'List installed dependencies'
   - script: |
       source activate test_env
+      export NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml --cov --cov-report=xml --cov-report=html
     displayName: 'pytest'
   # - script: |
@@ -126,6 +129,7 @@ jobs:
     displayName: 'List installed dependencies'
   - script: |
       call activate test_env
+      set NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml
     displayName: 'pytest'
   - task: PublishTestResults@2
@@ -153,8 +157,9 @@ jobs:
       versionSpec: '$(python.version)'
 
   - script: |
-      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines
+      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures
       pip install -e .
+      export NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml --cov=pvlib --cov-report=xml --cov-report=html
     displayName: 'Test with pytest'
 

ci/requirements-py35-min.yml

@@ -15,3 +15,4 @@ dependencies:
     - pip:
         - numpy==1.12.0
         - pandas==0.18.1
+        - pytest-rerunfailures # conda version is >3.6

ci/requirements-py35.yml

@@ -26,3 +26,4 @@ dependencies:
     - pip:
         - nrel-pysam>=2.0
         - pvfactors==1.0.1
+        - pytest-rerunfailures # conda version is >3.6

ci/requirements-py36.yml

@@ -16,6 +16,7 @@ dependencies:
     - pytest
     - pytest-cov
     - pytest-mock
+    - pytest-rerunfailures
     - pytest-timeout
     - python=3.6
     - pytz

ci/requirements-py37.yml

@@ -17,6 +17,7 @@ dependencies:
     - pytest-cov
     - pytest-mock
     - pytest-timeout
+    - pytest-rerunfailures
     - python=3.7
     - pytz
     - requests

docs/sphinx/source/whatsnew/v0.7.2.rst

@@ -13,6 +13,8 @@ Bug fixes
 ~~~~~~~~~
 * Fix :py:func:`~pvlib.iotools.read_tmy3` parsing when February contains
   a leap year (:pull:`866`)
+* Implement NREL Developer Network API key for consistent success with API
+  calls in :py:mod:`pvlib.tests.iotools.test_psm3` (:pull:`873`)
 
 Documentation
 ~~~~~~~~~~~~~
@@ -26,3 +28,4 @@ Contributors
 ~~~~~~~~~~~~
 * Mark Mikofski (:ghuser:`mikofski`)
 * Cliff Hansen (:ghuser:`cwhanse`)
+* Cameron T. Stark (:ghuser:`camerontstark`)

pvlib/tests/iotools/test_psm3.py

@@ -2,13 +2,15 @@
 test iotools for PSM3
 """
 
+import os
 from pvlib.iotools import psm3
 from conftest import needs_pandas_0_22, DATA_DIR
 import numpy as np
 import pandas as pd
 import pytest
 from requests import HTTPError
 from io import StringIO
+import warnings
 
 TMY_TEST_DATA = DATA_DIR / 'test_psm3_tmy-2017.csv'
 YEAR_TEST_DATA = DATA_DIR / 'test_psm3_2017.csv'
@@ -21,7 +23,26 @@
     'Temperature Units', 'Pressure Units', 'Wind Direction Units',
     'Wind Speed', 'Surface Albedo Units', 'Version']
 PVLIB_EMAIL = '[email protected]'
-DEMO_KEY = 'DEMO_KEY'
+
+
+@pytest.fixture(scope="module")
+def nrel_api_key():
+    """Supplies pvlib-python's NREL Developer Network API key.
+
+    Azure Pipelines CI utilizes a secret variable set to NREL_API_KEY
+    to mitigate failures associated with using the default key of
+    "DEMO_KEY". A user is capable of using their own key this way if
+    desired however the default key should suffice for testing purposes.
+    """
+    try:
+        demo_key = os.environ["NREL_API_KEY"]
+    except KeyError:
+        warnings.warn(
+            "WARNING: NREL API KEY environment variable not set! "
+            "Using DEMO_KEY instead. Unexpected failures may occur."
+        )
+        demo_key = 'DEMO_KEY'
+    return demo_key
 
 
 def assert_psm3_equal(header, data, expected):
@@ -50,47 +71,49 @@ def assert_psm3_equal(header, data, expected):
 
 
 @needs_pandas_0_22
-def test_get_psm3_tmy():
+@pytest.mark.flaky(reruns=5, reruns_delay=2)
+def test_get_psm3_tmy(nrel_api_key):
     """test get_psm3 with a TMY"""
-    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL,
-                                 names='tmy-2017')
+    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key,
+                                 PVLIB_EMAIL, names='tmy-2017')
     expected = pd.read_csv(TMY_TEST_DATA)
     assert_psm3_equal(header, data, expected)
-    # check errors
-    with pytest.raises(HTTPError):
-        # HTTP 403 forbidden because api_key is rejected
-        psm3.get_psm3(LATITUDE, LONGITUDE, api_key='BAD', email=PVLIB_EMAIL)
-    with pytest.raises(HTTPError):
-        # coordinates were not found in the NSRDB
-        psm3.get_psm3(51, -5, DEMO_KEY, PVLIB_EMAIL)
-    with pytest.raises(HTTPError):
-        # names is not one of the available options
-        psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL, names='bad')
 
 
 @needs_pandas_0_22
-def test_get_psm3_singleyear():
+@pytest.mark.flaky(reruns=5, reruns_delay=2)
+def test_get_psm3_singleyear(nrel_api_key):
     """test get_psm3 with a single year"""
-    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL,
-                                 names='2017', interval=30)
+    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key,
+                                 PVLIB_EMAIL, names='2017', interval=30)
     expected = pd.read_csv(YEAR_TEST_DATA)
     assert_psm3_equal(header, data, expected)
-    # check leap day
-    _, data_2012 = psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL,
-                                 names='2012', interval=60, leap_day=True)
-    assert len(data_2012) == (8760+24)
-    # check errors
-    with pytest.raises(HTTPError):
-        # HTTP 403 forbidden because api_key is rejected
-        psm3.get_psm3(LATITUDE, LONGITUDE, api_key='BAD', email=PVLIB_EMAIL,
-                      names='2017')
-    with pytest.raises(HTTPError):
-        # coordinates were not found in the NSRDB
-        psm3.get_psm3(51, -5, DEMO_KEY, PVLIB_EMAIL, names='2017')
-    with pytest.raises(HTTPError):
-        # intervals can only be 30 or 60 minutes
-        psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL, names='2017',
-                      interval=15)
+
+
+@pytest.mark.parametrize('latitude, longitude, api_key, names, interval',
+                         [(LATITUDE, LONGITUDE, 'BAD', 'tmy-2017', 60),
+                          (51, -5, nrel_api_key, 'tmy-2017', 60),
+                          (LATITUDE, LONGITUDE, nrel_api_key, 'bad', 60),
+                          (LATITUDE, LONGITUDE, nrel_api_key, '2017', 15),
+                          ])
+@needs_pandas_0_22
+@pytest.mark.flaky(reruns=5, reruns_delay=2)
+def test_get_psm3_tmy_errors(
+    latitude, longitude, api_key, names, interval
+):
+    """Test get_psm3() for multiple erroneous input scenarios.
+
+    These scenarios include:
+    * Bad API key -> HTTP 403 forbidden because api_key is rejected
+    * Bad latitude/longitude -> Coordinates were not found in the NSRDB.
+    * Bad name -> Name is not one of the available options.
+    * Bad interval, single year -> Intervals can only be 30 or 60 minutes.
+    """
+    with pytest.raises(HTTPError) as excinfo:
+        psm3.get_psm3(latitude, longitude, api_key, PVLIB_EMAIL,
+                      names=names, interval=interval)
+    # ensure the HTTPError caught isn't due to overuse of the API key
+    assert "OVER_RATE_LIMIT" not in str(excinfo.value)
 
 
 @pytest.fixture

setup.py

@@ -42,7 +42,7 @@
                     'pytz',
                     'requests']
 TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock',
-                 'pytest-timeout']
+                 'pytest-timeout', 'pytest-rerunfailures']
 EXTRAS_REQUIRE = {
     'optional': ['ephem', 'cython', 'netcdf4', 'nrel-pysam', 'numba',
                  'pvfactors', 'scipy', 'siphon', 'tables'],