Add urllib3 Retry mechanism, documentation and tests (#536)

Add support for urllib3 Retry adapter.

Co-authored-by: Mark Story <[email protected]>

Author: beliaev-maksim

CHANGES

@@ -2,6 +2,7 @@
 ------
 
 * Add `threading.Lock()` to allow `responses` working with `threading` module.
+* Add `urllib3` `Retry` mechanism. See #135
 * Removed internal `_cookies_from_headers` function
 * Now `add`, `upsert`, `replace` methods return registered response.
   `remove` method returns list of removed responses.

README.rst

@@ -997,6 +997,44 @@ in the execution chain and contain the history of redirects.
         assert rsp2.url in exc_info.value.response.history[1].url
 
 
+Validate ``Retry`` mechanism
+----------------------------
+
+If you are using the ``Retry`` features of ``urllib3`` and want to cover scenarios that test your retry limits, you can test those scenarios with ``responses`` as well. The best approach will be to use an `Ordered Registry`_
+
+..  code-block:: python
+
+    import requests
+
+    import responses
+    from responses import registries
+
+    @responses.activate(registry=registries.OrderedRegistry)
+    def test_max_retries():
+        url = 'https://example.com'
+        rsp1 = responses.get(url, body='Error', status=500)
+        rsp2 = responses.get(url, body='Error', status=500)
+        rsp3 = responses.get(url, body='Error', status=500)
+        rsp4 = responses.get(url, body='OK', status=200)
+
+        session = requests.Session()
+
+        adapter = requests.adapters.HTTPAdapter(max_retries=Retry(
+            total=4,
+            backoff_factor=0.1,
+            status_forcelist=[500],
+            method_whitelist=['GET', 'POST', 'PATCH']
+        ))
+        session.mount('https://', adapter)
+
+        resp = session.get(url)
+
+        assert resp.status_code == 200
+        assert rsp1.call_count == 1
+        assert rsp2.call_count == 1
+        assert rsp3.call_count == 1
+        assert rsp4.call_count == 1
+
 
 Using a callback to modify the response
 ---------------------------------------

responses/__init__.py

@@ -19,6 +19,7 @@
 from warnings import warn
 
 from requests.adapters import HTTPAdapter
+from requests.adapters import MaxRetryError
 from requests.exceptions import ConnectionError
 
 from responses.matchers import json_params_matcher as _json_params_matcher
@@ -846,7 +847,7 @@ def _parse_request_params(self, url):
             params[key] = values
         return params
 
-    def _on_request(self, adapter, request, **kwargs):
+    def _on_request(self, adapter, request, *, retries=None, **kwargs):
         # add attributes params and req_kwargs to 'request' object for further match comparison
         # original request object does not have these attributes
         request.params = self._parse_request_params(request.path_url)
@@ -904,6 +905,22 @@ def _on_request(self, adapter, request, **kwargs):
         response = resp_callback(response) if resp_callback else response
         match.call_count += 1
         self._calls.add(request, response)
+
+        retries = retries or adapter.max_retries
+        # first validate that current request is eligible to be retried.
+        # See ``requests.packages.urllib3.util.retry.Retry`` documentation.
+        if retries.is_retry(
+            method=response.request.method, status_code=response.status_code
+        ):
+            try:
+                retries = retries.increment(
+                    method=response.request.method, url=response.url, response=response
+                )
+                return self._on_request(adapter, request, retries=retries, **kwargs)
+            except MaxRetryError:
+                if retries.raise_on_status:
+                    raise
+                return response
         return response
 
     def start(self):

responses/tests/test_responses.py

@@ -13,16 +13,19 @@
 
 import pytest
 import requests
+from requests.adapters import MaxRetryError
 from requests.exceptions import ChunkedEncodingError
 from requests.exceptions import ConnectionError
 from requests.exceptions import HTTPError
+from urllib3.util.retry import Retry
 
 import responses
 from responses import BaseResponse
 from responses import CallbackResponse
 from responses import PassthroughResponse
 from responses import Response
 from responses import matchers
+from responses import registries
 
 
 def assert_reset():
@@ -2342,3 +2345,114 @@ def run():
 
     run()
     assert_reset()
+
+
+class TestMaxRetry:
+    def test_max_retries(self):
+        """This example is present in README.rst"""
+
+        @responses.activate(registry=registries.OrderedRegistry)
+        def run():
+            url = "https://example.com"
+            rsp1 = responses.get(url, body="Error", status=500)
+            rsp2 = responses.get(url, body="Error", status=500)
+            rsp3 = responses.get(url, body="Error", status=500)
+            rsp4 = responses.get(url, body="OK", status=200)
+
+            session = requests.Session()
+
+            adapter = requests.adapters.HTTPAdapter(
+                max_retries=Retry(
+                    total=4,
+                    backoff_factor=0.1,
+                    status_forcelist=[500],
+                    method_whitelist=["GET", "POST", "PATCH"],
+                )
+            )
+            session.mount("https://", adapter)
+
+            resp = session.get(url)
+
+            assert resp.status_code == 200
+            assert rsp1.call_count == 1
+            assert rsp2.call_count == 1
+            assert rsp3.call_count == 1
+            assert rsp4.call_count == 1
+
+        run()
+        assert_reset()
+
+    @pytest.mark.parametrize("raise_on_status", (True, False))
+    def test_max_retries_exceed(self, raise_on_status):
+        @responses.activate(registry=registries.OrderedRegistry)
+        def run():
+            url = "https://example.com"
+            rsp1 = responses.get(url, body="Error", status=500)
+            rsp2 = responses.get(url, body="Error", status=500)
+            rsp3 = responses.get(url, body="Error", status=500)
+
+            session = requests.Session()
+
+            adapter = requests.adapters.HTTPAdapter(
+                max_retries=Retry(
+                    total=2,
+                    backoff_factor=0.1,
+                    status_forcelist=[500],
+                    method_whitelist=["GET", "POST", "PATCH"],
+                    raise_on_status=raise_on_status,
+                )
+            )
+            session.mount("https://", adapter)
+
+            if raise_on_status:
+                with pytest.raises(MaxRetryError):
+                    session.get(url)
+            else:
+                resp = session.get(url)
+                assert resp.status_code == 500
+
+            assert rsp1.call_count == 1
+            assert rsp2.call_count == 1
+            assert rsp3.call_count == 1
+
+        run()
+        assert_reset()
+
+    def test_adapter_retry_untouched(self):
+        """Validate that every new request uses brand-new Retry object"""
+
+        @responses.activate(registry=registries.OrderedRegistry)
+        def run():
+            url = "https://example.com"
+            error_rsp = responses.get(url, body="Error", status=500)
+            responses.add(error_rsp)
+            responses.add(error_rsp)
+            ok_rsp = responses.get(url, body="OK", status=200)
+
+            responses.add(error_rsp)
+            responses.add(error_rsp)
+            responses.add(error_rsp)
+            responses.add(ok_rsp)
+
+            session = requests.Session()
+
+            adapter = requests.adapters.HTTPAdapter(
+                max_retries=Retry(
+                    total=4,
+                    backoff_factor=0.1,
+                    status_forcelist=[500],
+                    method_whitelist=["GET", "POST", "PATCH"],
+                )
+            )
+            session.mount("https://", adapter)
+
+            resp = session.get(url)
+            assert resp.status_code == 200
+
+            resp = session.get(url)
+            assert resp.status_code == 200
+
+            assert len(responses.calls) == 8
+
+        run()
+        assert_reset()