Merge branch 'fix-integ-ws'

PR #1372.

* fix-integ-ws:
  Retry any HTTP requests if we see 5xx errors
  Retry get_json failures
  Make ws integ test more robust

Author: jamesls

tests/aws/test_features.py

@@ -36,6 +36,10 @@ def _wrapped_with_retry(*args, **kwargs):
     return _create_wrapped_retry_function
 
 
+class InternalServerError(Exception):
+    pass
+
+
 class SmokeTestApplication(object):
 
     # Number of seconds to wait after redeploy before starting
@@ -87,13 +91,59 @@ def websocket_connect_url(self):
             )
         )
 
+    @retry(max_attempts=10, delay=5)
     def get_json(self, url):
+        try:
+            return self._get_json(url)
+        except requests.exceptions.HTTPError:
+            pass
+
+    def _get_json(self, url):
         if not url.startswith('/'):
             url = '/' + url
         response = requests.get(self.url + url)
         response.raise_for_status()
         return response.json()
 
+    @retry(max_attempts=10, delay=5)
+    def get_response(self, url, headers=None):
+        try:
+            return self._send_request('GET', url, headers=headers)
+        except InternalServerError:
+            pass
+
+    def _send_request(self, http_method, url, headers=None, data=None):
+        kwargs = {}
+        if headers is not None:
+            kwargs['headers'] = headers
+        if data is not None:
+            kwargs['data'] = data
+        response = requests.request(http_method, self.url + url, **kwargs)
+        if response.status_code >= 500:
+            raise InternalServerError()
+        return response
+
+    @retry(max_attempts=10, delay=5)
+    def post_response(self, url, headers=None, data=None):
+        try:
+            return self._send_request('POST', url, headers=headers, data=data)
+        except InternalServerError:
+            pass
+
+    @retry(max_attempts=10, delay=5)
+    def put_response(self, url):
+        try:
+            return self._send_request('PUT', url)
+        except InternalServerError:
+            pass
+
+    @retry(max_attempts=10, delay=5)
+    def options_response(self, url):
+        try:
+            return self._send_request('OPTIONS', url)
+        except InternalServerError:
+            pass
+
     def redeploy_once(self):
         # Redeploy the application once.  If a redeploy
         # has already happened, this function is a noop.
@@ -111,16 +161,12 @@ def redeploy_once(self):
             self._wait_for_stablize()
             time.sleep(self._POLLING_DELAY)
 
-    @retry(max_attempts=10, delay=5)
     def _wait_for_stablize(self):
         # After a deployment we sometimes need to wait for
         # API Gateway to propagate all of its changes.
         # We're going to give it num_attempts to give us a
         # 200 response before failing.
-        try:
-            return self.get_json('/')
-        except requests.exceptions.HTTPError:
-            pass
+        return self.get_json('/')
 
 
 @pytest.fixture
@@ -284,77 +330,75 @@ def _get_resource_id(apig_client, rest_api_id, path):
 
 
 def test_supports_post(smoke_test_app):
-    app_url = smoke_test_app.url
-    response = requests.post(app_url + '/post')
+    response = smoke_test_app.post_response('/post')
     response.raise_for_status()
     assert response.json() == {'success': True}
     with pytest.raises(requests.HTTPError):
         # Only POST is supported.
-        response = requests.get(app_url + '/post')
+        response = smoke_test_app.get_response('/post')
         response.raise_for_status()
 
 
 def test_supports_put(smoke_test_app):
-    app_url = smoke_test_app.url
-    response = requests.put(app_url + '/put')
+    response = smoke_test_app.put_response('/put')
     response.raise_for_status()
     assert response.json() == {'success': True}
     with pytest.raises(requests.HTTPError):
         # Only PUT is supported.
-        response = requests.get(app_url + '/put')
+        response = smoke_test_app.get_response('/put')
         response.raise_for_status()
 
 
 def test_supports_shared_routes(smoke_test_app):
-    app_url = smoke_test_app.url
-    response = requests.get(app_url + '/shared')
-    assert response.json() == {'method': 'GET'}
-    response = requests.post(app_url + '/shared')
+    response = smoke_test_app.get_json('/shared')
+    assert response == {'method': 'GET'}
+    response = smoke_test_app.post_response('/shared')
     assert response.json() == {'method': 'POST'}
 
 
 def test_can_read_json_body_on_post(smoke_test_app):
-    app_url = smoke_test_app.url
-    response = requests.post(
-        app_url + '/jsonpost', data=json.dumps({'hello': 'world'}),
+    response = smoke_test_app.post_response(
+        '/jsonpost', data=json.dumps({'hello': 'world'}),
         headers={'Content-Type': 'application/json'})
     response.raise_for_status()
     assert response.json() == {'json_body': {'hello': 'world'}}
 
 
 def test_can_raise_bad_request(smoke_test_app):
-    response = requests.get(smoke_test_app.url + '/badrequest')
+    response = smoke_test_app.get_response('/badrequest')
     assert response.status_code == 400
     assert response.json()['Code'] == 'BadRequestError'
     assert response.json()['Message'] == 'BadRequestError: Bad request.'
 
 
 def test_can_raise_not_found(smoke_test_app):
-    response = requests.get(smoke_test_app.url + '/notfound')
+    response = smoke_test_app.get_response('/notfound')
     assert response.status_code == 404
     assert response.json()['Code'] == 'NotFoundError'
 
 
 def test_unexpected_error_raises_500_in_prod_mode(smoke_test_app):
+    # Can't use smoke_test_app.get_response() because we're explicitly
+    # testing for a 500.
     response = requests.get(smoke_test_app.url + '/arbitrary-error')
     assert response.status_code == 500
     assert response.json()['Code'] == 'InternalServerError'
     assert 'internal server error' in response.json()['Message']
 
 
 def test_can_route_multiple_methods_in_one_view(smoke_test_app):
-    response = requests.get(smoke_test_app.url + '/multimethod')
+    response = smoke_test_app.get_response('/multimethod')
     response.raise_for_status()
     assert response.json()['method'] == 'GET'
 
-    response = requests.post(smoke_test_app.url + '/multimethod')
+    response = smoke_test_app.post_response('/multimethod')
     response.raise_for_status()
     assert response.json()['method'] == 'POST'
 
 
 def test_form_encoded_content_type(smoke_test_app):
-    response = requests.post(smoke_test_app.url + '/formencoded',
-                             data={'foo': 'bar'})
+    response = smoke_test_app.post_response('/formencoded',
+                                            data={'foo': 'bar'})
     response.raise_for_status()
     assert response.json() == {'parsed': {'foo': ['bar']}}
 
@@ -363,34 +407,31 @@ def test_can_round_trip_binary(smoke_test_app):
     # xde xed xbe xef will fail unicode decoding because xbe is an invalid
     # start byte in utf-8.
     bin_data = b'\xDE\xAD\xBE\xEF'
-    response = requests.post(smoke_test_app.url + '/binary',
-                             headers={
-                                 'Content-Type': 'application/octet-stream',
-                                 'Accept': 'application/octet-stream',
-                             },
-                             data=bin_data)
+    response = smoke_test_app.post_response(
+        '/binary',
+        headers={'Content-Type': 'application/octet-stream',
+                 'Accept': 'application/octet-stream'},
+        data=bin_data)
     response.raise_for_status()
     assert response.content == bin_data
 
 
 def test_can_round_trip_binary_custom_content_type(smoke_test_app):
     bin_data = b'\xDE\xAD\xBE\xEF'
-    response = requests.post(smoke_test_app.url + '/custom-binary',
-                             headers={
-                                 'Content-Type': 'application/binary',
-                                 'Accept': 'application/binary',
-                             },
-                             data=bin_data)
+    response = smoke_test_app.post_response(
+        '/custom-binary',
+        headers={'Content-Type': 'application/binary',
+                 'Accept': 'application/binary'},
+        data=bin_data
+    )
     assert response.content == bin_data
 
 
 def test_can_return_default_binary_data_to_a_browser(smoke_test_app):
     base64encoded_response = b'3q2+7w=='
     accept = 'text/html,application/xhtml+xml;q=0.9,image/webp,*/*;q=0.8'
-    response = requests.get(smoke_test_app.url + '/get-binary',
-                            headers={
-                                'Accept': accept,
-                             })
+    response = smoke_test_app.get_response(
+        '/get-binary', headers={'Accept': accept})
     response.raise_for_status()
     assert response.content == base64encoded_response
 
@@ -403,12 +444,12 @@ def _assert_contains_access_control_allow_methods(headers, methods):
 
 
 def test_can_support_cors(smoke_test_app):
-    response = requests.get(smoke_test_app.url + '/cors')
+    response = smoke_test_app.get_response('/cors')
     response.raise_for_status()
     assert response.headers['Access-Control-Allow-Origin'] == '*'
 
     # Should also have injected an OPTIONs request.
-    response = requests.options(smoke_test_app.url + '/cors')
+    response = smoke_test_app.options_response('/cors')
     response.raise_for_status()
     headers = response.headers
     assert headers['Access-Control-Allow-Origin'] == '*'
@@ -420,14 +461,14 @@ def test_can_support_cors(smoke_test_app):
 
 
 def test_can_support_custom_cors(smoke_test_app):
-    response = requests.get(smoke_test_app.url + '/custom_cors')
+    response = smoke_test_app.get_response('/custom_cors')
     response.raise_for_status()
     expected_allow_origin = 'https://foo.example.com'
     assert response.headers[
         'Access-Control-Allow-Origin'] == expected_allow_origin
 
     # Should also have injected an OPTIONs request.
-    response = requests.options(smoke_test_app.url + '/custom_cors')
+    response = smoke_test_app.options_response('/custom_cors')
     response.raise_for_status()
     headers = response.headers
     assert headers['Access-Control-Allow-Origin'] == expected_allow_origin
@@ -451,8 +492,7 @@ def test_multifile_support(smoke_test_app):
 
 
 def test_custom_response(smoke_test_app):
-    url = smoke_test_app.url + '/custom-response'
-    response = requests.get(url)
+    response = smoke_test_app.get_response('/custom-response')
     response.raise_for_status()
     # Custom header
     assert response.headers['Content-Type'] == 'text/plain'
@@ -463,28 +503,30 @@ def test_custom_response(smoke_test_app):
 
 
 def test_api_key_required_fails_with_no_key(smoke_test_app):
-    url = smoke_test_app.url + '/api-key-required'
-    response = requests.get(url)
+    response = smoke_test_app.get_response('/api-key-required')
     # Request should fail because we're not providing
     # an API key.
     assert response.status_code == 403
 
 
 def test_can_handle_charset(smoke_test_app):
-    url = smoke_test_app.url + '/json-only'
     # Should pass content type validation even with charset specified.
-    response = requests.get(
-        url, headers={'Content-Type': 'application/json; charset=utf-8'})
+    response = smoke_test_app.get_response(
+        '/json-only',
+        headers={'Content-Type': 'application/json; charset=utf-8'}
+    )
     assert response.status_code == 200
 
 
 def test_can_use_builtin_custom_auth(smoke_test_app):
-    url = smoke_test_app.url + '/builtin-auth'
+    url = '/builtin-auth'
     # First time without an Auth header, we should fail.
-    response = requests.get(url)
+    response = smoke_test_app.get_response(url)
     assert response.status_code == 401
     # Now with the proper auth header, things should work.
-    response = requests.get(url, headers={'Authorization': 'yes'})
+    response = smoke_test_app.get_response(
+        url, headers={'Authorization': 'yes'}
+    )
     assert response.status_code == 200
     context = response.json()['context']
     assert 'authorizer' in context
@@ -494,24 +536,23 @@ def test_can_use_builtin_custom_auth(smoke_test_app):
 
 
 def test_can_use_shared_auth(smoke_test_app):
-    url = smoke_test_app.url + '/fake-profile'
-    response = requests.get(url)
+    response = smoke_test_app.get_response('/fake-profile')
     # GETs are allowed
     assert response.status_code == 200
     # However, POSTs require auth.
     # This has the same auth config as /builtin-auth,
     # so we're testing the auth handler can be shared.
-    assert requests.post(url).status_code == 401
-    response = requests.post(url, headers={'Authorization': 'yes'})
+    assert smoke_test_app.post_response('/fake-profile').status_code == 401
+    response = smoke_test_app.post_response('/fake-profile',
+                                            headers={'Authorization': 'yes'})
     assert response.status_code == 200
     context = response.json()['context']
     assert 'authorizer' in context
     assert context['authorizer']['foo'] == 'bar'
 
 
 def test_empty_raw_body(smoke_test_app):
-    url = smoke_test_app.url + '/repr-raw-body'
-    response = requests.post(url)
+    response = smoke_test_app.post_response('/repr-raw-body')
     response.raise_for_status()
     assert response.json() == {'repr-raw-body': ''}
 
@@ -556,18 +597,16 @@ def test_redeploy_new_function(smoke_test_app):
 @pytest.mark.on_redeploy
 def test_redeploy_change_route_info(smoke_test_app):
     smoke_test_app.redeploy_once()
-    url = smoke_test_app.url + '/multimethod'
     # POST is no longer allowed:
-    assert requests.post(url).status_code == 403
+    assert smoke_test_app.post_response('/multimethod').status_code == 403
     # But PUT is now allowed in the redeployed app.py
-    assert requests.put(url).status_code == 200
+    assert smoke_test_app.put_response('/multimethod').status_code == 200
 
 
 @pytest.mark.on_redeploy
 def test_redeploy_view_deleted(smoke_test_app):
     smoke_test_app.redeploy_once()
-    url = smoke_test_app.url + '/path/foo'
-    response = requests.get(url)
+    response = smoke_test_app.get_response('/path/foo')
     # Request should fail because it's not in the redeployed
     # app.py
     assert response.status_code == 403