Skip to content

fix: Setting a default timeout on all HTTP connections #397

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jan 30, 2020
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions firebase_admin/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ def initialize_app(credential=None, options=None, name=_DEFAULT_APP_NAME):
Google Application Default Credentials are used.
options: A dictionary of configuration options (optional). Supported options include
``databaseURL``, ``storageBucket``, ``projectId``, ``databaseAuthVariableOverride``,
``serviceAccountId`` and ``httpTimeout``. If ``httpTimeout`` is not set, HTTP
connections initiated by client modules such as ``db`` will not time out.
``serviceAccountId`` and ``httpTimeout``. If ``httpTimeout`` is not set, the SDK
uses a default timeout of 120 seconds.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a behavioural change. If you want to keep the behaviour the same, we could instead explicitly set a None timeout and defer using the new default of 120s until the next major release of firebase-admin-python. (Although defaulting to an indefinite timeout does seem a little broken, so I'd be happy to call this a bug fix instead.)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

google-auth shipped the same change as minor version bump. Given our next version is going to be a major version bump this should be ok. And yes, I also prefer to consider this a bug fix than anything else.

name: Name of the app (optional).
Returns:
App: A newly initialized instance of App.
Expand Down
18 changes: 15 additions & 3 deletions firebase_admin/_http_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
raise_on_status=False, backoff_factor=0.5)


DEFAULT_TIMEOUT_SECONDS = 120


class HttpClient:
"""Base HTTP client used to make HTTP calls.

Expand All @@ -41,7 +44,7 @@ class HttpClient:

def __init__(
self, credential=None, session=None, base_url='', headers=None,
retries=DEFAULT_RETRY_CONFIG):
retries=DEFAULT_RETRY_CONFIG, timeout=DEFAULT_TIMEOUT_SECONDS):
"""Creates a new HttpClient instance from the provided arguments.

If a credential is provided, initializes a new HTTP session authorized with it. If neither
Expand All @@ -55,6 +58,8 @@ def __init__(
retries: A urllib retry configuration. Default settings would retry once for low-level
connection and socket read errors, and up to 4 times for HTTP 500 and 503 errors.
Pass a False value to disable retries (optional).
timeout: HTTP timeout in seconds. Defaults to 120 seconds when not specified. Set to
None to disable timeouts (optional).
"""
if credential:
self._session = transport.requests.AuthorizedSession(credential)
Expand All @@ -69,6 +74,7 @@ def __init__(
self._session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
self._session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))
self._base_url = base_url
self._timeout = timeout

@property
def session(self):
Expand All @@ -78,6 +84,10 @@ def session(self):
def base_url(self):
return self._base_url

@property
def timeout(self):
return self._timeout

def parse_body(self, resp):
raise NotImplementedError

Expand All @@ -93,15 +103,17 @@ class call this method to send HTTP requests out. Refer to
method: HTTP method name as a string (e.g. get, post).
url: URL of the remote endpoint.
kwargs: An additional set of keyword arguments to be passed into the requests API
(e.g. json, params).
(e.g. json, params, timeout).

Returns:
Response: An HTTP response object.

Raises:
RequestException: Any requests exceptions encountered while making the HTTP call.
"""
resp = self._session.request(method, self._base_url + url, **kwargs)
if 'timeout' not in kwargs:
kwargs['timeout'] = self.timeout
resp = self._session.request(method, self.base_url + url, **kwargs)
resp.raise_for_status()
return resp

Expand Down
13 changes: 5 additions & 8 deletions firebase_admin/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -775,7 +775,7 @@ def __init__(self, app):
self._auth_override = json.dumps(auth_override, separators=(',', ':'))
else:
self._auth_override = None
self._timeout = app.options.get('httpTimeout')
self._timeout = app.options.get('httpTimeout', _http_client.DEFAULT_TIMEOUT_SECONDS)
self._clients = {}

emulator_host = os.environ.get(_EMULATOR_HOST_ENV_VAR)
Expand Down Expand Up @@ -900,14 +900,13 @@ def __init__(self, credential, base_url, timeout, params=None):
credential: A Google credential that can be used to authenticate requests.
base_url: A URL prefix to be added to all outgoing requests. This is typically the
Firebase Realtime Database URL.
timeout: HTTP request timeout in seconds. If not set connections will never
timeout: HTTP request timeout in seconds. If set to None connections will never
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: the 'which is the default behaviour ...' part is no longer true.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If set to None connections will never
timeout, which is the default behavior of the underlying requests library.

That is still true. The requests library behavior hasn't yet changed. The default timeout=None behavior still comes from there.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops; yeah I missed the change from 'if unset' to 'if set to none' (despite suggesting exactly that earlier. 😳)

timeout, which is the default behavior of the underlying requests library.
params: Dict of query parameters to add to all outgoing requests.
"""
_http_client.JsonHttpClient.__init__(
self, credential=credential, base_url=base_url, headers={'User-Agent': _USER_AGENT})
self.credential = credential
self.timeout = timeout
super().__init__(
credential=credential, base_url=base_url,
timeout=timeout, headers={'User-Agent': _USER_AGENT})
self.params = params if params else {}

def request(self, method, url, **kwargs):
Expand Down Expand Up @@ -937,8 +936,6 @@ def request(self, method, url, **kwargs):
query = extra_params
kwargs['params'] = query

if self.timeout:
kwargs['timeout'] = self.timeout
try:
return super(_Client, self).request(method, url, **kwargs)
except requests.exceptions.RequestException as error:
Expand Down
11 changes: 5 additions & 6 deletions firebase_admin/messaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,8 +330,9 @@ def __init__(self, app):
'X-GOOG-API-FORMAT-VERSION': '2',
'X-FIREBASE-CLIENT': 'fire-admin-python/{0}'.format(firebase_admin.__version__),
}
self._client = _http_client.JsonHttpClient(credential=app.credential.get_credential())
self._timeout = app.options.get('httpTimeout')
timeout = app.options.get('httpTimeout', _http_client.DEFAULT_TIMEOUT_SECONDS)
self._client = _http_client.JsonHttpClient(
credential=app.credential.get_credential(), timeout=timeout)
self._transport = _auth.authorized_http(app.credential.get_credential())

@classmethod
Expand All @@ -348,8 +349,7 @@ def send(self, message, dry_run=False):
'post',
url=self._fcm_url,
headers=self._fcm_headers,
json=data,
timeout=self._timeout
json=data
)
except requests.exceptions.RequestException as error:
raise self._handle_fcm_error(error)
Expand Down Expand Up @@ -416,8 +416,7 @@ def make_topic_management_request(self, tokens, topic, operation):
'post',
url=url,
json=data,
headers=_MessagingService.IID_HEADERS,
timeout=self._timeout
headers=_MessagingService.IID_HEADERS
)
except requests.exceptions.RequestException as error:
raise self._handle_iid_error(error)
Expand Down
8 changes: 4 additions & 4 deletions firebase_admin/project_management.py
Original file line number Diff line number Diff line change
Expand Up @@ -478,11 +478,12 @@ def __init__(self, app):
'the GOOGLE_CLOUD_PROJECT environment variable.')
self._project_id = project_id
version_header = 'Python/Admin/{0}'.format(firebase_admin.__version__)
timeout = app.options.get('httpTimeout', _http_client.DEFAULT_TIMEOUT_SECONDS)
self._client = _http_client.JsonHttpClient(
credential=app.credential.get_credential(),
base_url=_ProjectManagementService.BASE_URL,
headers={'X-Client-Version': version_header})
self._timeout = app.options.get('httpTimeout')
headers={'X-Client-Version': version_header},
timeout=timeout)

def get_android_app_metadata(self, app_id):
return self._get_app_metadata(
Expand Down Expand Up @@ -658,7 +659,6 @@ def _make_request(self, method, url, json=None):

def _body_and_response(self, method, url, json=None):
try:
return self._client.body_and_response(
method=method, url=url, json=json, timeout=self._timeout)
return self._client.body_and_response(method=method, url=url, json=json)
except requests.exceptions.RequestException as error:
raise _utils.handle_platform_error_from_requests(error)
45 changes: 24 additions & 21 deletions tests/test_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import firebase_admin
from firebase_admin import db
from firebase_admin import exceptions
from firebase_admin import _http_client
from firebase_admin import _sseclient
from tests import testutils

Expand Down Expand Up @@ -731,15 +732,8 @@ def test_parse_db_url_errors(self, url, emulator_host):
def test_valid_db_url(self, url):
firebase_admin.initialize_app(testutils.MockCredential(), {'databaseURL' : url})
ref = db.reference()
recorder = []
adapter = MockAdapter('{}', 200, recorder)
ref._client.session.mount(url, adapter)
assert ref._client.base_url == 'https://test.firebaseio.com'
assert 'auth_variable_override' not in ref._client.params
assert ref._client.timeout is None
assert ref.get() == {}
assert len(recorder) == 1
assert recorder[0]._extra_kwargs.get('timeout') is None

@pytest.mark.parametrize('url', [
None, '', 'foo', 'http://test.firebaseio.com', 'https://google.com',
Expand All @@ -761,15 +755,13 @@ def test_multi_db_support(self):
ref = db.reference()
assert ref._client.base_url == default_url
assert 'auth_variable_override' not in ref._client.params
assert ref._client.timeout is None
assert ref._client is db.reference()._client
assert ref._client is db.reference(url=default_url)._client

other_url = 'https://other.firebaseio.com'
other_ref = db.reference(url=other_url)
assert other_ref._client.base_url == other_url
assert 'auth_variable_override' not in ref._client.params
assert other_ref._client.timeout is None
assert other_ref._client is db.reference(url=other_url)._client
assert other_ref._client is db.reference(url=other_url + '/')._client

Expand All @@ -782,7 +774,6 @@ def test_valid_auth_override(self, override):
default_ref = db.reference()
other_ref = db.reference(url='https://other.firebaseio.com')
for ref in [default_ref, other_ref]:
assert ref._client.timeout is None
if override == {}:
assert 'auth_variable_override' not in ref._client.params
else:
Expand All @@ -804,22 +795,22 @@ def test_invalid_auth_override(self, override):
with pytest.raises(ValueError):
db.reference(app=other_app, url='https://other.firebaseio.com')

def test_http_timeout(self):
@pytest.mark.parametrize('options, timeout', [
({'httpTimeout': 4}, 4),
({'httpTimeout': None}, None),
({}, _http_client.DEFAULT_TIMEOUT_SECONDS),
])
def test_http_timeout(self, options, timeout):
test_url = 'https://test.firebaseio.com'
firebase_admin.initialize_app(testutils.MockCredential(), {
all_options = {
'databaseURL' : test_url,
'httpTimeout': 60
})
}
all_options.update(options)
firebase_admin.initialize_app(testutils.MockCredential(), all_options)
default_ref = db.reference()
other_ref = db.reference(url='https://other.firebaseio.com')
for ref in [default_ref, other_ref]:
recorder = []
adapter = MockAdapter('{}', 200, recorder)
ref._client.session.mount(ref._client.base_url, adapter)
assert ref._client.timeout == 60
assert ref.get() == {}
assert len(recorder) == 1
assert recorder[0]._extra_kwargs['timeout'] == pytest.approx(60, 0.001)
self._check_timeout(ref, timeout)

def test_app_delete(self):
app = firebase_admin.initialize_app(
Expand All @@ -841,6 +832,18 @@ def test_user_agent_format(self):
firebase_admin.__version__, sys.version_info.major, sys.version_info.minor)
assert db._USER_AGENT == expected

def _check_timeout(self, ref, timeout):
assert ref._client.timeout == timeout
recorder = []
adapter = MockAdapter('{}', 200, recorder)
ref._client.session.mount(ref._client.base_url, adapter)
assert ref.get() == {}
assert len(recorder) == 1
if timeout is None:
assert recorder[0]._extra_kwargs['timeout'] is None
else:
assert recorder[0]._extra_kwargs['timeout'] == pytest.approx(timeout, 0.001)


@pytest.fixture(params=['foo', '$key', '$value'])
def initquery(request):
Expand Down
22 changes: 22 additions & 0 deletions tests/test_http_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,28 @@ def test_credential():
assert recorder[0].url == _TEST_URL
assert recorder[0].headers['Authorization'] == 'Bearer mock-token'

def test_default_timeout():
client = _http_client.HttpClient()
assert client.timeout == _http_client.DEFAULT_TIMEOUT_SECONDS
recorder = _instrument(client, 'body')
client.request('get', _TEST_URL)
assert len(recorder) == 1
assert recorder[0]._extra_kwargs['timeout'] == pytest.approx(
_http_client.DEFAULT_TIMEOUT_SECONDS, 0.001)

@pytest.mark.parametrize('timeout', [7, 0, None])
def test_custom_timeout(timeout):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function looks mostly identical to test_default_timeout. You could optionally refactor to something like this:

def _helper_test_timeout(**kwargs):
    client = _http_client.HttpClient()
    timeout = kwargs.get('timeout', _http_client.DEFAULT_TIMEOUT_SECONDS)
    assert client.timeout == timeout
    ...
    if timeout is None
        assert recorder[0]._extra_kwargs['timeout'] is None
    else:
        assert ... == pytest.approx(timeout, 0.001)

def test_default_timeout():
    _helper_test_timeout()

@parameterize(...)
def test_custom_timeout(timeout)
    _helper_test_timeout(timeout=timeout)

(Or you could parameterize the helper directly.)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refactored with parameterization.

client = _http_client.HttpClient(timeout=timeout)
assert client.timeout == timeout
recorder = _instrument(client, 'body')
client.request('get', _TEST_URL)
assert len(recorder) == 1
if timeout is None:
assert recorder[0]._extra_kwargs['timeout'] is None
else:
assert recorder[0]._extra_kwargs['timeout'] == pytest.approx(timeout, 0.001)


def _instrument(client, payload, status=200):
recorder = []
adapter = testutils.MockAdapter(payload, status, recorder)
Expand Down
7 changes: 7 additions & 0 deletions tests/test_instance_id.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import firebase_admin
from firebase_admin import exceptions
from firebase_admin import instance_id
from firebase_admin import _http_client
from tests import testutils


Expand Down Expand Up @@ -73,6 +74,12 @@ def evaluate():
instance_id.delete_instance_id('test')
testutils.run_without_project_id(evaluate)

def test_default_timeout(self):
cred = testutils.MockCredential()
app = firebase_admin.initialize_app(cred, {'projectId': 'explicit-project-id'})
iid_service = instance_id._get_iid_service(app)
assert iid_service._client.timeout == _http_client.DEFAULT_TIMEOUT_SECONDS

def test_delete_instance_id(self):
cred = testutils.MockCredential()
app = firebase_admin.initialize_app(cred, {'projectId': 'explicit-project-id'})
Expand Down
Loading