Skip to content

Commit c9d8fce

Browse files
Jon Wayne Parrottlandrito
authored andcommitted
Add google.api.core.retry with base retry functionality (googleapis#3819)
Add google.api.core.retry with base retry functionality Additionally: * Add google.api.core.exceptions.RetryError * Add google.api.core.helpers package * Add google.api.core.helpers.datetime_helpers module
1 parent 993db02 commit c9d8fce

File tree

7 files changed

+344
-0
lines changed

7 files changed

+344
-0
lines changed

core/google/api/core/exceptions.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,29 @@ class GoogleAPIError(Exception):
4040
pass
4141

4242

43+
@six.python_2_unicode_compatible
44+
class RetryError(GoogleAPIError):
45+
"""Raised when a function has exhausted all of its available retries.
46+
47+
Args:
48+
message (str): The exception message.
49+
cause (Exception): The last exception raised when retring the
50+
function.
51+
"""
52+
def __init__(self, message, cause):
53+
super(RetryError, self).__init__(message)
54+
self.message = message
55+
self._cause = cause
56+
57+
@property
58+
def cause(self):
59+
"""The last exception raised when retrying the function."""
60+
return self._cause
61+
62+
def __str__(self):
63+
return '{}, last exception: {}'.format(self.message, self.cause)
64+
65+
4366
class _GoogleAPICallErrorMeta(type):
4467
"""Metaclass for registering GoogleAPICallError subclasses."""
4568
def __new__(mcs, name, bases, class_dict):

core/google/api/core/helpers/__init__.py

Whitespace-only changes.
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Copyright 2017 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Helpers for :mod:`datetime`."""
16+
17+
import datetime
18+
19+
20+
def utcnow():
21+
"""A :meth:`datetime.datetime.utcnow()` alias to allow mocking in tests."""
22+
return datetime.datetime.utcnow()

core/google/api/core/retry.py

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
# Copyright 2017 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Helpers for retrying functions with exponential back-off."""
16+
17+
import datetime
18+
import logging
19+
import random
20+
import time
21+
22+
import six
23+
24+
from google.api.core import exceptions
25+
from google.api.core.helpers import datetime_helpers
26+
27+
_LOGGER = logging.getLogger(__name__)
28+
_DEFAULT_MAX_JITTER = 0.2
29+
30+
31+
def if_exception_type(*exception_types):
32+
"""Creates a predicate to check if the exception is of a given type.
33+
34+
Args:
35+
exception_types (Sequence[type]): The exception types to check for.
36+
37+
Returns:
38+
Callable[Exception]: A predicate that returns True if the provided
39+
exception is of the given type(s).
40+
"""
41+
def inner(exception):
42+
"""Bound predicate for checking an exception type."""
43+
return isinstance(exception, exception_types)
44+
return inner
45+
46+
47+
# pylint: disable=invalid-name
48+
# Pylint sees this as a constant, but it is also an alias that should be
49+
# considered a function.
50+
if_transient_error = if_exception_type((
51+
exceptions.InternalServerError,
52+
exceptions.TooManyRequests))
53+
"""A predicate that checks if an exception is a transient API error.
54+
55+
The following server errors are considered transient:
56+
57+
- :class:`google.api.core.exceptions.InternalServerError` - HTTP 500, gRPC
58+
``INTERNAL(13)`` and its subclasses.
59+
- :class:`google.api.core.exceptions.TooManyRequests` - HTTP 429
60+
- :class:`google.api.core.exceptions.ResourceExhausted` - gRPC
61+
``RESOURCE_EXHAUSTED(8)``
62+
"""
63+
# pylint: enable=invalid-name
64+
65+
66+
def exponential_sleep_generator(
67+
initial, maximum, multiplier=2, jitter=_DEFAULT_MAX_JITTER):
68+
"""Generates sleep intervals based on the exponential back-off algorithm.
69+
70+
This implements the `Truncated Exponential Back-off`_ algorithm.
71+
72+
.. _Truncated Exponential Back-off:
73+
https://cloud.google.com/storage/docs/exponential-backoff
74+
75+
Args:
76+
initial (float): The minimum about of time to delay. This must
77+
be greater than 0.
78+
maximum (float): The maximum about of time to delay.
79+
multiplier (float): The multiplier applied to the delay.
80+
jitter (float): The maximum about of randomness to apply to the delay.
81+
82+
Yields:
83+
float: successive sleep intervals.
84+
"""
85+
delay = initial
86+
while True:
87+
yield delay
88+
delay = min(
89+
delay * multiplier + random.uniform(0, jitter), maximum)
90+
91+
92+
def retry_target(target, predicate, sleep_generator, deadline):
93+
"""Call a function and retry if it fails.
94+
95+
This is the lowest-level retry helper. Generally, you'll use the
96+
higher-level retry helper :class:`Retry`.
97+
98+
Args:
99+
target(Callable): The function to call and retry. This must be a
100+
nullary function - apply arguments with `functools.partial`.
101+
predicate (Callable[Exception]): A callable used to determine if an
102+
exception raised by the target should be considered retryable.
103+
It should return True to retry or False otherwise.
104+
sleep_generator (Iterator[float]): An infinite iterator that determines
105+
how long to sleep between retries.
106+
deadline (float): How long to keep retrying the target.
107+
108+
Returns:
109+
Any: the return value of the target function.
110+
111+
Raises:
112+
google.api.core.RetryError: If the deadline is exceeded while retrying.
113+
ValueError: If the sleep generator stops yielding values.
114+
Exception: If the target raises a method that isn't retryable.
115+
"""
116+
if deadline is not None:
117+
deadline_datetime = (
118+
datetime_helpers.utcnow() + datetime.timedelta(seconds=deadline))
119+
else:
120+
deadline_datetime = None
121+
122+
last_exc = None
123+
124+
for sleep in sleep_generator:
125+
try:
126+
return target()
127+
128+
# pylint: disable=broad-except
129+
# This function explicitly must deal with broad exceptions.
130+
except Exception as exc:
131+
if not predicate(exc):
132+
raise
133+
last_exc = exc
134+
135+
now = datetime_helpers.utcnow()
136+
if deadline_datetime is not None and deadline_datetime < now:
137+
six.raise_from(
138+
exceptions.RetryError(
139+
'Deadline of {:.1f}s exceeded while calling {}'.format(
140+
deadline, target),
141+
last_exc),
142+
last_exc)
143+
144+
_LOGGER.debug('Retrying due to {}, sleeping {:.1f}s ...'.format(
145+
last_exc, sleep))
146+
time.sleep(sleep)
147+
148+
raise ValueError('Sleep generator stopped yielding sleep values.')

core/tests/unit/api_core/helpers/__init__.py

Whitespace-only changes.
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Copyright 2017, Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import datetime
16+
17+
from google.api.core.helpers import datetime_helpers
18+
19+
20+
def test_utcnow():
21+
result = datetime_helpers.utcnow()
22+
assert isinstance(result, datetime.datetime)
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# Copyright 2017 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import datetime
16+
import itertools
17+
18+
import mock
19+
import pytest
20+
21+
from google.api.core import exceptions
22+
from google.api.core import retry
23+
24+
25+
def test_if_exception_type():
26+
predicate = retry.if_exception_type(ValueError)
27+
28+
assert predicate(ValueError())
29+
assert not predicate(TypeError())
30+
31+
32+
def test_if_exception_type_multiple():
33+
predicate = retry.if_exception_type(ValueError, TypeError)
34+
35+
assert predicate(ValueError())
36+
assert predicate(TypeError())
37+
assert not predicate(RuntimeError())
38+
39+
40+
def test_if_transient_error():
41+
assert retry.if_transient_error(exceptions.InternalServerError(''))
42+
assert retry.if_transient_error(exceptions.TooManyRequests(''))
43+
assert not retry.if_transient_error(exceptions.InvalidArgument(''))
44+
45+
46+
def test_exponential_sleep_generator_base_2():
47+
gen = retry.exponential_sleep_generator(
48+
1, 60, 2, jitter=0.0)
49+
50+
result = list(itertools.islice(gen, 8))
51+
assert result == [1, 2, 4, 8, 16, 32, 60, 60]
52+
53+
54+
@mock.patch('random.uniform')
55+
def test_exponential_sleep_generator_jitter(uniform):
56+
uniform.return_value = 1
57+
gen = retry.exponential_sleep_generator(
58+
1, 60, 2, jitter=2.2)
59+
60+
result = list(itertools.islice(gen, 7))
61+
assert result == [1, 3, 7, 15, 31, 60, 60]
62+
uniform.assert_called_with(0.0, 2.2)
63+
64+
65+
@mock.patch('time.sleep')
66+
@mock.patch(
67+
'google.api.core.helpers.datetime_helpers.utcnow',
68+
return_value=datetime.datetime.min)
69+
def test_retry_target_success(utcnow, sleep):
70+
predicate = retry.if_exception_type(ValueError)
71+
call_count = [0]
72+
73+
def target():
74+
call_count[0] += 1
75+
if call_count[0] < 3:
76+
raise ValueError()
77+
return 42
78+
79+
result = retry.retry_target(target, predicate, range(10), None)
80+
81+
assert result == 42
82+
assert call_count[0] == 3
83+
sleep.assert_has_calls([mock.call(0), mock.call(1)])
84+
85+
86+
@mock.patch('time.sleep')
87+
@mock.patch(
88+
'google.api.core.helpers.datetime_helpers.utcnow',
89+
return_value=datetime.datetime.min)
90+
def test_retry_target_non_retryable_error(utcnow, sleep):
91+
predicate = retry.if_exception_type(ValueError)
92+
exception = TypeError()
93+
target = mock.Mock(side_effect=exception)
94+
95+
with pytest.raises(TypeError) as exc_info:
96+
retry.retry_target(target, predicate, range(10), None)
97+
98+
assert exc_info.value == exception
99+
sleep.assert_not_called()
100+
101+
102+
@mock.patch('time.sleep')
103+
@mock.patch(
104+
'google.api.core.helpers.datetime_helpers.utcnow')
105+
def test_retry_target_deadline_exceeded(utcnow, sleep):
106+
predicate = retry.if_exception_type(ValueError)
107+
exception = ValueError('meep')
108+
target = mock.Mock(side_effect=exception)
109+
# Setup the timeline so that the first call takes 5 seconds but the second
110+
# call takes 6, which puts the retry over the deadline.
111+
utcnow.side_effect = [
112+
# The first call to utcnow establishes the start of the timeline.
113+
datetime.datetime.min,
114+
datetime.datetime.min + datetime.timedelta(seconds=5),
115+
datetime.datetime.min + datetime.timedelta(seconds=11)]
116+
117+
with pytest.raises(exceptions.RetryError) as exc_info:
118+
retry.retry_target(target, predicate, range(10), deadline=10)
119+
120+
assert exc_info.value.cause == exception
121+
assert exc_info.match('Deadline of 10.0s exceeded')
122+
assert exc_info.match('last exception: meep')
123+
assert target.call_count == 2
124+
125+
126+
def test_retry_target_bad_sleep_generator():
127+
with pytest.raises(ValueError, match='Sleep generator'):
128+
retry.retry_target(
129+
mock.sentinel.target, mock.sentinel.predicate, [], None)

0 commit comments

Comments
 (0)