Skip to content

Commit 49008e3

Browse files
shaffeeullahgcf-owl-bot[bot]ddelgrosso1
authored
feat: customize retry behavior implementation (#1474) (#1493)
* feat: customize retry behavior implementation (#1474) * feat: customize retry behavior implementation * 🦉 Updates from OwlBot * fixed != * 🦉 Updates from OwlBot * updated names to match gogle gax * 🦉 Updates from OwlBot * refactored retryOptions into its own config * 🦉 Updates from OwlBot * fixed linting error * 🦉 Updates from OwlBot * added retry delay explanation * 🦉 Updates from OwlBot * refactored constants * 🦉 Updates from OwlBot * removed const assignment Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> * feat: implemented retry function (#1476) * implemented retry function * 🦉 Updates from OwlBot * fixed import * 🦉 Updates from OwlBot * resolved merge conflict * passed retry function to common * 🦉 Updates from OwlBot * refactored code to retryableErrFn * removed unused import * fixed typo * fixed typo * fixed failing tests * made retryableErrorFn configurable * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * wrote unit tests * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * changed reason check to be less brittle * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> * depend on common 3.7.0 for retry changes * feat: remove gaxios dependency (#1503) * feat: remove gaxios dependency * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * moved callbackFunction inline * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * assigned types to any * changed any to string Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> * feat: applied customization to multipart filesave (#1504) * feat: applied customization to multipart filesave * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * fixed failing tests * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md * removed unused import Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> * Update storage.ts added comment next to EAI_AGAIN * feat: pass retryOptions to gcs-resumable-upload (#1506) * feat: pass retryOptions to gcs-resumable-upload * linter fixes * additional test assertions for retryOptions * add additional asserts to resumable operation tests Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: Denis DelGrosso <[email protected]>
1 parent 742567b commit 49008e3

File tree

5 files changed

+456
-45
lines changed

5 files changed

+456
-45
lines changed

‎package.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
"precompile": "gts clean"
5151
},
5252
"dependencies": {
53-
"@google-cloud/common": "^3.6.0",
53+
"@google-cloud/common": "^3.7.0",
5454
"@google-cloud/paginator": "^3.0.0",
5555
"@google-cloud/promisify": "^2.0.0",
5656
"arrify": "^2.0.0",
@@ -59,8 +59,7 @@
5959
"date-and-time": "^1.0.0",
6060
"duplexify": "^4.0.0",
6161
"extend": "^3.0.2",
62-
"gaxios": "^4.0.0",
63-
"gcs-resumable-upload": "^3.1.4",
62+
"gcs-resumable-upload": "^3.3.0",
6463
"get-stream": "^6.0.0",
6564
"hash-stream-validation": "^0.2.2",
6665
"mime": "^2.2.0",

‎src/file.ts‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ import {
6363
// eslint-disable-next-line @typescript-eslint/no-var-requires
6464
const duplexify: DuplexifyConstructor = require('duplexify');
6565
import {normalize, objectKeyToLowercase, unicodeJSONStringify} from './util';
66-
import {GaxiosError, Headers, request as gaxiosRequest} from 'gaxios';
6766
import retry = require('async-retry');
6867

6968
export type GetExpirationDateResponse = [Date];
@@ -1254,6 +1253,10 @@ class File extends ServiceObject<File> {
12541253
query.userProject = options.userProject;
12551254
}
12561255

1256+
interface Headers {
1257+
[index: string]: string;
1258+
}
1259+
12571260
const headers = {
12581261
'Accept-Encoding': 'gzip',
12591262
'Cache-Control': 'no-store',
@@ -1560,6 +1563,7 @@ class File extends ServiceObject<File> {
15601563
private: options.private,
15611564
public: options.public,
15621565
userProject: options.userProject || this.userProject,
1566+
retryOptions: this.storage.retryOptions,
15631567
},
15641568
callback!
15651569
);
@@ -1962,6 +1966,7 @@ class File extends ServiceObject<File> {
19621966
bucket: this.bucket.name,
19631967
file: this.name,
19641968
generation: this.generation,
1969+
retryOptions: this.storage.retryOptions,
19651970
});
19661971
uploadStream.deleteConfig();
19671972
}
@@ -2993,18 +2998,26 @@ class File extends ServiceObject<File> {
29932998
*/
29942999

29953000
isPublic(callback?: IsPublicCallback): Promise<IsPublicResponse> | void {
2996-
gaxiosRequest({
2997-
method: 'HEAD',
2998-
url: `http://${
2999-
this.bucket.name
3000-
}.storage.googleapis.com/${encodeURIComponent(this.name)}`,
3001-
}).then(
3002-
() => callback!(null, true),
3003-
(err: GaxiosError) => {
3004-
if (err.code === '403') {
3005-
callback!(null, false);
3001+
util.makeRequest(
3002+
{
3003+
method: 'HEAD',
3004+
uri: `http://${
3005+
this.bucket.name
3006+
}.storage.googleapis.com/${encodeURIComponent(this.name)}`,
3007+
},
3008+
{
3009+
retryOptions: this.storage.retryOptions,
3010+
},
3011+
(err: Error | ApiError | null) => {
3012+
if (err) {
3013+
const apiError = err as ApiError;
3014+
if (apiError.code === 403) {
3015+
callback!(null, false);
3016+
} else {
3017+
callback!(err);
3018+
}
30063019
} else {
3007-
callback!(err);
3020+
callback!(null, true);
30083021
}
30093022
}
30103023
);
@@ -3611,7 +3624,11 @@ class File extends ServiceObject<File> {
36113624
await new Promise<void>((resolve, reject) => {
36123625
const writable = this.createWriteStream(options)
36133626
.on('error', err => {
3614-
if (isMultipart && util.shouldRetryRequest(err)) {
3627+
if (
3628+
isMultipart &&
3629+
this.storage.retryOptions.autoRetry &&
3630+
this.storage.retryOptions.retryableErrorFn!(err)
3631+
) {
36153632
return reject(err);
36163633
} else {
36173634
return bail(err);
@@ -3627,7 +3644,10 @@ class File extends ServiceObject<File> {
36273644
});
36283645
},
36293646
{
3630-
retries: 3,
3647+
retries: this.storage.retryOptions.maxRetries,
3648+
factor: this.storage.retryOptions.retryDelayMultiplier,
3649+
maxTimeout: this.storage.retryOptions.maxRetryDelay! * 1000, //convert to milliseconds
3650+
maxRetryTime: this.storage.retryOptions.totalTimeout! * 1000, //convert to milliseconds
36313651
}
36323652
);
36333653
if (!callback) {
@@ -3793,6 +3813,7 @@ class File extends ServiceObject<File> {
37933813
public: options.public,
37943814
uri: options.uri,
37953815
userProject: options.userProject || this.userProject,
3816+
retryOptions: this.storage.retryOptions,
37963817
});
37973818

37983819
uploadStream

‎src/storage.ts‎

Lines changed: 150 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,14 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
import {Metadata, Service, ServiceOptions} from '@google-cloud/common';
15+
import {
16+
ApiError,
17+
Metadata,
18+
Service,
19+
ServiceOptions,
20+
} from '@google-cloud/common';
1621
import {paginator} from '@google-cloud/paginator';
1722
import {promisifyAll} from '@google-cloud/promisify';
18-
1923
import arrify = require('arrify');
2024
import {Readable} from 'stream';
2125

@@ -45,9 +49,19 @@ export interface CreateBucketQuery {
4549
userProject: string;
4650
}
4751

48-
export interface StorageOptions extends ServiceOptions {
52+
export interface RetryOptions {
53+
retryDelayMultiplier?: number;
54+
totalTimeout?: number;
55+
maxRetryDelay?: number;
4956
autoRetry?: boolean;
5057
maxRetries?: number;
58+
retryableErrorFn?: (err: ApiError) => boolean;
59+
}
60+
61+
export interface StorageOptions extends ServiceOptions {
62+
retryOptions?: RetryOptions;
63+
autoRetry?: boolean; //Deprecated. Use retryOptions instead.
64+
maxRetries?: number; //Deprecated. Use retryOptions instead.
5165
/**
5266
* **This option is deprecated.**
5367
* @todo Remove in next major release.
@@ -163,6 +177,77 @@ export type GetHmacKeysResponse = [HmacKey[]];
163177

164178
export const PROTOCOL_REGEX = /^(\w*):\/\//;
165179

180+
/**
181+
* Default behavior: Automatically retry retriable server errors.
182+
*
183+
* @const {boolean}
184+
* @private
185+
*/
186+
const AUTO_RETRY_DEFAULT = true;
187+
188+
/**
189+
* Default behavior: Only attempt to retry retriable errors 3 times.
190+
*
191+
* @const {number}
192+
* @private
193+
*/
194+
const MAX_RETRY_DEFAULT = 3;
195+
196+
/**
197+
* Default behavior: Wait twice as long as previous retry before retrying.
198+
*
199+
* @const {number}
200+
* @private
201+
*/
202+
const RETRY_DELAY_MULTIPLIER_DEFAULT = 2;
203+
204+
/**
205+
* Default behavior: If the operation doesn't succeed after 600 seconds,
206+
* stop retrying.
207+
*
208+
* @const {number}
209+
* @private
210+
*/
211+
const TOTAL_TIMEOUT_DEFAULT = 600;
212+
213+
/**
214+
* Default behavior: Wait no more than 64 seconds between retries.
215+
*
216+
* @const {number}
217+
* @private
218+
*/
219+
const MAX_RETRY_DELAY_DEFAULT = 64;
220+
221+
/**
222+
* Returns true if the API request should be retried, given the error that was
223+
* given the first time the request was attempted.
224+
* @const
225+
* @private
226+
* @param {error} err - The API error to check if it is appropriate to retry.
227+
* @return {boolean} True if the API request should be retried, false otherwise.
228+
*/
229+
const RETRYABLE_ERR_FN_DEFAULT = function (err?: ApiError) {
230+
if (err) {
231+
if ([408, 429, 500, 502, 503, 504].indexOf(err.code!) !== -1) {
232+
return true;
233+
}
234+
235+
if (err.errors) {
236+
for (const e of err.errors) {
237+
const reason = e.reason?.toLowerCase();
238+
if (
239+
(reason && reason.includes('eai_again')) || //DNS lookup error
240+
reason === 'connection reset by peer' ||
241+
reason === 'unexpected connection closure'
242+
) {
243+
return true;
244+
}
245+
}
246+
}
247+
}
248+
return false;
249+
};
250+
166251
/*! Developer Documentation
167252
*
168253
* Invoke this method to create a new Storage object bound with pre-determined
@@ -350,6 +435,8 @@ export class Storage extends Service {
350435
getBucketsStream: () => Readable;
351436
getHmacKeysStream: () => Readable;
352437

438+
retryOptions: RetryOptions;
439+
353440
/**
354441
* @typedef {object} StorageOptions
355442
* @property {string} [projectId] The project ID from the Google Developer's
@@ -368,11 +455,26 @@ export class Storage extends Service {
368455
* @property {object} [credentials] Credentials object.
369456
* @property {string} [credentials.client_email]
370457
* @property {string} [credentials.private_key]
371-
* @property {boolean} [autoRetry=true] Automatically retry requests if the
458+
* @property {object} [retryOptions] Options for customizing retries. Retriable server errors
459+
* will be retried with exponential delay between them dictated by the formula
460+
* max(maxRetryDelay, retryDelayMultiplier*retryNumber) until maxRetries or totalTimeout
461+
* has been reached. Retries will only happen if autoRetry is set to true.
462+
* @property {boolean} [retryOptions.autoRetry=true] Automatically retry requests if the
372463
* response is related to rate limits or certain intermittent server
373464
* errors. We will exponentially backoff subsequent requests by default.
374-
* @property {number} [maxRetries=3] Maximum number of automatic retries
465+
* @property {number} [retryOptions.retryDelayMultiplier = 2] the multiplier by which to
466+
* increase the delay time between the completion of failed requests, and the
467+
* initiation of the subsequent retrying request.
468+
* @property {number} [retryOptions.totalTimeout = 600] The total time, starting from
469+
* when the initial request is sent, after which an error will
470+
* be returned, regardless of the retrying attempts made meanwhile.
471+
* @property {number} [retryOptions.maxRetryDelay = 64] The maximum delay time between requests.
472+
* When this value is reached, ``retryDelayMultiplier`` will no longer be used to
473+
* increase delay time.
474+
* @property {number} [retryOptions.maxRetries=3] Maximum number of automatic retries
375475
* attempted before returning the error.
476+
* @property {function} [retryOptions.retryableErrorFn] Function that returns true if a given
477+
* error should be retried and false otherwise.
376478
* @property {string} [userAgent] The value to be prepended to the User-Agent
377479
* header in API requests.
378480
*/
@@ -413,10 +515,49 @@ export class Storage extends Service {
413515
// Note: EMULATOR_HOST is an experimental configuration variable. Use apiEndpoint instead.
414516
const baseUrl = EMULATOR_HOST || `${options.apiEndpoint}/storage/v1`;
415517

518+
let autoRetryValue = AUTO_RETRY_DEFAULT;
519+
if (
520+
options.autoRetry !== undefined &&
521+
options.retryOptions?.autoRetry !== undefined
522+
) {
523+
throw new ApiError(
524+
'autoRetry is deprecated. Use retryOptions.autoRetry instead.'
525+
);
526+
} else if (options.autoRetry !== undefined) {
527+
autoRetryValue = options.autoRetry;
528+
} else if (options.retryOptions?.autoRetry !== undefined) {
529+
autoRetryValue = options.retryOptions.autoRetry;
530+
}
531+
532+
let maxRetryValue = MAX_RETRY_DEFAULT;
533+
if (options.maxRetries && options.retryOptions?.maxRetries) {
534+
throw new ApiError(
535+
'maxRetries is deprecated. Use retryOptions.maxRetries instead.'
536+
);
537+
} else if (options.maxRetries) {
538+
maxRetryValue = options.maxRetries;
539+
} else if (options.retryOptions?.maxRetries) {
540+
maxRetryValue = options.retryOptions.maxRetries;
541+
}
542+
416543
const config = {
417544
apiEndpoint: options.apiEndpoint!,
418-
autoRetry: options.autoRetry,
419-
maxRetries: options.maxRetries,
545+
retryOptions: {
546+
autoRetry: autoRetryValue,
547+
maxRetries: maxRetryValue,
548+
retryDelayMultiplier: options.retryOptions?.retryDelayMultiplier
549+
? options.retryOptions?.retryDelayMultiplier
550+
: RETRY_DELAY_MULTIPLIER_DEFAULT,
551+
totalTimeout: options.retryOptions?.totalTimeout
552+
? options.retryOptions?.totalTimeout
553+
: TOTAL_TIMEOUT_DEFAULT,
554+
maxRetryDelay: options.retryOptions?.maxRetryDelay
555+
? options.retryOptions?.maxRetryDelay
556+
: MAX_RETRY_DELAY_DEFAULT,
557+
retryableErrorFn: options.retryOptions?.retryableErrorFn
558+
? options.retryOptions?.retryableErrorFn
559+
: RETRYABLE_ERR_FN_DEFAULT,
560+
},
420561
baseUrl,
421562
customEndpoint,
422563
projectIdRequired: false,
@@ -438,6 +579,8 @@ export class Storage extends Service {
438579
*/
439580
this.acl = Storage.acl;
440581

582+
this.retryOptions = config.retryOptions;
583+
441584
this.getBucketsStream = paginator.streamify('getBuckets');
442585
this.getHmacKeysStream = paginator.streamify('getHmacKeys');
443586
}

0 commit comments

Comments
 (0)