[SEND] Widen upload retry window: exponential backoff + jitter and an extra attempt

packages/send/frontend/.env.sample

@@ -14,6 +14,12 @@ VITE_ALLOW_PUBLIC_LOGIN=true
 # This value is only used to override the default 100MB
 VITE_SPLIT_SIZE_IN_MB=500
 
+# Upload PUT retry tuning (optional; baked in at build time).
+# Number of retries after the first attempt (default 3 => 4 total attempts).
+VITE_UPLOAD_HTTP_RETRY_LIMIT=3
+# Base delay in ms for exponential backoff with jitter (default 1000).
+VITE_UPLOAD_HTTP_RETRY_BASE_DELAY_MS=1000
+
 # OIDC Configuration
 VITE_OIDC_CLIENT_ID=
 VITE_OIDC_ROOT_URL=

packages/send/frontend/src/lib/helpers.retry.test.ts

@@ -18,10 +18,20 @@ import {
   it,
   vi,
 } from 'vitest';
-import { uploadWithTracker } from './helpers';
+import {
+  getUploadRetryDelayMs,
+  UPLOAD_HTTP_RETRY_LIMIT,
+  uploadWithTracker,
+} from './helpers';
 
 const TEST_URL = 'http://test.local/api/upload';
 
+// Real setTimeout, captured before any spy replaces the global. Retry tests
+// mock setTimeout to run on the next tick (no wall-clock wait) so exponential
+// backoff doesn't make the suite take seconds, while still recording the
+// delays that were requested.
+const realSetTimeout = globalThis.setTimeout.bind(globalThis);
+
 function makeStream(content = 'test-payload') {
   return new ReadableStream({
     start(controller) {
@@ -33,17 +43,29 @@ function makeStream(content = 'test-payload') {
 
 describe('uploadWithTracker — HTTP-level retry', () => {
   const server = setupServer();
+  let scheduledDelays: number[];
 
   beforeAll(() => server.listen());
   afterEach(() => {
     server.resetHandlers();
-    vi.clearAllMocks();
+    vi.restoreAllMocks();
   });
   afterAll(() => server.close());
 
   beforeEach(() => {
     // Re-reset the mock per test so call counts are clean
     mockProgressTracker.setProgress = vi.fn();
+
+    // Record each requested backoff delay and fire the callback on the next
+    // tick so retries don't incur real exponential-backoff waits.
+    scheduledDelays = [];
+    vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+      cb: () => void,
+      ms?: number
+    ) => {
+      scheduledDelays.push(ms ?? 0);
+      return realSetTimeout(cb, 0);
+    }) as typeof setTimeout);
   });
 
   it('recovers from a transient 500 on the first attempt', async () => {
@@ -92,7 +114,7 @@ describe('uploadWithTracker — HTTP-level retry', () => {
     expect(calls).toBe(3);
   });
 
-  it('throws UPLOAD_FAILED after exhausting all 3 attempts', async () => {
+  it('throws UPLOAD_FAILED after exhausting all attempts (default 4)', async () => {
     let calls = 0;
     server.use(
       http.put(TEST_URL, () => {
@@ -109,7 +131,32 @@ describe('uploadWithTracker — HTTP-level retry', () => {
       })
     ).rejects.toThrow(/UPLOAD_FAILED/);
 
-    expect(calls).toBe(3);
+    // limit retries + the initial attempt
+    expect(calls).toBe(UPLOAD_HTTP_RETRY_LIMIT + 1);
+    // One backoff delay scheduled per retry
+    expect(scheduledDelays).toHaveLength(UPLOAD_HTTP_RETRY_LIMIT);
+  });
+
+  it('schedules retries with strictly increasing (exponential) backoff', async () => {
+    server.use(
+      http.put(TEST_URL, () =>
+        HttpResponse.text('persistent error', { status: 500 })
+      )
+    );
+
+    await expect(
+      uploadWithTracker({
+        url: TEST_URL,
+        readableStream: makeStream(),
+        progressTracker: mockProgressTracker,
+      })
+    ).rejects.toThrow(/UPLOAD_FAILED/);
+
+    // Jitter ranges per attempt don't overlap (each is [base*2^a*0.5,
+    // base*2^a)), so successive delays are always strictly increasing.
+    for (let i = 1; i < scheduledDelays.length; i++) {
+      expect(scheduledDelays[i]).toBeGreaterThan(scheduledDelays[i - 1]);
+    }
   });
 
   it('retries on a network error (msw: HttpResponse.error)', async () => {
@@ -181,3 +228,29 @@ describe('uploadWithTracker — HTTP-level retry', () => {
     expect(zeroResetCalls).toBeGreaterThanOrEqual(2);
   });
 });
+
+describe('getUploadRetryDelayMs', () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it('grows exponentially with the attempt index (jitter floor)', () => {
+    // Math.random() === 0 => jitter factor 0.5
+    vi.spyOn(Math, 'random').mockReturnValue(0);
+    expect(getUploadRetryDelayMs(0, 1000)).toBe(500);
+    expect(getUploadRetryDelayMs(1, 1000)).toBe(1000);
+    expect(getUploadRetryDelayMs(2, 1000)).toBe(2000);
+  });
+
+  it('keeps each delay within [base*2^attempt*0.5, base*2^attempt)', () => {
+    const base = 1000;
+    for (const r of [0, 0.25, 0.5, 0.75, 0.999]) {
+      vi.spyOn(Math, 'random').mockReturnValue(r);
+      for (let attempt = 0; attempt < 4; attempt++) {
+        const exp = base * 2 ** attempt;
+        const delay = getUploadRetryDelayMs(attempt, base);
+        expect(delay).toBeGreaterThanOrEqual(exp * 0.5);
+        expect(delay).toBeLessThan(exp);
+      }
+      vi.restoreAllMocks();
+    }
+  });
+});

packages/send/frontend/src/lib/helpers.test.ts

@@ -8,6 +8,10 @@ const TEST_URL = `${import.meta.env.VITE_SEND_SERVER_URL}/api`;
 const TEST_CONTENT = 'test-content';
 const TEST_BLOB = new Blob([TEST_CONTENT]);
 
+// Real setTimeout captured before any spy; retry tests fire the callback on
+// the next tick so the upload's exponential backoff adds no wall-clock wait.
+const realSetTimeout = globalThis.setTimeout.bind(globalThis);
+
 describe('helpers', () => {
   const restHandlers = [
     http.get(`${TEST_URL}/download/*`, () =>
@@ -28,8 +32,15 @@ describe('helpers', () => {
     server.close();
   });
 
+  beforeEach(() => {
+    vi.spyOn(globalThis, 'setTimeout').mockImplementation(((
+      cb: () => void
+    ) => realSetTimeout(cb, 0)) as typeof setTimeout);
+  });
+
   afterEach(() => {
     server.resetHandlers();
+    vi.restoreAllMocks();
   });
 
   describe('uploadWithTracker', () => {

packages/send/frontend/src/lib/helpers.ts

@@ -264,14 +264,46 @@ type UploadOptions = {
   progressTracker: ProgressTracker;
 };
 
+// Upload PUT retry policy. The hard B2 failures seen in production
+// (Sentry SEND-SUITE-FRONTEND-24H) are multi-minute stalls, so we retry a
+// handful of times with exponentially-growing, jittered delays to widen the
+// recovery window before giving up. Defaults are overridable via Vite env
+// vars — note these are baked in at build time, so a change still needs a
+// frontend deploy; they exist for tuning, not live runtime config.
+// UPLOAD_HTTP_RETRY_LIMIT is the number of *retries*; total attempts is
+// limit + 1 (default 3 retries => 4 attempts).
+export const UPLOAD_HTTP_RETRY_LIMIT: number =
+  Number(import.meta.env.VITE_UPLOAD_HTTP_RETRY_LIMIT) || 3;
+export const UPLOAD_HTTP_RETRY_BASE_DELAY_MS: number =
+  Number(import.meta.env.VITE_UPLOAD_HTTP_RETRY_BASE_DELAY_MS) || 1000;
+
+/**
+ * Exponential backoff with jitter for the upload PUT retry schedule:
+ *   delay = base * 2^attempt * (0.5 + Math.random() / 2)
+ * The jitter factor is in [0.5, 1.0), so with the default 1000ms base the
+ * per-attempt delays grow roughly ~1s, ~2s, ~4s while staying de-synchronized
+ * across clients (avoids a thundering herd when B2 recovers).
+ *
+ * @param attempt - zero-based index of the attempt that just failed
+ * @param baseDelayMs - base delay; defaults to UPLOAD_HTTP_RETRY_BASE_DELAY_MS
+ */
+export function getUploadRetryDelayMs(
+  attempt: number,
+  baseDelayMs: number = UPLOAD_HTTP_RETRY_BASE_DELAY_MS
+): number {
+  const exponential = baseDelayMs * 2 ** attempt;
+  const jitter = 0.5 + Math.random() / 2; // [0.5, 1.0)
+  // floor (not round) keeps each attempt's range strictly below the next's
+  // floor: [base*2^a*0.5, base*2^a), so successive delays never collide.
+  return Math.floor(exponential * jitter);
+}
+
 export const uploadWithTracker = ({
   url,
   readableStream,
   progressTracker,
 }: UploadOptions) => {
   const { setProgress } = progressTracker;
-  const HTTP_RETRY_LIMIT = 2;
-  const HTTP_RETRY_DELAY_MS = 1000;
   const XHR_TIMEOUT_MS = 30000;
 
   const attemptPut = (blob: Blob, attempt: number): Promise<string> => {
@@ -309,13 +341,14 @@ export const uploadWithTracker = ({
 
       xhr.send(blob);
     }).catch((error) => {
-      if (attempt < HTTP_RETRY_LIMIT) {
+      if (attempt < UPLOAD_HTTP_RETRY_LIMIT) {
+        const delayMs = getUploadRetryDelayMs(attempt);
         console.warn(
-          `HTTP PUT attempt ${attempt + 1} failed, retrying...`,
+          `HTTP PUT attempt ${attempt + 1} failed, retrying in ${delayMs}ms...`,
           error.message
         );
         return new Promise<string>((resolve) =>
-          setTimeout(resolve, HTTP_RETRY_DELAY_MS)
+          setTimeout(resolve, delayMs)
         ).then(() => attemptPut(blob, attempt + 1));
       }
       throw error;