feat(replay): Throttle breadcrumbs to max 300/5s

mydea · mydea · commit e61da64daa59 · 2023-05-10T14:16:51.000+02:00
diff --git a/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/init.js b/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/init.js
@@ -0,0 +1,17 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+window.Replay = new Sentry.Replay({
+  flushMinDelay: 5000,
+  flushMaxDelay: 5000,
+  useCompression: false,
+});
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  sampleRate: 0,
+  replaysSessionSampleRate: 1.0,
+  replaysOnErrorSampleRate: 0.0,
+
+  integrations: [window.Replay],
+});
diff --git a/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/subject.js b/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/subject.js
@@ -0,0 +1,35 @@
+window.loaded = [];
+const head = document.querySelector('head');
+
+const COUNT = 250;
+
+window.__isLoaded = (run = 1) => {
+  return window.loaded.length === COUNT * 2 * run;
+};
+
+document.querySelector('[data-network]').addEventListener('click', () => {
+  const offset = window.loaded.length;
+
+  // Create many scripts
+  for (let i = offset; i < offset + COUNT; i++) {
+    const script = document.createElement('script');
+    script.src = `/virtual-assets/script-${i}.js`;
+    script.setAttribute('crossorigin', 'anonymous');
+    head.appendChild(script);
+
+    script.addEventListener('load', () => {
+      window.loaded.push(`script-${i}`);
+    });
+  }
+});
+
+document.querySelector('[data-fetch]').addEventListener('click', () => {
+  const offset = window.loaded.length;
+
+  // Make many fetch requests
+  for (let i = offset; i < offset + COUNT; i++) {
+    fetch(`/virtual-assets/fetch-${i}.json`).then(() => {
+      window.loaded.push(`fetch-${i}`);
+    });
+  }
+});
diff --git a/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/template.html b/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/template.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+  </head>
+  <body>
+    <button data-fetch>Trigger fetch requests</button>
+    <button data-network>Trigger network requests</button>
+  </body>
+</html>
diff --git a/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/test.ts b/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/test.ts
@@ -0,0 +1,134 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../utils/fixtures';
+import { getCustomRecordingEvents, shouldSkipReplayTest, waitForReplayRequest } from '../../../utils/replayHelpers';
+
+const COUNT = 250;
+const THROTTLE_LIMIT = 300;
+
+sentryTest(
+  'throttles breadcrumbs when many requests are made at the same time',
+  async ({ getLocalTestUrl, page, forceFlushReplay, browserName }) => {
+    if (shouldSkipReplayTest() || browserName !== 'chromium') {
+      sentryTest.skip();
+    }
+
+    const reqPromise0 = waitForReplayRequest(page, 0);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
+    });
+
+    let scriptsLoaded = 0;
+    let fetchLoaded = 0;
+
+    await page.route('**/virtual-assets/script-**', route => {
+      scriptsLoaded++;
+      return route.fulfill({
+        status: 200,
+        contentType: 'text/javascript',
+        body: `const aha = ${'xx'.repeat(20_000)};`,
+      });
+    });
+
+    await page.route('**/virtual-assets/fetch-**', route => {
+      fetchLoaded++;
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ fetchResponse: 'aa'.repeat(20_000) }),
+      });
+    });
+
+    const url = await getLocalTestUrl({ testDir: __dirname });
+
+    await page.goto(url);
+    await reqPromise0;
+
+    const reqPromise1 = waitForReplayRequest(
+      page,
+      (_event, res) => {
+        const { performanceSpans } = getCustomRecordingEvents(res);
+
+        return performanceSpans.some(span => span.op === 'resource.script');
+      },
+      10_000,
+    );
+    const reqPromise1Breadcrumbs = waitForReplayRequest(
+      page,
+      (_event, res) => {
+        const { breadcrumbs } = getCustomRecordingEvents(res);
+
+        return breadcrumbs.some(breadcrumb => breadcrumb.category === 'replay.throttled');
+      },
+      10_000,
+    );
+
+    await page.click('[data-network]');
+    await page.click('[data-fetch]');
+
+    await page.waitForFunction('window.__isLoaded()');
+    await forceFlushReplay();
+
+    const { performanceSpans } = getCustomRecordingEvents(await reqPromise1);
+    const { breadcrumbs } = getCustomRecordingEvents(await reqPromise1Breadcrumbs);
+
+    // All assets have been _loaded_
+    expect(scriptsLoaded).toBe(COUNT);
+    expect(fetchLoaded).toBe(COUNT);
+
+    // But only some have been captured by replay
+    // We check for <= THROTTLE_LIMIT, as there have been some captured before, which take up some of the throttle limit
+    expect(performanceSpans.length).toBeLessThanOrEqual(THROTTLE_LIMIT);
+    expect(performanceSpans.length).toBeGreaterThan(THROTTLE_LIMIT - 50);
+
+    expect(breadcrumbs.filter(({ category }) => category === 'replay.throttled').length).toBe(1);
+
+    // Now we wait for 6s (5s + some wiggle room), and make some requests again
+    await page.waitForTimeout(6_000);
+    await forceFlushReplay();
+
+    const reqPromise2 = waitForReplayRequest(
+      page,
+      (_event, res) => {
+        const { performanceSpans } = getCustomRecordingEvents(res);
+
+        return performanceSpans.some(span => span.op === 'resource.script');
+      },
+      10_000,
+    );
+    const reqPromise2Breadcrumbs = waitForReplayRequest(
+      page,
+      (_event, res) => {
+        const { breadcrumbs } = getCustomRecordingEvents(res);
+
+        return breadcrumbs.some(breadcrumb => breadcrumb.category === 'replay.throttled');
+      },
+      10_000,
+    );
+
+    await page.click('[data-network]');
+    await page.click('[data-fetch]');
+
+    await page.waitForFunction('window.__isLoaded(2)');
+    await forceFlushReplay();
+
+    const { performanceSpans: performanceSpans2 } = getCustomRecordingEvents(await reqPromise2);
+    const { breadcrumbs: breadcrumbs2 } = getCustomRecordingEvents(await reqPromise2Breadcrumbs);
+
+    // All assets have been _loaded_
+    expect(scriptsLoaded).toBe(COUNT * 2);
+    expect(fetchLoaded).toBe(COUNT * 2);
+
+    // But only some have been captured by replay
+    // We check for <= THROTTLE_LIMIT, as there have been some captured before, which take up some of the throttle limit
+    expect(performanceSpans2.length).toBeLessThanOrEqual(THROTTLE_LIMIT);
+    expect(performanceSpans2.length).toBeGreaterThan(THROTTLE_LIMIT - 50);
+
+    expect(breadcrumbs2.filter(({ category }) => category === 'replay.throttled').length).toBe(1);
+  },
+);
diff --git a/packages/replay/.eslintrc.js b/packages/replay/.eslintrc.js
@@ -8,7 +8,9 @@ module.exports = {
   overrides: [
     {
       files: ['src/**/*.ts'],
-      rules: {},
+      rules: {
+        '@sentry-internal/sdk/no-unsupported-es6-methods': 'off',
+      },
     },
     {
       files: ['jest.setup.ts', 'jest.config.ts'],
diff --git a/packages/replay/src/coreHandlers/util/addBreadcrumbEvent.ts b/packages/replay/src/coreHandlers/util/addBreadcrumbEvent.ts
@@ -3,7 +3,6 @@ import type { Breadcrumb } from '@sentry/types';
 import { normalize } from '@sentry/utils';
 
 import type { ReplayContainer } from '../../types';
-import { addEvent } from '../../util/addEvent';
 
 /**
  * Add a breadcrumb event to replay.
@@ -20,7 +19,7 @@ export function addBreadcrumbEvent(replay: ReplayContainer, breadcrumb: Breadcru
   }
 
   replay.addUpdate(() => {
-    void addEvent(replay, {
+    void replay.throttledAddEvent({
       type: EventType.Custom,
       // TODO: We were converting from ms to seconds for breadcrumbs, spans,
       // but maybe we should just keep them as milliseconds
diff --git a/packages/replay/src/replay.ts b/packages/replay/src/replay.ts
@@ -24,6 +24,7 @@ import type {
   EventBuffer,
   InternalEventContext,
   PopEventContext,
+  RecordingEvent,
   RecordingOptions,
   ReplayContainer as ReplayContainerInterface,
   ReplayPluginOptions,
@@ -42,6 +43,8 @@ import { getHandleRecordingEmit } from './util/handleRecordingEmit';
 import { isExpired } from './util/isExpired';
 import { isSessionExpired } from './util/isSessionExpired';
 import { sendReplay } from './util/sendReplay';
+import type { SKIPPED } from './util/throttle';
+import { throttle, THROTTLED } from './util/throttle';
 
 /**
  * The main replay container class, which holds all the state and methods for recording and sending replays.
@@ -75,6 +78,11 @@ export class ReplayContainer implements ReplayContainerInterface {
     maxSessionLife: MAX_SESSION_LIFE,
   } as const;
 
+  private _throttledAddEvent: (
+    event: RecordingEvent,
+    isCheckout?: boolean,
+  ) => typeof THROTTLED | typeof SKIPPED | Promise<AddEventResult | null>;
+
   /**
    * Options to pass to `rrweb.record()`
    */
@@ -136,6 +144,14 @@ export class ReplayContainer implements ReplayContainerInterface {
     this._debouncedFlush = debounce(() => this._flush(), this._options.flushMinDelay, {
       maxWait: this._options.flushMaxDelay,
     });
+
+    this._throttledAddEvent = throttle(
+      (event: RecordingEvent, isCheckout?: boolean) => addEvent(this, event, isCheckout),
+      // Max 300 events...
+      300,
+      // ... per 5s
+      5,
+    );
   }
 
   /** Get the event context. */
@@ -546,6 +562,38 @@ export class ReplayContainer implements ReplayContainerInterface {
     this._context.urls.push(url);
   }
 
+  /**
+   * Add a breadcrumb event, that may be throttled.
+   * If it was throttled, we add a custom breadcrumb to indicate that.
+   */
+  public throttledAddEvent(
+    event: RecordingEvent,
+    isCheckout?: boolean,
+  ): typeof THROTTLED | typeof SKIPPED | Promise<AddEventResult | null> {
+    const res = this._throttledAddEvent(event, isCheckout);
+
+    // If this is THROTTLED, it means we have throttled the event for the first time
+    // In this case, we want to add a breadcrumb indicating that something was skipped
+    if (res === THROTTLED) {
+      const breadcrumb = createBreadcrumb({
+        category: 'replay.throttled',
+      });
+
+      this.addUpdate(() => {
+        void addEvent(this, {
+          type: EventType.Custom,
+          timestamp: breadcrumb.timestamp || 0,
+          data: {
+            tag: 'breadcrumb',
+            payload: breadcrumb,
+          },
+        });
+      });
+    }
+
+    return res;
+  }
+
   /**
    * Initialize and start all listeners to varying events (DOM,
    * Performance Observer, Recording, Sentry SDK, etc)
@@ -784,7 +832,7 @@ export class ReplayContainer implements ReplayContainerInterface {
    */
   private _createCustomBreadcrumb(breadcrumb: Breadcrumb): void {
     this.addUpdate(() => {
-      void addEvent(this, {
+      void this.throttledAddEvent({
         type: EventType.Custom,
         timestamp: breadcrumb.timestamp || 0,
         data: {
diff --git a/packages/replay/src/types.ts b/packages/replay/src/types.ts
@@ -8,6 +8,7 @@ import type {
 } from '@sentry/types';
 
 import type { eventWithTime, recordOptions } from './types/rrweb';
+import type { SKIPPED, THROTTLED } from './util/throttle';
 
 export type RecordingEvent = eventWithTime;
 export type RecordingOptions = recordOptions;
@@ -498,6 +499,10 @@ export interface ReplayContainer {
   session: Session | undefined;
   recordingMode: ReplayRecordingMode;
   timeouts: Timeouts;
+  throttledAddEvent: (
+    event: RecordingEvent,
+    isCheckout?: boolean,
+  ) => typeof THROTTLED | typeof SKIPPED | Promise<AddEventResult | null>;
   isEnabled(): boolean;
   isPaused(): boolean;
   getContext(): InternalEventContext;
diff --git a/packages/replay/src/util/createPerformanceSpans.ts b/packages/replay/src/util/createPerformanceSpans.ts
@@ -1,17 +1,16 @@
 import { EventType } from '@sentry-internal/rrweb';
 
 import type { AddEventResult, AllEntryData, ReplayContainer, ReplayPerformanceEntry } from '../types';
-import { addEvent } from './addEvent';
 
 /**
- * Create a "span" for each performance entry. The parent transaction is `this.replayEvent`.
+ * Create a "span" for each performance entry.
  */
 export function createPerformanceSpans(
   replay: ReplayContainer,
   entries: ReplayPerformanceEntry<AllEntryData>[],
 ): Promise<AddEventResult | null>[] {
-  return entries.map(({ type, start, end, name, data }) =>
-    addEvent(replay, {
+  return entries.map(({ type, start, end, name, data }) => {
+    const response = replay.throttledAddEvent({
       type: EventType.Custom,
       timestamp: start,
       data: {
@@ -24,6 +23,9 @@ export function createPerformanceSpans(
           data,
         },
       },
-    }),
-  );
+    });
+
+    // If response is a string, it means its either THROTTLED or SKIPPED
+    return typeof response === 'string' ? Promise.resolve(null) : response;
+  });
 }
diff --git a/packages/replay/src/util/throttle.ts b/packages/replay/src/util/throttle.ts
diff --git a/packages/replay/test/unit/util/throttle.test.ts b/packages/replay/test/unit/util/throttle.test.ts

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,9 @@ module.exports = {`
`8`	`8`	`overrides: [`
`9`	`9`	`{`
`10`	`10`	`files: ['src/*/.ts'],`
`11`		`- rules: {},`
	`11`	`+ rules: {`
	`12`	`+ '@sentry-internal/sdk/no-unsupported-es6-methods': 'off',`
	`13`	`+ },`
`12`	`14`	`},`
`13`	`15`	`{`
`14`	`16`	`files: ['jest.setup.ts', 'jest.config.ts'],`