test(replay): Add Playwright tests for error-mode and error linking (#7251)

Lms24 · web-flow · commit 9c2c018c7daf · 2023-02-22T16:26:43.000+01:00
Add four integration tests for replays where errors occur:

* error mode:
  * error to session transition once the error occurs
  * if the error is dropped, it shouldn't be linked in the replay event (see test comment about the problem with this scenario)
* session mode
  * if an error occurs, make sure it's linked in the replay event
  * unless the error is dropped
diff --git a/packages/integration-tests/suites/replay/errors/droppedError/init.js b/packages/integration-tests/suites/replay/errors/droppedError/init.js
@@ -0,0 +1,18 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+window.Replay = new Sentry.Replay({
+  flushMinDelay: 500,
+  flushMaxDelay: 500,
+});
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  sampleRate: 1,
+  replaysSessionSampleRate: 0.0,
+  replaysOnErrorSampleRate: 1.0,
+  beforeSend(_) {
+    return null;
+  },
+  integrations: [window.Replay],
+});
diff --git a/packages/integration-tests/suites/replay/errors/droppedError/test.ts b/packages/integration-tests/suites/replay/errors/droppedError/test.ts
@@ -0,0 +1,65 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../../utils/fixtures';
+import { getExpectedReplayEvent } from '../../../../utils/replayEventTemplates';
+import { getReplayEvent, shouldSkipReplayTest, waitForReplayRequest } from '../../../../utils/replayHelpers';
+
+/*
+ * This scenario currently shows somewhat unexpected behavior from the PoV of a user:
+ * The error is dropped, but the recording is started and continued anyway.
+ * If folks only sample error replays, this will lead to a lot of confusion as the resulting replay
+ * won't contain the error that started it (possibly none or only additional errors that occurred later on).
+ *
+ * This is because in error-mode, we start recording as soon as replay's eventProcessor is called with an error.
+ * If later event processors or beforeSend drop the error, the recording is already started.
+ *
+ * We'll need a proper SDK lifecycle hook (WIP) to fix this properly.
+ * TODO: Once we have lifecycle hooks, we should revisit this test and make sure it behaves as expected.
+ *       This means that the recording should not be started or stopped if the error that triggered it is not sent.
+ */
+sentryTest(
+  '[error-mode] should start recording if an error occurred although the error was dropped',
+  async ({ getLocalTestPath, page }) => {
+    if (shouldSkipReplayTest()) {
+      sentryTest.skip();
+    }
+
+    let callsToSentry = 0;
+    const reqPromise0 = waitForReplayRequest(page, 0);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      callsToSentry++;
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
+    });
+
+    const url = await getLocalTestPath({ testDir: __dirname });
+
+    await page.goto(url);
+    await page.click('#go-background');
+    expect(callsToSentry).toEqual(0);
+
+    await page.click('#error');
+    const req0 = await reqPromise0;
+
+    await page.click('#go-background');
+    expect(callsToSentry).toEqual(2); // 2 replay events
+
+    await page.click('#log');
+    await page.click('#go-background');
+
+    const event0 = getReplayEvent(req0);
+
+    expect(event0).toEqual(
+      getExpectedReplayEvent({
+        contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
+        // This is by design. A dropped error shouldn't be in this list.
+        error_ids: [],
+        replay_type: 'error',
+      }),
+    );
+  },
+);
diff --git a/packages/integration-tests/suites/replay/errors/errorMode/test.ts b/packages/integration-tests/suites/replay/errors/errorMode/test.ts
@@ -0,0 +1,133 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../../utils/fixtures';
+import { envelopeRequestParser } from '../../../../utils/helpers';
+import {
+  expectedClickBreadcrumb,
+  expectedConsoleBreadcrumb,
+  getExpectedReplayEvent,
+} from '../../../../utils/replayEventTemplates';
+import {
+  getReplayEvent,
+  getReplayRecordingContent,
+  shouldSkipReplayTest,
+  waitForReplayRequest,
+} from '../../../../utils/replayHelpers';
+
+sentryTest(
+  '[error-mode] should start recording and switch to session mode once an error is thrown',
+  async ({ getLocalTestPath, page }) => {
+    if (shouldSkipReplayTest()) {
+      sentryTest.skip();
+    }
+
+    let callsToSentry = 0;
+    let errorEventId: string | undefined;
+    const reqPromise0 = waitForReplayRequest(page, 0);
+    const reqPromise1 = waitForReplayRequest(page, 1);
+    const reqPromise2 = waitForReplayRequest(page, 2);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      const event = envelopeRequestParser(route.request());
+      // error events have no type field
+      if (event && !event.type && event.event_id) {
+        errorEventId = event.event_id;
+      }
+      callsToSentry++;
+
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
+    });
+
+    const url = await getLocalTestPath({ testDir: __dirname });
+
+    await page.goto(url);
+    await page.click('#go-background');
+    expect(callsToSentry).toEqual(0);
+
+    await page.click('#error');
+    const req0 = await reqPromise0;
+
+    await page.click('#go-background');
+    const req1 = await reqPromise1;
+
+    expect(callsToSentry).toEqual(3); // 1 error, 2 replay events
+
+    await page.click('#log');
+    await page.click('#go-background');
+    const req2 = await reqPromise2;
+
+    const event0 = getReplayEvent(req0);
+    const content0 = getReplayRecordingContent(req0);
+
+    const event1 = getReplayEvent(req1);
+    const content1 = getReplayRecordingContent(req1);
+
+    const event2 = getReplayEvent(req2);
+    const content2 = getReplayRecordingContent(req2);
+
+    expect(event0).toEqual(
+      getExpectedReplayEvent({
+        contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
+        // @ts-ignore this is fine
+        error_ids: [errorEventId],
+        replay_type: 'error',
+      }),
+    );
+
+    // The first event should have both, full and incremental snapshots,
+    // as we recorded and kept all events in the buffer
+    expect(content0.fullSnapshots).toHaveLength(1);
+    // We don't know how many incremental snapshots we'll have (also browser-dependent),
+    // but we know that we have at least 5
+    expect(content0.incrementalSnapshots.length).toBeGreaterThan(5);
+    // We want to make sure that the event that triggered the error was recorded.
+    expect(content0.breadcrumbs).toEqual(
+      expect.arrayContaining([
+        {
+          ...expectedClickBreadcrumb,
+          message: 'body > button#error',
+        },
+      ]),
+    );
+
+    expect(event1).toEqual(
+      getExpectedReplayEvent({
+        contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
+        // @ts-ignore this is fine
+        replay_type: 'error', // although we're in session mode, we still send 'error' as replay_type
+        replay_start_timestamp: undefined,
+        segment_id: 1,
+        urls: [],
+      }),
+    );
+
+    // Also the second snapshot should have a full snapshot, as we switched from error to session
+    // mode which triggers another checkout
+    expect(content1.fullSnapshots).toHaveLength(1);
+    expect(content1.incrementalSnapshots).toHaveLength(0);
+
+    // The next event should just be a normal replay event as we're now in session mode and
+    // we continue recording everything
+    expect(event2).toEqual(
+      getExpectedReplayEvent({
+        contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
+        // @ts-ignore this is fine
+        replay_type: 'error',
+        replay_start_timestamp: undefined,
+        segment_id: 2,
+        urls: [],
+      }),
+    );
+
+    expect(content2.breadcrumbs).toEqual(
+      expect.arrayContaining([
+        { ...expectedClickBreadcrumb, message: 'body > button#log' },
+        { ...expectedConsoleBreadcrumb, level: 'log', message: 'Some message' },
+      ]),
+    );
+  },
+);
diff --git a/packages/integration-tests/suites/replay/errors/errorsInSession/init.js b/packages/integration-tests/suites/replay/errors/errorsInSession/init.js
@@ -0,0 +1,22 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+window.Replay = new Sentry.Replay({
+  flushMinDelay: 500,
+  flushMaxDelay: 500,
+});
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  sampleRate: 1,
+  replaysSessionSampleRate: 1.0,
+  replaysOnErrorSampleRate: 0.0,
+  beforeSend(event, hint) {
+    if (hint.originalException.message.includes('[drop]')) {
+      return null;
+    }
+    return event;
+  },
+  integrations: [window.Replay],
+  debug: true,
+});
diff --git a/packages/integration-tests/suites/replay/errors/errorsInSession/test.ts b/packages/integration-tests/suites/replay/errors/errorsInSession/test.ts
@@ -0,0 +1,115 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../../utils/fixtures';
+import { envelopeRequestParser } from '../../../../utils/helpers';
+import { expectedClickBreadcrumb, getExpectedReplayEvent } from '../../../../utils/replayEventTemplates';
+import {
+  getReplayEvent,
+  getReplayRecordingContent,
+  shouldSkipReplayTest,
+  waitForReplayRequest,
+} from '../../../../utils/replayHelpers';
+
+sentryTest(
+  '[session-mode] replay event should contain an error id of an error that occurred during session recording',
+  async ({ getLocalTestPath, page }) => {
+    if (shouldSkipReplayTest()) {
+      sentryTest.skip();
+    }
+
+    let errorEventId: string = 'invalid_id';
+
+    const reqPromise0 = waitForReplayRequest(page, 0);
+    const reqPromise1 = waitForReplayRequest(page, 1);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      const event = envelopeRequestParser(route.request());
+      // error events have no type field
+      if (event && !event.type && event.event_id) {
+        errorEventId = event.event_id;
+      }
+
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
+    });
+
+    const url = await getLocalTestPath({ testDir: __dirname });
+
+    await page.goto(url);
+    await page.click('#go-background');
+    const req0 = await reqPromise0;
+
+    await page.click('#error');
+    await page.click('#go-background');
+    const req1 = await reqPromise1;
+
+    const event0 = getReplayEvent(req0);
+
+    const event1 = getReplayEvent(req1);
+    const content1 = getReplayRecordingContent(req1);
+
+    expect(event0).toEqual(getExpectedReplayEvent());
+
+    expect(event1).toEqual(
+      getExpectedReplayEvent({
+        replay_start_timestamp: undefined,
+        segment_id: 1,
+        // @ts-ignore this is fine
+        error_ids: [errorEventId],
+        urls: [],
+      }),
+    );
+
+    expect(content1.breadcrumbs).toEqual(
+      expect.arrayContaining([{ ...expectedClickBreadcrumb, message: 'body > button#error' }]),
+    );
+  },
+);
+
+sentryTest(
+  '[session-mode] replay event should not contain an error id of a dropped error while recording',
+  async ({ getLocalTestPath, page }) => {
+    if (shouldSkipReplayTest()) {
+      sentryTest.skip();
+    }
+
+    const reqPromise1 = waitForReplayRequest(page, 1);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
+    });
+
+    const url = await getLocalTestPath({ testDir: __dirname });
+
+    await page.goto(url);
+    await page.click('#go-background');
+
+    await page.click('#drop');
+    await page.click('#go-background');
+    const req1 = await reqPromise1;
+
+    const event1 = getReplayEvent(req1);
+    const content1 = getReplayRecordingContent(req1);
+
+    expect(event1).toEqual(
+      getExpectedReplayEvent({
+        replay_start_timestamp: undefined,
+        segment_id: 1,
+        error_ids: [], // <-- no error id
+        urls: [],
+      }),
+    );
+
+    // The button click that triggered the error should still be recorded
+    expect(content1.breadcrumbs).toEqual(
+      expect.arrayContaining([{ ...expectedClickBreadcrumb, message: 'body > button#drop' }]),
+    );
+  },
+);
diff --git a/packages/integration-tests/suites/replay/errors/init.js b/packages/integration-tests/suites/replay/errors/init.js
@@ -0,0 +1,16 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+window.Replay = new Sentry.Replay({
+  flushMinDelay: 500,
+  flushMaxDelay: 500,
+});
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  sampleRate: 1,
+  replaysSessionSampleRate: 0.0,
+  replaysOnErrorSampleRate: 1.0,
+
+  integrations: [window.Replay],
+});
diff --git a/packages/integration-tests/suites/replay/errors/subject.js b/packages/integration-tests/suites/replay/errors/subject.js
@@ -0,0 +1,18 @@
+document.getElementById('go-background').addEventListener('click', () => {
+  Object.defineProperty(document, 'hidden', { value: true, writable: true });
+  const ev = document.createEvent('Event');
+  ev.initEvent('visibilitychange');
+  document.dispatchEvent(ev);
+});
+
+document.getElementById('error').addEventListener('click', () => {
+  throw new Error('Ooops');
+});
+
+document.getElementById('drop').addEventListener('click', () => {
+  throw new Error('[drop] Ooops');
+});
+
+document.getElementById('log').addEventListener('click', () => {
+  console.log('Some message');
+});
diff --git a/packages/integration-tests/suites/replay/errors/template.html b/packages/integration-tests/suites/replay/errors/template.html
diff --git a/packages/integration-tests/utils/replayEventTemplates.ts b/packages/integration-tests/utils/replayEventTemplates.ts
