feat: initial commit

Thanks to @vaind for their initial work on this at getsentry/sentry-javascript#6611. We wanted to extract this to a separate tool so it can be run outside of the `sentry-javascript` repo.

Author: billyvg

.eslintrc.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  extends: ['../../.eslintrc.js'],
+  ignorePatterns: ['test-apps'],
+  overrides: [
+    {
+      files: ['*.ts'],
+      rules: {
+        'no-console': 'off',
+        '@typescript-eslint/no-non-null-assertion': 'off',
+        'import/no-unresolved': 'off',
+        '@sentry-internal/sdk/no-optional-chaining': 'off',
+        '@sentry-internal/sdk/no-nullish-coalescing': 'off',
+        '@sentry-internal/sdk/no-unsupported-es6-methods': 'off',
+        'jsdoc/require-jsdoc': 'off',
+      },
+    },
+  ],
+};

.gitignore

@@ -0,0 +1,4 @@
+out
+node_modules
+build
+lib

.prettierrc.json

@@ -0,0 +1,7 @@
+{
+  "arrowParens": "avoid",
+  "printWidth": 120,
+  "proseWrap": "always",
+  "singleQuote": true,
+  "trailingComma": "all"
+}

README.md

@@ -0,0 +1,21 @@
+# Overhead performance metrics
+
+Evaluates Sentry & Replay impact on website performance by running a web app in Chromium via Playwright and collecting various metrics.
+
+The general idea is to run a web app without Sentry, and then run the same app again with Sentry and another one with Sentry+Replay included.
+For the three scenarios, we collect some metrics (CPU, memory, vitals) and later compare them and post as a comment in a PR.
+Changes in the metrics, compared to previous runs from the main branch, should be evaluated on case-by-case basis when preparing and reviewing the PR.
+
+
+## Instructions
+
+WIP
+
+`npm install @server-internal/overhead`
+
+`./sentry-overhead -f myScenario.js`
+
+## Resources
+
+* https://github.com/addyosmani/puppeteer-webperf
+* https://web.dev/

bin/cli.mjs

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+
+import path from 'node:path';
+import { parseArgs } from "node:util";
+import * as url from 'node:url';
+
+import {MetricsCollector, MetricsStats, printStats} from '../lib/index.js';
+
+// From: https://exploringjs.com/nodejs-shell-scripting/ch_nodejs-path.html#detecting-if-module-is-main
+const modulePath = url.fileURLToPath(import.meta.url);
+if (import.meta.url.startsWith('file:')) { // (A)
+  if (process.argv[1] === modulePath) { // (B)
+    main();
+  }
+}
+
+
+async function main() {
+  const options = {
+    'headless': {
+      type: 'boolean',
+    },
+    'runs': {
+      type: 'string',
+      short: 'c'
+    },
+    'cpu': {
+      type: 'string',
+    },
+    'network': {
+      type: 'string',
+      short: 'n',
+    },
+    'file': {
+      type: 'string',
+      short: 'f',
+      multiple: true,
+    },
+  }
+
+  const {
+    values: { headless, runs, cpu, network, file },
+  } = parseArgs({options});
+
+  console.log({
+    headless,
+    runs,
+    cpu,
+    network,
+    file
+  })
+
+  if (!file || file.length < 1) {
+    // need file!
+    throw new Error('No scenario files given')
+  }
+
+
+  const modules = await Promise.all(file.map(async f => import(path.resolve('./', f))));
+  const scenarios = modules.flatMap(m => Object.values(m))
+
+  const collector = new MetricsCollector();
+  const result = await collector.execute({
+    name: 'default',
+    scenarios: scenarios.map(C => new C()),
+    runs: runs ?? 1,
+    tries: (runs ?? 1) * 5,
+    async shouldAccept(results) {
+      printStats(results);
+  
+      const cpuUsage = MetricsStats.mean(results, MetricsStats.cpu);
+      if (cpuUsage > 0.9) {
+        console.error(
+          `CPU usage too high to be accurate: ${(cpuUsage * 100).toFixed(2)} %.`,
+          'Consider simplifying the scenario or changing the CPU throttling factor.',
+        );
+        return false;
+      }
+      return true;
+    },
+  });
+  
+  result.writeToFile('out/latest-result.json');
+
+// const resultsSet = new ResultsSet(outDir);
+// const latestResult = Result.readFromFile(latestResultFile);
+//
+// const analysis = await ResultsAnalyzer.analyze(latestResult, resultsSet);
+// printAnalysis(analysis, latestResult);
+}
+

package.json

@@ -0,0 +1,45 @@
+{
+  "private": true,
+  "version": "7.54.0",
+  "name": "@sentry-internal/overhead",
+  "main": "lib/index.js",
+  "bin": {
+    "sentry-overhead": "./bin/cli.mjs"
+  },
+  "author": "Sentry",
+  "license": "MIT",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "dev:collect": "ts-node-esm ./configs/dev/collect.ts",
+    "dev:process": "ts-node-esm ./configs/dev/process.ts",
+    "dev:run:replay": "npx chrome ./test-apps/booking-app/with-replay.html",
+    "ci:collect": "ts-node-esm ./configs/ci/collect.ts",
+    "ci:process": "ts-node-esm ./configs/ci/process.ts",
+    "fix": "run-s fix:eslint fix:prettier",
+    "fix:eslint": "eslint . --format stylish --fix",
+    "fix:prettier": "prettier --write \"{src,test-apps,configs}/**/*.{ts,js,html,css}\"",
+    "lint": "run-s lint:prettier lint:eslint",
+    "lint:eslint": "eslint . --format stylish",
+    "lint:prettier": "prettier --check \"{src,test-apps,configs}/**/*.{ts,js,html,css}\""
+  },
+  "dependencies": {
+    "@types/node": "^18.11.17",
+    "axios": "^1.2.2",
+    "filesize": "^10.0.6",
+    "fs-extra": "^11.1.0",
+    "p-timeout": "^6.0.0",
+    "playwright": "^1.35.1",
+    "playwright-core": "^1.35.1",
+    "simple-git": "^3.16.0",
+    "simple-statistics": "^7.8.0",
+    "typescript": "^4.9.4"
+  },
+  "devDependencies": {
+    "prettier": "^2.8.8",
+    "ts-node": "^10.9.1"
+  },
+  "volta": {
+    "node": "18.16.0"
+  }
+}