Run performance tests daily (#552)

* Run and store perf test metric results in D1 everyday

* fix perf test ci setup

* update perf report aggregation

---------

Co-authored-by: scuffi <aferguson@cloudflare.com>

Author: scuffi

.github/workflows/performance.yml

@@ -14,10 +14,12 @@ on:
           - sustained-throughput
           - bursty-traffic
           - concurrent-creation
+          - burst-startup
+          - file-io
   release:
     types: [published]
   schedule:
-    - cron: '0 0 * * 0'
+    - cron: '0 0 * * *'
 
 permissions:
   contents: read
@@ -50,8 +52,35 @@ jobs:
           node-version: 24
           cache: 'npm'
 
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version-file: .bun-version
+
+      - uses: actions/cache/restore@v5
+        id: nm-cache
+        with:
+          path: |
+            node_modules
+            packages/*/node_modules
+            examples/*/node_modules
+            sites/*/node_modules
+          key: node-modules-${{ hashFiles('package-lock.json') }}
+
+      - run: npm ci --prefer-offline --no-audit --no-fund
+        if: steps.nm-cache.outputs.cache-hit != 'true'
+
+      - uses: actions/cache/save@v5
+        if: steps.nm-cache.outputs.cache-hit != 'true'
+        with:
+          path: |
+            node_modules
+            packages/*/node_modules
+            examples/*/node_modules
+            sites/*/node_modules
+          key: node-modules-${{ hashFiles('package-lock.json') }}
+
       - name: Build packages
-        run: npx turbo run build
+        run: npm run build
 
       - name: Get package version
         id: version
@@ -97,7 +126,9 @@ jobs:
       - name: Generate wrangler config
         run: |
           cd tests/e2e/test-worker
-          sed "s/{{WORKER_NAME}}/${{ env.PERF_WORKER_NAME }}/g; s/{{CONTAINER_NAME}}/${{ env.PERF_WORKER_NAME }}/g" \
+          sed -e "s|{{WORKER_NAME}}|${{ env.PERF_WORKER_NAME }}|g" \
+            -e "s|{{CONTAINER_NAME}}|${{ env.PERF_WORKER_NAME }}|g" \
+            -e "s|{{IMAGE_SANDBOX}}|registry.cloudflare.com/${{ secrets.CLOUDFLARE_ACCOUNT_ID }}/sandbox:perf|g" \
             wrangler.perf.template.jsonc > wrangler.jsonc
 
       - name: Deploy test worker
@@ -175,3 +206,12 @@ jobs:
           name: perf-baseline
           path: perf-results/latest.json
           retention-days: 7
+
+      - name: Ingest results into D1
+        if: always()
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          PERF_D1_DATABASE_ID: ${{ secrets.PERF_D1_DATABASE_ID }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+        run: npx tsx tests/perf/scripts/ingest-to-d1.ts perf-results/latest.json

package.json

@@ -20,6 +20,11 @@
     "test:perf:concurrent": "vitest run --config vitest.perf.config.ts tests/perf/scenarios/concurrent-creation.test.ts",
     "test:perf:sustained": "vitest run --config vitest.perf.config.ts tests/perf/scenarios/sustained-throughput.test.ts",
     "test:perf:burst": "vitest run --config vitest.perf.config.ts tests/perf/scenarios/bursty-traffic.test.ts",
+    "test:perf:burst-startup": "vitest run --config vitest.perf.config.ts tests/perf/scenarios/burst-startup.test.ts",
+    "test:perf:file-io": "vitest run --config vitest.perf.config.ts tests/perf/scenarios/file-io.test.ts",
+    "perf:ingest": "npx tsx tests/perf/scripts/ingest-to-d1.ts",
+    "perf:ingest:dry-run": "npx tsx tests/perf/scripts/ingest-to-d1.ts tests/perf/scripts/fixture-report.json --dry-run",
+    "perf:query": "wrangler d1 execute perf-results --remote --command",
     "docker:rebuild": "npm run build:clean && npm run docker:local -w @cloudflare/sandbox"
   },
   "keywords": [],

tests/e2e/test-worker/wrangler.perf.template.jsonc

@@ -16,7 +16,7 @@
   "containers": [
     {
       "class_name": "Sandbox",
-      "image": "./Dockerfile",
+      "image": "{{IMAGE_SANDBOX}}",
       "name": "{{CONTAINER_NAME}}"
     }
   ],

tests/perf/global-setup.ts

@@ -15,9 +15,11 @@ import {
   getTestWorkerUrl,
   type WranglerDevRunner
 } from '../e2e/helpers/wrangler-runner';
+import { teardown } from './global-teardown';
 import { GlobalMetricsStore } from './helpers/metrics-collector';
 
 export const PERF_STATE_FILE = join(tmpdir(), 'perf-test-state.json');
+export const PERF_SCENARIOS_FILE = join(tmpdir(), 'perf-test-scenarios.json');
 const TEST_WORKER_DIR = 'tests/e2e/test-worker';
 
 let runner: WranglerDevRunner | null = null;
@@ -43,6 +45,9 @@ export async function setup() {
   if (existsSync(PERF_STATE_FILE)) {
     unlinkSync(PERF_STATE_FILE);
   }
+  if (existsSync(PERF_SCENARIOS_FILE)) {
+    unlinkSync(PERF_SCENARIOS_FILE);
+  }
 
   // Ensure wrangler config exists for local mode
   if (!process.env.TEST_WORKER_URL) {
@@ -101,4 +106,6 @@ export async function setup() {
   console.log('[PerfSetup] Ready!\n');
 }
 
+export { teardown };
+
 export { runner };

tests/perf/global-teardown.ts

@@ -6,9 +6,12 @@
  */
 
 import { appendFileSync, existsSync, readFileSync, unlinkSync } from 'node:fs';
-import { PERF_STATE_FILE, runner } from './global-setup';
+import { PERF_SCENARIOS_FILE, PERF_STATE_FILE, runner } from './global-setup';
 import { GlobalMetricsStore } from './helpers/metrics-collector';
-import { ReportGenerator } from './helpers/report-generator';
+import {
+  ReportGenerator,
+  type ScenarioResult
+} from './helpers/report-generator';
 
 export async function teardown() {
   console.log('\n[PerfTeardown] Generating final reports...');
@@ -24,8 +27,19 @@ export async function teardown() {
     // Generate reports
     const store = GlobalMetricsStore.getInstance();
     const reporter = new ReportGenerator('./perf-results');
-
-    const report = reporter.generateJsonReport(store, workerUrl);
+    const runInfo = store.getRunInfo();
+    const scenarios: ScenarioResult[] = existsSync(PERF_SCENARIOS_FILE)
+      ? (JSON.parse(
+          readFileSync(PERF_SCENARIOS_FILE, 'utf-8')
+        ) as ScenarioResult[])
+      : Array.from(store.getAllScenarios().values()).map((collector) =>
+          reporter.generateScenarioResult(collector)
+        );
+    const report = reporter.generateJsonReportFromScenarios(
+      scenarios,
+      workerUrl,
+      runInfo.duration
+    );
     const filepath = reporter.writeJsonReport(report);
 
     console.log(`[PerfTeardown] JSON report written to: ${filepath}`);
@@ -44,6 +58,9 @@ export async function teardown() {
     if (existsSync(PERF_STATE_FILE)) {
       unlinkSync(PERF_STATE_FILE);
     }
+    if (existsSync(PERF_SCENARIOS_FILE)) {
+      unlinkSync(PERF_SCENARIOS_FILE);
+    }
   } catch (error) {
     console.error('[PerfTeardown] Error generating reports:', error);
   }

tests/perf/helpers/concurrent-runner.ts

@@ -96,10 +96,11 @@ export async function runBurst<T>(
   const maxDuration = options?.maxDuration || 120000;
 
   const start = performance.now();
-  const promises: Promise<
-    | { index: number; result: T; duration: number }
-    | { index: number; error: Error; duration: number }
-  >[] = [];
+  const results: ConcurrentResult<T>['results'] = [];
+  const promises: Promise<void>[] = [];
+  let successCount = 0;
+  let failureCount = 0;
+  let timedOut = false;
 
   for (let i = 0; i < count; i++) {
     if (i > 0) {
@@ -111,65 +112,42 @@ export async function runBurst<T>(
 
     promises.push(
       operation()
-        .then((result) => ({
-          index,
-          result,
-          duration: performance.now() - opStart
-        }))
-        .catch((error) => ({
-          index,
-          error: error as Error,
-          duration: performance.now() - opStart
-        }))
+        .then((result) => {
+          if (timedOut) return;
+          results.push({
+            index,
+            result,
+            duration: performance.now() - opStart
+          });
+          successCount++;
+        })
+        .catch((error) => {
+          if (timedOut) return;
+          results.push({
+            index,
+            error: error as Error,
+            duration: performance.now() - opStart
+          });
+          failureCount++;
+        })
     );
   }
 
-  // Wait for all with timeout
   let timeoutId: ReturnType<typeof setTimeout>;
-  const timeoutPromise = new Promise<never>((_, reject) => {
-    timeoutId = setTimeout(
-      () => reject(new Error('Burst timeout')),
-      maxDuration
-    );
+  const timeoutPromise = new Promise<boolean>((resolve) => {
+    timeoutId = setTimeout(() => {
+      timedOut = true;
+      resolve(true);
+    }, maxDuration);
   });
 
-  try {
-    const results = await Promise.race([Promise.all(promises), timeoutPromise]);
-    clearTimeout(timeoutId!);
-
-    let successCount = 0;
-    let failureCount = 0;
-    for (const r of results) {
-      if ('result' in r) successCount++;
-      else failureCount++;
-    }
+  await Promise.race([Promise.all(promises).then(() => false), timeoutPromise]);
+  clearTimeout(timeoutId!);
 
-    return {
-      results,
-      totalDuration: performance.now() - start,
-      successCount,
-      failureCount
-    };
-  } catch {
-    // Timeout - return partial results
-    const settled = await Promise.allSettled(promises);
-    let successCount = 0;
-    let failureCount = 0;
-    const results: ConcurrentResult<T>['results'] = [];
-
-    for (const s of settled) {
-      if (s.status === 'fulfilled') {
-        results.push(s.value);
-        if ('result' in s.value) successCount++;
-        else failureCount++;
-      }
-    }
-
-    return {
-      results,
-      totalDuration: performance.now() - start,
-      successCount,
-      failureCount
-    };
-  }
+  return {
+    results: [...results].sort((a, b) => a.index - b.index),
+    totalDuration: performance.now() - start,
+    successCount,
+    failureCount
+  };
 }

tests/perf/helpers/constants.ts

@@ -6,11 +6,13 @@ export const SCENARIOS = {
   COLD_START: 'cold-start',
   CONCURRENT: 'concurrent-creation',
   SUSTAINED: 'sustained-throughput',
-  BURST: 'bursty-traffic'
+  BURST: 'bursty-traffic',
+  BURST_STARTUP: 'burst-startup',
+  FILE_IO: 'file-io'
 } as const;
 
 export const METRICS = {
-  // Cold start
+  // Cold start / sequential startup (sequential create → first exec)
   COLD_START_LATENCY: 'cold-start-latency',
   WARM_COMMAND_LATENCY: 'warm-command-latency',
   // Concurrent creation
@@ -23,13 +25,23 @@ export const METRICS = {
   COMPLETED_COMMANDS: 'completed-commands',
   ACTUAL_THROUGHPUT: 'actual-throughput',
   LATENCY_DEGRADATION: 'latency-degradation',
-  // Burst
+  // Bursty traffic (commands on a warm sandbox)
   BURST_COMMAND: 'burst-command',
   BURST_DURATION: 'burst-duration',
   BURST_SUCCESS_RATE: 'burst-success-rate',
   BASELINE_LATENCY: 'baseline-latency',
   RECOVERY_LATENCY: 'recovery-latency',
-  RECOVERY_OVERHEAD: 'recovery-overhead'
+  RECOVERY_OVERHEAD: 'recovery-overhead',
+  // Burst startup (rapid sandbox creations)
+  BURST_STARTUP_LATENCY: 'burst-startup-latency',
+  BURST_STARTUP_SUCCESS_RATE: 'burst-startup-success-rate',
+  BURST_STARTUP_TOTAL_TIME: 'burst-startup-total-time',
+  // File I/O — used as prefixes, appended with '-<size>' (e.g. 'file-write-latency-10kb')
+  FILE_WRITE_LATENCY: 'file-write-latency',
+  FILE_READ_LATENCY: 'file-read-latency',
+  FILE_ROUNDTRIP_LATENCY: 'file-roundtrip-latency',
+  FILE_CONCURRENT_WRITE: 'file-concurrent-write',
+  FILE_CONCURRENT_READ: 'file-concurrent-read'
 } as const;
 
 /** Minimum success rate to pass a scenario (percentage) */