feat(vitest): write run summary when using GitHub Actions Reporter

Author: macarie

packages/vitest/src/node/reporters/github-actions.ts

@@ -4,6 +4,8 @@ import type { Vitest } from '../core'
 import type { TestProject } from '../project'
 import type { Reporter } from '../types/reporter'
 import type { TestCase, TestModule } from './reported-tasks'
+import { writeFileSync } from 'node:fs'
+import { relative } from 'node:path'
 import { stripVTControlCharacters } from 'node:util'
 import { getFullName, getTasks } from '@vitest/runner/utils'
 import { capturePrintError } from '../printError'
@@ -109,6 +111,19 @@ export class GithubActionsReporter implements Reporter {
       })
       this.ctx.logger.log(`\n${formatted}`)
     }
+
+    if (process.env.GITHUB_STEP_SUMMARY) {
+      try {
+        writeFileSync(
+          process.env.GITHUB_STEP_SUMMARY,
+          renderSummary(collectSummaryData(testModules)),
+          { flag: 'a' },
+        )
+      }
+      catch (error) {
+        this.ctx.logger.warn('Could not write summary to $GITHUB_STEP_SUMMARY', error)
+      }
+    }
   }
 }
 
@@ -165,3 +180,111 @@ function escapeProperty(s: string): string {
     .replace(/:/g, '%3A')
     .replace(/,/g, '%2C')
 }
+
+interface SummaryData {
+  flakyTests: Array<{
+    path: {
+      relative: string
+      absolute: string
+    }
+    tests: Array<{
+      testName: string
+      line: number | undefined
+      retries: {
+        allowed: number
+        count: number
+        ratio: number
+      }
+    }>
+  }>
+}
+
+function collectSummaryData(testModules: ReadonlyArray<TestModule>): SummaryData {
+  const summaryData: SummaryData = { flakyTests: [] }
+
+  for (const module of testModules) {
+    const flakyTests: SummaryData['flakyTests'][number] = {
+      path: { relative: module.relativeModuleId, absolute: module.moduleId },
+      tests: [],
+    }
+
+    for (const test of module.children.allTests()) {
+      const diagnostic = test.diagnostic()
+
+      if (diagnostic?.flaky) {
+        const retriesAllowed = typeof test.options.retry === 'number'
+          ? test.options.retry
+          : (test.options.retry?.count
+            // falling back to `retryCount` as this is used as the denominator to compute `retryRatio`
+            ?? diagnostic.retryCount)
+        const retriesRatio = diagnostic.retryCount / retriesAllowed
+
+        flakyTests.tests.push({
+          retries: {
+            allowed: retriesAllowed,
+            count: diagnostic.retryCount,
+            ratio: retriesRatio,
+          },
+          line: test.task.location?.line,
+          testName: test.task.fullTestName,
+        })
+      }
+    }
+
+    if (flakyTests.tests.length > 0) {
+      flakyTests.tests.sort((a, b) => b.retries.ratio - a.retries.ratio)
+
+      summaryData.flakyTests.push(flakyTests)
+    }
+  }
+
+  return summaryData
+}
+
+function createGitHubFileLinkCreator(): (path: string, line?: number) => string | null {
+  const repository = process.env.GITHUB_REPOSITORY
+  const commitHash = process.env.GITHUB_SHA
+  const rootPath = process.env.GITHUB_WORKSPACE
+
+  if (repository !== undefined && commitHash !== undefined && rootPath !== undefined) {
+    return (path, line) => {
+      const lineFragment = line !== undefined ? `#L${line}` : ''
+
+      return `https://github.com/${repository}/blob/${commitHash}/${relative(rootPath, path)}${lineFragment}`
+    }
+  }
+
+  return () => null
+}
+
+function mdLink(text: string, url: string | null): string {
+  return url === null ? text : `[${text}](${url})`
+}
+
+function renderSummary(summaryData: SummaryData): string {
+  const fileLinkCreator = createGitHubFileLinkCreator()
+
+  let summary = '## Vitest Test Report\n'
+
+  if (summaryData.flakyTests.length > 0) {
+    summary += '\n### Flaky Tests\n\nThese tests passed only after one or more retries, indicating potential instability.\n'
+
+    for (const flakyTests of summaryData.flakyTests) {
+      summary += `\n##### \`${flakyTests.path.relative}\` (${flakyTests.tests.length} flaky tests)\n`
+
+      for (const flakyTest of flakyTests.tests) {
+        const retriesText = `passed on retry ${flakyTest.retries.count} out of ${flakyTest.retries.allowed}`
+
+        summary += `\n- ${mdLink(`\`${flakyTest.testName}\``, fileLinkCreator(flakyTests.path.absolute, flakyTest.line))} (${flakyTest.retries.ratio >= 0.8 ? `**${retriesText}**` : retriesText})`
+      }
+
+      summary += '\n'
+    }
+  }
+
+  if (!summary.endsWith('\n')) {
+    summary += '\n'
+  }
+
+  return summary
+}