[test] Add demo-based axe-core accessibility tests for mui-material

.circleci/config.yml

@@ -273,6 +273,9 @@ jobs:
       - run:
           name: Run visual regression tests
           command: xvfb-run pnpm test:regressions
+      - run:
+          name: A11y results committed?
+          command: git add -A && git diff --exit-code --staged
       - run:
           name: Build packages for fixtures
           command: pnpm release:build

AGENTS.md

@@ -153,6 +153,28 @@ describe('Button', () => {
 });
 ```
 
+### Accessibility Testing
+
+Automated axe-core coverage piggybacks on the visual-regression Playwright loop in `test/regressions/index.test.js`. Each screenshot render for an enrolled demo is followed by `axe.run` on the same rendered `[data-testid="testcase"]` element, so no separate browser session is spun up.
+
+- `test/regressions/a11y/a11yConfig.ts` — test roster. Each entry maps a docs slug to a canonical component name; pending components live as `// TODO:` comments with the blocker noted inline.
+- `test/regressions/a11y/axe.ts` — `recordA11y` records per-demo results onto `ctx.task.meta.a11y` and asserts visual rules (`color-contrast`, `link-in-text-block`) unless listed in `skipAssertions`.
+- `test/regressions/a11y/a11yReporter.ts` — Vitest reporter (attached in `test/regressions/vitest.config.ts`) that aggregates `task.meta.a11y` into one JSON per component at `test/regressions/a11y/results/{Component}.json` (per-component aggregates + per-demo breakdown). One file per component so downstream docs consumers can import only what they need.
+
+Enroll a component: uncomment its `TODO` line in `a11yConfig.ts` into a real entry (or add a new one).
+
+```ts
+// test/regressions/a11y/a11yConfig.ts
+{
+  component: 'Alert',
+  slug: 'alert',
+  demos: ['BasicAlerts', 'ColorAlerts'],                // optional: defaults to every VRT demo
+  skipAssertions: ['color-contrast'],                   // optional: record known issues without failing CI
+},
+```
+
+Then run `pnpm test:regressions` to refresh `test/regressions/a11y/results/` (axe runs inline with the screenshot loop; the Vitest reporter writes per-component JSONs). CI enforces the directory is up to date via a git-diff check.
+
 ### Imports
 
 Use one-level deep imports to avoid bundling entire packages:

package.json

@@ -58,7 +58,7 @@
     "test:e2e:dev": "pnpm -F ./test/e2e dev",
     "test:e2e-website": "playwright test test/e2e-website --config test/e2e-website/playwright.config.ts",
     "test:e2e-website:dev": "cross-env PLAYWRIGHT_TEST_BASE_URL=http://localhost:3000 playwright test test/e2e-website --config test/e2e-website/playwright.config.ts",
-    "test:regressions": "cross-env NODE_ENV=production pnpm test:regressions:build && concurrently --success first --kill-others \"pnpm test:regressions:run\" \"pnpm test:regressions:server\"",
+    "test:regressions": "cross-env NODE_ENV=production pnpm test:regressions:build && concurrently --success first --kill-others \"pnpm test:regressions:run\" \"pnpm test:regressions:server\" && prettier --write test/regressions/a11y/results",
     "test:regressions:build": "vite build test/regressions",
     "test:regressions:dev": "vite test/regressions --port 5001",
     "test:regressions:run": "vitest run -r ./test/regressions/",
@@ -115,6 +115,7 @@
     "@vitejs/plugin-react": "^5.1.1",
     "@vitest/browser-playwright": "^4.0.13",
     "@vitest/coverage-v8": "^4.0.13",
+    "axe-core": "4.11.1",
     "babel-plugin-istanbul": "7.0.1",
     "babel-plugin-module-resolver": "5.0.3",
     "chalk": "5.6.2",

test/regressions/a11y/a11yConfig.ts

@@ -0,0 +1,90 @@
+/**
+ * A11y testing roster for `@mui/material`.
+ *
+ * Each entry maps a docs page (`docs/data/material/components/{slug}`) to a
+ * canonical component name (used as the key in the `results/{Component}.json`
+ * output file). Components not yet ready to test live as commented `TODO`
+ * entries with the blocker noted inline.
+ *
+ * - `demos` — filenames under the slug directory (no extension). Each maps to
+ *   the VRT route `/docs-components-{slug}/{demo}`. Omit to auto-discover
+ *   every VRT-exposed demo for this slug (inherits VRT's exclusion list).
+ * - `skipAssertions` — axe rule ids whose violations are recorded but not
+ *   asserted on. Used to track known issues without failing CI.
+ *
+ * See AGENTS.md → "Accessibility Testing" for the full workflow.
+ */
+export interface ComponentA11yConfig {
+  /** PascalCase component name. Keys the `results/{Component}.json` output file. */
+  component: string;
+  /** Directory under `docs/data/material/components/`. */
+  slug: string;
+  /**
+   * Demo filenames (no extension) under the slug directory. Each entry maps to
+   * the VRT route `/docs-components-{slug}/{demo}`. When omitted, the test
+   * auto-discovers every VRT-exposed demo for this slug from the fixture nav
+   * (i.e. inherits VRT's exclusion list).
+   */
+  demos?: string[];
+  /**
+   * Axe rule ids whose violations are recorded but not asserted on. The rule
+   * still runs; only the test-failing assertion is suppressed. Used to track
+   * known issues without failing CI.
+   */
+  skipAssertions?: string[];
+}
+
+// Components whose demos currently trip `color-contrast` (labels/icons near
+// the 4.5:1 threshold, or overlapped elements axe can't analyze). Recorded as
+// failedRules in the results JSON so the team can see what needs fixing;
+// asserting on them would block every CI run.
+const PARTIAL_SKIP = ['color-contrast'];
+
+export const COMPONENTS: ComponentA11yConfig[] = [
+  { component: 'Accordion', slug: 'accordion', skipAssertions: PARTIAL_SKIP },
+  { component: 'Alert', slug: 'alert', skipAssertions: PARTIAL_SKIP },
+  { component: 'AppBar', slug: 'app-bar', skipAssertions: PARTIAL_SKIP },
+  { component: 'Autocomplete', slug: 'autocomplete', skipAssertions: PARTIAL_SKIP },
+  { component: 'Avatar', slug: 'avatars', skipAssertions: PARTIAL_SKIP },
+  { component: 'Badge', slug: 'badges' },
+  { component: 'BottomNavigation', slug: 'bottom-navigation' },
+  { component: 'Breadcrumbs', slug: 'breadcrumbs', skipAssertions: PARTIAL_SKIP },
+  { component: 'Button', slug: 'buttons', demos: ['BasicButtons', 'ColorButtons'] },
+  { component: 'ButtonGroup', slug: 'button-group' },
+  { component: 'Card', slug: 'cards', demos: ['BasicCard', 'OutlinedCard'] },
+  { component: 'Checkbox', slug: 'checkboxes', skipAssertions: PARTIAL_SKIP },
+  { component: 'Chip', slug: 'chips', skipAssertions: PARTIAL_SKIP },
+  // TODO: Dialog — VRT excludes demos (need interaction)
+  { component: 'Divider', slug: 'dividers', skipAssertions: PARTIAL_SKIP },
+  { component: 'Drawer', slug: 'drawers', skipAssertions: PARTIAL_SKIP },
+  { component: 'Fab', slug: 'floating-action-button' },
+  { component: 'Icon', slug: 'icons' },
+  // TODO: ImageList — VRT excludes demos (images don't load)
+  { component: 'Link', slug: 'links' },
+  { component: 'List', slug: 'lists', skipAssertions: PARTIAL_SKIP },
+  // TODO: Menu — VRT excludes demos (need interaction)
+  { component: 'Menubar', slug: 'menubar' },
+  { component: 'Modal', slug: 'modal', skipAssertions: PARTIAL_SKIP },
+  { component: 'NumberField', slug: 'number-field', skipAssertions: PARTIAL_SKIP },
+  { component: 'Pagination', slug: 'pagination', skipAssertions: PARTIAL_SKIP },
+  { component: 'Popover', slug: 'popover' },
+  // TODO: Popper — VRT excludes demos (need interaction)
+  // TODO: Progress — VRT excludes demos (flaky)
+  { component: 'Radio', slug: 'radio-buttons', skipAssertions: PARTIAL_SKIP },
+  { component: 'Rating', slug: 'rating' },
+  { component: 'Select', slug: 'selects', skipAssertions: PARTIAL_SKIP },
+  { component: 'Skeleton', slug: 'skeleton' },
+  { component: 'Slider', slug: 'slider', skipAssertions: PARTIAL_SKIP },
+  { component: 'Snackbar', slug: 'snackbars', skipAssertions: PARTIAL_SKIP },
+  // TODO: SpeedDial — VRT excludes demos (need interaction)
+  { component: 'Stepper', slug: 'steppers', skipAssertions: PARTIAL_SKIP },
+  { component: 'Switch', slug: 'switches', skipAssertions: PARTIAL_SKIP },
+  { component: 'Table', slug: 'table', skipAssertions: PARTIAL_SKIP },
+  { component: 'Tabs', slug: 'tabs', skipAssertions: PARTIAL_SKIP },
+  { component: 'TextField', slug: 'text-fields', skipAssertions: PARTIAL_SKIP },
+  // TODO: TextareaAutosize — superseded by dedicated regression test
+  { component: 'Timeline', slug: 'timeline' },
+  { component: 'ToggleButton', slug: 'toggle-button', skipAssertions: PARTIAL_SKIP },
+  // TODO: Tooltip — VRT excludes demos (need interaction)
+  { component: 'Typography', slug: 'typography' },
+];

test/regressions/a11y/a11yReporter.ts

@@ -0,0 +1,128 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import chalk from 'chalk';
+import type { Reporter, TestCase, TestModule, TestSuite } from 'vitest/node';
+import type { A11yMeta } from './axe';
+
+const OUT_DIR = path.resolve(__dirname, 'results');
+
+interface DemoResult {
+  passedRules: string[];
+  failedRules: string[];
+  testedRules: Record<string, string[]>;
+}
+
+interface ComponentResult {
+  passed: number;
+  failed: number;
+  total: number;
+  passedRules: string[];
+  failedRules: Record<string, string[]>;
+  testedRules: Record<string, string[]>;
+  demos: Record<string, DemoResult>;
+}
+
+function* walkTests(node: TestModule | TestSuite): Generator<TestCase, undefined, void> {
+  for (const child of node.children) {
+    if (child.type === 'test') {
+      yield child;
+    } else if (child.type === 'suite') {
+      yield* walkTests(child);
+    }
+  }
+}
+
+function aggregate(entries: A11yMeta[]): ComponentResult {
+  const collected = new Set<string>();
+  const tested: Record<string, Set<string>> = {};
+  const failed = new Map<string, string[]>();
+  const demos: Record<string, DemoResult> = {};
+
+  for (const entry of entries) {
+    for (const rule of entry.collectedRules) {
+      collected.add(rule);
+    }
+    for (const [tag, ids] of Object.entries(entry.testedRules)) {
+      if (!tested[tag]) {
+        tested[tag] = new Set();
+      }
+      for (const id of ids) {
+        tested[tag].add(id);
+      }
+    }
+    for (const rule of entry.violations) {
+      const list = failed.get(rule) ?? [];
+      list.push(entry.demo);
+      failed.set(rule, list);
+    }
+
+    const violationSet = new Set(entry.violations);
+    demos[entry.demo] = {
+      passedRules: entry.collectedRules.filter((r) => !violationSet.has(r)).sort(),
+      failedRules: [...entry.violations].sort(),
+      testedRules: entry.testedRules,
+    };
+  }
+
+  const failedIds = new Set(failed.keys());
+  const passedRules = [...collected].filter((r) => !failedIds.has(r)).sort();
+
+  return {
+    passed: passedRules.length,
+    failed: failedIds.size,
+    total: collected.size,
+    passedRules,
+    failedRules: Object.fromEntries(failed),
+    testedRules: Object.fromEntries(Object.entries(tested).map(([tag, ids]) => [tag, [...ids]])),
+    demos,
+  };
+}
+
+export default class A11yReporter implements Reporter {
+  onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
+    const byComponent = new Map<string, A11yMeta[]>();
+    for (const mod of testModules) {
+      for (const test of walkTests(mod)) {
+        const meta = (test.meta() as { a11y?: A11yMeta }).a11y;
+        if (meta) {
+          const list = byComponent.get(meta.component) ?? [];
+          list.push(meta);
+          byComponent.set(meta.component, list);
+        }
+      }
+    }
+
+    if (byComponent.size === 0) {
+      return;
+    }
+
+    fs.mkdirSync(OUT_DIR, { recursive: true });
+
+    const names = [...byComponent.keys()].sort();
+    const results: Record<string, ComponentResult> = {};
+    for (const component of names) {
+      const result = aggregate(byComponent.get(component)!);
+      results[component] = result;
+      fs.writeFileSync(
+        path.join(OUT_DIR, `${component}.json`),
+        `${JSON.stringify(result, null, 2)}\n`,
+      );
+    }
+
+    const pass = names.filter((n) => results[n].failed === 0);
+    const partial = names.filter((n) => results[n].failed > 0);
+    // eslint-disable-next-line no-console
+    console.log(
+      [
+        '',
+        chalk.bold(
+          `a11y results (${names.length} components) -> ${path.relative(process.cwd(), OUT_DIR)}/`,
+        ),
+        '',
+        `  ✅ Pass (${pass.length}):     ${pass.join(', ') || '—'}`,
+        `  ⚠️  Partial (${partial.length}):  ${partial.join(', ') || '—'}`,
+        '',
+      ].join('\n'),
+    );
+  }
+}