chore: Add visual regression testing

.github/workflows/deploy.yml

@@ -65,3 +65,13 @@ jobs:
     with:
       artifact-name: dev-pages-react${{ matrix.react }}
       deployment-path: pages/lib/static-default
+
+  visual:
+    name: Visual regression
+    needs: quick-build
+    if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+    uses: ./.github/workflows/visual-regression.yml
+    secrets: inherit
+    with:
+      pr-artifact-name: dev-pages-react18
+      caller-run-id: ${{ github.run_id }}

.github/workflows/visual-regression.yml

@@ -0,0 +1,184 @@
+name: Visual Regression Tests
+
+on:
+  workflow_call:
+    inputs:
+      pr-artifact-name:
+        description: 'Name of the artifact containing PR pages (built by quick-build job). If not provided, pages will be built locally.'
+        required: false
+        type: string
+      caller-run-id:
+        description: 'The run ID of the calling workflow, used to download artifacts it uploaded.'
+        required: false
+        type: string
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+jobs:
+  # Stage the PR pages within this run so matrix jobs can download them without
+  # needing cross-run artifact access. Runs in parallel with build-baseline.
+  stage-pr-pages:
+    name: Stage PR pages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Install dependencies
+        run: npm i
+
+      - name: Build PR pages locally
+        if: ${{ !inputs.pr-artifact-name }}
+        run: |
+          npx gulp quick-build
+          node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default
+        env:
+          NODE_ENV: production
+
+      - name: Download PR pages artifact from caller run
+        if: ${{ inputs.pr-artifact-name }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.pr-artifact-name }}
+          path: pages/lib/static-default
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.caller-run-id }}
+
+      - name: Upload PR pages artifact (for matrix jobs)
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-pr-pages
+          path: pages/lib/static-default
+          retention-days: 1
+
+  # Build the baseline (main branch) pages once and share them across all browser jobs.
+  # Runs in parallel with stage-pr-pages.
+  build-baseline:
+    name: Build baseline pages
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Install dependencies
+        run: npm i
+
+      # Use a git worktree so the baseline has its own directory and its own
+      # node_modules. This means a PR that changes package-lock.json will still
+      # produce a correct baseline: the baseline installs from main's lockfile
+      # and the PR build installs from the PR's lockfile, so both sides use the
+      # dependency versions that are correct for their respective source trees.
+      - name: Create baseline worktree from origin/main
+        run: git worktree add /tmp/baseline origin/main
+
+      - name: Install baseline dependencies
+        run: npm i
+        working-directory: /tmp/baseline
+
+      - name: Build baseline pages
+        run: npx gulp quick-build
+        working-directory: /tmp/baseline
+        env:
+          NODE_ENV: production
+
+      - name: Bundle baseline pages
+        run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline
+        working-directory: /tmp/baseline
+        env:
+          NODE_ENV: production
+
+      - name: Upload baseline artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-baseline-pages
+          path: pages/lib/static-visual-baseline
+          retention-days: 1
+
+  visual:
+    name: Visual regression (${{ matrix.browser }})
+    needs: [stage-pr-pages, build-baseline]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - browser: chrome
+            os: ubuntu-latest
+          - browser: safari
+            os: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Setup Chrome and ChromeDriver
+        if: matrix.browser == 'chrome'
+        uses: browser-actions/setup-chrome@v1
+        with:
+          chrome-version: stable
+
+      - name: Enable SafariDriver
+        if: matrix.browser == 'safari'
+        run: sudo safaridriver --enable
+
+      - name: Install dependencies
+        run: npm i
+
+      - name: Download PR pages artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: visual-pr-pages
+          path: pages/lib/static-default
+
+      - name: Download baseline artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: visual-baseline-pages
+          path: pages/lib/static-visual-baseline
+
+      # ── Run tests ─────────────────────────────────────────────────────────
+      - name: Start test server (port 8080)
+        run: npx serve --no-clipboard --listen 8080 pages/lib/static-default &
+
+      - name: Start baseline server (port 8081)
+        run: npx serve --no-clipboard --listen 8081 pages/lib/static-visual-baseline &
+
+      - name: Wait for servers to be ready
+        run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081
+
+      - name: Run visual regression tests
+        run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js
+        env:
+          TZ: UTC
+          BROWSER: ${{ matrix.browser }}
+
+      - name: Upload diff artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-regression-diffs-${{ matrix.browser }}
+          path: visual-regression-output/
+          retention-days: 14

build-tools/visual/global-setup.js

@@ -0,0 +1,18 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+const { spawn } = require('child_process');
+const waitOn = require('wait-on');
+
+module.exports = async () => {
+  if (process.env.BROWSER === 'safari') {
+    const driverProcess = spawn('safaridriver', ['--port', '4444']);
+    driverProcess.on('error', err => {
+      throw err;
+    });
+    await waitOn({ resources: ['http-get://localhost:4444/status'], timeout: 10000 });
+    global.__DRIVER_PROCESS__ = driverProcess;
+  } else {
+    const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher');
+    await startWebdriver();
+  }
+};

build-tools/visual/global-teardown.js

@@ -0,0 +1,12 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+module.exports = () => {
+  if (process.env.BROWSER === 'safari') {
+    if (global.__DRIVER_PROCESS__) {
+      global.__DRIVER_PROCESS__.kill();
+    }
+  } else {
+    const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher');
+    shutdownWebdriver();
+  }
+};

build-tools/visual/setup.js

@@ -0,0 +1,25 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+/* global jest */
+const { configure } = require('@cloudscape-design/browser-test-tools/use-browser');
+
+const isSafari = process.env.BROWSER === 'safari';
+
+// The PR build (the code under test) is served on port 8080.
+// The baseline build (main branch, same node_modules) is served on port 8081.
+configure({
+  browserName: isSafari ? 'Safari' : 'ChromeHeadlessIntegration',
+  browserCreatorOptions: {
+    seleniumUrl: isSafari ? 'http://localhost:4444' : 'http://localhost:9515',
+  },
+  webdriverOptions: {
+    baseUrl: 'http://localhost:8080',
+  },
+});
+
+// Retries help with flaky tests, but Safari's single-session constraint means
+// a retry can hit "already paired" if the previous attempt's session hasn't
+// fully released. Disable retries for Safari.
+if (!isSafari) {
+  jest.retryTimes(2, { logErrorsBeforeRetry: true });
+}

docs/RUNNING_TESTS.md

@@ -60,11 +60,60 @@ TZ=UTC npx jest -u -c jest.unit.config.js src/
 ```
 ## Visual Regression Tests
 
-> **Note:** The components repository does not have visual regression tests on GitHub. This section applies to other repositories such as chat-components, code-view, chart-components, and board-components.
+Visual regression tests run automatically when opening a pull request in GitHub (see `.github/workflows/visual-regression.yml`).
 
-Visual regression tests for permutation pages run automatically when opening a pull request in GitHub.
+They compare permutation pages between the PR build and a baseline build of `main`, both served locally in the same CI job. Each side installs from its own `package-lock.json` via a git worktree, so dependency changes in the PR are handled correctly and unpinned updates in sister repositories affect both sides equally.
 
-To check results: look at the "Visual Regression Tests" action in the PR. The "Test for regressions" step logs which pages failed. For a full report, download the `visual-regression-snapshots-results` artifact from the action summary.
+### How it works
 
-If there are unexpected regressions, fix your pull request.
-If the changes are expected, call this out in your pull request comments.
+1. The PR pages are built and served on port 8080.
+2. A git worktree of `origin/main` is created, its dependencies installed, and its pages built and served on port 8081.
+3. The single test runner (`test/visual/visual.test.ts`) iterates over all test definitions, captures the `.screenshot-area` element from both servers for each test, and fails if any pixels differ.
+
+### Running locally
+
+```
+npm run test:visual
+```
+
+This handles the full build and comparison in one command. If both outputs are already built, skip the build step:
+
+```
+NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js
+```
+
+(Requires both servers to be running — start the PR build with `npm run start:integ` on port 8080 and the baseline build on port 8081, or set `NEW_HOST` / `OLD_HOST` env vars to point at different hosts.)
+
+### Adding tests for a new component
+
+Create `test/definitions/visual/<component>.ts`:
+
+```ts
+import { TestSuite } from '../types';
+
+const suite: TestSuite = {
+  description: 'my-component',
+  tests: [
+    {
+      description: 'permutations',
+      path: 'my-component/permutations',
+    },
+  ],
+};
+
+export default suite;
+```
+
+Then import and add it to `test/definitions/visual/index.ts`:
+
+```ts
+import myComponent from './my-component';
+
+export const allSuites: TestSuite[] = [..., myComponent];
+```
+
+### Reviewing failures
+
+If the CI job fails, download the `visual-regression-diffs` artifact from the Actions summary.
+
+If the diff is expected (intentional visual change), note it in your PR description.

eslint.config.mjs

@@ -225,7 +225,7 @@ export default tsEslint.config(
     },
   },
   {
-    files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**'],
+    files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**', 'test/visual/**'],
     rules: {
       // useBrowser is not a hook
       'react-hooks/rules-of-hooks': 'off',

jest.visual.config.js

@@ -0,0 +1,27 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+const path = require('path');
+const os = require('os');
+
+module.exports = {
+  verbose: true,
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        tsconfig: 'tsconfig.integ.json',
+      },
+    ],
+  },
+  reporters: ['default', 'github-actions'],
+  testTimeout: 120_000, // 2min — pages can be tall and slow to capture
+  // Safari's WebDriver only supports one concurrent session, so tests must run serially.
+  // Chrome can run multiple workers to speed things up.
+  maxWorkers: process.env.BROWSER === 'safari' ? 1 : os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1),
+  globalSetup: '<rootDir>/build-tools/visual/global-setup.js',
+  globalTeardown: '<rootDir>/build-tools/visual/global-teardown.js',
+  setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')],
+  moduleFileExtensions: ['js', 'ts'],
+  testMatch: ['<rootDir>/test/visual/visual.test.ts'],
+};