Feature/vrt fixes

.github/workflows/pull-request.yml

@@ -7,42 +7,9 @@ on:
       - next
 
 jobs:
-  test:
-    name: Lint & Test & Build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-
-      # @see https://www.voorhoede.nl/en/blog/super-fast-npm-install-on-github-actions/
-      - uses: actions/cache@v2
-        id: cache
-        with:
-          path: ./node_modules
-          key: ${{ runner.os }}-modules-${{ hashFiles('package-lock.json') }}
-
-      - name: Install packages
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: npm ci
-
-      - name: Run postinstall
-        if: steps.cache.outputs.cache-hit == 'true'
-        run: npm run --if-present postinstall && npm rebuild && npm run prepare --if-present
-
-      - name: Lint
-        run: npm run lint
-
-      - name: Test
-        run: sudo npx playwright install-deps && npm run test:ci
-
-      - name: Build
-        run: npm run storybook:build
-
-#  vrt:
-#    name: Visual Regression Test
-#    runs-on: macos-12
+#  test:
+#    name: Lint & Test & Build
+#    runs-on: ubuntu-latest
 #    steps:
 #      - uses: actions/checkout@v3
 #      - uses: actions/setup-node@v3
@@ -64,13 +31,46 @@ jobs:
 #        if: steps.cache.outputs.cache-hit == 'true'
 #        run: npm run --if-present postinstall && npm rebuild && npm run prepare --if-present
 #
+#      - name: Lint
+#        run: npm run lint
+#
 #      - name: Test
-#        run: npm run test:vrt
+#        run: sudo npx playwright install-deps && npm run test:ci
 #
-#      - name: Archive visual diff results
-#        uses: actions/upload-artifact@v3
-#        if: failure()
-#        with:
-#          name: visual-diffs
-#          path: screenshots
-#          retention-days: 7
+#      - name: Build
+#        run: npm run storybook:build
+
+  vrt:
+    name: Visual Regression Test
+    runs-on: macos-12
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+
+      # @see https://www.voorhoede.nl/en/blog/super-fast-npm-install-on-github-actions/
+      - uses: actions/cache@v2
+        id: cache
+        with:
+          path: ./node_modules
+          key: ${{ runner.os }}-modules-${{ hashFiles('package-lock.json') }}
+
+      - name: Install packages
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: npm ci
+
+      - name: Run postinstall
+        if: steps.cache.outputs.cache-hit == 'true'
+        run: npm run --if-present postinstall && npm rebuild && npm run prepare --if-present
+
+      - name: Test
+        run: npm run test:vrt
+
+      - name: Archive visual diff results
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: visual-diffs
+          path: screenshots
+          retention-days: 7

.github/workflows/release.yml

@@ -47,48 +47,47 @@ jobs:
         env:
           CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
 
-#  vrt:
-#      name: Visual Regression Test
-#      runs-on: macos-12
-#      steps:
-#        - uses: actions/checkout@v3
-#        - uses: actions/setup-node@v3
-#          with:
-#            node-version: '16'
-#
-#        # @see https://www.voorhoede.nl/en/blog/super-fast-npm-install-on-github-actions/
-#        - uses: actions/cache@v2
-#          id: cache
-#          with:
-#            path: ./node_modules
-#            key: ${{ runner.os }}-modules-${{ hashFiles('package-lock.json') }}
-#
-#        - name: Install packages
-#          if: steps.cache.outputs.cache-hit != 'true'
-#          run: npm ci
-#
-#        - name: Run postinstall
-#          if: steps.cache.outputs.cache-hit == 'true'
-#          run: npm run --if-present postinstall && npm rebuild && npm run prepare --if-present
-#
-#        - name: Test
-#          run: npm run test:vrt
-#
-#        - name: Archive visual diff results
-#          uses: actions/upload-artifact@v3
-#          if: failure()
-#          with:
-#            name: visual-diffs
-#            path: screenshots
-#            retention-days: 7
+  vrt:
+      name: Visual Regression Test
+      runs-on: macos-12
+      steps:
+        - uses: actions/checkout@v3
+        - uses: actions/setup-node@v3
+          with:
+            node-version: '16'
+
+        # @see https://www.voorhoede.nl/en/blog/super-fast-npm-install-on-github-actions/
+        - uses: actions/cache@v2
+          id: cache
+          with:
+            path: ./node_modules
+            key: ${{ runner.os }}-modules-${{ hashFiles('package-lock.json') }}
+
+        - name: Install packages
+          if: steps.cache.outputs.cache-hit != 'true'
+          run: npm ci
+
+        - name: Run postinstall
+          if: steps.cache.outputs.cache-hit == 'true'
+          run: npm run --if-present postinstall && npm rebuild && npm run prepare --if-present
+
+        - name: Test
+          run: npm run test:vrt
+
+        - name: Archive visual diff results
+          uses: actions/upload-artifact@v3
+          if: failure()
+          with:
+            name: visual-diffs
+            path: screenshots
+            retention-days: 7
 
   release:
     name: Release & Deploy
     # Prevents changesets action from creating a PR on forks
     if: github.repository == 'inventage/web-components'
     runs-on: ubuntu-latest
-#    needs: [ test, vrt ]
-    needs: [ test ]
+    needs: [ test, vrt ]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3

package.json

@@ -79,7 +79,7 @@
     "@web/storybook-prebuilt": "^0.1.34",
     "@web/test-runner": "^0.14.0",
     "@web/test-runner-playwright": "^0.8.10",
-    "@web/test-runner-visual-regression": "^0.6.6",
+    "@web/test-runner-visual-regression": "^0.7.0",
     "commitlint-plugin-workspace-scopes": "^1.1.0",
     "cpy-cli": "^4.2.0",
     "cz-conventional-changelog": "^3.3.0",

web-test-runner.config-vrt.mjs

@@ -4,4 +4,8 @@ import config from './web-test-runner.config.mjs';
 config.testsFinishTimeout = 200000;
 config.testFramework.config.timeout = 100000;
 
+// Increase threshold to prevent regression errors because of 1-2 pixel differences
+config.failureThreshold = 2;
+config.failureThresholdType = 'pixel';
+
 export default config;

web-test-runner.config.mjs

@@ -101,7 +101,6 @@ export default {
       // @see https://github.com/mapbox/pixelmatch#api
       diffOptions: {
         threshold: 0.2,
-        includeAA: false,
       },
     }),
   ],