feat(vrt): add Canvas2D visual regression tests

.github/workflows/tests.yml

@@ -63,4 +63,5 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: failed-results
-          path: visual-regression/failed-results/
+          path: |
+            visual-regression/failed-results/

examples/common/ExampleSettings.ts

@@ -41,6 +41,13 @@ export interface SnapshotOptions {
     w: number;
     h: number;
   };
+  /**
+   * Maximum number of pixels allowed to differ before the snapshot is
+   * considered a failure. Defaults to 10 to absorb sub-pixel rounding
+   * differences across platforms (e.g. macOS vs Linux headless Chromium).
+   * Set to 0 to require a pixel-perfect match.
+   */
+  maxDiffPixels?: number;
 }
 
 export interface ExampleSettings {

examples/index.ts

@@ -472,9 +472,17 @@ async function runAutomation(
           },
           memMonitor: null,
         };
-        await automation(exampleSettings);
-        testRoot.parent = null;
-        testRoot.destroy();
+        try {
+          await automation(exampleSettings);
+        } catch (err) {
+          console.error(
+            `Automation error in test '${testName}':`,
+            (err as Error).message,
+          );
+        } finally {
+          testRoot.parent = null;
+          testRoot.destroy();
+        }
       }
     }
   }

visual-regression/src/index.ts

@@ -112,6 +112,14 @@ const argv = yargs(hideBin(process.argv))
       default: '*',
       description: 'Tests to run ("*" wildcard pattern)',
     },
+    renderMode: {
+      type: 'string',
+      alias: 'r',
+      default: runtimeEnv === 'ci' ? 'all' : 'webgl',
+      choices: ['webgl', 'canvas', 'all'],
+      description:
+        'Renderer mode to test ("webgl", "canvas", or "all" for both)',
+    },
   })
   .parseSync();
 
@@ -148,6 +156,7 @@ async function dockerCiMode(): Promise<number> {
     argv.skipBuild ? '--skipBuild' : '',
     argv.port ? `--port ${argv.port}` : '',
     argv.filter ? `--filter "${argv.filter}"` : '',
+    argv.renderMode ? `--renderMode ${argv.renderMode}` : '',
   ].join(' ');
 
   // Get the directory of the current file
@@ -220,7 +229,17 @@ async function compareCaptureMode(): Promise<number> {
     }
 
     // Run the tests
-    exitCode = await runTest('chromium');
+    const renderModes: ('webgl' | 'canvas')[] =
+      argv.renderMode === 'all'
+        ? ['webgl', 'canvas']
+        : [argv.renderMode as 'webgl' | 'canvas'];
+    exitCode = 0;
+    for (const mode of renderModes) {
+      const result = await runTest('chromium', mode);
+      if (result !== 0) {
+        exitCode = result;
+      }
+    }
   } finally {
     // Kill the serve-examples process
     serveExamplesChildProc.kill();
@@ -232,9 +251,13 @@ async function compareCaptureMode(): Promise<number> {
  * Run the tests in capture or compare mode depending on the `argv.capture` flag
  * for a specific browser type.
  */
-async function runTest(browserType: 'chromium') {
+async function runTest(
+  browserType: 'chromium',
+  renderMode: 'webgl' | 'canvas',
+) {
   const paramString = Object.entries({
     browser: browserType,
+    renderMode,
     overwrite: argv.overwrite,
     filter: argv.filter,
     RUNTIME_ENV: runtimeEnv,
@@ -249,7 +272,9 @@ async function runTest(browserType: 'chromium') {
     ),
   );
 
-  const snapshotSubDirName = `${browserType}-${runtimeEnv}`;
+  const snapshotSubDirName = `${browserType}-${runtimeEnv}${
+    renderMode === 'canvas' ? '-canvas' : ''
+  }`;
 
   const snapshotSubDir = path.join(certifiedSnapshotDir, snapshotSubDirName);
 
@@ -449,7 +474,7 @@ async function runTest(browserType: 'chromium') {
 
   // Go to the examples page
   await page.goto(
-    `http://localhost:${argv.port}/?automation=true&test=${argv.filter}`,
+    `http://localhost:${argv.port}/?automation=true&test=${argv.filter}&renderMode=${renderMode}`,
   );
 
   return donePromise;

visual-regression/src/snapshot.ts

@@ -19,6 +19,13 @@ export interface SnapshotOptions {
     width: number;
     height: number;
   };
+  /**
+   * Maximum number of pixels allowed to differ before the snapshot is
+   * considered a failure. Defaults to 10 to absorb sub-pixel rounding
+   * differences across platforms (e.g. macOS vs Linux headless Chromium).
+   * Set to 0 to require a pixel-perfect match.
+   */
+  maxDiffPixels?: number;
 }
 
 interface CompareResult {
@@ -95,7 +102,13 @@ export async function compareSnapshot(
   const expectedPng = await fs.promises.readFile(snapshotPath);
   const width = options.clip?.width || (1080 as number);
   const height = options.clip?.height || (1920 as number);
-  const result = compareBuffers(actualPng, expectedPng, width, height);
+  const result = compareBuffers(
+    actualPng,
+    expectedPng,
+    width,
+    height,
+    options.maxDiffPixels,
+  );
 
   if (result.doesMatch) {
     console.log(chalk.green.bold('PASS!'));
@@ -170,6 +183,7 @@ export function compareBuffers(
   expectedImageBuffer: Buffer,
   width: number,
   height: number,
+  maxDiffPixels = 10,
 ): CompareResult {
   const diff = new PNG({ width: width as number, height: height as number });
   const actualImage = PNG.sync.read(actualImageBuffer);
@@ -196,12 +210,14 @@ export function compareBuffers(
     { threshold: 0.8 }, // Adjust threshold for sensitivity
   );
 
-  const doesMatch = count === 0;
+  const doesMatch = count <= maxDiffPixels;
 
   return {
     doesMatch,
 
     diffImageBuffer: doesMatch ? undefined : diff,
-    reason: doesMatch ? undefined : `${count} pixels differ`,
+    reason: doesMatch
+      ? undefined
+      : `${count} pixels differ (max allowed: ${maxDiffPixels})`,
   };
 }