From 1202f439d56ae13fe98cc45d57530d4ddabe771f Mon Sep 17 00:00:00 2001 From: Dan Ichim Date: Wed, 3 Jun 2026 16:27:51 +0300 Subject: [PATCH 01/28] add poc playwright --- .gitignore | 8 ++ package-lock.json | 68 +++++++++- package.json | 14 ++- tests/screenshot/playwright/README.md | 114 +++++++++++++++++ tests/screenshot/playwright/benchmark.mjs | 96 +++++++++++++++ tests/screenshot/playwright/global-setup.js | 85 +++++++++++++ tests/screenshot/playwright/package.json | 3 + tests/screenshot/playwright/paths.js | 25 ++++ .../playwright/playwright.config.mjs | 62 ++++++++++ .../playwright/specs/light/Light-spec.js | 8 ++ .../playwright/specs/light/Light2-spec.js | 8 ++ .../playwright/specs/light/Light3-spec.js | 8 ++ .../playwright/specs/light/Light4-spec.js | 8 ++ .../playwright/specs/light/Light5-spec.js | 8 ++ .../playwright/specs/light/Light6-spec.js | 8 ++ .../playwright/specs/light/Light7-spec.js | 8 ++ .../playwright/specs/light/Light8-spec.js | 8 ++ .../playwright/specs/light/Light9-spec.js | 8 ++ .../playwright/specs/neutral/Default-spec.js | 8 ++ .../playwright/specs/neutral/Default2-spec.js | 8 ++ .../playwright/specs/neutral/Default3-spec.js | 8 ++ .../playwright/specs/neutral/Default4-spec.js | 8 ++ .../playwright/specs/neutral/Default5-spec.js | 8 ++ .../playwright/specs/neutral/Default6-spec.js | 8 ++ .../playwright/specs/neutral/Default7-spec.js | 8 ++ .../playwright/specs/neutral/Default8-spec.js | 8 ++ .../playwright/specs/neutral/Default9-spec.js | 8 ++ .../specs/neutral/HighContrast-spec.js | 8 ++ .../specs/neutral/HighContrast2-spec.js | 8 ++ .../specs/neutral/HighContrast3-spec.js | 8 ++ .../specs/neutral/HighContrast4-spec.js | 8 ++ .../specs/neutral/HighContrast5-spec.js | 8 ++ .../specs/neutral/HighContrast6-spec.js | 8 ++ .../specs/neutral/HighContrast7-spec.js | 8 ++ .../specs/neutral/HighContrast8-spec.js | 8 ++ .../specs/neutral/HighContrast9-spec.js | 8 ++ .../playwright/utils/limestone-page.js | 28 +++++ .../playwright/utils/run-screenshot-tests.js | 105 ++++++++++++++++ .../playwright/utils/screenshot-name.js | 19 +++ .../screenshot/scripts/run-component-wdio.mjs | 116 ++++++++++++++++++ .../scripts/run-playwright-component.mjs | 78 ++++++++++++ 41 files changed, 1032 insertions(+), 5 deletions(-) create mode 100644 tests/screenshot/playwright/README.md create mode 100644 tests/screenshot/playwright/benchmark.mjs create mode 100644 tests/screenshot/playwright/global-setup.js create mode 100644 tests/screenshot/playwright/package.json create mode 100644 tests/screenshot/playwright/paths.js create mode 100644 tests/screenshot/playwright/playwright.config.mjs create mode 100644 tests/screenshot/playwright/specs/light/Light-spec.js create mode 100644 tests/screenshot/playwright/specs/light/Light2-spec.js create mode 100644 tests/screenshot/playwright/specs/light/Light3-spec.js create mode 100644 tests/screenshot/playwright/specs/light/Light4-spec.js create mode 100644 tests/screenshot/playwright/specs/light/Light5-spec.js create mode 100644 tests/screenshot/playwright/specs/light/Light6-spec.js create mode 100644 tests/screenshot/playwright/specs/light/Light7-spec.js create mode 100644 tests/screenshot/playwright/specs/light/Light8-spec.js create mode 100644 tests/screenshot/playwright/specs/light/Light9-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default2-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default3-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default4-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default5-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default6-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default7-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default8-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/Default9-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast2-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast3-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast4-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast5-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast6-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast7-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast8-spec.js create mode 100644 tests/screenshot/playwright/specs/neutral/HighContrast9-spec.js create mode 100644 tests/screenshot/playwright/utils/limestone-page.js create mode 100644 tests/screenshot/playwright/utils/run-screenshot-tests.js create mode 100644 tests/screenshot/playwright/utils/screenshot-name.js create mode 100644 tests/screenshot/scripts/run-component-wdio.mjs create mode 100644 tests/screenshot/scripts/run-playwright-component.mjs diff --git a/.gitignore b/.gitignore index b92af6988..87ec9668a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,5 +20,13 @@ errorShots coverage .eslintcache +# Screenshot tests (tests/screenshot/) — generated artifacts only +# Playwright baselines: snapshots///*.png (same paths as WDIO reference/) +tests/screenshot/playwright/snapshots +tests/screenshot/playwright/test-results +tests/screenshot/playwright/reports +tests/screenshot/playwright/blob-report +tests/screenshot/playwright/.test-data.json + # Mac .DS_Store diff --git a/package-lock.json b/package-lock.json index 308eb4182..2f3189bc1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@enovaui/core-tokens": "^2.0.0", "@enovaui/webos-tokens": "^2.0.0", "classnames": "^2.5.1", - "ilib": "^14.22.0", + "ilib": "^14.21.0 || ^14.21.0-webos1", "invariant": "^2.2.4", "prop-types": "^15.8.1", "ramda": "^0.32.0", @@ -27,7 +27,8 @@ }, "devDependencies": { "@enact/docs-utils": "^0.4.17", - "@enact/ui-test-utils": "^4.0.2" + "@enact/ui-test-utils": "^4.0.2", + "@playwright/test": "^1.57.0" }, "peerDependencies": { "ilib": "^14.21.0 || ^14.21.0-webos1" @@ -16192,6 +16193,22 @@ "node": ">= 8" } }, + "node_modules/@playwright/test": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.0.tgz", + "integrity": "sha512-cKA5B6lpFEMyMGjxF54QihfYpB4FkEGH+qZhtArDEG+wezQAJY8Pq6C7T1SjWz+FFzt3TbyoXBQYk/0292TdJA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -19599,6 +19616,53 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/playwright": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", + "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", + "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.15", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", diff --git a/package.json b/package.json index 43deda659..16c6139a1 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,13 @@ "test-ss": "start-tests tests/screenshot/config/wdio.conf.js", "test-ss-docker": "start-tests tests/screenshot/config/wdio.docker.conf.js", "test-ss-tv": "start-tests tests/screenshot/config/wdio.tv.conf.js", + "test-playwright": "playwright test --config tests/screenshot/playwright/playwright.config.mjs", + "test-playwright:update": "playwright test --config tests/screenshot/playwright/playwright.config.mjs --update-snapshots", + "test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", + "test-playwright:component": "node tests/screenshot/scripts/run-playwright-component.mjs", + "test-ss:component": "node tests/screenshot/scripts/run-component-wdio.mjs", + "prepare-playwright": "playwright install chrome", + "benchmark-screenshots": "node tests/screenshot/playwright/benchmark.mjs", "test-json": "enact test --json", "test-watch": "enact test --watch", "transpile": "enact transpile", @@ -47,7 +54,7 @@ "@enovaui/core-tokens": "^2.0.0", "@enovaui/webos-tokens": "^2.0.0", "classnames": "^2.5.1", - "ilib": "^14.22.0", + "ilib": "^14.21.0 || ^14.21.0-webos1", "invariant": "^2.2.4", "prop-types": "^15.8.1", "ramda": "^0.32.0", @@ -64,6 +71,7 @@ }, "devDependencies": { "@enact/docs-utils": "^0.4.17", - "@enact/ui-test-utils": "^4.0.2" + "@enact/ui-test-utils": "^4.0.2", + "@playwright/test": "^1.57.0" } -} \ No newline at end of file +} diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md new file mode 100644 index 000000000..1bd724ba4 --- /dev/null +++ b/tests/screenshot/playwright/README.md @@ -0,0 +1,114 @@ +# Playwright screenshot tests + +Visual regression with [Playwright](https://playwright.dev/) `toHaveScreenshot`, alongside WDIO (`npm run test-ss`). Scenarios live in `../apps/components/`; at test time Playwright only serves `../dist/` and picks cases via URL (`component` + `testId`), not by reading `.js` files on disk. + +## Layout + +``` +tests/screenshot/ + apps/components/ ← scenario source (shared with WDIO) + images/, videos/ + dist/ ← buildApps('screenshot') (shared) + specs/ ← WDIO *-specs.js + playwright/ ← this folder + specs/neutral|light/ *-spec.js shards + utils/ registerScreenshotTests, limestone-page, screenshot-name + snapshots/ PNG baselines: //.png (same layout as WDIO reference/) + benchmark.mjs + scripts/ run-playwright-component, run-component-wdio (npm shortcuts) + global-setup.js build, .test-data.json, serve :4568 (default) + playwright.config.mjs + paths.js +``` + +| | WDIO | Playwright | +|--|------|------------| +| Scenarios / build | `../apps/`, `../dist/` | Same | +| Static server port | **4567** | **4568** (default) | +| Specs | `../specs/*-specs.js` | `specs/*-spec.js` | +| Assertions / baselines | `dist/screenshots/reference/` | `playwright/snapshots/` (same `//` paths) | + +## How a scenario runs (e.g. Chip, testId 3) + +``` +Chip.js → LimestoneComponents / testMetadata → buildApps → ../dist/ + → global-setup ?request → .test-data.json + → registerScreenshotTests → GET …?component=Chip&testId=3&skin=… + → prepareTest in bundle → toHaveScreenshot +``` + +## Setup + +```bash +npm install +npm run prepare-playwright +``` + +Uses `@enact/ui-test-utils@^4.0.2` from npm (same as `test-ss`). + +## Commands + +Run from the **repository root**: + +| Command | Description | +|---------|-------------| +| `npm run test-playwright` | Full suite (27 shards) | +| `npm run test-playwright:update` | Update all baselines | +| `npm run test-playwright:report` | HTML report | +| `npm run test-playwright:component -- Sprite` | One component, all cases (e.g. Sprite ≈ 4, Spinner ≈ 6, Button ≈ 205) | +| `npm run test-playwright:component -- Sprite --test-id 0` | Single case by index | +| `npm run test-playwright:component -- Sprite --update` | Refresh baselines | +| `npm run test-ss:component -- Sprite` | WDIO, same component (Default shard) | +| `npm run benchmark-screenshots -- Chip` | Playwright vs WDIO timing (`--build` to rebuild dist) | + +Without baselines, add `--update` once. + +## Benchmark (local) + +Compare end-to-end runtime for one component (Default shard, `instances` / `workers` = 1, existing `tests/screenshot/dist`): + +```bash +npm run benchmark-screenshots -- Chip +# optional: PLAYWRIGHT_SKIP_BUILD=1 if dist is already built +``` + +Both runners filter a single component name (WDIO uses `--component ^Name$` so `Chip` does not include `Chips`). + +| Component | Cases | Playwright | WebdriverIO | Faster | +|-----------|------:|-----------:|--------------:|--------| +| Sprite | 4 | 16.7 s | 18.0 s | Playwright (~8%) | +| Chip | 51 | 99.2 s | 84.6 s | WDIO (~15%) | + +Measured on Windows (Chrome 132), June 2026; times include browser startup and screenshot compare. Chip ≈ 1.9 s/test (Playwright) vs ≈ 1.7 s/test (WDIO). Full suite (`Button` ≈ 205 cases) is much slower — use component scripts for day-to-day runs. + +**View results:** Playwright — `npm run test-playwright:report` and `snapshots//Limestone/`. WDIO — `tests/screenshot/dist/newFiles.html` / `failedTests.html` and `dist/screenshots/reference/`. + +**Parallel with WDIO:** default ports differ (WDIO **4567**, Playwright **4568**), both serve `../dist/`. Override with `PLAYWRIGHT_BASE_URL` if needed. + +## Maintenance + +- **New/changed UI case:** edit `../apps/components/.js`; register in `../apps/LimestoneComponents.js` if new; run tests (build unless `PLAYWRIGHT_SKIP_BUILD=1`); `--update` for PNGs. +- **New WDIO shard:** add matching `specs/**/-spec.js` with same `skin`, `highContrast`, `concurrency` as `../specs/`. Shards split by `testId % workers` (`Default` … `Default9`, `HighContrast*`, `Light*`). + +## Environment variables + +| Variable | Effect | +|----------|--------| +| `PLAYWRIGHT_SKIP_BUILD=1` | Skip `buildApps('screenshot')` | +| `PLAYWRIGHT_REFRESH_TEST_DATA=1` | Regenerate `.test-data.json` | +| `PLAYWRIGHT_COMPONENT` | One component (set by `../scripts/run-playwright-component.mjs`) | +| `PLAYWRIGHT_TEST_ID` / `PLAYWRIGHT_TITLE` | Filter cases (or `--test-id` / `--title` on npm script) | +| `PLAYWRIGHT_INSTANCES` / `PLAYWRIGHT_WORKERS` | Shard index / parallelism | +| `PLAYWRIGHT_BASE_URL` | Playwright static server (default `http://localhost:4568`) | + +## Troubleshooting + +| Issue | Fix | +|-------|-----| +| WebServer timeout | Build `../dist/` (unset `PLAYWRIGHT_SKIP_BUILD` or `npm run test-ss`) | +| Snapshot does not exist | `--update` | +| No tests found | Check `PLAYWRIGHT_COMPONENT`; refresh `.test-data.json` | +| Missing images/videos on build | Use `../images/`, `../videos/` | +| Port in use | Change `PLAYWRIGHT_BASE_URL` (port in URL must match `webServer`) | + +CI: `build-scripts/enact-playwright-tests.sh`. diff --git a/tests/screenshot/playwright/benchmark.mjs b/tests/screenshot/playwright/benchmark.mjs new file mode 100644 index 000000000..59147a640 --- /dev/null +++ b/tests/screenshot/playwright/benchmark.mjs @@ -0,0 +1,96 @@ +#!/usr/bin/env node +/** + * Compare Playwright vs WebdriverIO runtime for one component (all its screenshot cases). + * + * Usage: + * npm run benchmark-screenshots -- Sprite + * npm run benchmark-screenshots -- Sprite --build + */ +import {spawn} from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +import {assertComponentSource, SCREENSHOT_DIST} from './paths.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); + +const args = process.argv.slice(2).filter(a => a !== '--'); +const component = args.find(a => !a.startsWith('--')); +const withBuild = args.includes('--build'); + +if (!component) { + console.error('Usage: npm run benchmark-screenshots -- [--build]'); + console.error('Example: npm run benchmark-screenshots -- Sprite'); + process.exit(1); +} + +assertComponentSource(component); + +const distIndex = path.join(SCREENSHOT_DIST, 'Limestone-View', 'index.html'); +const skipBuild = !withBuild && fs.existsSync(distIndex); + +function runCommand (label, command, args, env = {}) { + return new Promise((resolve, reject) => { + const start = Date.now(); + const child = spawn(command, args, { + cwd: repoRoot, + env: {...process.env, ...env}, + shell: true, + stdio: 'inherit' + }); + + child.on('close', code => { + const durationMs = Date.now() - start; + if (code === 0) { + resolve({label, durationMs}); + } else { + reject(new Error(`${label} failed with exit code ${code} after ${(durationMs / 1000).toFixed(1)}s`)); + } + }); + }); +} + +async function main () { + console.log(`\nBenchmark: ${component} (all cases, 1 worker / instances 1)`); + if (skipBuild) { + console.log('Using existing tests/screenshot/dist (pass --build to rebuild)\n'); + } else { + console.log('Will build tests/screenshot/dist before run\n'); + } + + const playwright = await runCommand( + 'Playwright', + 'node', + ['tests/screenshot/scripts/run-playwright-component.mjs', component], + skipBuild ? {PLAYWRIGHT_SKIP_BUILD: '1'} : {} + ); + + const wdioArgv = ['tests/screenshot/scripts/run-component-wdio.mjs', component]; + if (skipBuild) { + wdioArgv.push('--skip-build'); + } + + const wdio = await runCommand( + 'WebdriverIO', + 'node', + wdioArgv, + {} + ); + + const pwSec = playwright.durationMs / 1000; + const wdioSec = wdio.durationMs / 1000; + const delta = wdioSec - pwSec; + + console.log('\n--- Results ---'); + console.log(`Component: ${component}`); + console.log(`Playwright: ${pwSec.toFixed(1)}s`); + console.log(`WebdriverIO: ${wdioSec.toFixed(1)}s`); + console.log(`Difference: ${Math.abs(delta).toFixed(1)}s (${delta > 0 ? 'Playwright faster' : delta < 0 ? 'WDIO faster' : 'tie'})\n`); +} + +main().catch(err => { + console.error(err.message); + process.exit(1); +}); diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js new file mode 100644 index 000000000..2b3bc2b5f --- /dev/null +++ b/tests/screenshot/playwright/global-setup.js @@ -0,0 +1,85 @@ +import fs from 'fs'; +import path from 'path'; +import {spawn} from 'child_process'; +import {chromium} from '@playwright/test'; + +import buildApps from '../../../node_modules/@enact/ui-test-utils/src/build-apps.js'; + +import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST, TEST_DATA_FILE} from './paths.js'; + +const baseURL = PLAYWRIGHT_BASE_URL; +const distPath = SCREENSHOT_DIST; +const testDataFile = TEST_DATA_FILE; +const port = PLAYWRIGHT_PORT; + +async function waitForServer (url, timeout = 120000) { + const start = Date.now(); + + while (Date.now() - start < timeout) { + try { + const response = await fetch(url); + if (response.ok || response.status === 404) { + return; + } + } catch { + // retry until the static server is ready + } + await new Promise(resolve => setTimeout(resolve, 500)); + } + + throw new Error(`Static server at ${url} did not start within ${timeout}ms`); +} + +function startStaticServer () { + const command = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + spawn(command, ['--yes', 'serve', distPath, '-l', port], { + stdio: 'pipe', + shell: process.platform === 'win32' + }); +} + +function shouldRefreshTestData () { + if (process.env.PLAYWRIGHT_REFRESH_TEST_DATA === '1') { + return true; + } + if (!fs.existsSync(testDataFile)) { + return true; + } + const distIndex = path.join(distPath, 'Limestone-View', 'index.html'); + if (!fs.existsSync(distIndex)) { + return true; + } + return fs.statSync(distIndex).mtimeMs > fs.statSync(testDataFile).mtimeMs; +} + +export default async function globalSetup () { + if (!process.env.PLAYWRIGHT_SKIP_BUILD) { + await buildApps('screenshot'); + } + + try { + await fetch(baseURL); + } catch { + startStaticServer(); + await waitForServer(baseURL); + } + + if (!shouldRefreshTestData()) { + return; + } + + const browser = await chromium.launch({ + channel: 'chrome', + headless: true + }); + + try { + const page = await browser.newPage(); + await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'networkidle'}); + const testData = await page.evaluate(() => window.__TEST_DATA); + + fs.writeFileSync(testDataFile, JSON.stringify(testData)); + } finally { + await browser.close(); + } +} diff --git a/tests/screenshot/playwright/package.json b/tests/screenshot/playwright/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/tests/screenshot/playwright/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/tests/screenshot/playwright/paths.js b/tests/screenshot/playwright/paths.js new file mode 100644 index 000000000..b22fd618f --- /dev/null +++ b/tests/screenshot/playwright/paths.js @@ -0,0 +1,25 @@ +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const screenshotRoot = path.join(__dirname, '..'); + +/** Default 4568 so WDIO can keep 4567 when both run in parallel. */ +export const PLAYWRIGHT_BASE_URL = process.env.PLAYWRIGHT_BASE_URL || 'http://localhost:4568'; +export const PLAYWRIGHT_PORT = new URL(PLAYWRIGHT_BASE_URL).port || '4568'; + +export const SCREENSHOT_DIST = path.join(screenshotRoot, 'dist'); +export const SCREENSHOT_COMPONENTS = path.join(screenshotRoot, 'apps', 'components'); +export const TEST_DATA_FILE = path.join(__dirname, '.test-data.json'); + +function componentSourcePath (componentName) { + return path.join(SCREENSHOT_COMPONENTS, `${componentName}.js`); +} + +export function assertComponentSource (componentName) { + const source = componentSourcePath(componentName); + if (!fs.existsSync(source)) { + throw new Error(`Missing ${source} — add the component under tests/screenshot/apps/components/`); + } +} diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs new file mode 100644 index 000000000..a4e2b2454 --- /dev/null +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -0,0 +1,62 @@ +import path from 'path'; +import {fileURLToPath} from 'url'; +import {defineConfig, devices} from '@playwright/test'; + +import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT} from './paths.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const distDir = path.join(__dirname, '..', 'dist'); + +export default defineConfig({ + testDir: path.join(__dirname, 'specs'), + testMatch: '**/*-spec.js', + testIgnore: ['**/utils/**', '**/scripts/**'], + globalSetup: path.join(__dirname, 'global-setup.js'), + snapshotPathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}'), + expect: { + toHaveScreenshot: { + pathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}') + } + }, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.PLAYWRIGHT_WORKERS ? Number.parseInt(process.env.PLAYWRIGHT_WORKERS, 10) : 5, + reporter: [ + ['list'], + ['html', {outputFolder: path.join(__dirname, 'reports', 'html'), open: 'never'}] + ], + outputDir: path.join(__dirname, 'test-results'), + use: { + baseURL: PLAYWRIGHT_BASE_URL, + trace: 'on-first-retry', + viewport: {width: 1920, height: 1167}, + launchOptions: { + args: [ + '--disable-infobars', + '--disable-search-engine-choice-screen', + '--disable-notifications', + '--disable-popup-blocking', + '--disable-lcd-text', + '--force-device-scale-factor=1', + '--disable-gpu', + '--window-size=1920,1080' + ] + } + }, + webServer: { + command: `npx --yes serve ${distDir} -l ${PLAYWRIGHT_PORT}`, + url: PLAYWRIGHT_BASE_URL, + reuseExistingServer: true, + timeout: 120000 + }, + projects: [ + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + channel: 'chrome' + } + } + ] +}); diff --git a/tests/screenshot/playwright/specs/light/Light-spec.js b/tests/screenshot/playwright/specs/light/Light-spec.js new file mode 100644 index 000000000..e4f4e9758 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 1 +}); diff --git a/tests/screenshot/playwright/specs/light/Light2-spec.js b/tests/screenshot/playwright/specs/light/Light2-spec.js new file mode 100644 index 000000000..b43afb1cc --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light2-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 2 +}); diff --git a/tests/screenshot/playwright/specs/light/Light3-spec.js b/tests/screenshot/playwright/specs/light/Light3-spec.js new file mode 100644 index 000000000..9a752cd96 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light3-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 3 +}); diff --git a/tests/screenshot/playwright/specs/light/Light4-spec.js b/tests/screenshot/playwright/specs/light/Light4-spec.js new file mode 100644 index 000000000..7173e2054 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light4-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 4 +}); diff --git a/tests/screenshot/playwright/specs/light/Light5-spec.js b/tests/screenshot/playwright/specs/light/Light5-spec.js new file mode 100644 index 000000000..0e47df980 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light5-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 5 +}); diff --git a/tests/screenshot/playwright/specs/light/Light6-spec.js b/tests/screenshot/playwright/specs/light/Light6-spec.js new file mode 100644 index 000000000..6c31e0d00 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light6-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 6 +}); diff --git a/tests/screenshot/playwright/specs/light/Light7-spec.js b/tests/screenshot/playwright/specs/light/Light7-spec.js new file mode 100644 index 000000000..87bbe08b8 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light7-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 7 +}); diff --git a/tests/screenshot/playwright/specs/light/Light8-spec.js b/tests/screenshot/playwright/specs/light/Light8-spec.js new file mode 100644 index 000000000..28fbefab0 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light8-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 8 +}); diff --git a/tests/screenshot/playwright/specs/light/Light9-spec.js b/tests/screenshot/playwright/specs/light/Light9-spec.js new file mode 100644 index 000000000..215d84363 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light9-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 9 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default-spec.js b/tests/screenshot/playwright/specs/neutral/Default-spec.js new file mode 100644 index 000000000..4d9c64902 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 1 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default2-spec.js b/tests/screenshot/playwright/specs/neutral/Default2-spec.js new file mode 100644 index 000000000..89e00d887 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default2-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 2 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default3-spec.js b/tests/screenshot/playwright/specs/neutral/Default3-spec.js new file mode 100644 index 000000000..82162dad7 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default3-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 3 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default4-spec.js b/tests/screenshot/playwright/specs/neutral/Default4-spec.js new file mode 100644 index 000000000..733ecdaff --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default4-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 4 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default5-spec.js b/tests/screenshot/playwright/specs/neutral/Default5-spec.js new file mode 100644 index 000000000..3ea5b09f6 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default5-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 5 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default6-spec.js b/tests/screenshot/playwright/specs/neutral/Default6-spec.js new file mode 100644 index 000000000..081f8ea3d --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default6-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 6 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default7-spec.js b/tests/screenshot/playwright/specs/neutral/Default7-spec.js new file mode 100644 index 000000000..134b7c7c0 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default7-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 7 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default8-spec.js b/tests/screenshot/playwright/specs/neutral/Default8-spec.js new file mode 100644 index 000000000..3de7684c6 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default8-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 8 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default9-spec.js b/tests/screenshot/playwright/specs/neutral/Default9-spec.js new file mode 100644 index 000000000..a1485f337 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default9-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 9 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast-spec.js new file mode 100644 index 000000000..f607c25a4 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 1 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast2-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast2-spec.js new file mode 100644 index 000000000..724696a6b --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast2-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 2 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast3-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast3-spec.js new file mode 100644 index 000000000..c42c328d7 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast3-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 3 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast4-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast4-spec.js new file mode 100644 index 000000000..ca4ff6a8d --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast4-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 4 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast5-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast5-spec.js new file mode 100644 index 000000000..c4bfaeba8 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast5-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 5 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast6-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast6-spec.js new file mode 100644 index 000000000..1f257a5ec --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast6-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 6 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast7-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast7-spec.js new file mode 100644 index 000000000..a3433b018 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast7-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 7 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast8-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast8-spec.js new file mode 100644 index 000000000..1b5e901ba --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast8-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 8 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast9-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast9-spec.js new file mode 100644 index 000000000..cded4447a --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast9-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 9 +}); diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js new file mode 100644 index 000000000..5cbcc983d --- /dev/null +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -0,0 +1,28 @@ +const VIEW = 'Limestone-View'; + +function serializeParams (params) { + const searchParams = new URLSearchParams(); + + for (const [key, value] of Object.entries(params)) { + if (value != null) { + searchParams.set(key, String(value)); + } + } + + return searchParams.toString(); +} + +function buildUrl (urlExtra = '?locale=en-US') { + return `/${VIEW}/${urlExtra}`; +} + +async function open (page, urlExtra = '?locale=en-US') { + await page.goto(buildUrl(urlExtra), {waitUntil: 'networkidle'}); + await page.waitForTimeout(200); +} + +export async function openComponent (page, params) { + const query = serializeParams(Object.assign({locale: 'en-US'}, params)); + await open(page, `?${query}`); + await page.locator('[data-ui-test-id="test"]').waitFor({state: 'visible', timeout: 30000}); +} diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js new file mode 100644 index 000000000..7413d52d8 --- /dev/null +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -0,0 +1,105 @@ +import fs from 'fs'; + +import {test, expect} from '@playwright/test'; + +import {assertComponentSource, TEST_DATA_FILE} from '../paths.js'; +import {getScreenshotBasename, getScreenshotName} from './screenshot-name.js'; +import {openComponent} from './limestone-page.js'; + +const testIdFilter = process.env.PLAYWRIGHT_TEST_ID != null + ? Number.parseInt(process.env.PLAYWRIGHT_TEST_ID, 10) + : null; +const titleFilter = process.env.PLAYWRIGHT_TITLE; +const maxInstances = process.env.PLAYWRIGHT_INSTANCES + ? Number.parseInt(process.env.PLAYWRIGHT_INSTANCES, 10) + : 5; + +function loadTestData () { + return JSON.parse(fs.readFileSync(TEST_DATA_FILE, 'utf8')); +} + +function resolveComponentFilter (config) { + return config.component || process.env.PLAYWRIGHT_COMPONENT || null; +} + +function shouldIncludeTest (component, testId, title, configComponent) { + const componentFilter = resolveComponentFilter({component: configComponent}); + if (componentFilter) { + const pattern = componentFilter.startsWith('^') ? componentFilter : `^${componentFilter}$`; + if (!component.match(new RegExp(pattern))) { + return false; + } + } + if (testIdFilter != null && testIdFilter !== testId) { + return false; + } + if (titleFilter && !title.match(new RegExp(titleFilter))) { + return false; + } + return true; +} + +/** + * Cases from tests/screenshot/apps/components/*.js (built into tests/screenshot/dist). + */ +function getScreenshotTests ({testName, skin, highContrast, concurrency, component: configComponent, maxWorkers = maxInstances}) { + if (concurrency && concurrency > maxWorkers) { + return []; + } + + if (configComponent) { + assertComponentSource(configComponent); + } + + const testCases = loadTestData(); + const tests = []; + + for (const component in testCases) { + testCases[component].forEach((testCase, testId) => { + if (concurrency && testId % maxWorkers !== concurrency - 1) { + return; + } + if (!shouldIncludeTest(component, testId, testCase.title, configComponent)) { + return; + } + + tests.push({ + title: `${component}~/${testName}~/${testCase.title}~/${testId}`, + screenshotPath: getScreenshotName(component, testName, testCase.title), + snapshotBasename: getScreenshotBasename(testCase.title), + params: {component, testId, skin, highContrast} + }); + }); + } + + return tests; +} + +export function registerScreenshotTests (config) { + const component = resolveComponentFilter(config); + const suiteName = component + ? `${config.testName} — ${component}` + : (config.concurrency ? `${config.testName} (shard ${config.concurrency})` : config.testName); + const cases = getScreenshotTests(config); + + test.describe(suiteName, () => { + test.beforeEach(async ({page}) => { + await page.setViewportSize({width: 1920, height: 1167}); + }); + + for (const screenshotTest of cases) { + test(screenshotTest.title, async ({page}) => { + await openComponent(page, screenshotTest.params); + await expect(page).toHaveScreenshot([ + screenshotTest.params.component, + config.testName, + screenshotTest.snapshotBasename + ], { + animations: 'disabled', + caret: 'hide', + maxDiffPixelRatio: 0 + }); + }); + } + }); +} diff --git a/tests/screenshot/playwright/utils/screenshot-name.js b/tests/screenshot/playwright/utils/screenshot-name.js new file mode 100644 index 000000000..36a090581 --- /dev/null +++ b/tests/screenshot/playwright/utils/screenshot-name.js @@ -0,0 +1,19 @@ +import crypto from 'crypto'; +import path from 'path'; + +/** + * Case file name only (WDIO visual-service convention). + * @see ui-test-utils/utils/runTest.js, screenshot/utils/confHelpers.js + */ +export function getScreenshotBasename (title) { + let testCaseName = title.replace(/[/\\:?*"|<>]/g, '_'); + testCaseName = testCaseName.substring(0, 128) + '-' + crypto.createHash('md5').update(testCaseName).digest('hex'); + + return `${testCaseName}.png`.replace(/ /g, '_'); +} + +/** Relative path: //.png (matches WDIO reference layout). */ +export function getScreenshotName (component, testName, title) { + return path.posix.join(component, testName, getScreenshotBasename(title)); +} + \ No newline at end of file diff --git a/tests/screenshot/scripts/run-component-wdio.mjs b/tests/screenshot/scripts/run-component-wdio.mjs new file mode 100644 index 000000000..1abc47ebe --- /dev/null +++ b/tests/screenshot/scripts/run-component-wdio.mjs @@ -0,0 +1,116 @@ +#!/usr/bin/env node + +/** + + * Run WDIO screenshot tests for one component (Default shard, instances 1). + + * + + * Usage: + + * npm run test-ss:component -- Sprite + + * npm run test-ss:component -- Sprite --skip-build + + */ + +import {spawnSync} from 'child_process'; + +import fs from 'fs'; + +import path from 'path'; + +import {fileURLToPath} from 'url'; + + + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const repoRoot = path.join(__dirname, '..', '..', '..'); + + + +const args = process.argv.slice(2).filter(a => a !== '--'); + +const component = args.find(a => !a.startsWith('--')); + +const skipBuild = args.includes('--skip-build'); + + + +if (!component) { + + console.error('Usage: npm run test-ss:component -- [--skip-build]'); + + console.error('Example: npm run test-ss:component -- Sprite'); + + process.exit(1); + +} + + + +const screenshotSource = path.join(repoRoot, 'tests', 'screenshot', 'apps', 'components', `${component}.js`); + +if (!fs.existsSync(screenshotSource)) { + + console.error(`Missing ${screenshotSource}`); + + process.exit(1); + +} + + + +const spec = path.join(repoRoot, 'tests', 'screenshot', 'specs', 'neutral', 'Default-specs.js'); + +console.log(`WDIO screenshots: ${component} (${spec})`); + + + +const wdioArgs = [ + + 'run', 'test-ss', + + '--', + + '--component', `^${component}$`, + + '--instances', '1', + + '--spec', spec + +]; + + + +if (skipBuild) { + + wdioArgs.push('--skip-build'); + +} + + + +const result = spawnSync( + + 'npm', + + wdioArgs, + + { + + cwd: repoRoot, + + stdio: 'inherit', + + shell: true + + } + +); + + + +process.exit(result.status ?? 1); + diff --git a/tests/screenshot/scripts/run-playwright-component.mjs b/tests/screenshot/scripts/run-playwright-component.mjs new file mode 100644 index 000000000..1a5a92ed3 --- /dev/null +++ b/tests/screenshot/scripts/run-playwright-component.mjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node +/** + * Run Playwright screenshot tests for one component (all cases, single shard). + * + * Usage: + * npm run test-playwright:component -- Sprite + * npm run test-playwright:component -- Sprite --update + * npm run test-playwright:component -- Sprite --test-id 0 + */ +import {spawnSync} from 'child_process'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +import {assertComponentSource} from '../playwright/paths.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); +const playwrightRoot = path.join(__dirname, '..', 'playwright'); +const config = path.join(playwrightRoot, 'playwright.config.mjs'); +const spec = 'specs/neutral/Default-spec.js'; + +const args = process.argv.slice(2).filter(a => a !== '--'); + +function flagValue (name) { + const index = args.indexOf(name); + return index === -1 ? null : args[index + 1]; +} + +const component = args.find(a => !a.startsWith('--') && args.indexOf(a) === args.findIndex(x => !x.startsWith('--'))); +const update = args.includes('--update'); +const testId = flagValue('--test-id'); +const title = flagValue('--title'); + +if (!component) { + console.error('Usage: npm run test-playwright:component -- [options]'); + console.error('Options: --update, --test-id , --title '); + console.error('Example: npm run test-playwright:component -- Sprite --update'); + process.exit(1); +} + +assertComponentSource(component); + +const playwrightArgs = [ + 'test', + '--config', config, + spec +]; + +if (update) { + playwrightArgs.push('--update-snapshots'); +} + +const env = { + ...process.env, + PLAYWRIGHT_COMPONENT: component, + PLAYWRIGHT_INSTANCES: '1', + PLAYWRIGHT_WORKERS: '1' +}; + +if (testId != null) { + env.PLAYWRIGHT_TEST_ID = testId; +} +if (title != null) { + env.PLAYWRIGHT_TITLE = title; +} + +const result = spawnSync( + process.platform === 'win32' ? 'npx.cmd' : 'npx', + ['playwright', ...playwrightArgs], + { + cwd: repoRoot, + stdio: 'inherit', + env, + shell: process.platform === 'win32' + } +); + +process.exit(result.status ?? 1); From 8e8245c72d1297e4b058220aaeb2c4e491da2899 Mon Sep 17 00:00:00 2001 From: Dan Ichim Date: Wed, 3 Jun 2026 18:09:32 +0300 Subject: [PATCH 02/28] add spec --- tests/screenshot/playwright/README.md | 54 +++++- tests/screenshot/playwright/global-setup.js | 177 +++++++++--------- .../screenshot/playwright/global-teardown.js | 11 ++ .../playwright/playwright.config.mjs | 16 +- .../playwright/utils/run-screenshot-tests.js | 3 +- .../screenshot/scripts/run-component-wdio.mjs | 174 ++++++----------- 6 files changed, 226 insertions(+), 209 deletions(-) create mode 100644 tests/screenshot/playwright/global-teardown.js diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index 1bd724ba4..302b1853a 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -16,7 +16,8 @@ tests/screenshot/ snapshots/ PNG baselines: //.png (same layout as WDIO reference/) benchmark.mjs scripts/ run-playwright-component, run-component-wdio (npm shortcuts) - global-setup.js build, .test-data.json, serve :4568 (default) + global-setup.js build, .test-data.json, serve :4568 if needed + global-teardown.js stops serve only when global-setup started it playwright.config.mjs paths.js ``` @@ -52,7 +53,7 @@ Run from the **repository root**: | Command | Description | |---------|-------------| -| `npm run test-playwright` | Full suite (27 shards) | +| `npm run test-playwright` | Full suite — all components, 27 shards (neutral/light, Default/HighContrast) | | `npm run test-playwright:update` | Update all baselines | | `npm run test-playwright:report` | HTML report | | `npm run test-playwright:component -- Sprite` | One component, all cases (e.g. Sprite ≈ 4, Spinner ≈ 6, Button ≈ 205) | @@ -63,6 +64,48 @@ Run from the **repository root**: Without baselines, add `--update` once. +## Full suite (all components) + +Runs every registered component across all Playwright shards (`specs/neutral`, `specs/light`). Expect a long run (thousands of screenshots). + +```bash +npm install +npm run prepare-playwright + +# First time or after UI / scenario changes — creates/updates all baselines +npm run test-playwright:update + +# Verify against baselines (reuses dist build when possible) +$env:PLAYWRIGHT_SKIP_BUILD='1' # PowerShell; omit on first run +npm run test-playwright + +# Browse results (list, timings, failures with diffs) +npm run test-playwright:report +``` + +| Step | Output | +|------|--------| +| `--update` | `playwright/snapshots//Limestone/*.png` | +| test run | Terminal summary; failures in `playwright/test-results/` | +| `:report` | `playwright/reports/html/index.html` | + +Optional: more workers locally — `$env:PLAYWRIGHT_WORKERS='8'; npm run test-playwright` (default **5**). + +**Subset of shards** (same as Jenkins `SPEC` / TV): + +| `PLAYWRIGHT_SPEC` | Runs | +|-------------------|------| +| *(unset)* | All `*-spec.js` files (27 shards) | +| `Default` | `Default-spec.js` … `Default9-spec.js` | +| `HighContrast` | `HighContrast-spec.js` … `HighContrast9-spec.js` | +| `Light` | `Light-spec.js` … `Light9-spec.js` | +| `Default-spec` | Only `specs/neutral/Default-spec.js` (TV-style single shard) | + +```powershell +$env:PLAYWRIGHT_SPEC='Default' +npm run test-playwright +``` + ## Benchmark (local) Compare end-to-end runtime for one component (Default shard, `instances` / `workers` = 1, existing `tests/screenshot/dist`): @@ -83,7 +126,7 @@ Measured on Windows (Chrome 132), June 2026; times include browser startup and s **View results:** Playwright — `npm run test-playwright:report` and `snapshots//Limestone/`. WDIO — `tests/screenshot/dist/newFiles.html` / `failedTests.html` and `dist/screenshots/reference/`. -**Parallel with WDIO:** default ports differ (WDIO **4567**, Playwright **4568**), both serve `../dist/`. Override with `PLAYWRIGHT_BASE_URL` if needed. +**Parallel with WDIO:** default ports differ (WDIO **4567**, Playwright **4568**), both serve `../dist/`. Override with `PLAYWRIGHT_BASE_URL` if needed. `webServer` in config also uses `reuseExistingServer`; if `global-setup` had to start `serve`, `global-teardown` stops that process after the run. ## Maintenance @@ -99,6 +142,7 @@ Measured on Windows (Chrome 132), June 2026; times include browser startup and s | `PLAYWRIGHT_COMPONENT` | One component (set by `../scripts/run-playwright-component.mjs`) | | `PLAYWRIGHT_TEST_ID` / `PLAYWRIGHT_TITLE` | Filter cases (or `--test-id` / `--title` on npm script) | | `PLAYWRIGHT_INSTANCES` / `PLAYWRIGHT_WORKERS` | Shard index / parallelism | +| `PLAYWRIGHT_SPEC` | Limit spec files (`Default`, `Light`, `Default-spec`, …) — see [Full suite](#full-suite-all-components) | | `PLAYWRIGHT_BASE_URL` | Playwright static server (default `http://localhost:4568`) | ## Troubleshooting @@ -107,8 +151,8 @@ Measured on Windows (Chrome 132), June 2026; times include browser startup and s |-------|-----| | WebServer timeout | Build `../dist/` (unset `PLAYWRIGHT_SKIP_BUILD` or `npm run test-ss`) | | Snapshot does not exist | `--update` | -| No tests found | Check `PLAYWRIGHT_COMPONENT`; refresh `.test-data.json` | +| No tests found | Check `PLAYWRIGHT_COMPONENT` / `PLAYWRIGHT_SPEC`; refresh `.test-data.json` | | Missing images/videos on build | Use `../images/`, `../videos/` | | Port in use | Change `PLAYWRIGHT_BASE_URL` (port in URL must match `webServer`) | -CI: `build-scripts/enact-playwright-tests.sh`. +**CI (planned):** Jenkins job `enact-playwright-tests` via `build-scripts/enact-playwright-tests.sh` — sets `PLAYWRIGHT_SPEC` (`Default-spec` on TV, or `SPEC` shard group), `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_WORKERS`. Uploads `playwright/` to Nebula: `reports/html`, `snapshots/`, `test-results/`. diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js index 2b3bc2b5f..53f42b312 100644 --- a/tests/screenshot/playwright/global-setup.js +++ b/tests/screenshot/playwright/global-setup.js @@ -1,85 +1,92 @@ -import fs from 'fs'; -import path from 'path'; -import {spawn} from 'child_process'; -import {chromium} from '@playwright/test'; - -import buildApps from '../../../node_modules/@enact/ui-test-utils/src/build-apps.js'; - -import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST, TEST_DATA_FILE} from './paths.js'; - -const baseURL = PLAYWRIGHT_BASE_URL; -const distPath = SCREENSHOT_DIST; -const testDataFile = TEST_DATA_FILE; -const port = PLAYWRIGHT_PORT; - -async function waitForServer (url, timeout = 120000) { - const start = Date.now(); - - while (Date.now() - start < timeout) { - try { - const response = await fetch(url); - if (response.ok || response.status === 404) { - return; - } - } catch { - // retry until the static server is ready - } - await new Promise(resolve => setTimeout(resolve, 500)); - } - - throw new Error(`Static server at ${url} did not start within ${timeout}ms`); -} - -function startStaticServer () { - const command = process.platform === 'win32' ? 'npx.cmd' : 'npx'; - spawn(command, ['--yes', 'serve', distPath, '-l', port], { - stdio: 'pipe', - shell: process.platform === 'win32' - }); -} - -function shouldRefreshTestData () { - if (process.env.PLAYWRIGHT_REFRESH_TEST_DATA === '1') { - return true; - } - if (!fs.existsSync(testDataFile)) { - return true; - } - const distIndex = path.join(distPath, 'Limestone-View', 'index.html'); - if (!fs.existsSync(distIndex)) { - return true; - } - return fs.statSync(distIndex).mtimeMs > fs.statSync(testDataFile).mtimeMs; -} - -export default async function globalSetup () { - if (!process.env.PLAYWRIGHT_SKIP_BUILD) { - await buildApps('screenshot'); - } - - try { - await fetch(baseURL); - } catch { - startStaticServer(); - await waitForServer(baseURL); - } - - if (!shouldRefreshTestData()) { - return; - } - - const browser = await chromium.launch({ - channel: 'chrome', - headless: true - }); - - try { - const page = await browser.newPage(); - await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'networkidle'}); - const testData = await page.evaluate(() => window.__TEST_DATA); - - fs.writeFileSync(testDataFile, JSON.stringify(testData)); - } finally { - await browser.close(); - } -} +import fs from 'fs'; +import path from 'path'; +import {spawn} from 'child_process'; +import {chromium} from '@playwright/test'; + +import buildApps from '../../../node_modules/@enact/ui-test-utils/src/build-apps.js'; + +import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST, TEST_DATA_FILE} from './paths.js'; + +const baseURL = PLAYWRIGHT_BASE_URL; +const distPath = SCREENSHOT_DIST; +const testDataFile = TEST_DATA_FILE; +const port = PLAYWRIGHT_PORT; + +/** Set when this file starts `serve`; torn down in global-teardown.js */ +export const staticServer = { + process: null, + started: false +}; + +async function waitForServer (url, timeout = 120000) { + const start = Date.now(); + + while (Date.now() - start < timeout) { + try { + const response = await fetch(url); + if (response.ok || response.status === 404) { + return; + } + } catch { + // retry until the static server is ready + } + await new Promise(resolve => setTimeout(resolve, 500)); + } + + throw new Error(`Static server at ${url} did not start within ${timeout}ms`); +} + +function startStaticServer () { + const command = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + staticServer.process = spawn(command, ['--yes', 'serve', distPath, '-l', port], { + stdio: 'pipe', + shell: process.platform === 'win32' + }); + staticServer.started = true; +} + +function shouldRefreshTestData () { + if (process.env.PLAYWRIGHT_REFRESH_TEST_DATA === '1') { + return true; + } + if (!fs.existsSync(testDataFile)) { + return true; + } + const distIndex = path.join(distPath, 'Limestone-View', 'index.html'); + if (!fs.existsSync(distIndex)) { + return true; + } + return fs.statSync(distIndex).mtimeMs > fs.statSync(testDataFile).mtimeMs; +} + +export default async function globalSetup () { + if (!process.env.PLAYWRIGHT_SKIP_BUILD) { + await buildApps('screenshot'); + } + + try { + await fetch(baseURL); + } catch { + startStaticServer(); + await waitForServer(baseURL); + } + + if (!shouldRefreshTestData()) { + return; + } + + const browser = await chromium.launch({ + channel: 'chrome', + headless: true + }); + + try { + const page = await browser.newPage(); + await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'networkidle'}); + const testData = await page.evaluate(() => window.__TEST_DATA); + + fs.writeFileSync(testDataFile, JSON.stringify(testData)); + } finally { + await browser.close(); + } +} diff --git a/tests/screenshot/playwright/global-teardown.js b/tests/screenshot/playwright/global-teardown.js new file mode 100644 index 000000000..33530f23b --- /dev/null +++ b/tests/screenshot/playwright/global-teardown.js @@ -0,0 +1,11 @@ +import {staticServer} from './global-setup.js'; + +export default async function globalTeardown () { + if (!staticServer.started || !staticServer.process) { + return; + } + + staticServer.process.kill(); + staticServer.process = null; + staticServer.started = false; +} diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index a4e2b2454..9b9731c42 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -7,11 +7,25 @@ import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT} from './paths.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const distDir = path.join(__dirname, '..', 'dist'); +/** Jenkins: SPEC=Default → Default* shards; PLAYWRIGHT_SPEC=Default-spec → one file (TV). */ +function resolveTestMatch () { + const spec = process.env.PLAYWRIGHT_SPEC?.trim(); + if (!spec) { + return '**/*-spec.js'; + } + const base = spec.replace(/\.js$/, ''); + if (base.endsWith('-spec')) { + return `**/${base}.js`; + } + return `**/*${base}*-spec.js`; +} + export default defineConfig({ testDir: path.join(__dirname, 'specs'), - testMatch: '**/*-spec.js', + testMatch: resolveTestMatch(), testIgnore: ['**/utils/**', '**/scripts/**'], globalSetup: path.join(__dirname, 'global-setup.js'), + globalTeardown: path.join(__dirname, 'global-teardown.js'), snapshotPathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}'), expect: { toHaveScreenshot: { diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js index 7413d52d8..27a308cee 100644 --- a/tests/screenshot/playwright/utils/run-screenshot-tests.js +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -3,7 +3,7 @@ import fs from 'fs'; import {test, expect} from '@playwright/test'; import {assertComponentSource, TEST_DATA_FILE} from '../paths.js'; -import {getScreenshotBasename, getScreenshotName} from './screenshot-name.js'; +import {getScreenshotBasename} from './screenshot-name.js'; import {openComponent} from './limestone-page.js'; const testIdFilter = process.env.PLAYWRIGHT_TEST_ID != null @@ -65,7 +65,6 @@ function getScreenshotTests ({testName, skin, highContrast, concurrency, compone tests.push({ title: `${component}~/${testName}~/${testCase.title}~/${testId}`, - screenshotPath: getScreenshotName(component, testName, testCase.title), snapshotBasename: getScreenshotBasename(testCase.title), params: {component, testId, skin, highContrast} }); diff --git a/tests/screenshot/scripts/run-component-wdio.mjs b/tests/screenshot/scripts/run-component-wdio.mjs index 1abc47ebe..068491e5a 100644 --- a/tests/screenshot/scripts/run-component-wdio.mjs +++ b/tests/screenshot/scripts/run-component-wdio.mjs @@ -1,116 +1,58 @@ -#!/usr/bin/env node - -/** - - * Run WDIO screenshot tests for one component (Default shard, instances 1). - - * - - * Usage: - - * npm run test-ss:component -- Sprite - - * npm run test-ss:component -- Sprite --skip-build - - */ - -import {spawnSync} from 'child_process'; - -import fs from 'fs'; - -import path from 'path'; - -import {fileURLToPath} from 'url'; - - - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); - -const repoRoot = path.join(__dirname, '..', '..', '..'); - - - -const args = process.argv.slice(2).filter(a => a !== '--'); - -const component = args.find(a => !a.startsWith('--')); - -const skipBuild = args.includes('--skip-build'); - - - -if (!component) { - - console.error('Usage: npm run test-ss:component -- [--skip-build]'); - - console.error('Example: npm run test-ss:component -- Sprite'); - - process.exit(1); - -} - - - -const screenshotSource = path.join(repoRoot, 'tests', 'screenshot', 'apps', 'components', `${component}.js`); - -if (!fs.existsSync(screenshotSource)) { - - console.error(`Missing ${screenshotSource}`); - - process.exit(1); - -} - - - -const spec = path.join(repoRoot, 'tests', 'screenshot', 'specs', 'neutral', 'Default-specs.js'); - -console.log(`WDIO screenshots: ${component} (${spec})`); - - - -const wdioArgs = [ - - 'run', 'test-ss', - - '--', - - '--component', `^${component}$`, - - '--instances', '1', - - '--spec', spec - -]; - - - -if (skipBuild) { - - wdioArgs.push('--skip-build'); - -} - - - -const result = spawnSync( - - 'npm', - - wdioArgs, - - { - - cwd: repoRoot, - - stdio: 'inherit', - - shell: true - - } - -); - - - -process.exit(result.status ?? 1); - +#!/usr/bin/env node +/** + * Run WDIO screenshot tests for one component (Default shard, instances 1). + * + * Usage: + * npm run test-ss:component -- Sprite + * npm run test-ss:component -- Sprite --skip-build + */ +import {spawnSync} from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); + +const args = process.argv.slice(2).filter(a => a !== '--'); +const component = args.find(a => !a.startsWith('--')); +const skipBuild = args.includes('--skip-build'); + +if (!component) { + console.error('Usage: npm run test-ss:component -- [--skip-build]'); + console.error('Example: npm run test-ss:component -- Sprite'); + process.exit(1); +} + +const screenshotSource = path.join(repoRoot, 'tests', 'screenshot', 'apps', 'components', `${component}.js`); +if (!fs.existsSync(screenshotSource)) { + console.error(`Missing ${screenshotSource}`); + process.exit(1); +} + +const spec = path.join(repoRoot, 'tests', 'screenshot', 'specs', 'neutral', 'Default-specs.js'); +console.log(`WDIO screenshots: ${component} (${spec})`); + +const wdioArgs = [ + 'run', 'test-ss', + '--', + '--component', `^${component}$`, + '--instances', '1', + '--spec', spec +]; + +if (skipBuild) { + wdioArgs.push('--skip-build'); +} + +const result = spawnSync( + 'npm', + wdioArgs, + { + cwd: repoRoot, + stdio: 'inherit', + shell: true + } +); + +process.exit(result.status ?? 1); From d85581eb3c5f49f111c61bce11c5dd2a5242edff Mon Sep 17 00:00:00 2001 From: Dan Ichim Date: Thu, 4 Jun 2026 17:15:46 +0300 Subject: [PATCH 03/28] add parallel --- tests/screenshot/playwright/README.md | 27 ++- tests/screenshot/playwright/benchmark.mjs | 210 ++++++++++-------- tests/screenshot/playwright/global-setup.js | 184 +++++++-------- .../screenshot/playwright/global-teardown.js | 22 +- .../playwright/utils/run-screenshot-tests.js | 26 ++- .../screenshot/scripts/run-component-wdio.mjs | 129 ++++++----- .../scripts/run-playwright-component.mjs | 162 +++++++------- 7 files changed, 405 insertions(+), 355 deletions(-) diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index 302b1853a..efc1507c3 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -61,6 +61,7 @@ Run from the **repository root**: | `npm run test-playwright:component -- Sprite --update` | Refresh baselines | | `npm run test-ss:component -- Sprite` | WDIO, same component (Default shard) | | `npm run benchmark-screenshots -- Chip` | Playwright vs WDIO timing (`--build` to rebuild dist) | +| `npm run benchmark-screenshots -- Chip --parallel 5` | Same, up to 5 cases at a time per runner | Without baselines, add `--update` once. @@ -108,21 +109,31 @@ npm run test-playwright ## Benchmark (local) -Compare end-to-end runtime for one component (Default shard, `instances` / `workers` = 1, existing `tests/screenshot/dist`): +Compare end-to-end runtime for one component (Default shard, all cases, existing `tests/screenshot/dist`): ```bash npm run benchmark-screenshots -- Chip -# optional: PLAYWRIGHT_SKIP_BUILD=1 if dist is already built +npm run benchmark-screenshots -- Chip --parallel 5 # faster: 5 workers / 5 WDIO browsers +# optional: --build to rebuild dist; PLAYWRIGHT_SKIP_BUILD=1 when dist exists ``` -Both runners filter a single component name (WDIO uses `--component ^Name$` so `Chip` does not include `Chips`). +**Parallelism:** `--parallel N` (default `1`) runs every case for the component, but up to `N` at once. Playwright uses `PLAYWRIGHT_INSTANCES=1` (no shard filter) and `PLAYWRIGHT_WORKERS=N`. WDIO uses `--instances 1` (all cases) and `--parallel N` for `wdio:maxInstances` (requires `@enact/ui-test-utils` with `--parallel` support). -| Component | Cases | Playwright | WebdriverIO | Faster | -|-----------|------:|-----------:|--------------:|--------| -| Sprite | 4 | 16.7 s | 18.0 s | Playwright (~8%) | -| Chip | 51 | 99.2 s | 84.6 s | WDIO (~15%) | +Both runners filter a single component name (WDIO uses `--component ^Name$` so `Chip` does not include `Chips`). Benchmark still runs Playwright then WDIO sequentially; ports **4568** / **4567** stay separate. -Measured on Windows (Chrome 132), June 2026; times include browser startup and screenshot compare. Chip ≈ 1.9 s/test (Playwright) vs ≈ 1.7 s/test (WDIO). Full suite (`Button` ≈ 205 cases) is much slower — use component scripts for day-to-day runs. +| Component | Cases | Parallel | Playwright | WebdriverIO | Faster | +|-----------|------:|---------:|-----------:|--------------:|--------| +| Sprite | 4 | 1 | 16.7 s | 18.0 s | Playwright (~8%) | +| Chip | 51 | 1 | 99.2 s | 84.6 s | WDIO (~15%) | +| Chip | 51 | 5 | 58.9 s | 154.8 s | Playwright (~62%) | + +Measured on Windows (Chrome 132), June 2026; times include browser startup and screenshot compare. Benchmark runs Playwright then WDIO (not at the same time). + +**Sequential (`--parallel 1`, default):** Chip ≈ 1.9 s/test (Playwright) vs ≈ 1.7 s/test (WDIO). + +**Parallel 5 (`--parallel 5`):** Chip ≈ 1.2 s/test (Playwright) vs ≈ 3.0 s/test (WDIO). Playwright wall time drops (~41%); WDIO can be slower on one machine with many Chrome sessions (visual compare + resource contention) — CI uses sharded jobs plus `--parallel` instead of one large component run. + +Full suite (`Button` ≈ 205 cases) is much slower — use component scripts for day-to-day runs. **View results:** Playwright — `npm run test-playwright:report` and `snapshots//Limestone/`. WDIO — `tests/screenshot/dist/newFiles.html` / `failedTests.html` and `dist/screenshots/reference/`. diff --git a/tests/screenshot/playwright/benchmark.mjs b/tests/screenshot/playwright/benchmark.mjs index 59147a640..4cc0ba9af 100644 --- a/tests/screenshot/playwright/benchmark.mjs +++ b/tests/screenshot/playwright/benchmark.mjs @@ -1,96 +1,114 @@ -#!/usr/bin/env node -/** - * Compare Playwright vs WebdriverIO runtime for one component (all its screenshot cases). - * - * Usage: - * npm run benchmark-screenshots -- Sprite - * npm run benchmark-screenshots -- Sprite --build - */ -import {spawn} from 'child_process'; -import fs from 'fs'; -import path from 'path'; -import {fileURLToPath} from 'url'; - -import {assertComponentSource, SCREENSHOT_DIST} from './paths.js'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.join(__dirname, '..', '..', '..'); - -const args = process.argv.slice(2).filter(a => a !== '--'); -const component = args.find(a => !a.startsWith('--')); -const withBuild = args.includes('--build'); - -if (!component) { - console.error('Usage: npm run benchmark-screenshots -- [--build]'); - console.error('Example: npm run benchmark-screenshots -- Sprite'); - process.exit(1); -} - -assertComponentSource(component); - -const distIndex = path.join(SCREENSHOT_DIST, 'Limestone-View', 'index.html'); -const skipBuild = !withBuild && fs.existsSync(distIndex); - -function runCommand (label, command, args, env = {}) { - return new Promise((resolve, reject) => { - const start = Date.now(); - const child = spawn(command, args, { - cwd: repoRoot, - env: {...process.env, ...env}, - shell: true, - stdio: 'inherit' - }); - - child.on('close', code => { - const durationMs = Date.now() - start; - if (code === 0) { - resolve({label, durationMs}); - } else { - reject(new Error(`${label} failed with exit code ${code} after ${(durationMs / 1000).toFixed(1)}s`)); - } - }); - }); -} - -async function main () { - console.log(`\nBenchmark: ${component} (all cases, 1 worker / instances 1)`); - if (skipBuild) { - console.log('Using existing tests/screenshot/dist (pass --build to rebuild)\n'); - } else { - console.log('Will build tests/screenshot/dist before run\n'); - } - - const playwright = await runCommand( - 'Playwright', - 'node', - ['tests/screenshot/scripts/run-playwright-component.mjs', component], - skipBuild ? {PLAYWRIGHT_SKIP_BUILD: '1'} : {} - ); - - const wdioArgv = ['tests/screenshot/scripts/run-component-wdio.mjs', component]; - if (skipBuild) { - wdioArgv.push('--skip-build'); - } - - const wdio = await runCommand( - 'WebdriverIO', - 'node', - wdioArgv, - {} - ); - - const pwSec = playwright.durationMs / 1000; - const wdioSec = wdio.durationMs / 1000; - const delta = wdioSec - pwSec; - - console.log('\n--- Results ---'); - console.log(`Component: ${component}`); - console.log(`Playwright: ${pwSec.toFixed(1)}s`); - console.log(`WebdriverIO: ${wdioSec.toFixed(1)}s`); - console.log(`Difference: ${Math.abs(delta).toFixed(1)}s (${delta > 0 ? 'Playwright faster' : delta < 0 ? 'WDIO faster' : 'tie'})\n`); -} - -main().catch(err => { - console.error(err.message); - process.exit(1); -}); +#!/usr/bin/env node +/** + * Compare Playwright vs WebdriverIO runtime for one component (all its screenshot cases). + * + * Usage: + * npm run benchmark-screenshots -- Sprite + * npm run benchmark-screenshots -- Sprite --build + * npm run benchmark-screenshots -- Chip --parallel 5 + * (reuses tests/screenshot/dist when built; pass --build to rebuild) + */ +import {spawn} from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +import {assertComponentSource, SCREENSHOT_DIST} from './paths.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); + +const args = process.argv.slice(2).filter(a => a !== '--'); + +function flagValue (name) { + const index = args.indexOf(name); + return index === -1 ? null : args[index + 1]; +} + +const component = args.find(a => !a.startsWith('--')); +const withBuild = args.includes('--build'); +const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1', 10) || 1); + +if (!component) { + console.error('Usage: npm run benchmark-screenshots -- [--build] [--parallel ]'); + console.error('Example: npm run benchmark-screenshots -- Sprite'); + console.error('Example: npm run benchmark-screenshots -- Sprite --build'); + console.error('Example: npm run benchmark-screenshots -- Chip --parallel 5'); + process.exit(1); +} + +assertComponentSource(component); + +const distIndex = path.join(SCREENSHOT_DIST, 'Limestone-View', 'index.html'); +const skipBuild = !withBuild && fs.existsSync(distIndex); + +function runCommand (label, command, commandArgs, env = {}) { + return new Promise((resolve, reject) => { + const start = Date.now(); + const child = spawn(command, commandArgs, { + cwd: repoRoot, + env: {...process.env, ...env}, + shell: true, + stdio: 'inherit' + }); + + child.on('close', code => { + const durationMs = Date.now() - start; + if (code === 0) { + resolve({label, durationMs}); + } else { + reject(new Error(`${label} failed with exit code ${code} after ${(durationMs / 1000).toFixed(1)}s`)); + } + }); + }); +} + +async function main () { + const parallelLabel = parallel === 1 + ? '1 worker / 1 browser (sequential)' + : `parallel ${parallel} (PW: workers=${parallel}, instances=1; WDIO: --parallel ${parallel}, --instances 1)`; + + console.log(`\nBenchmark: ${component} (all cases, ${parallelLabel})`); + if (skipBuild) { + console.log('Using existing tests/screenshot/dist (pass --build to rebuild)\n'); + } else { + console.log('Will build tests/screenshot/dist before run\n'); + } + + const parallelArg = ['--parallel', String(parallel)]; + + const playwright = await runCommand( + 'Playwright', + 'node', + ['tests/screenshot/scripts/run-playwright-component.mjs', component, ...parallelArg], + skipBuild ? {PLAYWRIGHT_SKIP_BUILD: '1'} : {} + ); + + const wdioArgv = ['tests/screenshot/scripts/run-component-wdio.mjs', component, ...parallelArg]; + if (skipBuild) { + wdioArgv.push('--skip-build'); + } + + const wdio = await runCommand( + 'WebdriverIO', + 'node', + wdioArgv, + {} + ); + + const pwSec = playwright.durationMs / 1000; + const wdioSec = wdio.durationMs / 1000; + const delta = wdioSec - pwSec; + + console.log('\n--- Results ---'); + console.log(`Component: ${component}`); + console.log(`Parallelism: ${parallel}`); + console.log(`Playwright: ${pwSec.toFixed(1)}s`); + console.log(`WebdriverIO: ${wdioSec.toFixed(1)}s`); + console.log(`Difference: ${Math.abs(delta).toFixed(1)}s (${delta > 0 ? 'Playwright faster' : delta < 0 ? 'WDIO faster' : 'tie'})\n`); +} + +main().catch(err => { + console.error(err.message); + process.exit(1); +}); diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js index 53f42b312..fb84067a6 100644 --- a/tests/screenshot/playwright/global-setup.js +++ b/tests/screenshot/playwright/global-setup.js @@ -1,92 +1,92 @@ -import fs from 'fs'; -import path from 'path'; -import {spawn} from 'child_process'; -import {chromium} from '@playwright/test'; - -import buildApps from '../../../node_modules/@enact/ui-test-utils/src/build-apps.js'; - -import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST, TEST_DATA_FILE} from './paths.js'; - -const baseURL = PLAYWRIGHT_BASE_URL; -const distPath = SCREENSHOT_DIST; -const testDataFile = TEST_DATA_FILE; -const port = PLAYWRIGHT_PORT; - -/** Set when this file starts `serve`; torn down in global-teardown.js */ -export const staticServer = { - process: null, - started: false -}; - -async function waitForServer (url, timeout = 120000) { - const start = Date.now(); - - while (Date.now() - start < timeout) { - try { - const response = await fetch(url); - if (response.ok || response.status === 404) { - return; - } - } catch { - // retry until the static server is ready - } - await new Promise(resolve => setTimeout(resolve, 500)); - } - - throw new Error(`Static server at ${url} did not start within ${timeout}ms`); -} - -function startStaticServer () { - const command = process.platform === 'win32' ? 'npx.cmd' : 'npx'; - staticServer.process = spawn(command, ['--yes', 'serve', distPath, '-l', port], { - stdio: 'pipe', - shell: process.platform === 'win32' - }); - staticServer.started = true; -} - -function shouldRefreshTestData () { - if (process.env.PLAYWRIGHT_REFRESH_TEST_DATA === '1') { - return true; - } - if (!fs.existsSync(testDataFile)) { - return true; - } - const distIndex = path.join(distPath, 'Limestone-View', 'index.html'); - if (!fs.existsSync(distIndex)) { - return true; - } - return fs.statSync(distIndex).mtimeMs > fs.statSync(testDataFile).mtimeMs; -} - -export default async function globalSetup () { - if (!process.env.PLAYWRIGHT_SKIP_BUILD) { - await buildApps('screenshot'); - } - - try { - await fetch(baseURL); - } catch { - startStaticServer(); - await waitForServer(baseURL); - } - - if (!shouldRefreshTestData()) { - return; - } - - const browser = await chromium.launch({ - channel: 'chrome', - headless: true - }); - - try { - const page = await browser.newPage(); - await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'networkidle'}); - const testData = await page.evaluate(() => window.__TEST_DATA); - - fs.writeFileSync(testDataFile, JSON.stringify(testData)); - } finally { - await browser.close(); - } -} +import fs from 'fs'; +import path from 'path'; +import {spawn} from 'child_process'; + +import buildApps from '@enact/ui-test-utils/build-apps'; +import {chromium} from '@playwright/test'; + +import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST, TEST_DATA_FILE} from './paths.js'; + +const baseURL = PLAYWRIGHT_BASE_URL; +const distPath = SCREENSHOT_DIST; +const testDataFile = TEST_DATA_FILE; +const port = PLAYWRIGHT_PORT; + +/** Set when this file starts `serve`; torn down in global-teardown.js */ +export const staticServer = { + process: null, + started: false +}; + +async function waitForServer (url, timeout = 120000) { + const start = Date.now(); + + while (Date.now() - start < timeout) { + try { + const response = await fetch(url); + if (response.ok || response.status === 404) { + return; + } + } catch { + // retry until the static server is ready + } + await new Promise(resolve => setTimeout(resolve, 500)); + } + + throw new Error(`Static server at ${url} did not start within ${timeout}ms`); +} + +function startStaticServer () { + const command = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + staticServer.process = spawn(command, ['--yes', 'serve', distPath, '-l', port], { + stdio: 'pipe', + shell: process.platform === 'win32' + }); + staticServer.started = true; +} + +function shouldRefreshTestData () { + if (process.env.PLAYWRIGHT_REFRESH_TEST_DATA === '1') { + return true; + } + if (!fs.existsSync(testDataFile)) { + return true; + } + const distIndex = path.join(distPath, 'Limestone-View', 'index.html'); + if (!fs.existsSync(distIndex)) { + return true; + } + return fs.statSync(distIndex).mtimeMs > fs.statSync(testDataFile).mtimeMs; +} + +export default async function globalSetup () { + if (!process.env.PLAYWRIGHT_SKIP_BUILD) { + await buildApps('screenshot'); + } + + try { + await fetch(baseURL); + } catch { + startStaticServer(); + await waitForServer(baseURL); + } + + if (!shouldRefreshTestData()) { + return; + } + + const browser = await chromium.launch({ + channel: 'chrome', + headless: true + }); + + try { + const page = await browser.newPage(); + await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'networkidle'}); + const testData = await page.evaluate(() => window.__TEST_DATA); + + fs.writeFileSync(testDataFile, JSON.stringify(testData)); + } finally { + await browser.close(); + } +} diff --git a/tests/screenshot/playwright/global-teardown.js b/tests/screenshot/playwright/global-teardown.js index 33530f23b..18eb75e44 100644 --- a/tests/screenshot/playwright/global-teardown.js +++ b/tests/screenshot/playwright/global-teardown.js @@ -1,11 +1,11 @@ -import {staticServer} from './global-setup.js'; - -export default async function globalTeardown () { - if (!staticServer.started || !staticServer.process) { - return; - } - - staticServer.process.kill(); - staticServer.process = null; - staticServer.started = false; -} +import {staticServer} from './global-setup.js'; + +export default async function globalTeardown () { + if (!staticServer.started || !staticServer.process) { + return; + } + + staticServer.process.kill(); + staticServer.process = null; + staticServer.started = false; +} diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js index 27a308cee..88f7189f1 100644 --- a/tests/screenshot/playwright/utils/run-screenshot-tests.js +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -3,7 +3,7 @@ import fs from 'fs'; import {test, expect} from '@playwright/test'; import {assertComponentSource, TEST_DATA_FILE} from '../paths.js'; -import {getScreenshotBasename} from './screenshot-name.js'; +import {getScreenshotName} from './screenshot-name.js'; import {openComponent} from './limestone-page.js'; const testIdFilter = process.env.PLAYWRIGHT_TEST_ID != null @@ -65,8 +65,7 @@ function getScreenshotTests ({testName, skin, highContrast, concurrency, compone tests.push({ title: `${component}~/${testName}~/${testCase.title}~/${testId}`, - snapshotBasename: getScreenshotBasename(testCase.title), - params: {component, testId, skin, highContrast} + params: {component, testId, skin, highContrast, caseTitle: testCase.title} }); }); } @@ -89,15 +88,18 @@ export function registerScreenshotTests (config) { for (const screenshotTest of cases) { test(screenshotTest.title, async ({page}) => { await openComponent(page, screenshotTest.params); - await expect(page).toHaveScreenshot([ - screenshotTest.params.component, - config.testName, - screenshotTest.snapshotBasename - ], { - animations: 'disabled', - caret: 'hide', - maxDiffPixelRatio: 0 - }); + await expect(page).toHaveScreenshot( + getScreenshotName( + screenshotTest.params.component, + config.testName, + screenshotTest.params.caseTitle + ), + { + animations: 'disabled', + caret: 'hide', + maxDiffPixelRatio: 0 + } + ); }); } }); diff --git a/tests/screenshot/scripts/run-component-wdio.mjs b/tests/screenshot/scripts/run-component-wdio.mjs index 068491e5a..52023577c 100644 --- a/tests/screenshot/scripts/run-component-wdio.mjs +++ b/tests/screenshot/scripts/run-component-wdio.mjs @@ -1,58 +1,71 @@ -#!/usr/bin/env node -/** - * Run WDIO screenshot tests for one component (Default shard, instances 1). - * - * Usage: - * npm run test-ss:component -- Sprite - * npm run test-ss:component -- Sprite --skip-build - */ -import {spawnSync} from 'child_process'; -import fs from 'fs'; -import path from 'path'; -import {fileURLToPath} from 'url'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.join(__dirname, '..', '..', '..'); - -const args = process.argv.slice(2).filter(a => a !== '--'); -const component = args.find(a => !a.startsWith('--')); -const skipBuild = args.includes('--skip-build'); - -if (!component) { - console.error('Usage: npm run test-ss:component -- [--skip-build]'); - console.error('Example: npm run test-ss:component -- Sprite'); - process.exit(1); -} - -const screenshotSource = path.join(repoRoot, 'tests', 'screenshot', 'apps', 'components', `${component}.js`); -if (!fs.existsSync(screenshotSource)) { - console.error(`Missing ${screenshotSource}`); - process.exit(1); -} - -const spec = path.join(repoRoot, 'tests', 'screenshot', 'specs', 'neutral', 'Default-specs.js'); -console.log(`WDIO screenshots: ${component} (${spec})`); - -const wdioArgs = [ - 'run', 'test-ss', - '--', - '--component', `^${component}$`, - '--instances', '1', - '--spec', spec -]; - -if (skipBuild) { - wdioArgs.push('--skip-build'); -} - -const result = spawnSync( - 'npm', - wdioArgs, - { - cwd: repoRoot, - stdio: 'inherit', - shell: true - } -); - -process.exit(result.status ?? 1); +#!/usr/bin/env node +/** + * Run WDIO screenshot tests for one component (Default shard, instances 1 = all cases). + * + * Usage: + * npm run test-ss:component -- Sprite + * npm run test-ss:component -- Sprite --skip-build + * npm run test-ss:component -- Sprite --parallel 5 + */ +import {spawnSync} from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); + +const args = process.argv.slice(2).filter(a => a !== '--'); + +function flagValue (name) { + const index = args.indexOf(name); + return index === -1 ? null : args[index + 1]; +} + +const component = args.find(a => !a.startsWith('--')); +const skipBuild = args.includes('--skip-build'); +const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1', 10) || 1); + +if (!component) { + console.error('Usage: npm run test-ss:component -- [--skip-build] [--parallel ]'); + console.error('Example: npm run test-ss:component -- Sprite'); + console.error('Example: npm run test-ss:component -- Sprite --skip-build'); + console.error('Example: npm run test-ss:component -- Sprite --parallel 5'); + process.exit(1); +} + +const screenshotSource = path.join(repoRoot, 'tests', 'screenshot', 'apps', 'components', `${component}.js`); +if (!fs.existsSync(screenshotSource)) { + console.error(`Missing ${screenshotSource}`); + process.exit(1); +} + +const spec = path.join(repoRoot, 'tests', 'screenshot', 'specs', 'neutral', 'Default-specs.js'); + +console.log(`WDIO screenshots: ${component} (${spec}, parallel=${parallel})`); + +const wdioArgs = [ + 'run', 'test-ss', + '--', + '--component', `^${component}$`, + '--instances', '1', + '--spec', spec +]; + +wdioArgs.push('--parallel', String(parallel)); + +if (skipBuild) { + wdioArgs.push('--skip-build'); +} + +const result = spawnSync( + 'npm', + wdioArgs, + { + cwd: repoRoot, + stdio: 'inherit', + shell: true + } +); + +process.exit(result.status ?? 1); diff --git a/tests/screenshot/scripts/run-playwright-component.mjs b/tests/screenshot/scripts/run-playwright-component.mjs index 1a5a92ed3..1afd6816b 100644 --- a/tests/screenshot/scripts/run-playwright-component.mjs +++ b/tests/screenshot/scripts/run-playwright-component.mjs @@ -1,78 +1,84 @@ -#!/usr/bin/env node -/** - * Run Playwright screenshot tests for one component (all cases, single shard). - * - * Usage: - * npm run test-playwright:component -- Sprite - * npm run test-playwright:component -- Sprite --update - * npm run test-playwright:component -- Sprite --test-id 0 - */ -import {spawnSync} from 'child_process'; -import path from 'path'; -import {fileURLToPath} from 'url'; - -import {assertComponentSource} from '../playwright/paths.js'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.join(__dirname, '..', '..', '..'); -const playwrightRoot = path.join(__dirname, '..', 'playwright'); -const config = path.join(playwrightRoot, 'playwright.config.mjs'); -const spec = 'specs/neutral/Default-spec.js'; - -const args = process.argv.slice(2).filter(a => a !== '--'); - -function flagValue (name) { - const index = args.indexOf(name); - return index === -1 ? null : args[index + 1]; -} - -const component = args.find(a => !a.startsWith('--') && args.indexOf(a) === args.findIndex(x => !x.startsWith('--'))); -const update = args.includes('--update'); -const testId = flagValue('--test-id'); -const title = flagValue('--title'); - -if (!component) { - console.error('Usage: npm run test-playwright:component -- [options]'); - console.error('Options: --update, --test-id , --title '); - console.error('Example: npm run test-playwright:component -- Sprite --update'); - process.exit(1); -} - -assertComponentSource(component); - -const playwrightArgs = [ - 'test', - '--config', config, - spec -]; - -if (update) { - playwrightArgs.push('--update-snapshots'); -} - -const env = { - ...process.env, - PLAYWRIGHT_COMPONENT: component, - PLAYWRIGHT_INSTANCES: '1', - PLAYWRIGHT_WORKERS: '1' -}; - -if (testId != null) { - env.PLAYWRIGHT_TEST_ID = testId; -} -if (title != null) { - env.PLAYWRIGHT_TITLE = title; -} - -const result = spawnSync( - process.platform === 'win32' ? 'npx.cmd' : 'npx', - ['playwright', ...playwrightArgs], - { - cwd: repoRoot, - stdio: 'inherit', - env, - shell: process.platform === 'win32' - } -); - -process.exit(result.status ?? 1); +#!/usr/bin/env node +/** + * Run Playwright screenshot tests for one component (all cases; INSTANCES=1, WORKERS from --parallel). + * + * Usage: + * npm run test-playwright:component -- Sprite + * npm run test-playwright:component -- Sprite --update + * npm run test-playwright:component -- Sprite --test-id 0 + * npm run test-playwright:component -- Sprite --title + * npm run test-playwright:component -- Sprite --parallel 5 + */ +import {spawnSync} from 'child_process'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +import {assertComponentSource} from '../playwright/paths.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); +const playwrightRoot = path.join(__dirname, '..', 'playwright'); +const config = path.join(playwrightRoot, 'playwright.config.mjs'); +const spec = 'specs/neutral/Default-spec.js'; + +const args = process.argv.slice(2).filter(a => a !== '--'); + +function flagValue (name) { + const index = args.indexOf(name); + return index === -1 ? null : args[index + 1]; +} + +const component = args.find(a => !a.startsWith('--') && args.indexOf(a) === args.findIndex(x => !x.startsWith('--'))); +const update = args.includes('--update'); +const testId = flagValue('--test-id'); +const title = flagValue('--title'); +const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1', 10) || 1); + +if (!component) { + console.error('Usage: npm run test-playwright:component -- [options]'); + console.error('Options: --update, --test-id , --title , --parallel '); + console.error('Example: npm run test-playwright:component -- Sprite'); + console.error('Example: npm run test-playwright:component -- Sprite --update'); + console.error('Example: npm run test-playwright:component -- Sprite --test-id 0'); + console.error('Example: npm run test-playwright:component -- Sprite --parallel 5'); + process.exit(1); +} + +assertComponentSource(component); + +const playwrightArgs = [ + 'test', + '--config', config, + spec +]; + +if (update) { + playwrightArgs.push('--update-snapshots'); +} + +const env = { + ...process.env, + PLAYWRIGHT_COMPONENT: component, + PLAYWRIGHT_INSTANCES: '1', + PLAYWRIGHT_WORKERS: String(parallel) +}; + +if (testId != null) { + env.PLAYWRIGHT_TEST_ID = testId; +} +if (title != null) { + env.PLAYWRIGHT_TITLE = title; +} + +const result = spawnSync( + process.platform === 'win32' ? 'npx.cmd' : 'npx', + ['playwright', ...playwrightArgs], + { + cwd: repoRoot, + stdio: 'inherit', + env, + shell: process.platform === 'win32' + } +); + +process.exit(result.status ?? 1); From 44d2249ad1c6a2f787a7dd081a43f608aa422adf Mon Sep 17 00:00:00 2001 From: Dan Ichim Date: Thu, 4 Jun 2026 17:21:16 +0300 Subject: [PATCH 04/28] update readme --- tests/screenshot/playwright/README.md | 85 +++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index efc1507c3..702b7c1ed 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -29,13 +29,90 @@ tests/screenshot/ | Specs | `../specs/*-specs.js` | `specs/*-spec.js` | | Assertions / baselines | `dist/screenshots/reference/` | `playwright/snapshots/` (same `//` paths) | -## How a scenario runs (e.g. Chip, testId 3) +## How a component reaches Playwright (e.g. Chip, testId 3) +Playwright never opens `apps/components/Chip.js` at test time. That file is **source** for the screenshot app; the runner only serves the **built** bundle under `../dist/` and drives cases by URL query (`component` + `testId`). + +### 1. Author scenarios in `apps/components/` + +`Chip.js` exports an array of JSX cases (order = `testId`): + +```js +const ChipTests = [ + Default Chip, // testId 0 + …, // testId 1 + … + Chip with checkmark Icon, // testId 3 + … +]; +export default ChipTests; +``` + +`withConfig({…}, […])` expands into more entries (focus, `largeText`, etc.), so Chip ends up with ~51 cases — same list WDIO uses. + +### 2. Register the component + +`apps/LimestoneComponents.js` imports `Chip` from `./components/Chip` and puts it in the `components` map. For each key it runs `generateTestData` (`@enact/ui-test-utils`) to build `testMetadata[Chip]`: one `{title, …}` per array index (titles come from props/children when the JSX has no explicit `title`). + +### 3. Build into `../dist/` + +`global-setup` (unless `PLAYWRIGHT_SKIP_BUILD=1`) calls `buildApps('screenshot')` from `@enact/ui-test-utils/build-apps`. That packs `Limestone-View` (entry `screenshot/index.js`) into `dist/Limestone-View/`. The bundle includes **all** registered components and `testMetadata` from step 2. Assets from `../images/` and `../videos/` are copied as part of the same build WDIO uses. + +### 4. Cache case list for the runner + +On first run (or when dist is newer than the cache), `global-setup` opens: + +`http://localhost:4568/Limestone-View/?request` + +With `?request`, the app sets `window.__TEST_DATA = testMetadata` and does not render a case. Playwright writes that JSON to `playwright/.test-data.json`, e.g.: + +```json +{ "Chip": [ { "title": "…" }, { "title": "…" }, … ] } ``` -Chip.js → LimestoneComponents / testMetadata → buildApps → ../dist/ + +`registerScreenshotTests` reads this file to know **which** `(component, testId, title)` pairs exist — it does not parse `Chip.js` again. + +### 5. Playwright spec → one test per case + +`specs/neutral/Default-spec.js` calls `registerScreenshotTests({ testName: 'Limestone', skin: 'neutral', … })`. That loops `.test-data.json` and, for each included case, registers a Playwright test. + +For **Chip, testId 3** on the Default shard (`concurrency: 1`, `PLAYWRIGHT_INSTANCES=5` → only ids where `testId % 5 === 0` on that shard; component runs use `PLAYWRIGHT_INSTANCES=1` so **all** Chip ids run): + +| Filter | Example | +|--------|---------| +| Shard | `testId % PLAYWRIGHT_INSTANCES === concurrency - 1` | +| Component | `PLAYWRIGHT_COMPONENT=Chip` (from `run-playwright-component.mjs`) | +| Single case | `PLAYWRIGHT_TEST_ID=3` | + +Playwright test title: `Chip~/Limestone~/~/3`. + +### 6. Open the case in the browser + +`openComponent` navigates to the built app (base URL port **4568**): + +`/Limestone-View/?locale=en-US&component=Chip&testId=3&skin=neutral&highContrast=false` + +`Limestone-View` calls `prepareTest('Chip', 3)`, clones `components.Chip[3]` from the bundle, applies theme/skin from query + per-case metadata, and renders `[data-ui-test-id="test"]`. + +### 7. Screenshot baseline + +`toHaveScreenshot` uses `getScreenshotName('Chip', 'Limestone', )` → +`snapshots/Chip/Limestone/.png` (same path layout as WDIO `dist/screenshots/reference/`). + +```text +apps/components/Chip.js + → LimestoneComponents.js (import + generateTestData) + → buildApps → tests/screenshot/dist/Limestone-View/ → global-setup ?request → .test-data.json - → registerScreenshotTests → GET …?component=Chip&testId=3&skin=… - → prepareTest in bundle → toHaveScreenshot + → Default-spec.js → registerScreenshotTests + → page: ?component=Chip&testId=3&skin=neutral&… + → prepareTest(Chip, 3) in bundle → toHaveScreenshot +``` + +**Quick run for that one case:** + +```bash +npm run test-playwright:component -- Chip --test-id 3 ``` ## Setup From 499bd522cd6c4bbcb22ab2f6ca41a1fa9e873769 Mon Sep 17 00:00:00 2001 From: Dan Ichim Date: Thu, 4 Jun 2026 17:23:28 +0300 Subject: [PATCH 05/28] revert ilib --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2f3189bc1..40041183e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@enovaui/core-tokens": "^2.0.0", "@enovaui/webos-tokens": "^2.0.0", "classnames": "^2.5.1", - "ilib": "^14.21.0 || ^14.21.0-webos1", + "ilib": "^14.22.0", "invariant": "^2.2.4", "prop-types": "^15.8.1", "ramda": "^0.32.0", diff --git a/package.json b/package.json index 16c6139a1..9e3fd8e22 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "@enovaui/core-tokens": "^2.0.0", "@enovaui/webos-tokens": "^2.0.0", "classnames": "^2.5.1", - "ilib": "^14.21.0 || ^14.21.0-webos1", + "ilib": "^14.22.0", "invariant": "^2.2.4", "prop-types": "^15.8.1", "ramda": "^0.32.0", From 43c4504670be0f47cf5942690642e9ef83977d18 Mon Sep 17 00:00:00 2001 From: Dan Ichim Date: Fri, 5 Jun 2026 11:11:19 +0300 Subject: [PATCH 06/28] update readme --- tests/screenshot/playwright/README.md | 421 +++++++++++++++++--------- 1 file changed, 286 insertions(+), 135 deletions(-) diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index 702b7c1ed..6b9491021 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -1,246 +1,397 @@ -# Playwright screenshot tests +# Playwright screenshot tests — developer guide -Visual regression with [Playwright](https://playwright.dev/) `toHaveScreenshot`, alongside WDIO (`npm run test-ss`). Scenarios live in `../apps/components/`; at test time Playwright only serves `../dist/` and picks cases via URL (`component` + `testId`), not by reading `.js` files on disk. +Visual regression for Limestone using [Playwright](https://playwright.dev/) `toHaveScreenshot`. This runs **alongside** the existing WebdriverIO suite (`npm run test-ss`). Both share the same scenario source, build output, and case indices — only the runner, port, spec files, and baseline folder differ. -## Layout +--- + +## Quick start + +From the **repository root**: + +```bash +npm install +npm run prepare-playwright # installs Chrome for Playwright (once per machine) + +# First time for a component — create PNG baselines (use parallel 1 for stability) +npm run test-playwright:component -- Sprite --update + +# Verify screenshots against baselines +npm run test-playwright:component -- Sprite + +# Compare Playwright vs WDIO runtime on one component +npm run benchmark-screenshots -- Chip +``` + +**Day-to-day:** prefer `test-playwright:component` over the full suite. Button alone has **205** cases; the full suite has **thousands** of screenshots across 27 shards. + +--- + +## How it fits with WDIO + +| | WebdriverIO | Playwright | +|--|-------------|------------| +| Scenario source | `tests/screenshot/apps/components/*.js` | Same | +| Build output | `tests/screenshot/dist/` via `buildApps('screenshot')` | Same | +| Static server | port **4567** | port **4568** (default) | +| Spec files | `tests/screenshot/specs/**/*-specs.js` | `tests/screenshot/playwright/specs/**/*-spec.js` | +| Baselines | `dist/screenshots/reference/` | `playwright/snapshots/` | +| Compare API | `@wdio/visual-service` | `expect(page).toHaveScreenshot()` | + +Playwright does **not** read `apps/components/*.js` at test time. It serves the **built** app from `dist/` and selects cases with URL query parameters (`component`, `testId`, `skin`, …). + +--- + +## Repository layout ``` tests/screenshot/ - apps/components/ ← scenario source (shared with WDIO) + apps/ + components/ ← scenario JSX (shared with WDIO) + LimestoneComponents.js + Limestone-View.js images/, videos/ - dist/ ← buildApps('screenshot') (shared) + dist/ ← buildApps('screenshot') output (shared) specs/ ← WDIO *-specs.js + scripts/ + run-playwright-component.mjs + run-component-wdio.mjs playwright/ ← this folder - specs/neutral|light/ *-spec.js shards + specs/neutral|light/ *-spec.js (27 shards) utils/ registerScreenshotTests, limestone-page, screenshot-name - snapshots/ PNG baselines: //.png (same layout as WDIO reference/) - benchmark.mjs - scripts/ run-playwright-component, run-component-wdio (npm shortcuts) + snapshots/ PNG baselines global-setup.js build, .test-data.json, serve :4568 if needed - global-teardown.js stops serve only when global-setup started it + global-teardown.js stops serve when global-setup started it playwright.config.mjs paths.js + benchmark.mjs ``` -| | WDIO | Playwright | -|--|------|------------| -| Scenarios / build | `../apps/`, `../dist/` | Same | -| Static server port | **4567** | **4568** (default) | -| Specs | `../specs/*-specs.js` | `specs/*-spec.js` | -| Assertions / baselines | `dist/screenshots/reference/` | `playwright/snapshots/` (same `//` paths) | - -## How a component reaches Playwright (e.g. Chip, testId 3) +--- -Playwright never opens `apps/components/Chip.js` at test time. That file is **source** for the screenshot app; the runner only serves the **built** bundle under `../dist/` and drives cases by URL query (`component` + `testId`). +## How a component reaches Playwright (example: Chip, testId 3) -### 1. Author scenarios in `apps/components/` +### 1. Author scenarios — `apps/components/Chip.js` -`Chip.js` exports an array of JSX cases (order = `testId`): +Each exported array entry is one case; **array index = `testId`**: ```js const ChipTests = [ - Default Chip, // testId 0 - …, // testId 1 - … - Chip with checkmark Icon, // testId 3 + Default Chip, // testId 0 + Chip with Icon, // testId 1 + Chip with checkmark Icon, // testId 3 … ]; export default ChipTests; ``` -`withConfig({…}, […])` expands into more entries (focus, `largeText`, etc.), so Chip ends up with ~51 cases — same list WDIO uses. +`withConfig({ focus: true }, […])` and `withConfig({ skinVariants: ['largeText'] }, […])` append more cases. Chip has **51** cases total (same as WDIO). -### 2. Register the component +### 2. Register — `apps/LimestoneComponents.js` -`apps/LimestoneComponents.js` imports `Chip` from `./components/Chip` and puts it in the `components` map. For each key it runs `generateTestData` (`@enact/ui-test-utils`) to build `testMetadata[Chip]`: one `{title, …}` per array index (titles come from props/children when the JSX has no explicit `title`). +`Chip` is imported into the `components` map. `generateTestData()` (`@enact/ui-test-utils`) produces `testMetadata.Chip`: one `{ title, … }` per index (titles are derived from JSX props when no explicit `title` is set). -### 3. Build into `../dist/` +### 3. Build — `buildApps('screenshot')` -`global-setup` (unless `PLAYWRIGHT_SKIP_BUILD=1`) calls `buildApps('screenshot')` from `@enact/ui-test-utils/build-apps`. That packs `Limestone-View` (entry `screenshot/index.js`) into `dist/Limestone-View/`. The bundle includes **all** registered components and `testMetadata` from step 2. Assets from `../images/` and `../videos/` are copied as part of the same build WDIO uses. +Triggered from `global-setup.js` unless `PLAYWRIGHT_SKIP_BUILD=1`. Packs `Limestone-View` into `dist/Limestone-View/`. The bundle contains all components and metadata. WDIO uses the same build. -### 4. Cache case list for the runner +### 4. Case index — `.test-data.json` -On first run (or when dist is newer than the cache), `global-setup` opens: +`global-setup` loads `http://localhost:4568/Limestone-View/?request`. That sets `window.__TEST_DATA = testMetadata` without rendering a case. The JSON is cached at `playwright/.test-data.json`. -`http://localhost:4568/Limestone-View/?request` +`registerScreenshotTests()` reads this file to register Playwright tests — it never parses `Chip.js` on disk. -With `?request`, the app sets `window.__TEST_DATA = testMetadata` and does not render a case. Playwright writes that JSON to `playwright/.test-data.json`, e.g.: +### 5. Spec shard — `specs/neutral/Default-spec.js` -```json -{ "Chip": [ { "title": "…" }, { "title": "…" }, … ] } +```js +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 1 // shard 1 of N (see sharding below) +}); ``` -`registerScreenshotTests` reads this file to know **which** `(component, testId, title)` pairs exist — it does not parse `Chip.js` again. +### 6. Browser URL -### 5. Playwright spec → one test per case +``` +/Limestone-View/?locale=en-US&component=Chip&testId=3&skin=neutral&highContrast=false +``` -`specs/neutral/Default-spec.js` calls `registerScreenshotTests({ testName: 'Limestone', skin: 'neutral', … })`. That loops `.test-data.json` and, for each included case, registers a Playwright test. +`Limestone-View` → `prepareTest('Chip', 3)` → renders `[data-ui-test-id="test"]`. -For **Chip, testId 3** on the Default shard (`concurrency: 1`, `PLAYWRIGHT_INSTANCES=5` → only ids where `testId % 5 === 0` on that shard; component runs use `PLAYWRIGHT_INSTANCES=1` so **all** Chip ids run): +### 7. Screenshot -| Filter | Example | -|--------|---------| -| Shard | `testId % PLAYWRIGHT_INSTANCES === concurrency - 1` | -| Component | `PLAYWRIGHT_COMPONENT=Chip` (from `run-playwright-component.mjs`) | -| Single case | `PLAYWRIGHT_TEST_ID=3` | +`toHaveScreenshot(getScreenshotName('Chip', 'Limestone', ))` writes/compares under `playwright/snapshots/` (filename encodes component, test name, and case title — same naming rules as WDIO reference PNGs). -Playwright test title: `Chip~/Limestone~/<case title>~/3`. +**One case only:** -### 6. Open the case in the browser +```bash +npm run test-playwright:component -- Chip --test-id 3 +``` -`openComponent` navigates to the built app (base URL port **4568**): +--- -`/Limestone-View/?locale=en-US&component=Chip&testId=3&skin=neutral&highContrast=false` +## Setup (detailed) -`Limestone-View` calls `prepareTest('Chip', 3)`, clones `components.Chip[3]` from the bundle, applies theme/skin from query + per-case metadata, and renders `[data-ui-test-id="test"]`. +### Prerequisites -### 7. Screenshot baseline +- Node.js + npm (same as the rest of Limestone) +- Chrome (Playwright uses `channel: 'chrome'` from `playwright.config.mjs`) +- `@enact/ui-test-utils@^4.0.2` (declared in root `package.json`, same as `test-ss`) -`toHaveScreenshot` uses `getScreenshotName('Chip', 'Limestone', <case title>)` → -`snapshots/Chip/Limestone/<sanitized-title>.png` (same path layout as WDIO `dist/screenshots/reference/`). +### Install -```text -apps/components/Chip.js - → LimestoneComponents.js (import + generateTestData) - → buildApps → tests/screenshot/dist/Limestone-View/ - → global-setup ?request → .test-data.json - → Default-spec.js → registerScreenshotTests - → page: ?component=Chip&testId=3&skin=neutral&… - → prepareTest(Chip, 3) in bundle → toHaveScreenshot +```bash +npm install +npm run prepare-playwright ``` -**Quick run for that one case:** +`prepare-playwright` runs `playwright install chrome`. It is **not** hooked to `postinstall` — run it once per machine (CI runs it via `build-scripts/enact-playwright-tests.sh`). -```bash -npm run test-playwright:component -- Chip --test-id 3 +### Build screenshot dist + +The first Playwright run builds `tests/screenshot/dist/` automatically. To reuse an existing build: + +```powershell +$env:PLAYWRIGHT_SKIP_BUILD='1' # PowerShell +# export PLAYWRIGHT_SKIP_BUILD=1 # bash ``` -## Setup +Or build via WDIO / `buildApps` before running Playwright. + +### Baselines + +Playwright needs PNG baselines under `playwright/snapshots/`. Without them, tests fail with “snapshot doesn't exist”. ```bash -npm install -npm run prepare-playwright +# One component (recommended) +npm run test-playwright:component -- Sprite --update + +# Entire suite (long — thousands of PNGs) +npm run test-playwright:update ``` -Uses `@enact/ui-test-utils@^4.0.2` from npm (same as `test-ss`). +**Tip:** use `--parallel 1` when running `--update` on components with **focus** cases (Chip, Button). Parallel updates can produce slightly different focus pixels and cause flaky compares on later parallel runs. -## Commands +--- -Run from the **repository root**: +## Running tests -| Command | Description | -|---------|-------------| -| `npm run test-playwright` | Full suite — all components, 27 shards (neutral/light, Default/HighContrast) | -| `npm run test-playwright:update` | Update all baselines | -| `npm run test-playwright:report` | HTML report | -| `npm run test-playwright:component -- Sprite` | One component, all cases (e.g. Sprite ≈ 4, Spinner ≈ 6, Button ≈ 205) | -| `npm run test-playwright:component -- Sprite --test-id 0` | Single case by index | -| `npm run test-playwright:component -- Sprite --update` | Refresh baselines | -| `npm run test-ss:component -- Sprite` | WDIO, same component (Default shard) | -| `npm run benchmark-screenshots -- Chip` | Playwright vs WDIO timing (`--build` to rebuild dist) | -| `npm run benchmark-screenshots -- Chip --parallel 5` | Same, up to 5 cases at a time per runner | +All commands run from the **repository root**. -Without baselines, add `--update` once. +### Command reference -## Full suite (all components) +| Command | What it does | +|---------|----------------| +| `npm run test-playwright` | Full suite — all components, all Playwright shards | +| `npm run test-playwright:update` | Update all Playwright baselines | +| `npm run test-playwright:report` | Open HTML report (`playwright/reports/html/`) | +| `npm run test-playwright:component -- <Name>` | One component, all its cases (Default neutral shard) | +| `npm run test-playwright:component -- <Name> --update` | Refresh baselines for that component | +| `npm run test-playwright:component -- <Name> --test-id <n>` | Single case by index | +| `npm run test-playwright:component -- <Name> --parallel <n>` | Same cases, `n` Playwright workers | +| `npm run test-ss:component -- <Name>` | WDIO equivalent (for comparison) | +| `npm run benchmark-screenshots -- <Name>` | Time Playwright then WDIO for one component | -Runs every registered component across all Playwright shards (`specs/neutral`, `specs/light`). Expect a long run (thousands of screenshots). +### Single component (recommended) ```bash -npm install -npm run prepare-playwright +# Verify (skip rebuild if dist exists) +PLAYWRIGHT_SKIP_BUILD=1 npm run test-playwright:component -- Button + +# Faster local run +PLAYWRIGHT_SKIP_BUILD=1 npm run test-playwright:component -- Button --parallel 5 + +# Filter by title regex +npm run test-playwright:component -- Button --title "Focused" +``` + +Component script sets `PLAYWRIGHT_INSTANCES=1` (all `testId`s for that component) and `PLAYWRIGHT_WORKERS` from `--parallel` (default **1**). + +### Full suite -# First time or after UI / scenario changes — creates/updates all baselines +Runs every registered component across **27** spec shards (`Default`…`Default9`, `HighContrast*`, `Light*` under `specs/neutral` and `specs/light`). + +```bash +npm install && npm run prepare-playwright + +# First time or after UI changes npm run test-playwright:update -# Verify against baselines (reuses dist build when possible) -$env:PLAYWRIGHT_SKIP_BUILD='1' # PowerShell; omit on first run -npm run test-playwright +# Verify +PLAYWRIGHT_SKIP_BUILD=1 npm run test-playwright -# Browse results (list, timings, failures with diffs) +# HTML report npm run test-playwright:report ``` -| Step | Output | -|------|--------| -| `--update` | `playwright/snapshots/<Component>/Limestone/*.png` | -| test run | Terminal summary; failures in `playwright/test-results/` | -| `:report` | `playwright/reports/html/index.html` | - -Optional: more workers locally — `$env:PLAYWRIGHT_WORKERS='8'; npm run test-playwright` (default **5**). +**Parallelism in full suite:** `playwright.config.mjs` defaults to **5 workers** (`PLAYWRIGHT_WORKERS`). Shards split cases by `testId % PLAYWRIGHT_INSTANCES` (default instances **5**). -**Subset of shards** (same as Jenkins `SPEC` / TV): +**Limit shards** (same idea as Jenkins `SPEC`): | `PLAYWRIGHT_SPEC` | Runs | |-------------------|------| -| *(unset)* | All `*-spec.js` files (27 shards) | +| *(unset)* | All `*-spec.js` files | | `Default` | `Default-spec.js` … `Default9-spec.js` | | `HighContrast` | `HighContrast-spec.js` … `HighContrast9-spec.js` | | `Light` | `Light-spec.js` … `Light9-spec.js` | -| `Default-spec` | Only `specs/neutral/Default-spec.js` (TV-style single shard) | +| `Default-spec` | Only `specs/neutral/Default-spec.js` (TV-style) | ```powershell $env:PLAYWRIGHT_SPEC='Default' +$env:PLAYWRIGHT_SKIP_BUILD='1' npm run test-playwright ``` -## Benchmark (local) +### Sharding vs workers -Compare end-to-end runtime for one component (Default shard, all cases, existing `tests/screenshot/dist`): +| Concept | Env / flag | Meaning | +|---------|------------|---------| +| **Shard** | `concurrency` in spec + `PLAYWRIGHT_INSTANCES` | Which `testId`s this spec file owns (`testId % instances === concurrency - 1`) | +| **Workers** | `PLAYWRIGHT_WORKERS` or `--parallel` on component script | How many tests run at once in one Playwright process | +| **Component filter** | `PLAYWRIGHT_COMPONENT` | Only one component's cases | + +Full suite: many spec files × 5 workers each. Component run: one spec file, all cases, N workers. + +--- + +## Viewing results + +| Output | Location | +|--------|----------| +| Playwright HTML report | `npm run test-playwright:report` → `playwright/reports/html/index.html` | +| Playwright baselines | `playwright/snapshots/` | +| Playwright failures | `playwright/test-results/` (actual, diff, error-context) | +| WDIO failures | `tests/screenshot/dist/failedTests.html`, `newFiles.html` | +| WDIO baselines | `tests/screenshot/dist/screenshots/reference/` | + +--- + +## Benchmarking + +Compare end-to-end runtime for **one component** (Playwright first, then WDIO, same `dist/`): ```bash -npm run benchmark-screenshots -- Chip -npm run benchmark-screenshots -- Chip --parallel 5 # faster: 5 workers / 5 WDIO browsers -# optional: --build to rebuild dist; PLAYWRIGHT_SKIP_BUILD=1 when dist exists +npm run benchmark-screenshots -- Sprite +npm run benchmark-screenshots -- Chip --parallel 5 +npm run benchmark-screenshots -- Button --parallel 5 + +# Rebuild dist first if missing +npm run benchmark-screenshots -- Chip --build ``` -**Parallelism:** `--parallel N` (default `1`) runs every case for the component, but up to `N` at once. Playwright uses `PLAYWRIGHT_INSTANCES=1` (no shard filter) and `PLAYWRIGHT_WORKERS=N`. WDIO uses `--instances 1` (all cases) and `--parallel N` for `wdio:maxInstances` (requires `@enact/ui-test-utils` with `--parallel` support). +- **`--parallel N`** (default `1`): every case for the component, up to `N` at once. +- Playwright: `PLAYWRIGHT_INSTANCES=1`, `PLAYWRIGHT_WORKERS=N`. +- WDIO: `--instances 1` (all cases), `--parallel N` (requires `@enact/ui-test-utils` with `--parallel` support). +- Ports stay separate: Playwright **4568**, WDIO **4567**. -Both runners filter a single component name (WDIO uses `--component ^Name$` so `Chip` does not include `Chips`). Benchmark still runs Playwright then WDIO sequentially; ports **4568** / **4567** stay separate. +Benchmark requires **existing baselines** for both runners (Playwright `snapshots/`, WDIO `dist/screenshots/reference/`). Create Playwright baselines with `--update` before benchmarking. -| Component | Cases | Parallel | Playwright | WebdriverIO | Faster | -|-----------|------:|---------:|-----------:|--------------:|--------| -| Sprite | 4 | 1 | 16.7 s | 18.0 s | Playwright (~8%) | -| Chip | 51 | 1 | 99.2 s | 84.6 s | WDIO (~15%) | -| Chip | 51 | 5 | 58.9 s | 154.8 s | Playwright (~62%) | +--- -Measured on Windows (Chrome 132), June 2026; times include browser startup and screenshot compare. Benchmark runs Playwright then WDIO (not at the same time). +## Performance analysis -**Sequential (`--parallel 1`, default):** Chip ≈ 1.9 s/test (Playwright) vs ≈ 1.7 s/test (WDIO). +Measured **locally on Windows, Chrome 132, June 2026**. Times include browser startup, page load, and screenshot compare. `PLAYWRIGHT_SKIP_BUILD=1` (reused `tests/screenshot/dist`). -**Parallel 5 (`--parallel 5`):** Chip ≈ 1.2 s/test (Playwright) vs ≈ 3.0 s/test (WDIO). Playwright wall time drops (~41%); WDIO can be slower on one machine with many Chrome sessions (visual compare + resource contention) — CI uses sharded jobs plus `--parallel` instead of one large component run. +### Results -Full suite (`Button` ≈ 205 cases) is much slower — use component scripts for day-to-day runs. +| Component | Cases | Parallel | Playwright | WebdriverIO | Faster | PW s/case | WDIO s/case | +|-----------|------:|---------:|-----------:|--------------:|--------|----------:|------------:| +| Sprite | 4 | 1 | 16.2 s | 19.7 s | Playwright (~18%) | 4.1 | 4.9 | +| Sprite | 4 | 5 | 15.7 s | 17.9 s | Playwright (~12%) | 3.9 | 4.5 | +| Chip | 51 | 1 | 103.8 s | 83.9 s | WDIO (~19%) | 2.0 | 1.6 | +| Chip | 51 | 5 | 50.2 s † | 137.7 s | Playwright (~64%) | 1.0 | 2.7 | +| Button | 205 | 1 | 525.4 s | 1427.2 s | Playwright (~63%) | 2.6 | 7.0 | +| Button | 205 | 5 | 111.4 s † | 979.3 s | Playwright (~89%) | 0.5 | 4.8 | -**View results:** Playwright — `npm run test-playwright:report` and `snapshots/<Component>/Limestone/`. WDIO — `tests/screenshot/dist/newFiles.html` / `failedTests.html` and `dist/screenshots/reference/`. +† Parallel Playwright runs can **flake on focus cases** (spotlight timing). Wall time is still representative; use `--parallel 1` for `--update` on focus-heavy components, or re-run failed ids with `--test-id`. -**Parallel with WDIO:** default ports differ (WDIO **4567**, Playwright **4568**), both serve `../dist/`. Override with `PLAYWRIGHT_BASE_URL` if needed. `webServer` in config also uses `reuseExistingServer`; if `global-setup` had to start `serve`, `global-teardown` stops that process after the run. +### Takeaways -## Maintenance +1. **Small components (Sprite):** similar cost; Playwright slightly faster even at parallel 1. +2. **Medium (Chip), sequential:** WDIO is ~15–20% faster per case — one browser, mature visual-service path. +3. **Medium/large, parallel 5:** Playwright scales much better — shared server, lighter workers. Chip ~2× faster wall time; Button ~9× faster wall time vs WDIO parallel 5. +4. **Large (Button), sequential:** Playwright ~2.6 s/case vs WDIO ~7.0 s/case — WDIO session overhead dominates at 205 cases. +5. **Full suite:** not benchmarked here; use **component scripts** for daily work. CI should shard by `PLAYWRIGHT_SPEC` / `PLAYWRIGHT_INSTANCES` and set `PLAYWRIGHT_WORKERS` per agent (see `build-scripts/enact-playwright-tests.sh`). -- **New/changed UI case:** edit `../apps/components/<Name>.js`; register in `../apps/LimestoneComponents.js` if new; run tests (build unless `PLAYWRIGHT_SKIP_BUILD=1`); `--update` for PNGs. -- **New WDIO shard:** add matching `specs/**/<Name>-spec.js` with same `skin`, `highContrast`, `concurrency` as `../specs/`. Shards split by `testId % workers` (`Default` … `Default9`, `HighContrast*`, `Light*`). +### Making Playwright faster locally + +```powershell +$env:PLAYWRIGHT_SKIP_BUILD='1' +$env:PLAYWRIGHT_WORKERS='8' # full suite only; component uses --parallel +npm run test-playwright:component -- Button --parallel 8 +``` + +Diminishing returns appear when too many Chrome instances contend for CPU/RAM (WDIO shows this clearly at `--parallel 5` on one machine). + +--- ## Environment variables | Variable | Effect | |----------|--------| -| `PLAYWRIGHT_SKIP_BUILD=1` | Skip `buildApps('screenshot')` | -| `PLAYWRIGHT_REFRESH_TEST_DATA=1` | Regenerate `.test-data.json` | -| `PLAYWRIGHT_COMPONENT` | One component (set by `../scripts/run-playwright-component.mjs`) | -| `PLAYWRIGHT_TEST_ID` / `PLAYWRIGHT_TITLE` | Filter cases (or `--test-id` / `--title` on npm script) | -| `PLAYWRIGHT_INSTANCES` / `PLAYWRIGHT_WORKERS` | Shard index / parallelism | -| `PLAYWRIGHT_SPEC` | Limit spec files (`Default`, `Light`, `Default-spec`, …) — see [Full suite](#full-suite-all-components) | -| `PLAYWRIGHT_BASE_URL` | Playwright static server (default `http://localhost:4568`) | +| `PLAYWRIGHT_SKIP_BUILD=1` | Skip `buildApps('screenshot')` in global-setup | +| `PLAYWRIGHT_REFRESH_TEST_DATA=1` | Force-regenerate `.test-data.json` | +| `PLAYWRIGHT_COMPONENT` | Filter to one component (set by `run-playwright-component.mjs`) | +| `PLAYWRIGHT_TEST_ID` | Filter to one case index | +| `PLAYWRIGHT_TITLE` | Filter cases by title regex | +| `PLAYWRIGHT_INSTANCES` | Shard divisor (`testId % instances`); component script sets `1` | +| `PLAYWRIGHT_WORKERS` | Parallel workers in `playwright.config.mjs` (default **5**) | +| `PLAYWRIGHT_SPEC` | Limit which `*-spec.js` files run | +| `PLAYWRIGHT_BASE_URL` | Static server URL (default `http://localhost:4568`) | + +--- ## Troubleshooting | Issue | Fix | |-------|-----| -| WebServer timeout | Build `../dist/` (unset `PLAYWRIGHT_SKIP_BUILD` or `npm run test-ss`) | -| Snapshot does not exist | `--update` | -| No tests found | Check `PLAYWRIGHT_COMPONENT` / `PLAYWRIGHT_SPEC`; refresh `.test-data.json` | -| Missing images/videos on build | Use `../images/`, `../videos/` | -| Port in use | Change `PLAYWRIGHT_BASE_URL` (port in URL must match `webServer`) | +| WebServer / server timeout | Build `tests/screenshot/dist/` (unset `PLAYWRIGHT_SKIP_BUILD`) | +| Snapshot does not exist | `npm run test-playwright:component -- <Name> --update` | +| Snapshot diff on focus cases in parallel | Re-baseline with `--parallel 1`, or run failing `--test-id` alone | +| No tests found | Check `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_SPEC`; delete stale `.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1` | +| Missing images on build | Place assets under `tests/screenshot/images/`, `videos/` | +| Port already in use | Set `PLAYWRIGHT_BASE_URL` to another port (must match `webServer` in config) | +| `build-apps` import error | Requires `@enact/ui-test-utils` with export `./build-apps` (local link or published version) | +| WDIO `--parallel` ignored | Update `@enact/ui-test-utils` (see ui-test-utils PR for `--parallel` CLI) | + +--- + +## CI (planned) + +Jenkins job via `build-scripts/enact-playwright-tests.sh`: + +1. `npm run prepare-playwright` +2. Optionally seed `playwright/snapshots/` from Nebula reference +3. `npm run test-playwright` with `PLAYWRIGHT_SPEC`, `PLAYWRIGHT_WORKERS`, `PLAYWRIGHT_COMPONENT` +4. Upload `playwright/reports/html`, `snapshots/`, `test-results/` + +On TV: typically `PLAYWRIGHT_SPEC=Default-spec` (single shard). + +--- -**CI (planned):** Jenkins job `enact-playwright-tests` via `build-scripts/enact-playwright-tests.sh` — sets `PLAYWRIGHT_SPEC` (`Default-spec` on TV, or `SPEC` shard group), `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_WORKERS`. Uploads `playwright/` to Nebula: `reports/html`, `snapshots/`, `test-results/`. +## Maintenance + +### New or changed UI case + +1. Edit `tests/screenshot/apps/components/<Name>.js` +2. Register in `tests/screenshot/apps/LimestoneComponents.js` if new component +3. Run tests (build runs unless `PLAYWRIGHT_SKIP_BUILD=1`) +4. `npm run test-playwright:component -- <Name> --update` for new PNGs + +### New WDIO shard + +Add matching `playwright/specs/**/<Name>-spec.js` with the same `skin`, `highContrast`, and `concurrency` as the WDIO `*-specs.js` file. + +--- + +## Related npm scripts (root `package.json`) + +```json +"test-playwright": "playwright test --config tests/screenshot/playwright/playwright.config.mjs", +"test-playwright:update": "... --update-snapshots", +"test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", +"test-playwright:component": "node tests/screenshot/scripts/run-playwright-component.mjs", +"prepare-playwright": "playwright install chrome", +"benchmark-screenshots": "node tests/screenshot/playwright/benchmark.mjs" +``` From e484cb7d7d5d54d7d7815887fcce23b1d8aff0fc Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Fri, 5 Jun 2026 12:07:12 +0300 Subject: [PATCH 07/28] fix lint --- tests/screenshot/playwright/benchmark.mjs | 18 ++++++--- tests/screenshot/playwright/global-setup.js | 6 +-- .../playwright/playwright.config.mjs | 2 +- .../playwright/utils/run-screenshot-tests.js | 32 +++++++++------- .../playwright/utils/screenshot-name.js | 38 +++++++++---------- .../screenshot/scripts/run-component-wdio.mjs | 3 +- .../scripts/run-playwright-component.mjs | 3 +- 7 files changed, 58 insertions(+), 44 deletions(-) diff --git a/tests/screenshot/playwright/benchmark.mjs b/tests/screenshot/playwright/benchmark.mjs index 4cc0ba9af..eaf436ced 100644 --- a/tests/screenshot/playwright/benchmark.mjs +++ b/tests/screenshot/playwright/benchmark.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +/* eslint-disable no-console */ /** * Compare Playwright vs WebdriverIO runtime for one component (all its screenshot cases). * @@ -27,7 +28,7 @@ function flagValue (name) { const component = args.find(a => !a.startsWith('--')); const withBuild = args.includes('--build'); -const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1', 10) || 1); +const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1') || 1); if (!component) { console.error('Usage: npm run benchmark-screenshots -- <ComponentName> [--build] [--parallel <n>]'); @@ -64,9 +65,9 @@ function runCommand (label, command, commandArgs, env = {}) { } async function main () { - const parallelLabel = parallel === 1 - ? '1 worker / 1 browser (sequential)' - : `parallel ${parallel} (PW: workers=${parallel}, instances=1; WDIO: --parallel ${parallel}, --instances 1)`; + const parallelLabel = parallel === 1 ? + '1 worker / 1 browser (sequential)' : + `parallel ${parallel} (PW: workers=${parallel}, instances=1; WDIO: --parallel ${parallel}, --instances 1)`; console.log(`\nBenchmark: ${component} (all cases, ${parallelLabel})`); if (skipBuild) { @@ -99,13 +100,20 @@ async function main () { const pwSec = playwright.durationMs / 1000; const wdioSec = wdio.durationMs / 1000; const delta = wdioSec - pwSec; + let winner = 'tie'; + + if (delta > 0) { + winner = 'Playwright faster'; + } else if (delta < 0) { + winner = 'WDIO faster'; + } console.log('\n--- Results ---'); console.log(`Component: ${component}`); console.log(`Parallelism: ${parallel}`); console.log(`Playwright: ${pwSec.toFixed(1)}s`); console.log(`WebdriverIO: ${wdioSec.toFixed(1)}s`); - console.log(`Difference: ${Math.abs(delta).toFixed(1)}s (${delta > 0 ? 'Playwright faster' : delta < 0 ? 'WDIO faster' : 'tie'})\n`); + console.log(`Difference: ${Math.abs(delta).toFixed(1)}s (${winner})\n`); } main().catch(err => { diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js index fb84067a6..6d01683ff 100644 --- a/tests/screenshot/playwright/global-setup.js +++ b/tests/screenshot/playwright/global-setup.js @@ -75,18 +75,18 @@ export default async function globalSetup () { return; } - const browser = await chromium.launch({ + const pwBrowser = await chromium.launch({ channel: 'chrome', headless: true }); try { - const page = await browser.newPage(); + const page = await pwBrowser.newPage(); await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'networkidle'}); const testData = await page.evaluate(() => window.__TEST_DATA); fs.writeFileSync(testDataFile, JSON.stringify(testData)); } finally { - await browser.close(); + await pwBrowser.close(); } } diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index 9b9731c42..02ff09d0a 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -35,7 +35,7 @@ export default defineConfig({ fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, - workers: process.env.PLAYWRIGHT_WORKERS ? Number.parseInt(process.env.PLAYWRIGHT_WORKERS, 10) : 5, + workers: process.env.PLAYWRIGHT_WORKERS ? Number.parseInt(process.env.PLAYWRIGHT_WORKERS) : 5, reporter: [ ['list'], ['html', {outputFolder: path.join(__dirname, 'reports', 'html'), open: 'never'}] diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js index 88f7189f1..f839d4a95 100644 --- a/tests/screenshot/playwright/utils/run-screenshot-tests.js +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -1,18 +1,18 @@ import fs from 'fs'; -import {test, expect} from '@playwright/test'; +import {test as baseTest, expect as baseExpect} from '@playwright/test'; import {assertComponentSource, TEST_DATA_FILE} from '../paths.js'; import {getScreenshotName} from './screenshot-name.js'; import {openComponent} from './limestone-page.js'; -const testIdFilter = process.env.PLAYWRIGHT_TEST_ID != null - ? Number.parseInt(process.env.PLAYWRIGHT_TEST_ID, 10) - : null; +const testIdFilter = process.env.PLAYWRIGHT_TEST_ID != null ? + Number.parseInt(process.env.PLAYWRIGHT_TEST_ID) : + null; const titleFilter = process.env.PLAYWRIGHT_TITLE; -const maxInstances = process.env.PLAYWRIGHT_INSTANCES - ? Number.parseInt(process.env.PLAYWRIGHT_INSTANCES, 10) - : 5; +const maxInstances = process.env.PLAYWRIGHT_INSTANCES ? + Number.parseInt(process.env.PLAYWRIGHT_INSTANCES) : + 5; function loadTestData () { return JSON.parse(fs.readFileSync(TEST_DATA_FILE, 'utf8')); @@ -75,20 +75,24 @@ function getScreenshotTests ({testName, skin, highContrast, concurrency, compone export function registerScreenshotTests (config) { const component = resolveComponentFilter(config); - const suiteName = component - ? `${config.testName} — ${component}` - : (config.concurrency ? `${config.testName} (shard ${config.concurrency})` : config.testName); + let suiteName = config.testName; + + if (component) { + suiteName = `${config.testName} — ${component}`; + } else if (config.concurrency) { + suiteName = `${config.testName} (shard ${config.concurrency})`; + } const cases = getScreenshotTests(config); - test.describe(suiteName, () => { - test.beforeEach(async ({page}) => { + baseTest.describe(suiteName, () => { + baseTest.beforeEach(async ({page}) => { await page.setViewportSize({width: 1920, height: 1167}); }); for (const screenshotTest of cases) { - test(screenshotTest.title, async ({page}) => { + baseTest(screenshotTest.title, async ({page}) => { await openComponent(page, screenshotTest.params); - await expect(page).toHaveScreenshot( + await baseExpect(page).toHaveScreenshot( getScreenshotName( screenshotTest.params.component, config.testName, diff --git a/tests/screenshot/playwright/utils/screenshot-name.js b/tests/screenshot/playwright/utils/screenshot-name.js index 36a090581..d44cd84a4 100644 --- a/tests/screenshot/playwright/utils/screenshot-name.js +++ b/tests/screenshot/playwright/utils/screenshot-name.js @@ -1,19 +1,19 @@ -import crypto from 'crypto'; -import path from 'path'; - -/** - * Case file name only (WDIO visual-service convention). - * @see ui-test-utils/utils/runTest.js, screenshot/utils/confHelpers.js - */ -export function getScreenshotBasename (title) { - let testCaseName = title.replace(/[/\\:?*"|<>]/g, '_'); - testCaseName = testCaseName.substring(0, 128) + '-' + crypto.createHash('md5').update(testCaseName).digest('hex'); - - return `${testCaseName}.png`.replace(/ /g, '_'); -} - -/** Relative path: <Component>/<TestName>/<case>.png (matches WDIO reference layout). */ -export function getScreenshotName (component, testName, title) { - return path.posix.join(component, testName, getScreenshotBasename(title)); -} - \ No newline at end of file +import {createHash} from 'node:crypto'; +import path from 'path'; + +/** + * Case file name only (WDIO visual-service convention). + * @see ui-test-utils/utils/runTest.js, screenshot/utils/confHelpers.js + */ +export function getScreenshotBasename (title) { + let testCaseName = title.replace(/[/\\:?*"|<>]/g, '_'); + testCaseName = testCaseName.substring(0, 128) + '-' + createHash('md5').update(testCaseName).digest('hex'); + + return `${testCaseName}.png`.replace(/ /g, '_'); +} + +/** Relative path: <Component>/<TestName>/<case>.png (matches WDIO reference layout). */ +export function getScreenshotName (component, testName, title) { + return path.posix.join(component, testName, getScreenshotBasename(title)); +} + diff --git a/tests/screenshot/scripts/run-component-wdio.mjs b/tests/screenshot/scripts/run-component-wdio.mjs index 52023577c..7be5b3787 100644 --- a/tests/screenshot/scripts/run-component-wdio.mjs +++ b/tests/screenshot/scripts/run-component-wdio.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +/* eslint-disable no-console */ /** * Run WDIO screenshot tests for one component (Default shard, instances 1 = all cases). * @@ -24,7 +25,7 @@ function flagValue (name) { const component = args.find(a => !a.startsWith('--')); const skipBuild = args.includes('--skip-build'); -const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1', 10) || 1); +const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1') || 1); if (!component) { console.error('Usage: npm run test-ss:component -- <ComponentName> [--skip-build] [--parallel <n>]'); diff --git a/tests/screenshot/scripts/run-playwright-component.mjs b/tests/screenshot/scripts/run-playwright-component.mjs index 1afd6816b..85ba6998a 100644 --- a/tests/screenshot/scripts/run-playwright-component.mjs +++ b/tests/screenshot/scripts/run-playwright-component.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +/* eslint-disable no-console */ /** * Run Playwright screenshot tests for one component (all cases; INSTANCES=1, WORKERS from --parallel). * @@ -32,7 +33,7 @@ const component = args.find(a => !a.startsWith('--') && args.indexOf(a) === args const update = args.includes('--update'); const testId = flagValue('--test-id'); const title = flagValue('--title'); -const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1', 10) || 1); +const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1') || 1); if (!component) { console.error('Usage: npm run test-playwright:component -- <ComponentName> [options]'); From 8ba5d1b3a1e5cebb937216831bc99d464e133aee Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Mon, 8 Jun 2026 18:04:32 +0200 Subject: [PATCH 08/28] fix timeout --- tests/screenshot/playwright/playwright.config.mjs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index 02ff09d0a..880ce10a8 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -59,10 +59,13 @@ export default defineConfig({ } }, webServer: { - command: `npx --yes serve ${distDir} -l ${PLAYWRIGHT_PORT}`, + command: process.platform === 'win32' + ? `npx.cmd --yes serve "${distDir}" -l ${PLAYWRIGHT_PORT}` + : `npx --yes serve ${distDir} -l ${PLAYWRIGHT_PORT}`, url: PLAYWRIGHT_BASE_URL, reuseExistingServer: true, - timeout: 120000 + timeout: 120000, + shell: process.platform === 'win32' }, projects: [ { From a1af9c319c55418f757b9c0867124e67a068fbc5 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Tue, 9 Jun 2026 10:18:12 +0200 Subject: [PATCH 09/28] add review changes --- .gitignore | 1 + tests/screenshot/playwright/README.md | 116 ++++++++++++++++-- tests/screenshot/playwright/benchmark.mjs | 16 +-- tests/screenshot/playwright/global-setup.js | 12 +- .../screenshot/playwright/global-teardown.js | 19 ++- tests/screenshot/playwright/paths.js | 19 +++ .../playwright/playwright.config.mjs | 8 +- .../playwright/utils/limestone-page.js | 3 +- .../playwright/utils/run-screenshot-tests.js | 4 +- .../playwright/utils/shard-registry.js | 65 ++++++++++ .../scripts/parse-component-args.mjs | 41 +++++++ .../scripts/run-playwright-component.mjs | 14 +-- 12 files changed, 272 insertions(+), 46 deletions(-) create mode 100644 tests/screenshot/playwright/utils/shard-registry.js create mode 100644 tests/screenshot/scripts/parse-component-args.mjs diff --git a/.gitignore b/.gitignore index 87ec9668a..bcac744df 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ tests/screenshot/playwright/test-results tests/screenshot/playwright/reports tests/screenshot/playwright/blob-report tests/screenshot/playwright/.test-data.json +tests/screenshot/playwright/.shard-registry.jsonl # Mac .DS_Store diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index 6b9491021..4fee828a1 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -10,7 +10,9 @@ From the **repository root**: ```bash npm install -npm run prepare-playwright # installs Chrome for Playwright (once per machine) +npm run bootstrap # enact CLI — required for the screenshot build +npm link @enact/ui-test-utils # see “Link local ui-test-utils” below until npm publishes ./build-apps +npm run prepare-playwright # optional if Chrome is already available (see below) # First time for a component — create PNG baselines (use parallel 1 for stability) npm run test-playwright:component -- Sprite --update @@ -36,6 +38,7 @@ npm run benchmark-screenshots -- Chip | Spec files | `tests/screenshot/specs/**/*-specs.js` | `tests/screenshot/playwright/specs/**/*-spec.js` | | Baselines | `dist/screenshots/reference/` | `playwright/snapshots/` | | Compare API | `@wdio/visual-service` | `expect(page).toHaveScreenshot()` | +| Viewport | 1920×**1167** (WDIO window) | 1920×**1080** (FHD) | Playwright does **not** read `apps/components/*.js` at test time. It serves the **built** app from `dist/` and selects cases with URL query parameters (`component`, `testId`, `skin`, …). @@ -55,12 +58,14 @@ tests/screenshot/ scripts/ run-playwright-component.mjs run-component-wdio.mjs + parse-component-args.mjs playwright/ ← this folder specs/neutral|light/ *-spec.js (27 shards) - utils/ registerScreenshotTests, limestone-page, screenshot-name - snapshots/ PNG baselines - global-setup.js build, .test-data.json, serve :4568 if needed - global-teardown.js stops serve when global-setup started it + utils/ registerScreenshotTests, limestone-page, screenshot-name, shard-registry + snapshots/ PNG baselines (gitignored — generate with --update) + .shard-registry.jsonl run-time shard log (gitignored; see shard-registry.js) + global-setup.js build, dist check, .test-data.json, serve :4568 if needed + global-teardown.js shard coverage check, stops serve when global-setup started it playwright.config.mjs paths.js benchmark.mjs @@ -143,10 +148,13 @@ npm run test-playwright:component -- Chip --test-id 3 ```bash npm install -npm run prepare-playwright +npm run bootstrap +npm run prepare-playwright # optional on machines that already have Chrome (see below) ``` -`prepare-playwright` runs `playwright install chrome`. It is **not** hooked to `postinstall` — run it once per machine (CI runs it via `build-scripts/enact-playwright-tests.sh`). +`prepare-playwright` runs `playwright install chrome`. It is **not** hooked to `postinstall`. Run it on **fresh machines and CI** (`build-scripts/enact-playwright-tests.sh`). On a machine where Chrome is already installed, Playwright prints *"chrome" is already installed on the system!"* — that is success; you can skip this step on later runs. + +`global-setup.js` calls `assertScreenshotDist()` after the build step. If `tests/screenshot/dist/Limestone-View/index.html` is missing, the run fails immediately with a clear error (not a 120s webServer timeout). ### Build screenshot dist @@ -159,6 +167,40 @@ $env:PLAYWRIGHT_SKIP_BUILD='1' # PowerShell Or build via WDIO / `buildApps` before running Playwright. +### Link local `@enact/ui-test-utils` (until `./build-apps` is published) + +Playwright `global-setup.js` imports `@enact/ui-test-utils/build-apps`. The npm release **4.0.2** does not export that path yet; the local `ui-test-utils` repo does (see `exports["./build-apps"]` in its `package.json`). Until a new version is published, link the local package: + +```bash +# From your ui-test-utils clone (must include "./build-apps": "./src/build-apps.js" in exports) +cd ../ui-test-utils +npm install +npm link + +# From limestone repo root +cd ../limestone +npm link @enact/ui-test-utils +npm install # refresh lockfile/node_modules if needed +``` + +Alternative without global link — in limestone root `package.json`: + +```json +"@enact/ui-test-utils": "file:../ui-test-utils" +``` + +Then `npm install` from limestone root. + +To undo `npm link`: + +```bash +cd limestone +npm unlink @enact/ui-test-utils +npm install +``` + +When `@enact/ui-test-utils` is published with the `./build-apps` export, remove the link and use the registry version from `package.json` again. + ### Baselines Playwright needs PNG baselines under `playwright/snapshots/`. Without them, tests fail with “snapshot doesn't exist”. @@ -206,14 +248,41 @@ PLAYWRIGHT_SKIP_BUILD=1 npm run test-playwright:component -- Button --parallel 5 npm run test-playwright:component -- Button --title "Focused" ``` -Component script sets `PLAYWRIGHT_INSTANCES=1` (all `testId`s for that component) and `PLAYWRIGHT_WORKERS` from `--parallel` (default **1**). +Component script sets `PLAYWRIGHT_INSTANCES=1` (all `testId`s for that component), `PLAYWRIGHT_WORKERS` from `--parallel` (default **1**), and runs only **`specs/neutral/Default-spec.js`** (neutral skin, no high contrast). Light / high-contrast cases require the full suite or a different spec shard. + +#### `scripts/parse-component-args.mjs` — why it exists + +npm passes arguments after `--` to the Node script. A naive parser such as `args.find(a => !a.startsWith('--'))` treats **flag values** as the component name — e.g. `… --test-id 0 Sprite` would resolve component `"0"` instead of `"Sprite"`. + +`parseComponentArgs()` is a small shared parser used by: + +- `run-playwright-component.mjs` — `--update`, `--test-id`, `--title`, `--parallel` +- `benchmark.mjs` — `--build`, `--parallel` + +**Rules:** + +- The **component name** is the first positional token (non-flag argument). +- Flags with values (`--test-id`, `--title`, `--parallel`) consume the next token as their value, not as the component. +- Boolean flags: `--update`, `--build`, `--skip-build`. + +**Examples:** + +| Command | Component resolved | +|---------|-------------------| +| `npm run test-playwright:component -- Button --parallel 5` | `Button` | +| `npm run test-playwright:component -- Button --test-id 0` | `Button` | +| `npm run benchmark-screenshots -- Chip --build` | `Chip` | + +Put the component name **first** after `--` (as in the examples above). `run-component-wdio.mjs` still uses the older parser — use the same name-first order there too. ### Full suite Runs every registered component across **27** spec shards (`Default`…`Default9`, `HighContrast*`, `Light*` under `specs/neutral` and `specs/light`). ```bash -npm install && npm run prepare-playwright +npm install && npm run bootstrap +npm link @enact/ui-test-utils # until ./build-apps is on npm (see above) +npm run prepare-playwright # First time or after UI changes npm run test-playwright:update @@ -251,6 +320,26 @@ npm run test-playwright | **Workers** | `PLAYWRIGHT_WORKERS` or `--parallel` on component script | How many tests run at once in one Playwright process | | **Component filter** | `PLAYWRIGHT_COMPONENT` | Only one component's cases | +**Shard coverage:** `PLAYWRIGHT_INSTANCES` must match the shard files you actually run. If you set `PLAYWRIGHT_INSTANCES=9` but only invoke shards 1–5, cases with `testId % 9` in `{5,6,7,8}` are silently skipped. `global-teardown` fails the run when it detects this partial coverage (single-shard runs such as TV-style `Default-spec` only are exempt). Shards with `concurrency > PLAYWRIGHT_INSTANCES` are inert (empty) — expected when the default instances is **5** but nine shard files exist per skin. + +#### `utils/shard-registry.js` — why it exists + +Each `*-spec.js` file is independent: it only knows its own `concurrency` value and cannot see which other spec files ran in the same Playwright process. That makes **partial shard runs look successful** — the suite is green even when whole slices of `testId`s were never rendered. + +`shard-registry.js` collects which shards actually loaded and validates coverage at the end of the run: + +| Step | Where | What | +|------|--------|------| +| Reset | `global-setup.js` → `clearShardRegistry()` | Deletes `playwright/.shard-registry.jsonl` from any previous run | +| Record | `registerScreenshotTests()` → `recordShard()` | Appends one JSON line per spec file: skin, highContrast, `concurrency`, `PLAYWRIGHT_INSTANCES` | +| Validate | `global-teardown.js` → `validateShardCoverage()` | For each skin/HC group, ensures shards `1…PLAYWRIGHT_INSTANCES` all ran when more than one shard was invoked | + +**Skipped when** `PLAYWRIGHT_COMPONENT` is set (component script uses `PLAYWRIGHT_INSTANCES=1` and a single spec — no cross-shard risk). + +**Example failure:** `PLAYWRIGHT_INSTANCES=9` but only `Default-spec.js` … `Default5-spec.js` are on the command line → teardown throws *Incomplete Playwright shard coverage* listing missing shards. + +The registry file is **gitignored** (`.shard-registry.jsonl`); it is a run-time artifact only. + Full suite: many spec files × 5 workers each. Component run: one spec file, all cases, N workers. --- @@ -302,7 +391,7 @@ Measured **locally on Windows, Chrome 132, June 2026**. Times include browser st | Chip | 51 | 1 | 103.8 s | 83.9 s | WDIO (~19%) | 2.0 | 1.6 | | Chip | 51 | 5 | 50.2 s † | 137.7 s | Playwright (~64%) | 1.0 | 2.7 | | Button | 205 | 1 | 525.4 s | 1427.2 s | Playwright (~63%) | 2.6 | 7.0 | -| Button | 205 | 5 | 111.4 s † | 979.3 s | Playwright (~89%) | 0.5 | 4.8 | +| Button | 205 | 5 | 96 s † | 979.3 s | Playwright (~89%) | 0.5 | 4.8 | † Parallel Playwright runs can **flake on focus cases** (spotlight timing). Wall time is still representative; use `--parallel 1` for `--update` on focus-heavy components, or re-run failed ids with `--test-id`. @@ -346,13 +435,15 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | Issue | Fix | |-------|-----| -| WebServer / server timeout | Build `tests/screenshot/dist/` (unset `PLAYWRIGHT_SKIP_BUILD`) | +| WebServer / server timeout | Ensure `tests/screenshot/dist/Limestone-View/index.html` exists (unset `PLAYWRIGHT_SKIP_BUILD`, run `npm run bootstrap`) | +| Missing dist / build failed | Read the `assertScreenshotDist()` error; link `@enact/ui-test-utils` if you see `./build-apps` export error | | Snapshot does not exist | `npm run test-playwright:component -- <Name> --update` | | Snapshot diff on focus cases in parallel | Re-baseline with `--parallel 1`, or run failing `--test-id` alone | | No tests found | Check `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_SPEC`; delete stale `.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1` | | Missing images on build | Place assets under `tests/screenshot/images/`, `videos/` | | Port already in use | Set `PLAYWRIGHT_BASE_URL` to another port (must match `webServer` in config) | -| `build-apps` import error | Requires `@enact/ui-test-utils` with export `./build-apps` (local link or published version) | +| `build-apps` import error | Link or pin `@enact/ui-test-utils` with export `./build-apps` (see **Link local @enact/ui-test-utils** above) | +| Incomplete shard coverage | Ensure `PLAYWRIGHT_INSTANCES` matches invoked shard files (see **Sharding vs workers**) | | WDIO `--parallel` ignored | Update `@enact/ui-test-utils` (see ui-test-utils PR for `--parallel` CLI) | --- @@ -392,6 +483,7 @@ Add matching `playwright/specs/**/<Name>-spec.js` with the same `skin`, `highCon "test-playwright:update": "... --update-snapshots", "test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", "test-playwright:component": "node tests/screenshot/scripts/run-playwright-component.mjs", +"test-ss:component": "node tests/screenshot/scripts/run-component-wdio.mjs", "prepare-playwright": "playwright install chrome", "benchmark-screenshots": "node tests/screenshot/playwright/benchmark.mjs" ``` diff --git a/tests/screenshot/playwright/benchmark.mjs b/tests/screenshot/playwright/benchmark.mjs index eaf436ced..b23327ba1 100644 --- a/tests/screenshot/playwright/benchmark.mjs +++ b/tests/screenshot/playwright/benchmark.mjs @@ -14,21 +14,13 @@ import fs from 'fs'; import path from 'path'; import {fileURLToPath} from 'url'; -import {assertComponentSource, SCREENSHOT_DIST} from './paths.js'; +import {assertComponentSource, SCREENSHOT_VIEW_INDEX} from './paths.js'; +import {parseComponentArgs} from '../scripts/parse-component-args.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.join(__dirname, '..', '..', '..'); -const args = process.argv.slice(2).filter(a => a !== '--'); - -function flagValue (name) { - const index = args.indexOf(name); - return index === -1 ? null : args[index + 1]; -} - -const component = args.find(a => !a.startsWith('--')); -const withBuild = args.includes('--build'); -const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1') || 1); +const {component, withBuild, parallel} = parseComponentArgs(); if (!component) { console.error('Usage: npm run benchmark-screenshots -- <ComponentName> [--build] [--parallel <n>]'); @@ -40,7 +32,7 @@ if (!component) { assertComponentSource(component); -const distIndex = path.join(SCREENSHOT_DIST, 'Limestone-View', 'index.html'); +const distIndex = SCREENSHOT_VIEW_INDEX; const skipBuild = !withBuild && fs.existsSync(distIndex); function runCommand (label, command, commandArgs, env = {}) { diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js index 6d01683ff..71ea6c8f5 100644 --- a/tests/screenshot/playwright/global-setup.js +++ b/tests/screenshot/playwright/global-setup.js @@ -1,11 +1,11 @@ import fs from 'fs'; -import path from 'path'; import {spawn} from 'child_process'; import buildApps from '@enact/ui-test-utils/build-apps'; import {chromium} from '@playwright/test'; -import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST, TEST_DATA_FILE} from './paths.js'; +import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST, SCREENSHOT_VIEW_INDEX, TEST_DATA_FILE, assertScreenshotDist} from './paths.js'; +import {clearShardRegistry} from './utils/shard-registry.js'; const baseURL = PLAYWRIGHT_BASE_URL; const distPath = SCREENSHOT_DIST; @@ -52,7 +52,7 @@ function shouldRefreshTestData () { if (!fs.existsSync(testDataFile)) { return true; } - const distIndex = path.join(distPath, 'Limestone-View', 'index.html'); + const distIndex = SCREENSHOT_VIEW_INDEX; if (!fs.existsSync(distIndex)) { return true; } @@ -60,10 +60,16 @@ function shouldRefreshTestData () { } export default async function globalSetup () { + clearShardRegistry(); + if (!process.env.PLAYWRIGHT_SKIP_BUILD) { await buildApps('screenshot'); } + assertScreenshotDist(); + + // Reuse an existing server on PLAYWRIGHT_BASE_URL when present; otherwise start `serve` + // so global-setup can fetch test metadata before Playwright's webServer hook runs. try { await fetch(baseURL); } catch { diff --git a/tests/screenshot/playwright/global-teardown.js b/tests/screenshot/playwright/global-teardown.js index 18eb75e44..cf91fb2f7 100644 --- a/tests/screenshot/playwright/global-teardown.js +++ b/tests/screenshot/playwright/global-teardown.js @@ -1,11 +1,28 @@ +import {spawnSync} from 'child_process'; + import {staticServer} from './global-setup.js'; +import {validateShardCoverage} from './utils/shard-registry.js'; + +function killProcessTree (childProcess) { + if (process.platform === 'win32') { + spawnSync('taskkill', ['/T', '/F', '/PID', String(childProcess.pid)], { + stdio: 'ignore', + shell: true + }); + return; + } + + childProcess.kill(); +} export default async function globalTeardown () { + validateShardCoverage(); + if (!staticServer.started || !staticServer.process) { return; } - staticServer.process.kill(); + killProcessTree(staticServer.process); staticServer.process = null; staticServer.started = false; } diff --git a/tests/screenshot/playwright/paths.js b/tests/screenshot/playwright/paths.js index b22fd618f..23c4a8163 100644 --- a/tests/screenshot/playwright/paths.js +++ b/tests/screenshot/playwright/paths.js @@ -10,6 +10,7 @@ export const PLAYWRIGHT_BASE_URL = process.env.PLAYWRIGHT_BASE_URL || 'http://lo export const PLAYWRIGHT_PORT = new URL(PLAYWRIGHT_BASE_URL).port || '4568'; export const SCREENSHOT_DIST = path.join(screenshotRoot, 'dist'); +export const SCREENSHOT_VIEW_INDEX = path.join(SCREENSHOT_DIST, 'Limestone-View', 'index.html'); export const SCREENSHOT_COMPONENTS = path.join(screenshotRoot, 'apps', 'components'); export const TEST_DATA_FILE = path.join(__dirname, '.test-data.json'); @@ -17,6 +18,24 @@ function componentSourcePath (componentName) { return path.join(SCREENSHOT_COMPONENTS, `${componentName}.js`); } +export function assertScreenshotDist () { + if (fs.existsSync(SCREENSHOT_VIEW_INDEX)) { + return; + } + + if (process.env.PLAYWRIGHT_SKIP_BUILD) { + throw new Error( + `Missing ${SCREENSHOT_VIEW_INDEX}. ` + + 'Unset PLAYWRIGHT_SKIP_BUILD to build automatically, or run npm run test-ss once to populate tests/screenshot/dist/.' + ); + } + + throw new Error( + `Screenshot build did not produce ${SCREENSHOT_VIEW_INDEX}. ` + + 'Run npm run bootstrap, then retry (buildApps(\'screenshot\') requires the enact CLI).' + ); +} + export function assertComponentSource (componentName) { const source = componentSourcePath(componentName); if (!fs.existsSync(source)) { diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index 880ce10a8..b0a5bf1c6 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -44,7 +44,7 @@ export default defineConfig({ use: { baseURL: PLAYWRIGHT_BASE_URL, trace: 'on-first-retry', - viewport: {width: 1920, height: 1167}, + viewport: {width: 1920, height: 1080}, launchOptions: { args: [ '--disable-infobars', @@ -59,9 +59,9 @@ export default defineConfig({ } }, webServer: { - command: process.platform === 'win32' - ? `npx.cmd --yes serve "${distDir}" -l ${PLAYWRIGHT_PORT}` - : `npx --yes serve ${distDir} -l ${PLAYWRIGHT_PORT}`, + command: process.platform === 'win32' ? + `npx.cmd --yes serve "${distDir}" -l ${PLAYWRIGHT_PORT}` : + `npx --yes serve ${distDir} -l ${PLAYWRIGHT_PORT}`, url: PLAYWRIGHT_BASE_URL, reuseExistingServer: true, timeout: 120000, diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index 5cbcc983d..d6ed3a60b 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -18,11 +18,12 @@ function buildUrl (urlExtra = '?locale=en-US') { async function open (page, urlExtra = '?locale=en-US') { await page.goto(buildUrl(urlExtra), {waitUntil: 'networkidle'}); - await page.waitForTimeout(200); } export async function openComponent (page, params) { const query = serializeParams(Object.assign({locale: 'en-US'}, params)); await open(page, `?${query}`); await page.locator('[data-ui-test-id="test"]').waitFor({state: 'visible', timeout: 30000}); + // Wait for web fonts to finish loading so glyphs are painted before the screenshot. + await page.evaluate(() => document.fonts.ready); } diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js index f839d4a95..b4f08e9b3 100644 --- a/tests/screenshot/playwright/utils/run-screenshot-tests.js +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -5,6 +5,7 @@ import {test as baseTest, expect as baseExpect} from '@playwright/test'; import {assertComponentSource, TEST_DATA_FILE} from '../paths.js'; import {getScreenshotName} from './screenshot-name.js'; import {openComponent} from './limestone-page.js'; +import {recordShard} from './shard-registry.js'; const testIdFilter = process.env.PLAYWRIGHT_TEST_ID != null ? Number.parseInt(process.env.PLAYWRIGHT_TEST_ID) : @@ -83,10 +84,11 @@ export function registerScreenshotTests (config) { suiteName = `${config.testName} (shard ${config.concurrency})`; } const cases = getScreenshotTests(config); + recordShard(config, maxInstances); baseTest.describe(suiteName, () => { baseTest.beforeEach(async ({page}) => { - await page.setViewportSize({width: 1920, height: 1167}); + await page.setViewportSize({width: 1920, height: 1080}); }); for (const screenshotTest of cases) { diff --git a/tests/screenshot/playwright/utils/shard-registry.js b/tests/screenshot/playwright/utils/shard-registry.js new file mode 100644 index 000000000..9057ef614 --- /dev/null +++ b/tests/screenshot/playwright/utils/shard-registry.js @@ -0,0 +1,65 @@ +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const registryPath = path.join(__dirname, '..', '.shard-registry.jsonl'); + +export function clearShardRegistry () { + if (fs.existsSync(registryPath)) { + fs.unlinkSync(registryPath); + } +} + +export function recordShard ({skin, highContrast, concurrency}, maxWorkers) { + if (process.env.PLAYWRIGHT_COMPONENT || !concurrency) { + return; + } + + const entry = JSON.stringify({ + groupKey: `${skin}-${highContrast}`, + concurrency, + maxWorkers + }); + + fs.appendFileSync(registryPath, `${entry}\n`); +} + +export function validateShardCoverage () { + if (process.env.PLAYWRIGHT_COMPONENT || !fs.existsSync(registryPath)) { + return; + } + + const groups = new Map(); + + for (const line of fs.readFileSync(registryPath, 'utf8').split('\n')) { + if (!line.trim()) { + continue; + } + + const {groupKey, concurrency, maxWorkers} = JSON.parse(line); + + if (!groups.has(groupKey)) { + groups.set(groupKey, {maxWorkers, shards: new Set()}); + } + + const group = groups.get(groupKey); + group.maxWorkers = maxWorkers; + group.shards.add(concurrency); + } + + for (const [groupKey, {maxWorkers, shards}] of groups) { + const covering = [...shards].filter(shard => shard <= maxWorkers).sort((a, b) => a - b); + const required = [...Array(maxWorkers)].map((_, index) => index + 1); + const missing = required.filter(shard => !covering.includes(shard)); + + if (covering.length > 1 && missing.length > 0) { + throw new Error( + `Incomplete Playwright shard coverage for ${groupKey}: ` + + `PLAYWRIGHT_INSTANCES=${maxWorkers} requires shards ${required.join(', ')}, ` + + `but only ${covering.join(', ')} ran. ` + + `Cases with testId % ${maxWorkers} in {${missing.map(shard => shard - 1).join(', ')}} were never tested.` + ); + } + } +} diff --git a/tests/screenshot/scripts/parse-component-args.mjs b/tests/screenshot/scripts/parse-component-args.mjs new file mode 100644 index 000000000..9b5c3e894 --- /dev/null +++ b/tests/screenshot/scripts/parse-component-args.mjs @@ -0,0 +1,41 @@ +const VALUE_FLAGS = new Set(['--test-id', '--title', '--parallel']); + +/** + * Parse component CLI args without treating flag values as the component name. + * Component name must be the first positional token (flags may follow). + */ +export function parseComponentArgs (argv = process.argv) { + const args = argv.slice(2).filter(a => a !== '--'); + const flags = new Set(); + const values = {}; + let component = null; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (!arg.startsWith('--')) { + if (!component) { + component = arg; + } + continue; + } + + if (VALUE_FLAGS.has(arg)) { + values[arg] = args[i + 1] ?? null; + i++; + continue; + } + + flags.add(arg); + } + + return { + component, + update: flags.has('--update'), + withBuild: flags.has('--build'), + skipBuild: flags.has('--skip-build'), + testId: values['--test-id'], + title: values['--title'], + parallel: Math.max(1, Number.parseInt(values['--parallel'] ?? '1') || 1) + }; +} diff --git a/tests/screenshot/scripts/run-playwright-component.mjs b/tests/screenshot/scripts/run-playwright-component.mjs index 85ba6998a..61318cd29 100644 --- a/tests/screenshot/scripts/run-playwright-component.mjs +++ b/tests/screenshot/scripts/run-playwright-component.mjs @@ -15,6 +15,7 @@ import path from 'path'; import {fileURLToPath} from 'url'; import {assertComponentSource} from '../playwright/paths.js'; +import {parseComponentArgs} from './parse-component-args.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.join(__dirname, '..', '..', '..'); @@ -22,18 +23,7 @@ const playwrightRoot = path.join(__dirname, '..', 'playwright'); const config = path.join(playwrightRoot, 'playwright.config.mjs'); const spec = 'specs/neutral/Default-spec.js'; -const args = process.argv.slice(2).filter(a => a !== '--'); - -function flagValue (name) { - const index = args.indexOf(name); - return index === -1 ? null : args[index + 1]; -} - -const component = args.find(a => !a.startsWith('--') && args.indexOf(a) === args.findIndex(x => !x.startsWith('--'))); -const update = args.includes('--update'); -const testId = flagValue('--test-id'); -const title = flagValue('--title'); -const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1') || 1); +const {component, update, testId, title, parallel} = parseComponentArgs(); if (!component) { console.error('Usage: npm run test-playwright:component -- <ComponentName> [options]'); From 98d1b86417deea9f005c3853250f41dcb4c85805 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Wed, 10 Jun 2026 12:28:22 +0200 Subject: [PATCH 10/28] update review changes --- package.json | 4 +- tests/screenshot/playwright/README.md | 97 +++++++++----- tests/screenshot/playwright/global-setup.js | 54 ++------ .../screenshot/playwright/global-teardown.js | 23 ---- tests/screenshot/playwright/paths.js | 14 +- .../playwright/playwright.config.mjs | 6 +- tests/screenshot/playwright/static-server.js | 126 ++++++++++++++++++ .../playwright/utils/limestone-page.js | 2 +- .../playwright/utils/run-screenshot-tests.js | 48 ++++++- .../playwright/utils/screenshot-name.js | 5 + .../scripts/parse-component-args.mjs | 4 +- .../scripts/parse-playwright-args.mjs | 39 ++++++ .../scripts/run-playwright-component.mjs | 67 +++++----- tests/screenshot/scripts/run-playwright.mjs | 59 ++++++++ tests/screenshot/scripts/spawn-playwright.mjs | 31 +++++ 15 files changed, 422 insertions(+), 157 deletions(-) create mode 100644 tests/screenshot/playwright/static-server.js create mode 100644 tests/screenshot/scripts/parse-playwright-args.mjs create mode 100644 tests/screenshot/scripts/run-playwright.mjs create mode 100644 tests/screenshot/scripts/spawn-playwright.mjs diff --git a/package.json b/package.json index 9e3fd8e22..5a782c475 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,8 @@ "test-ss": "start-tests tests/screenshot/config/wdio.conf.js", "test-ss-docker": "start-tests tests/screenshot/config/wdio.docker.conf.js", "test-ss-tv": "start-tests tests/screenshot/config/wdio.tv.conf.js", - "test-playwright": "playwright test --config tests/screenshot/playwright/playwright.config.mjs", - "test-playwright:update": "playwright test --config tests/screenshot/playwright/playwright.config.mjs --update-snapshots", + "test-playwright": "node tests/screenshot/scripts/run-playwright.mjs", + "test-playwright:update": "node tests/screenshot/scripts/run-playwright.mjs -- --update", "test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", "test-playwright:component": "node tests/screenshot/scripts/run-playwright-component.mjs", "test-ss:component": "node tests/screenshot/scripts/run-component-wdio.mjs", diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index 4fee828a1..a3e1e29dd 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -14,12 +14,12 @@ npm run bootstrap # enact CLI — required for the screens npm link @enact/ui-test-utils # see “Link local ui-test-utils” below until npm publishes ./build-apps npm run prepare-playwright # optional if Chrome is already available (see below) -# First time for a component — create PNG baselines (use parallel 1 for stability) -npm run test-playwright:component -- Sprite --update - -# Verify screenshots against baselines +# First time for a component — baselines are created automatically on pass npm run test-playwright:component -- Sprite +# Force refresh baselines after UI changes +npm run test-playwright:component -- Sprite --update + # Compare Playwright vs WDIO runtime on one component npm run benchmark-screenshots -- Chip ``` @@ -56,6 +56,7 @@ tests/screenshot/ dist/ ← buildApps('screenshot') output (shared) specs/ ← WDIO *-specs.js scripts/ + run-playwright.mjs run-playwright-component.mjs run-component-wdio.mjs parse-component-args.mjs @@ -126,7 +127,7 @@ registerScreenshotTests({ ### 7. Screenshot -`toHaveScreenshot(getScreenshotName('Chip', 'Limestone', <title>))` writes/compares under `playwright/snapshots/` (filename encodes component, test name, and case title — same naming rules as WDIO reference PNGs). +`toHaveScreenshot(getScreenshotPathSegments('Chip', 'Limestone', <title>))` writes/compares under `playwright/snapshots/<Component>/<TestName>/<case>.png` — same folder layout as WDIO `dist/screenshots/reference/`. **One case only:** @@ -154,18 +155,18 @@ npm run prepare-playwright # optional on machines that already have Chrome (se `prepare-playwright` runs `playwright install chrome`. It is **not** hooked to `postinstall`. Run it on **fresh machines and CI** (`build-scripts/enact-playwright-tests.sh`). On a machine where Chrome is already installed, Playwright prints *"chrome" is already installed on the system!"* — that is success; you can skip this step on later runs. -`global-setup.js` calls `assertScreenshotDist()` after the build step. If `tests/screenshot/dist/Limestone-View/index.html` is missing, the run fails immediately with a clear error (not a 120s webServer timeout). +`global-setup.js` calls `assertScreenshotDist()` after the build step. If `tests/screenshot/dist/Limestone-View/index.html` is missing, the run fails immediately with a clear error in global-setup (static server is started there after dist is verified). ### Build screenshot dist -The first Playwright run builds `tests/screenshot/dist/` automatically. To reuse an existing build: +The first Playwright run builds `tests/screenshot/dist/` automatically. To reuse an existing build, pass `--skip-build`: -```powershell -$env:PLAYWRIGHT_SKIP_BUILD='1' # PowerShell -# export PLAYWRIGHT_SKIP_BUILD=1 # bash +```bash +npm run test-playwright -- --skip-build +npm run test-playwright:component -- Button --skip-build ``` -Or build via WDIO / `buildApps` before running Playwright. +Or set `PLAYWRIGHT_SKIP_BUILD=1` (same effect as `--skip-build`). ### Link local `@enact/ui-test-utils` (until `./build-apps` is published) @@ -203,13 +204,16 @@ When `@enact/ui-test-utils` is published with the `./build-apps` export, remove ### Baselines -Playwright needs PNG baselines under `playwright/snapshots/`. Without them, tests fail with “snapshot doesn't exist”. +Missing PNG baselines under `playwright/snapshots/` are **created automatically** on the first run (same idea as WDIO `autoSaveBaseline`). Use `--update` to **force** overwriting existing baselines. ```bash -# One component (recommended) +# First run — baselines are written as tests pass +npm run test-playwright:component -- Sprite + +# Force refresh all baselines for one component npm run test-playwright:component -- Sprite --update -# Entire suite (long — thousands of PNGs) +# Force refresh entire suite npm run test-playwright:update ``` @@ -226,10 +230,16 @@ All commands run from the **repository root**. | Command | What it does | |---------|----------------| | `npm run test-playwright` | Full suite — all components, all Playwright shards | -| `npm run test-playwright:update` | Update all Playwright baselines | +| `npm run test-playwright -- --component Button` | All specs; components whose name contains `Button` | +| `npm run test-playwright -- --spec Light` | `Light*` specs only; all components | +| `npm run test-playwright -- --component Button --spec Light` | Both filters combined | +| `npm run test-playwright -- --skip-build` | Skip `buildApps('screenshot')` when `dist/` already exists | +| `npm run test-playwright:update` | Force-update all Playwright baselines (`--update`) | | `npm run test-playwright:report` | Open HTML report (`playwright/reports/html/`) | -| `npm run test-playwright:component -- <Name>` | One component, all its cases (Default neutral shard) | -| `npm run test-playwright:component -- <Name> --update` | Refresh baselines for that component | +| `npm run test-playwright:component -- <Name>` | One component, all spec shards | +| `npm run test-playwright:component -- <Name> --update` | Force refresh baselines for that component | +| `npm run test-playwright:component -- <Name> --skip-build` | Skip screenshot build | +| `npm run test-playwright:component -- <Name> --spec Light` | One component, `Light*` specs only | | `npm run test-playwright:component -- <Name> --test-id <n>` | Single case by index | | `npm run test-playwright:component -- <Name> --parallel <n>` | Same cases, `n` Playwright workers | | `npm run test-ss:component -- <Name>` | WDIO equivalent (for comparison) | @@ -239,16 +249,21 @@ All commands run from the **repository root**. ```bash # Verify (skip rebuild if dist exists) -PLAYWRIGHT_SKIP_BUILD=1 npm run test-playwright:component -- Button +npm run test-playwright:component -- Button --skip-build # Faster local run -PLAYWRIGHT_SKIP_BUILD=1 npm run test-playwright:component -- Button --parallel 5 +npm run test-playwright:component -- Button --skip-build --parallel 5 # Filter by title regex npm run test-playwright:component -- Button --title "Focused" + +# Light skin only +npm run test-playwright:component -- Button --spec Light ``` -Component script sets `PLAYWRIGHT_INSTANCES=1` (all `testId`s for that component), `PLAYWRIGHT_WORKERS` from `--parallel` (default **1**), and runs only **`specs/neutral/Default-spec.js`** (neutral skin, no high contrast). Light / high-contrast cases require the full suite or a different spec shard. +Each run prints total wall-clock time when finished (e.g. `Playwright (Button) finished in 42.3s`). + +Component script sets `PLAYWRIGHT_INSTANCES=1` (all `testId`s for that component), `PLAYWRIGHT_WORKERS` from `--parallel` (default **1**), and runs **all** matching spec shards unless `--spec` is set. #### `scripts/parse-component-args.mjs` — why it exists @@ -256,7 +271,8 @@ npm passes arguments after `--` to the Node script. A naive parser such as `args `parseComponentArgs()` is a small shared parser used by: -- `run-playwright-component.mjs` — `--update`, `--test-id`, `--title`, `--parallel` +- `run-playwright.mjs` — `--component`, `--spec`, `--update`, `--skip-build`, `--parallel` +- `run-playwright-component.mjs` — `--update`, `--skip-build`, `--spec`, `--test-id`, `--title`, `--parallel` - `benchmark.mjs` — `--build`, `--parallel` **Rules:** @@ -264,6 +280,7 @@ npm passes arguments after `--` to the Node script. A naive parser such as `args - The **component name** is the first positional token (non-flag argument). - Flags with values (`--test-id`, `--title`, `--parallel`) consume the next token as their value, not as the component. - Boolean flags: `--update`, `--build`, `--skip-build`. +- Value flags: `--spec` (component script and full suite via `run-playwright.mjs`). **Examples:** @@ -284,11 +301,19 @@ npm install && npm run bootstrap npm link @enact/ui-test-utils # until ./build-apps is on npm (see above) npm run prepare-playwright -# First time or after UI changes +# First run or after UI changes — force refresh all baselines npm run test-playwright:update -# Verify -PLAYWRIGHT_SKIP_BUILD=1 npm run test-playwright +# Verify (all components, all specs) +npm run test-playwright + +# Skip rebuild when dist/ already exists +npm run test-playwright -- --skip-build + +# Filter examples +npm run test-playwright -- --component Button +npm run test-playwright -- --spec Light +npm run test-playwright -- --component Button --spec Light # HTML report npm run test-playwright:report @@ -308,8 +333,7 @@ npm run test-playwright:report ```powershell $env:PLAYWRIGHT_SPEC='Default' -$env:PLAYWRIGHT_SKIP_BUILD='1' -npm run test-playwright +npm run test-playwright -- --skip-build ``` ### Sharding vs workers @@ -419,9 +443,11 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | Variable | Effect | |----------|--------| -| `PLAYWRIGHT_SKIP_BUILD=1` | Skip `buildApps('screenshot')` in global-setup | +| `PLAYWRIGHT_SKIP_BUILD=1` | Skip `buildApps('screenshot')` in global-setup (same as `--skip-build`) | +| `PLAYWRIGHT_FORCE_UPDATE=1` | Force overwrite baselines (same as `--update`) | +| `PLAYWRIGHT_COMPONENT_EXACT=1` | Exact component name match (set by component script) | +| `PLAYWRIGHT_COMPONENT` | Filter components (substring match from `--component`; exact when set by component script) | | `PLAYWRIGHT_REFRESH_TEST_DATA=1` | Force-regenerate `.test-data.json` | -| `PLAYWRIGHT_COMPONENT` | Filter to one component (set by `run-playwright-component.mjs`) | | `PLAYWRIGHT_TEST_ID` | Filter to one case index | | `PLAYWRIGHT_TITLE` | Filter cases by title regex | | `PLAYWRIGHT_INSTANCES` | Shard divisor (`testId % instances`); component script sets `1` | @@ -435,13 +461,12 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | Issue | Fix | |-------|-----| -| WebServer / server timeout | Ensure `tests/screenshot/dist/Limestone-View/index.html` exists (unset `PLAYWRIGHT_SKIP_BUILD`, run `npm run bootstrap`) | -| Missing dist / build failed | Read the `assertScreenshotDist()` error; link `@enact/ui-test-utils` if you see `./build-apps` export error | -| Snapshot does not exist | `npm run test-playwright:component -- <Name> --update` | +| Missing dist / build failed | Read the `assertScreenshotDist()` error (fail-fast in global-setup, not a webServer timeout); link `@enact/ui-test-utils` if build-apps fails | +| Snapshot diff | Re-baseline with `--update`, or run failing `--test-id` alone | | Snapshot diff on focus cases in parallel | Re-baseline with `--parallel 1`, or run failing `--test-id` alone | | No tests found | Check `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_SPEC`; delete stale `.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1` | | Missing images on build | Place assets under `tests/screenshot/images/`, `videos/` | -| Port already in use | Set `PLAYWRIGHT_BASE_URL` to another port (must match `webServer` in config) | +| Port already in use | Set `PLAYWRIGHT_BASE_URL` to another port; restart any stale `serve` on 4568 | | `build-apps` import error | Link or pin `@enact/ui-test-utils` with export `./build-apps` (see **Link local @enact/ui-test-utils** above) | | Incomplete shard coverage | Ensure `PLAYWRIGHT_INSTANCES` matches invoked shard files (see **Sharding vs workers**) | | WDIO `--parallel` ignored | Update `@enact/ui-test-utils` (see ui-test-utils PR for `--parallel` CLI) | @@ -467,8 +492,8 @@ On TV: typically `PLAYWRIGHT_SPEC=Default-spec` (single shard). 1. Edit `tests/screenshot/apps/components/<Name>.js` 2. Register in `tests/screenshot/apps/LimestoneComponents.js` if new component -3. Run tests (build runs unless `PLAYWRIGHT_SKIP_BUILD=1`) -4. `npm run test-playwright:component -- <Name> --update` for new PNGs +3. Run tests (build runs unless `--skip-build`); missing baselines are created automatically +4. `npm run test-playwright:component -- <Name> --update` to force refresh PNGs after UI changes ### New WDIO shard @@ -479,8 +504,8 @@ Add matching `playwright/specs/**/<Name>-spec.js` with the same `skin`, `highCon ## Related npm scripts (root `package.json`) ```json -"test-playwright": "playwright test --config tests/screenshot/playwright/playwright.config.mjs", -"test-playwright:update": "... --update-snapshots", +"test-playwright": "node tests/screenshot/scripts/run-playwright.mjs", +"test-playwright:update": "node tests/screenshot/scripts/run-playwright.mjs -- --update", "test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", "test-playwright:component": "node tests/screenshot/scripts/run-playwright-component.mjs", "test-ss:component": "node tests/screenshot/scripts/run-component-wdio.mjs", diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js index 71ea6c8f5..af4d685a3 100644 --- a/tests/screenshot/playwright/global-setup.js +++ b/tests/screenshot/playwright/global-setup.js @@ -1,49 +1,14 @@ import fs from 'fs'; -import {spawn} from 'child_process'; import buildApps from '@enact/ui-test-utils/build-apps'; import {chromium} from '@playwright/test'; -import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST, SCREENSHOT_VIEW_INDEX, TEST_DATA_FILE, assertScreenshotDist} from './paths.js'; +import {PLAYWRIGHT_BASE_URL, SCREENSHOT_VIEW_INDEX, TEST_DATA_FILE, assertScreenshotDist} from './paths.js'; +import {ensureStaticServer, stopStaticServer} from './static-server.js'; import {clearShardRegistry} from './utils/shard-registry.js'; const baseURL = PLAYWRIGHT_BASE_URL; -const distPath = SCREENSHOT_DIST; const testDataFile = TEST_DATA_FILE; -const port = PLAYWRIGHT_PORT; - -/** Set when this file starts `serve`; torn down in global-teardown.js */ -export const staticServer = { - process: null, - started: false -}; - -async function waitForServer (url, timeout = 120000) { - const start = Date.now(); - - while (Date.now() - start < timeout) { - try { - const response = await fetch(url); - if (response.ok || response.status === 404) { - return; - } - } catch { - // retry until the static server is ready - } - await new Promise(resolve => setTimeout(resolve, 500)); - } - - throw new Error(`Static server at ${url} did not start within ${timeout}ms`); -} - -function startStaticServer () { - const command = process.platform === 'win32' ? 'npx.cmd' : 'npx'; - staticServer.process = spawn(command, ['--yes', 'serve', distPath, '-l', port], { - stdio: 'pipe', - shell: process.platform === 'win32' - }); - staticServer.started = true; -} function shouldRefreshTestData () { if (process.env.PLAYWRIGHT_REFRESH_TEST_DATA === '1') { @@ -68,14 +33,10 @@ export default async function globalSetup () { assertScreenshotDist(); - // Reuse an existing server on PLAYWRIGHT_BASE_URL when present; otherwise start `serve` - // so global-setup can fetch test metadata before Playwright's webServer hook runs. - try { - await fetch(baseURL); - } catch { - startStaticServer(); - await waitForServer(baseURL); - } + // Playwright lifecycle: globalSetup → webServer (playwright.config.mjs) → tests → teardown. + // We start serve here only to fetch ?request metadata into .test-data.json, then stop our + // process so webServer can own the port for the full run (reuseExistingServer when not CI). + await ensureStaticServer(); if (!shouldRefreshTestData()) { return; @@ -88,11 +49,12 @@ export default async function globalSetup () { try { const page = await pwBrowser.newPage(); - await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'networkidle'}); + await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'load'}); const testData = await page.evaluate(() => window.__TEST_DATA); fs.writeFileSync(testDataFile, JSON.stringify(testData)); } finally { await pwBrowser.close(); + stopStaticServer(); } } diff --git a/tests/screenshot/playwright/global-teardown.js b/tests/screenshot/playwright/global-teardown.js index cf91fb2f7..19819c835 100644 --- a/tests/screenshot/playwright/global-teardown.js +++ b/tests/screenshot/playwright/global-teardown.js @@ -1,28 +1,5 @@ -import {spawnSync} from 'child_process'; - -import {staticServer} from './global-setup.js'; import {validateShardCoverage} from './utils/shard-registry.js'; -function killProcessTree (childProcess) { - if (process.platform === 'win32') { - spawnSync('taskkill', ['/T', '/F', '/PID', String(childProcess.pid)], { - stdio: 'ignore', - shell: true - }); - return; - } - - childProcess.kill(); -} - export default async function globalTeardown () { validateShardCoverage(); - - if (!staticServer.started || !staticServer.process) { - return; - } - - killProcessTree(staticServer.process); - staticServer.process = null; - staticServer.started = false; } diff --git a/tests/screenshot/playwright/paths.js b/tests/screenshot/playwright/paths.js index 23c4a8163..dc4921d57 100644 --- a/tests/screenshot/playwright/paths.js +++ b/tests/screenshot/playwright/paths.js @@ -13,6 +13,7 @@ export const SCREENSHOT_DIST = path.join(screenshotRoot, 'dist'); export const SCREENSHOT_VIEW_INDEX = path.join(SCREENSHOT_DIST, 'Limestone-View', 'index.html'); export const SCREENSHOT_COMPONENTS = path.join(screenshotRoot, 'apps', 'components'); export const TEST_DATA_FILE = path.join(__dirname, '.test-data.json'); +export const SNAPSHOTS_DIR = path.join(__dirname, 'snapshots'); function componentSourcePath (componentName) { return path.join(SCREENSHOT_COMPONENTS, `${componentName}.js`); @@ -23,16 +24,21 @@ export function assertScreenshotDist () { return; } + const distHint = !fs.existsSync(SCREENSHOT_DIST) ? + `Directory ${SCREENSHOT_DIST} does not exist. ` : + ''; + if (process.env.PLAYWRIGHT_SKIP_BUILD) { throw new Error( - `Missing ${SCREENSHOT_VIEW_INDEX}. ` + - 'Unset PLAYWRIGHT_SKIP_BUILD to build automatically, or run npm run test-ss once to populate tests/screenshot/dist/.' + `${distHint}Missing ${SCREENSHOT_VIEW_INDEX}. ` + + 'Remove --skip-build (or unset PLAYWRIGHT_SKIP_BUILD) to build automatically, ' + + 'or run npm run test-ss once to populate tests/screenshot/dist/.' ); } throw new Error( - `Screenshot build did not produce ${SCREENSHOT_VIEW_INDEX}. ` + - 'Run npm run bootstrap, then retry (buildApps(\'screenshot\') requires the enact CLI).' + `${distHint}Screenshot build did not produce ${SCREENSHOT_VIEW_INDEX}. ` + + 'Run npm run bootstrap, link @enact/ui-test-utils if build-apps is missing from exports, then retry.' ); } diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index b0a5bf1c6..c54798a04 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -6,6 +6,7 @@ import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT} from './paths.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const distDir = path.join(__dirname, '..', 'dist'); +const serverHealthUrl = `${PLAYWRIGHT_BASE_URL}/Limestone-View/index.html`; /** Jenkins: SPEC=Default → Default* shards; PLAYWRIGHT_SPEC=Default-spec → one file (TV). */ function resolveTestMatch () { @@ -26,6 +27,7 @@ export default defineConfig({ testIgnore: ['**/utils/**', '**/scripts/**'], globalSetup: path.join(__dirname, 'global-setup.js'), globalTeardown: path.join(__dirname, 'global-teardown.js'), + updateSnapshots: 'none', snapshotPathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}'), expect: { toHaveScreenshot: { @@ -62,8 +64,8 @@ export default defineConfig({ command: process.platform === 'win32' ? `npx.cmd --yes serve "${distDir}" -l ${PLAYWRIGHT_PORT}` : `npx --yes serve ${distDir} -l ${PLAYWRIGHT_PORT}`, - url: PLAYWRIGHT_BASE_URL, - reuseExistingServer: true, + url: serverHealthUrl, + reuseExistingServer: !process.env.CI, timeout: 120000, shell: process.platform === 'win32' }, diff --git a/tests/screenshot/playwright/static-server.js b/tests/screenshot/playwright/static-server.js new file mode 100644 index 000000000..1ab7b8660 --- /dev/null +++ b/tests/screenshot/playwright/static-server.js @@ -0,0 +1,126 @@ +import {spawn, spawnSync} from 'child_process'; + +import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST} from './paths.js'; + +const distPath = SCREENSHOT_DIST; +const port = PLAYWRIGHT_PORT; +const healthCheckUrl = `${PLAYWRIGHT_BASE_URL}/Limestone-View/index.html`; + +/** Set when global-setup starts `serve` for ?request metadata (stopped before tests). */ +export const staticServer = { + process: null, + started: false +}; + +function killProcessTree (childProcess) { + if (process.platform === 'win32') { + spawnSync('taskkill', ['/T', '/F', '/PID', String(childProcess.pid)], { + stdio: 'ignore', + shell: true + }); + return; + } + + childProcess.kill(); +} + +/** Free the Playwright port when a previous run left `serve` listening or wedged. */ +export function killProcessOnPort (listenPort = port) { + if (process.platform === 'win32') { + const result = spawnSync('netstat', ['-ano'], {encoding: 'utf8', shell: true}); + if (result.status !== 0) { + return; + } + + for (const line of result.stdout.split('\n')) { + if (!line.includes(`:${listenPort}`) || !line.includes('LISTENING')) { + continue; + } + + const pid = line.trim().split(/\s+/).at(-1); + if (pid && /^\d+$/.test(pid)) { + spawnSync('taskkill', ['/T', '/F', '/PID', pid], {stdio: 'ignore', shell: true}); + } + } + return; + } + + const result = spawnSync('lsof', ['-ti', `:${listenPort}`], {encoding: 'utf8'}); + if (result.status !== 0 || !result.stdout.trim()) { + return; + } + + for (const pid of result.stdout.trim().split('\n')) { + if (/^\d+$/.test(pid)) { + spawnSync('kill', ['-9', pid], {stdio: 'ignore'}); + } + } +} + +export function stopStaticServer () { + if (!staticServer.started || !staticServer.process) { + return; + } + + killProcessTree(staticServer.process); + staticServer.process = null; + staticServer.started = false; + killProcessOnPort(port); +} + +async function fetchWithTimeout (url, timeoutMs = 5000) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + return await fetch(url, {signal: controller.signal}); + } finally { + clearTimeout(timeout); + } +} + +async function isServerHealthy () { + try { + const response = await fetchWithTimeout(healthCheckUrl); + return response.ok; + } catch { + return false; + } +} + +function startStaticServer () { + const command = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + staticServer.process = spawn(command, ['--yes', 'serve', distPath, '-l', port], { + stdio: 'pipe', + shell: process.platform === 'win32' + }); + staticServer.started = true; +} + +async function waitForServer (url, timeout = 120000) { + const start = Date.now(); + + while (Date.now() - start < timeout) { + try { + const response = await fetchWithTimeout(url, 5000); + if (response.ok) { + return; + } + } catch { + // retry until the static server is ready + } + await new Promise(resolve => setTimeout(resolve, 500)); + } + + throw new Error(`Static server at ${url} did not start within ${timeout}ms`); +} + +export async function ensureStaticServer () { + if (await isServerHealthy()) { + return; + } + + stopStaticServer(); + startStaticServer(); + await waitForServer(healthCheckUrl); +} diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index d6ed3a60b..381e80340 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -17,7 +17,7 @@ function buildUrl (urlExtra = '?locale=en-US') { } async function open (page, urlExtra = '?locale=en-US') { - await page.goto(buildUrl(urlExtra), {waitUntil: 'networkidle'}); + await page.goto(buildUrl(urlExtra), {waitUntil: 'load'}); } export async function openComponent (page, params) { diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js index b4f08e9b3..c6eac1517 100644 --- a/tests/screenshot/playwright/utils/run-screenshot-tests.js +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -1,12 +1,16 @@ import fs from 'fs'; +import path from 'path'; import {test as baseTest, expect as baseExpect} from '@playwright/test'; import {assertComponentSource, TEST_DATA_FILE} from '../paths.js'; -import {getScreenshotName} from './screenshot-name.js'; +import {getScreenshotPathSegments} from './screenshot-name.js'; import {openComponent} from './limestone-page.js'; import {recordShard} from './shard-registry.js'; +const forceUpdate = process.env.PLAYWRIGHT_FORCE_UPDATE === '1'; +const componentExactMatch = process.env.PLAYWRIGHT_COMPONENT_EXACT === '1'; + const testIdFilter = process.env.PLAYWRIGHT_TEST_ID != null ? Number.parseInt(process.env.PLAYWRIGHT_TEST_ID) : null; @@ -26,8 +30,12 @@ function resolveComponentFilter (config) { function shouldIncludeTest (component, testId, title, configComponent) { const componentFilter = resolveComponentFilter({component: configComponent}); if (componentFilter) { - const pattern = componentFilter.startsWith('^') ? componentFilter : `^${componentFilter}$`; - if (!component.match(new RegExp(pattern))) { + if (componentExactMatch) { + const pattern = componentFilter.startsWith('^') ? componentFilter : `^${componentFilter}$`; + if (!component.match(new RegExp(pattern))) { + return false; + } + } else if (!component.includes(componentFilter)) { return false; } } @@ -40,6 +48,32 @@ function shouldIncludeTest (component, testId, title, configComponent) { return true; } +function resolveSnapshotPath (testInfo, segments) { + return testInfo.snapshotPath(...segments, {kind: 'screenshot'}); +} + +/** + * WDIO autoSaveBaseline: create missing baselines without --update. + * PLAYWRIGHT_FORCE_UPDATE (--update) always overwrites existing baselines. + * Snapshot path: snapshots/<Component>/<TestName>/<case>.png (same as WDIO). + */ +async function assertPageScreenshot (page, testInfo, segments, options) { + const filePath = resolveSnapshotPath(testInfo, segments); + + if (forceUpdate || !fs.existsSync(filePath)) { + fs.mkdirSync(path.dirname(filePath), {recursive: true}); + await page.screenshot({ + animations: 'disabled', + caret: 'hide', + type: 'png', + path: filePath + }); + return; + } + + await baseExpect(page).toHaveScreenshot(segments, options); +} + /** * Cases from tests/screenshot/apps/components/*.js (built into tests/screenshot/dist). */ @@ -92,10 +126,12 @@ export function registerScreenshotTests (config) { }); for (const screenshotTest of cases) { - baseTest(screenshotTest.title, async ({page}) => { + baseTest(screenshotTest.title, async ({page}, testInfo) => { await openComponent(page, screenshotTest.params); - await baseExpect(page).toHaveScreenshot( - getScreenshotName( + await assertPageScreenshot( + page, + testInfo, + getScreenshotPathSegments( screenshotTest.params.component, config.testName, screenshotTest.params.caseTitle diff --git a/tests/screenshot/playwright/utils/screenshot-name.js b/tests/screenshot/playwright/utils/screenshot-name.js index d44cd84a4..e1518e561 100644 --- a/tests/screenshot/playwright/utils/screenshot-name.js +++ b/tests/screenshot/playwright/utils/screenshot-name.js @@ -17,3 +17,8 @@ export function getScreenshotName (component, testName, title) { return path.posix.join(component, testName, getScreenshotBasename(title)); } +/** Playwright snapshot segments — array form keeps WDIO folder layout (not flat dashes). */ +export function getScreenshotPathSegments (component, testName, title) { + return getScreenshotName(component, testName, title).split('/'); +} + diff --git a/tests/screenshot/scripts/parse-component-args.mjs b/tests/screenshot/scripts/parse-component-args.mjs index 9b5c3e894..6ea6089fb 100644 --- a/tests/screenshot/scripts/parse-component-args.mjs +++ b/tests/screenshot/scripts/parse-component-args.mjs @@ -1,4 +1,4 @@ -const VALUE_FLAGS = new Set(['--test-id', '--title', '--parallel']); +const VALUE_FLAGS = new Set(['--test-id', '--title', '--parallel', '--spec']); /** * Parse component CLI args without treating flag values as the component name. @@ -32,10 +32,12 @@ export function parseComponentArgs (argv = process.argv) { return { component, update: flags.has('--update'), + // --build is used by benchmark.mjs only; Playwright runners use --skip-build instead. withBuild: flags.has('--build'), skipBuild: flags.has('--skip-build'), testId: values['--test-id'], title: values['--title'], + spec: values['--spec'], parallel: Math.max(1, Number.parseInt(values['--parallel'] ?? '1') || 1) }; } diff --git a/tests/screenshot/scripts/parse-playwright-args.mjs b/tests/screenshot/scripts/parse-playwright-args.mjs new file mode 100644 index 000000000..b986538bb --- /dev/null +++ b/tests/screenshot/scripts/parse-playwright-args.mjs @@ -0,0 +1,39 @@ +const VALUE_FLAGS = new Set(['--component', '--spec', '--parallel', '--test-id', '--title']); + +/** + * Parse CLI args for npm run test-playwright. + * All flags are optional; default run executes all specs on all components. + */ +export function parsePlaywrightArgs (argv = process.argv) { + const args = argv.slice(2).filter(a => a !== '--'); + const flags = new Set(); + const values = {}; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (!arg.startsWith('--')) { + continue; + } + + if (VALUE_FLAGS.has(arg)) { + values[arg] = args[i + 1] ?? null; + i++; + continue; + } + + flags.add(arg); + } + + return { + component: values['--component'] ?? null, + spec: values['--spec'] ?? null, + update: flags.has('--update'), + skipBuild: flags.has('--skip-build'), + testId: values['--test-id'], + title: values['--title'], + parallel: values['--parallel'] != null ? + Math.max(1, Number.parseInt(values['--parallel']) || 1) : + null + }; +} diff --git a/tests/screenshot/scripts/run-playwright-component.mjs b/tests/screenshot/scripts/run-playwright-component.mjs index 61318cd29..882cadc97 100644 --- a/tests/screenshot/scripts/run-playwright-component.mjs +++ b/tests/screenshot/scripts/run-playwright-component.mjs @@ -6,70 +6,65 @@ * Usage: * npm run test-playwright:component -- Sprite * npm run test-playwright:component -- Sprite --update + * npm run test-playwright:component -- Sprite --skip-build + * npm run test-playwright:component -- Sprite --spec Light * npm run test-playwright:component -- Sprite --test-id 0 * npm run test-playwright:component -- Sprite --title <regex> * npm run test-playwright:component -- Sprite --parallel 5 */ -import {spawnSync} from 'child_process'; -import path from 'path'; -import {fileURLToPath} from 'url'; - import {assertComponentSource} from '../playwright/paths.js'; import {parseComponentArgs} from './parse-component-args.mjs'; +import {spawnPlaywright} from './spawn-playwright.mjs'; -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const repoRoot = path.join(__dirname, '..', '..', '..'); -const playwrightRoot = path.join(__dirname, '..', 'playwright'); -const config = path.join(playwrightRoot, 'playwright.config.mjs'); -const spec = 'specs/neutral/Default-spec.js'; - -const {component, update, testId, title, parallel} = parseComponentArgs(); +const {component, update, skipBuild, spec, testId, title, parallel} = parseComponentArgs(); if (!component) { console.error('Usage: npm run test-playwright:component -- <ComponentName> [options]'); - console.error('Options: --update, --test-id <n>, --title <regex>, --parallel <n>'); + console.error('Options: --update, --skip-build, --spec <Name>, --test-id <n>, --title <regex>, --parallel <n>'); console.error('Example: npm run test-playwright:component -- Sprite'); console.error('Example: npm run test-playwright:component -- Sprite --update'); - console.error('Example: npm run test-playwright:component -- Sprite --test-id 0'); - console.error('Example: npm run test-playwright:component -- Sprite --parallel 5'); + console.error('Example: npm run test-playwright:component -- Sprite --skip-build'); + console.error('Example: npm run test-playwright:component -- Sprite --spec Light'); process.exit(1); } assertComponentSource(component); -const playwrightArgs = [ - 'test', - '--config', config, - spec -]; - -if (update) { - playwrightArgs.push('--update-snapshots'); -} - const env = { - ...process.env, PLAYWRIGHT_COMPONENT: component, + PLAYWRIGHT_COMPONENT_EXACT: '1', PLAYWRIGHT_INSTANCES: '1', PLAYWRIGHT_WORKERS: String(parallel) }; +if (update) { + env.PLAYWRIGHT_FORCE_UPDATE = '1'; +} + +if (skipBuild) { + env.PLAYWRIGHT_SKIP_BUILD = '1'; +} + +if (spec) { + env.PLAYWRIGHT_SPEC = spec; +} + if (testId != null) { env.PLAYWRIGHT_TEST_ID = testId; } + if (title != null) { env.PLAYWRIGHT_TITLE = title; } -const result = spawnSync( - process.platform === 'win32' ? 'npx.cmd' : 'npx', - ['playwright', ...playwrightArgs], - { - cwd: repoRoot, - stdio: 'inherit', - env, - shell: process.platform === 'win32' - } -); +const filters = [`component ${component}`]; +if (spec) { + filters.push(`spec ~ ${spec}`); +} +console.log(`Playwright: ${filters.join(', ')}`); -process.exit(result.status ?? 1); +const exitCode = spawnPlaywright({ + env, + label: `Playwright (${component})` +}); +process.exit(exitCode); diff --git a/tests/screenshot/scripts/run-playwright.mjs b/tests/screenshot/scripts/run-playwright.mjs new file mode 100644 index 000000000..fc35be240 --- /dev/null +++ b/tests/screenshot/scripts/run-playwright.mjs @@ -0,0 +1,59 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +/** + * Run Playwright screenshot tests with optional component/spec filters. + * + * Usage: + * npm run test-playwright + * npm run test-playwright -- --component Button + * npm run test-playwright -- --spec Light + * npm run test-playwright -- --component Button --spec Light + * npm run test-playwright -- --update + * npm run test-playwright -- --skip-build + * npm run test-playwright -- --parallel 5 + */ +import {spawnPlaywright} from './spawn-playwright.mjs'; +import {parsePlaywrightArgs} from './parse-playwright-args.mjs'; + +const {component, spec, update, skipBuild, parallel, testId, title} = parsePlaywrightArgs(); +const env = {}; +const filters = []; + +if (component) { + env.PLAYWRIGHT_COMPONENT = component; + filters.push(`component ~ ${component}`); +} + +if (spec) { + env.PLAYWRIGHT_SPEC = spec; + filters.push(`spec ~ ${spec}`); +} + +if (update) { + env.PLAYWRIGHT_FORCE_UPDATE = '1'; +} + +if (skipBuild) { + env.PLAYWRIGHT_SKIP_BUILD = '1'; +} + +if (parallel != null) { + env.PLAYWRIGHT_WORKERS = String(parallel); +} + +if (testId != null) { + env.PLAYWRIGHT_TEST_ID = testId; +} + +if (title != null) { + env.PLAYWRIGHT_TITLE = title; +} + +if (filters.length) { + console.log(`Playwright filters: ${filters.join(', ')}`); +} else { + console.log('Playwright: all components, all specs'); +} + +const exitCode = spawnPlaywright({env}); +process.exit(exitCode); diff --git a/tests/screenshot/scripts/spawn-playwright.mjs b/tests/screenshot/scripts/spawn-playwright.mjs new file mode 100644 index 000000000..28ebebd93 --- /dev/null +++ b/tests/screenshot/scripts/spawn-playwright.mjs @@ -0,0 +1,31 @@ +import {spawnSync} from 'child_process'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +/* eslint-disable no-console */ + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); +const config = path.join(__dirname, '..', 'playwright', 'playwright.config.mjs'); + +/** + * Run Playwright with optional env overrides and print wall-clock duration. + */ +export function spawnPlaywright ({playwrightArgs = [], env = {}, label = 'Playwright'}) { + const start = Date.now(); + const result = spawnSync( + process.platform === 'win32' ? 'npx.cmd' : 'npx', + ['playwright', 'test', '--config', config, ...playwrightArgs], + { + cwd: repoRoot, + stdio: 'inherit', + env: {...process.env, ...env}, + shell: process.platform === 'win32' + } + ); + const durationSec = (Date.now() - start) / 1000; + + console.log(`\n${label} finished in ${durationSec.toFixed(1)}s`); + + return result.status ?? 1; +} From 2fe2f128e701ecc895d52ddf0c149edaa8f5de89 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Wed, 10 Jun 2026 12:37:00 +0200 Subject: [PATCH 11/28] fix lint --- tests/screenshot/playwright/static-server.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/screenshot/playwright/static-server.js b/tests/screenshot/playwright/static-server.js index 1ab7b8660..067b47eae 100644 --- a/tests/screenshot/playwright/static-server.js +++ b/tests/screenshot/playwright/static-server.js @@ -27,12 +27,12 @@ function killProcessTree (childProcess) { /** Free the Playwright port when a previous run left `serve` listening or wedged. */ export function killProcessOnPort (listenPort = port) { if (process.platform === 'win32') { - const result = spawnSync('netstat', ['-ano'], {encoding: 'utf8', shell: true}); - if (result.status !== 0) { + const netstat = spawnSync('netstat', ['-ano'], {encoding: 'utf8', shell: true}); + if (netstat.status !== 0) { return; } - for (const line of result.stdout.split('\n')) { + for (const line of netstat.stdout.split('\n')) { if (!line.includes(`:${listenPort}`) || !line.includes('LISTENING')) { continue; } @@ -45,12 +45,12 @@ export function killProcessOnPort (listenPort = port) { return; } - const result = spawnSync('lsof', ['-ti', `:${listenPort}`], {encoding: 'utf8'}); - if (result.status !== 0 || !result.stdout.trim()) { + const lsof = spawnSync('lsof', ['-ti', `:${listenPort}`], {encoding: 'utf8'}); + if (lsof.status !== 0 || !lsof.stdout.trim()) { return; } - for (const pid of result.stdout.trim().split('\n')) { + for (const pid of lsof.stdout.trim().split('\n')) { if (/^\d+$/.test(pid)) { spawnSync('kill', ['-9', pid], {stdio: 'ignore'}); } From e672ff0bebd08a81b36861af9cb05aa98e27683b Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Wed, 10 Jun 2026 20:35:47 +0300 Subject: [PATCH 12/28] automate run to create dist folder --- .../scripts/ensure-screenshot-dist.mjs | 48 +++++++++++++++++++ .../scripts/run-playwright-component.mjs | 11 +++-- tests/screenshot/scripts/run-playwright.mjs | 13 +++-- 3 files changed, 63 insertions(+), 9 deletions(-) create mode 100644 tests/screenshot/scripts/ensure-screenshot-dist.mjs diff --git a/tests/screenshot/scripts/ensure-screenshot-dist.mjs b/tests/screenshot/scripts/ensure-screenshot-dist.mjs new file mode 100644 index 000000000..b6144c050 --- /dev/null +++ b/tests/screenshot/scripts/ensure-screenshot-dist.mjs @@ -0,0 +1,48 @@ +/* eslint-disable no-console */ +import fs from 'fs'; + +import buildApps from '@enact/ui-test-utils/build-apps'; + +import {SCREENSHOT_DIST, SCREENSHOT_VIEW_INDEX} from '../playwright/paths.js'; + +export function hasScreenshotDist () { + return fs.existsSync(SCREENSHOT_VIEW_INDEX); +} + +function assertBuildOutput () { + if (hasScreenshotDist()) { + return; + } + + const distHint = !fs.existsSync(SCREENSHOT_DIST) ? + `Directory ${SCREENSHOT_DIST} does not exist. ` : + ''; + + throw new Error( + `${distHint}Screenshot build did not produce ${SCREENSHOT_VIEW_INDEX}. ` + + 'Run npm run bootstrap, link @enact/ui-test-utils if build-apps is missing from exports, then retry.' + ); +} + +/** + * Ensure tests/screenshot/dist exists before Playwright starts its webServer. + * Builds when dist is missing (even with --skip-build) so runs do not fail on a cold checkout. + * + * @returns {Promise<{skipBuild: boolean}>} skipBuild=true → set PLAYWRIGHT_SKIP_BUILD for global-setup + */ +export async function ensureScreenshotDist ({skipBuild = false} = {}) { + if (hasScreenshotDist()) { + return {skipBuild}; + } + + if (skipBuild) { + console.log('tests/screenshot/dist is missing; building before Playwright (ignoring --skip-build)'); + } else { + console.log('Building tests/screenshot/dist...'); + } + + await buildApps('screenshot'); + assertBuildOutput(); + + return {skipBuild: true}; +} diff --git a/tests/screenshot/scripts/run-playwright-component.mjs b/tests/screenshot/scripts/run-playwright-component.mjs index 882cadc97..11daf5904 100644 --- a/tests/screenshot/scripts/run-playwright-component.mjs +++ b/tests/screenshot/scripts/run-playwright-component.mjs @@ -13,6 +13,7 @@ * npm run test-playwright:component -- Sprite --parallel 5 */ import {assertComponentSource} from '../playwright/paths.js'; +import {ensureScreenshotDist} from './ensure-screenshot-dist.mjs'; import {parseComponentArgs} from './parse-component-args.mjs'; import {spawnPlaywright} from './spawn-playwright.mjs'; @@ -41,10 +42,6 @@ if (update) { env.PLAYWRIGHT_FORCE_UPDATE = '1'; } -if (skipBuild) { - env.PLAYWRIGHT_SKIP_BUILD = '1'; -} - if (spec) { env.PLAYWRIGHT_SPEC = spec; } @@ -63,6 +60,12 @@ if (spec) { } console.log(`Playwright: ${filters.join(', ')}`); +const {skipBuild: skipBuildInSetup} = await ensureScreenshotDist({skipBuild}); + +if (skipBuild || skipBuildInSetup) { + env.PLAYWRIGHT_SKIP_BUILD = '1'; +} + const exitCode = spawnPlaywright({ env, label: `Playwright (${component})` diff --git a/tests/screenshot/scripts/run-playwright.mjs b/tests/screenshot/scripts/run-playwright.mjs index fc35be240..2880090b7 100644 --- a/tests/screenshot/scripts/run-playwright.mjs +++ b/tests/screenshot/scripts/run-playwright.mjs @@ -12,8 +12,9 @@ * npm run test-playwright -- --skip-build * npm run test-playwright -- --parallel 5 */ -import {spawnPlaywright} from './spawn-playwright.mjs'; +import {ensureScreenshotDist} from './ensure-screenshot-dist.mjs'; import {parsePlaywrightArgs} from './parse-playwright-args.mjs'; +import {spawnPlaywright} from './spawn-playwright.mjs'; const {component, spec, update, skipBuild, parallel, testId, title} = parsePlaywrightArgs(); const env = {}; @@ -33,10 +34,6 @@ if (update) { env.PLAYWRIGHT_FORCE_UPDATE = '1'; } -if (skipBuild) { - env.PLAYWRIGHT_SKIP_BUILD = '1'; -} - if (parallel != null) { env.PLAYWRIGHT_WORKERS = String(parallel); } @@ -55,5 +52,11 @@ if (filters.length) { console.log('Playwright: all components, all specs'); } +const {skipBuild: skipBuildInSetup} = await ensureScreenshotDist({skipBuild}); + +if (skipBuild || skipBuildInSetup) { + env.PLAYWRIGHT_SKIP_BUILD = '1'; +} + const exitCode = spawnPlaywright({env}); process.exit(exitCode); From 7d2a19ab6fc5b06d8f602f318033832444006ad7 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Thu, 11 Jun 2026 16:03:57 +0300 Subject: [PATCH 13/28] merge run and parse in one file --- package.json | 2 +- tests/screenshot/playwright/README.md | 68 +++++---- tests/screenshot/playwright/benchmark.mjs | 4 +- tests/screenshot/playwright/global-setup.js | 38 +++-- tests/screenshot/playwright/paths.js | 4 +- .../playwright/playwright.config.mjs | 5 +- tests/screenshot/playwright/static-server.js | 11 +- .../playwright/utils/limestone-page.js | 4 +- .../playwright/utils/run-screenshot-tests.js | 36 ++++- .../playwright/utils/shard-registry.js | 3 +- .../scripts/ensure-screenshot-dist.mjs | 19 +-- tests/screenshot/scripts/parse-cli-args.mjs | 91 +++++++++++ .../scripts/parse-component-args.mjs | 43 ------ .../scripts/parse-playwright-args.mjs | 39 ----- .../scripts/run-playwright-component.mjs | 73 --------- tests/screenshot/scripts/run-playwright.mjs | 142 ++++++++++++++---- tests/screenshot/scripts/spawn-playwright.mjs | 14 +- 17 files changed, 332 insertions(+), 264 deletions(-) create mode 100644 tests/screenshot/scripts/parse-cli-args.mjs delete mode 100644 tests/screenshot/scripts/parse-component-args.mjs delete mode 100644 tests/screenshot/scripts/parse-playwright-args.mjs delete mode 100644 tests/screenshot/scripts/run-playwright-component.mjs diff --git a/package.json b/package.json index 5a782c475..edad3fc2e 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "test-playwright": "node tests/screenshot/scripts/run-playwright.mjs", "test-playwright:update": "node tests/screenshot/scripts/run-playwright.mjs -- --update", "test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", - "test-playwright:component": "node tests/screenshot/scripts/run-playwright-component.mjs", + "test-playwright:component": "node tests/screenshot/scripts/run-playwright.mjs", "test-ss:component": "node tests/screenshot/scripts/run-component-wdio.mjs", "prepare-playwright": "playwright install chrome", "benchmark-screenshots": "node tests/screenshot/playwright/benchmark.mjs", diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index a3e1e29dd..b5d291754 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -56,19 +56,20 @@ tests/screenshot/ dist/ ← buildApps('screenshot') output (shared) specs/ ← WDIO *-specs.js scripts/ - run-playwright.mjs - run-playwright-component.mjs + run-playwright.mjs full suite + component (same entry; mode from argv) + parse-cli-args.mjs shared CLI parsing + ensure-screenshot-dist.mjs + spawn-playwright.mjs run-component-wdio.mjs - parse-component-args.mjs playwright/ ← this folder specs/neutral|light/ *-spec.js (27 shards) utils/ registerScreenshotTests, limestone-page, screenshot-name, shard-registry snapshots/ PNG baselines (gitignored — generate with --update) .shard-registry.jsonl run-time shard log (gitignored; see shard-registry.js) - global-setup.js build, dist check, .test-data.json, serve :4568 if needed - global-teardown.js shard coverage check, stops serve when global-setup started it + global-setup.js build, dist check, .test-data.json (serve :4568 only while fetching) + global-teardown.js shard coverage check playwright.config.mjs - paths.js + paths.js SCREENSHOT_VIEW, SCREENSHOT_HEALTH_URL, assertScreenshotDist() benchmark.mjs ``` @@ -102,7 +103,7 @@ Triggered from `global-setup.js` unless `PLAYWRIGHT_SKIP_BUILD=1`. Packs `Limest ### 4. Case index — `.test-data.json` -`global-setup` loads `http://localhost:4568/Limestone-View/?request`. That sets `window.__TEST_DATA = testMetadata` without rendering a case. The JSON is cached at `playwright/.test-data.json`. +When `.test-data.json` is missing or older than `dist/Limestone-View/index.html`, `global-setup` briefly starts `serve` on port **4568**, loads `…/Limestone-View/?request` (waits for `window.__TEST_DATA`), writes `playwright/.test-data.json`, then stops that server so Playwright’s `webServer` can bind the same port. Cached runs with `--skip-build` skip the fetch entirely. `registerScreenshotTests()` reads this file to register Playwright tests — it never parses `Chip.js` on disk. @@ -155,7 +156,7 @@ npm run prepare-playwright # optional on machines that already have Chrome (se `prepare-playwright` runs `playwright install chrome`. It is **not** hooked to `postinstall`. Run it on **fresh machines and CI** (`build-scripts/enact-playwright-tests.sh`). On a machine where Chrome is already installed, Playwright prints *"chrome" is already installed on the system!"* — that is success; you can skip this step on later runs. -`global-setup.js` calls `assertScreenshotDist()` after the build step. If `tests/screenshot/dist/Limestone-View/index.html` is missing, the run fails immediately with a clear error in global-setup (static server is started there after dist is verified). +`global-setup.js` and `ensure-screenshot-dist.mjs` call `assertScreenshotDist()` from `paths.js` when a build is needed. If `tests/screenshot/dist/Limestone-View/index.html` is missing, the run fails immediately with a clear error (not a webServer timeout). ### Build screenshot dist @@ -233,6 +234,9 @@ All commands run from the **repository root**. | `npm run test-playwright -- --component Button` | All specs; components whose name contains `Button` | | `npm run test-playwright -- --spec Light` | `Light*` specs only; all components | | `npm run test-playwright -- --component Button --spec Light` | Both filters combined | +| `npm run test-playwright -- --title "Focused"` | Filter cases by title (regex per term; case-insensitive fallback) | +| `npm run test-playwright -- --title Button Focused` | Shorthand: `Button` → `--component`, `Focused` → `--title` | +| `npm run test-playwright -- --test-id <n>` | Single case index (invalid values fail fast) | | `npm run test-playwright -- --skip-build` | Skip `buildApps('screenshot')` when `dist/` already exists | | `npm run test-playwright:update` | Force-update all Playwright baselines (`--update`) | | `npm run test-playwright:report` | Open HTML report (`playwright/reports/html/`) | @@ -261,36 +265,39 @@ npm run test-playwright:component -- Button --title "Focused" npm run test-playwright:component -- Button --spec Light ``` -Each run prints total wall-clock time when finished (e.g. `Playwright (Button) finished in 42.3s`). +Each run prints total wall-clock time when finished (e.g. `Playwright (Button) finished in 1m 2.3s`). -Component script sets `PLAYWRIGHT_INSTANCES=1` (all `testId`s for that component), `PLAYWRIGHT_WORKERS` from `--parallel` (default **1**), and runs **all** matching spec shards unless `--spec` is set. +`test-playwright` and `test-playwright:component` both invoke `scripts/run-playwright.mjs`. **Component mode** is selected when the first argument after `--` is a positional name (e.g. `Sprite`); **suite mode** uses flags only (`--component`, `--spec`, …). -#### `scripts/parse-component-args.mjs` — why it exists +Component mode sets `PLAYWRIGHT_INSTANCES=1` (all `testId`s for that component), `PLAYWRIGHT_WORKERS` from `--parallel` (default **1**), and runs **all** matching spec shards unless `--spec` is set. + +#### `scripts/parse-cli-args.mjs` — why it exists npm passes arguments after `--` to the Node script. A naive parser such as `args.find(a => !a.startsWith('--'))` treats **flag values** as the component name — e.g. `… --test-id 0 Sprite` would resolve component `"0"` instead of `"Sprite"`. -`parseComponentArgs()` is a small shared parser used by: +`parse-cli-args.mjs` exports `parsePlaywrightArgs()`, `parseComponentArgs()`, and `isComponentPlaywrightRun()` for: -- `run-playwright.mjs` — `--component`, `--spec`, `--update`, `--skip-build`, `--parallel` -- `run-playwright-component.mjs` — `--update`, `--skip-build`, `--spec`, `--test-id`, `--title`, `--parallel` +- `run-playwright.mjs` — full suite and component entry (same file) - `benchmark.mjs` — `--build`, `--parallel` **Rules:** -- The **component name** is the first positional token (non-flag argument). -- Flags with values (`--test-id`, `--title`, `--parallel`) consume the next token as their value, not as the component. +- **Component mode:** the component name is the first positional token (non-flag argument). +- **Suite mode:** use `--component <Name>`; no positional component name. +- Value flags (`--test-id`, `--title`, `--parallel`, `--spec`, `--component`) consume following tokens until the next `--flag` (multi-word values such as `--title Button Focused` are supported). - Boolean flags: `--update`, `--build`, `--skip-build`. -- Value flags: `--spec` (component script and full suite via `run-playwright.mjs`). **Examples:** -| Command | Component resolved | -|---------|-------------------| -| `npm run test-playwright:component -- Button --parallel 5` | `Button` | -| `npm run test-playwright:component -- Button --test-id 0` | `Button` | -| `npm run benchmark-screenshots -- Chip --build` | `Chip` | +| Command | Mode / component resolved | +|---------|---------------------------| +| `npm run test-playwright:component -- Button --parallel 5` | component → `Button` | +| `npm run test-playwright:component -- Button --test-id 0` | component → `Button` | +| `npm run test-playwright -- --component Button --spec Light` | suite; component filter `Button` | +| `npm run test-playwright -- --title Button Focused --spec Light` | suite; `Button` + title `Focused` | +| `npm run benchmark-screenshots -- Chip --build` | component → `Chip` | -Put the component name **first** after `--` (as in the examples above). `run-component-wdio.mjs` still uses the older parser — use the same name-first order there too. +Put the component name **first** after `--` in component mode (as in the examples above). `run-component-wdio.mjs` still uses its own parser — use the same name-first order there too. ### Full suite @@ -344,7 +351,7 @@ npm run test-playwright -- --skip-build | **Workers** | `PLAYWRIGHT_WORKERS` or `--parallel` on component script | How many tests run at once in one Playwright process | | **Component filter** | `PLAYWRIGHT_COMPONENT` | Only one component's cases | -**Shard coverage:** `PLAYWRIGHT_INSTANCES` must match the shard files you actually run. If you set `PLAYWRIGHT_INSTANCES=9` but only invoke shards 1–5, cases with `testId % 9` in `{5,6,7,8}` are silently skipped. `global-teardown` fails the run when it detects this partial coverage (single-shard runs such as TV-style `Default-spec` only are exempt). Shards with `concurrency > PLAYWRIGHT_INSTANCES` are inert (empty) — expected when the default instances is **5** but nine shard files exist per skin. +**Shard coverage:** `PLAYWRIGHT_INSTANCES` must match the shard files you actually run. If you set `PLAYWRIGHT_INSTANCES=9` but only invoke shards 1–5, cases with `testId % 9` in `{5,6,7,8}` are silently skipped. `global-teardown` fails the run when it detects partial coverage. The only exemption is a deliberate TV-style run where **only shard 1** ran (`Default-spec` alone). Running a single non-1 shard (e.g. only `Default3-spec`) still fails validation. Shards with `concurrency > PLAYWRIGHT_INSTANCES` are inert (empty) — expected when the default instances is **5** but nine shard files exist per skin. #### `utils/shard-registry.js` — why it exists @@ -356,7 +363,7 @@ Each `*-spec.js` file is independent: it only knows its own `concurrency` value |------|--------|------| | Reset | `global-setup.js` → `clearShardRegistry()` | Deletes `playwright/.shard-registry.jsonl` from any previous run | | Record | `registerScreenshotTests()` → `recordShard()` | Appends one JSON line per spec file: skin, highContrast, `concurrency`, `PLAYWRIGHT_INSTANCES` | -| Validate | `global-teardown.js` → `validateShardCoverage()` | For each skin/HC group, ensures shards `1…PLAYWRIGHT_INSTANCES` all ran when more than one shard was invoked | +| Validate | `global-teardown.js` → `validateShardCoverage()` | For each skin/HC group, ensures shards `1…PLAYWRIGHT_INSTANCES` all ran (exempt only when shard 1 alone ran) | **Skipped when** `PLAYWRIGHT_COMPONENT` is set (component script uses `PLAYWRIGHT_INSTANCES=1` and a single spec — no cross-shard risk). @@ -448,8 +455,8 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | `PLAYWRIGHT_COMPONENT_EXACT=1` | Exact component name match (set by component script) | | `PLAYWRIGHT_COMPONENT` | Filter components (substring match from `--component`; exact when set by component script) | | `PLAYWRIGHT_REFRESH_TEST_DATA=1` | Force-regenerate `.test-data.json` | -| `PLAYWRIGHT_TEST_ID` | Filter to one case index | -| `PLAYWRIGHT_TITLE` | Filter cases by title regex | +| `PLAYWRIGHT_TEST_ID` | Filter to one case index (must be a non-negative integer; invalid values throw) | +| `PLAYWRIGHT_TITLE` | Filter cases by title (regex per whitespace-separated term; invalid regex falls back to substring; case-insensitive) | | `PLAYWRIGHT_INSTANCES` | Shard divisor (`testId % instances`); component script sets `1` | | `PLAYWRIGHT_WORKERS` | Parallel workers in `playwright.config.mjs` (default **5**) | | `PLAYWRIGHT_SPEC` | Limit which `*-spec.js` files run | @@ -464,7 +471,10 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | Missing dist / build failed | Read the `assertScreenshotDist()` error (fail-fast in global-setup, not a webServer timeout); link `@enact/ui-test-utils` if build-apps fails | | Snapshot diff | Re-baseline with `--update`, or run failing `--test-id` alone | | Snapshot diff on focus cases in parallel | Re-baseline with `--parallel 1`, or run failing `--test-id` alone | -| No tests found | Check `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_SPEC`; delete stale `.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1` | +| No tests found | Check `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_SPEC`, `PLAYWRIGHT_TITLE`, `PLAYWRIGHT_TEST_ID` (case titles omit the component name — use `--component Button --title Focused` or `--title Button Focused`); delete stale `.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1` | +| Invalid `PLAYWRIGHT_TEST_ID` | Use a non-negative integer for `--test-id` (e.g. `--test-id 3`) | +| Stale `.test-data.json` (`undefined` parse error) | Delete `playwright/.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1`; rebuild dist if needed | +| Port conflict on CI after setup | Ensure global-setup stopped its temporary `serve` (fresh `.test-data.json` + `--skip-build` should not leave an orphan on :4568) | | Missing images on build | Place assets under `tests/screenshot/images/`, `videos/` | | Port already in use | Set `PLAYWRIGHT_BASE_URL` to another port; restart any stale `serve` on 4568 | | `build-apps` import error | Link or pin `@enact/ui-test-utils` with export `./build-apps` (see **Link local @enact/ui-test-utils** above) | @@ -507,7 +517,7 @@ Add matching `playwright/specs/**/<Name>-spec.js` with the same `skin`, `highCon "test-playwright": "node tests/screenshot/scripts/run-playwright.mjs", "test-playwright:update": "node tests/screenshot/scripts/run-playwright.mjs -- --update", "test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", -"test-playwright:component": "node tests/screenshot/scripts/run-playwright-component.mjs", +"test-playwright:component": "node tests/screenshot/scripts/run-playwright.mjs", "test-ss:component": "node tests/screenshot/scripts/run-component-wdio.mjs", "prepare-playwright": "playwright install chrome", "benchmark-screenshots": "node tests/screenshot/playwright/benchmark.mjs" diff --git a/tests/screenshot/playwright/benchmark.mjs b/tests/screenshot/playwright/benchmark.mjs index b23327ba1..7969cdc58 100644 --- a/tests/screenshot/playwright/benchmark.mjs +++ b/tests/screenshot/playwright/benchmark.mjs @@ -15,7 +15,7 @@ import path from 'path'; import {fileURLToPath} from 'url'; import {assertComponentSource, SCREENSHOT_VIEW_INDEX} from './paths.js'; -import {parseComponentArgs} from '../scripts/parse-component-args.mjs'; +import {parseComponentArgs} from '../scripts/parse-cli-args.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.join(__dirname, '..', '..', '..'); @@ -73,7 +73,7 @@ async function main () { const playwright = await runCommand( 'Playwright', 'node', - ['tests/screenshot/scripts/run-playwright-component.mjs', component, ...parallelArg], + ['tests/screenshot/scripts/run-playwright.mjs', component, ...parallelArg], skipBuild ? {PLAYWRIGHT_SKIP_BUILD: '1'} : {} ); diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js index af4d685a3..ab4140ea6 100644 --- a/tests/screenshot/playwright/global-setup.js +++ b/tests/screenshot/playwright/global-setup.js @@ -33,28 +33,40 @@ export default async function globalSetup () { assertScreenshotDist(); + if (!shouldRefreshTestData()) { + return; + } + // Playwright lifecycle: globalSetup → webServer (playwright.config.mjs) → tests → teardown. // We start serve here only to fetch ?request metadata into .test-data.json, then stop our // process so webServer can own the port for the full run (reuseExistingServer when not CI). await ensureStaticServer(); - if (!shouldRefreshTestData()) { - return; - } + try { + const pwBrowser = await chromium.launch({ + channel: 'chrome', + headless: true + }); - const pwBrowser = await chromium.launch({ - channel: 'chrome', - headless: true - }); + try { + const page = await pwBrowser.newPage(); + const requestUrl = `${baseURL}/Limestone-View/?request`; + await page.goto(requestUrl, {waitUntil: 'load'}); + await page.waitForFunction(() => window.__TEST_DATA != null, undefined, {timeout: 30000}); + const testData = await page.evaluate(() => window.__TEST_DATA); - try { - const page = await pwBrowser.newPage(); - await page.goto(`${baseURL}/Limestone-View/?request`, {waitUntil: 'load'}); - const testData = await page.evaluate(() => window.__TEST_DATA); + if (testData == null) { + throw new Error( + `No window.__TEST_DATA at ${requestUrl}. The screenshot build may be stale or broken — ` + + 'rebuild tests/screenshot/dist (unset PLAYWRIGHT_SKIP_BUILD) and retry.' + ); + } - fs.writeFileSync(testDataFile, JSON.stringify(testData)); + fs.writeFileSync(testDataFile, JSON.stringify(testData)); + } finally { + await pwBrowser.close(); + } } finally { - await pwBrowser.close(); stopStaticServer(); } } diff --git a/tests/screenshot/playwright/paths.js b/tests/screenshot/playwright/paths.js index dc4921d57..d7615428f 100644 --- a/tests/screenshot/playwright/paths.js +++ b/tests/screenshot/playwright/paths.js @@ -10,7 +10,9 @@ export const PLAYWRIGHT_BASE_URL = process.env.PLAYWRIGHT_BASE_URL || 'http://lo export const PLAYWRIGHT_PORT = new URL(PLAYWRIGHT_BASE_URL).port || '4568'; export const SCREENSHOT_DIST = path.join(screenshotRoot, 'dist'); -export const SCREENSHOT_VIEW_INDEX = path.join(SCREENSHOT_DIST, 'Limestone-View', 'index.html'); +export const SCREENSHOT_VIEW = 'Limestone-View'; +export const SCREENSHOT_VIEW_INDEX = path.join(SCREENSHOT_DIST, SCREENSHOT_VIEW, 'index.html'); +export const SCREENSHOT_HEALTH_URL = `${PLAYWRIGHT_BASE_URL}/${SCREENSHOT_VIEW}/index.html`; export const SCREENSHOT_COMPONENTS = path.join(screenshotRoot, 'apps', 'components'); export const TEST_DATA_FILE = path.join(__dirname, '.test-data.json'); export const SNAPSHOTS_DIR = path.join(__dirname, 'snapshots'); diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index c54798a04..58fbd0fa8 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -2,11 +2,10 @@ import path from 'path'; import {fileURLToPath} from 'url'; import {defineConfig, devices} from '@playwright/test'; -import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT} from './paths.js'; +import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_HEALTH_URL} from './paths.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const distDir = path.join(__dirname, '..', 'dist'); -const serverHealthUrl = `${PLAYWRIGHT_BASE_URL}/Limestone-View/index.html`; /** Jenkins: SPEC=Default → Default* shards; PLAYWRIGHT_SPEC=Default-spec → one file (TV). */ function resolveTestMatch () { @@ -64,7 +63,7 @@ export default defineConfig({ command: process.platform === 'win32' ? `npx.cmd --yes serve "${distDir}" -l ${PLAYWRIGHT_PORT}` : `npx --yes serve ${distDir} -l ${PLAYWRIGHT_PORT}`, - url: serverHealthUrl, + url: SCREENSHOT_HEALTH_URL, reuseExistingServer: !process.env.CI, timeout: 120000, shell: process.platform === 'win32' diff --git a/tests/screenshot/playwright/static-server.js b/tests/screenshot/playwright/static-server.js index 067b47eae..21484beb4 100644 --- a/tests/screenshot/playwright/static-server.js +++ b/tests/screenshot/playwright/static-server.js @@ -1,10 +1,9 @@ import {spawn, spawnSync} from 'child_process'; -import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_DIST} from './paths.js'; +import {PLAYWRIGHT_PORT, SCREENSHOT_DIST, SCREENSHOT_HEALTH_URL} from './paths.js'; const distPath = SCREENSHOT_DIST; const port = PLAYWRIGHT_PORT; -const healthCheckUrl = `${PLAYWRIGHT_BASE_URL}/Limestone-View/index.html`; /** Set when global-setup starts `serve` for ?request metadata (stopped before tests). */ export const staticServer = { @@ -32,8 +31,10 @@ export function killProcessOnPort (listenPort = port) { return; } + const portPattern = new RegExp(`:${String(listenPort)}(?!\\d)`); + for (const line of netstat.stdout.split('\n')) { - if (!line.includes(`:${listenPort}`) || !line.includes('LISTENING')) { + if (!portPattern.test(line) || !line.includes('LISTENING')) { continue; } @@ -81,7 +82,7 @@ async function fetchWithTimeout (url, timeoutMs = 5000) { async function isServerHealthy () { try { - const response = await fetchWithTimeout(healthCheckUrl); + const response = await fetchWithTimeout(SCREENSHOT_HEALTH_URL); return response.ok; } catch { return false; @@ -122,5 +123,5 @@ export async function ensureStaticServer () { stopStaticServer(); startStaticServer(); - await waitForServer(healthCheckUrl); + await waitForServer(SCREENSHOT_HEALTH_URL); } diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index 381e80340..5cbaa4b7e 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -1,4 +1,4 @@ -const VIEW = 'Limestone-View'; +import {SCREENSHOT_VIEW} from '../paths.js'; function serializeParams (params) { const searchParams = new URLSearchParams(); @@ -13,7 +13,7 @@ function serializeParams (params) { } function buildUrl (urlExtra = '?locale=en-US') { - return `/${VIEW}/${urlExtra}`; + return `/${SCREENSHOT_VIEW}/${urlExtra}`; } async function open (page, urlExtra = '?locale=en-US') { diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js index c6eac1517..ab9692bd9 100644 --- a/tests/screenshot/playwright/utils/run-screenshot-tests.js +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -11,9 +11,25 @@ import {recordShard} from './shard-registry.js'; const forceUpdate = process.env.PLAYWRIGHT_FORCE_UPDATE === '1'; const componentExactMatch = process.env.PLAYWRIGHT_COMPONENT_EXACT === '1'; -const testIdFilter = process.env.PLAYWRIGHT_TEST_ID != null ? - Number.parseInt(process.env.PLAYWRIGHT_TEST_ID) : - null; +function parseTestIdFilter () { + const raw = process.env.PLAYWRIGHT_TEST_ID; + + if (raw == null) { + return null; + } + + const trimmed = raw.trim(); + + if (!/^\d+$/.test(trimmed)) { + throw new Error( + `Invalid PLAYWRIGHT_TEST_ID "${raw}": must be a non-negative integer (e.g. --test-id 3).` + ); + } + + return Number.parseInt(trimmed, 10); +} + +const testIdFilter = parseTestIdFilter(); const titleFilter = process.env.PLAYWRIGHT_TITLE; const maxInstances = process.env.PLAYWRIGHT_INSTANCES ? Number.parseInt(process.env.PLAYWRIGHT_INSTANCES) : @@ -27,6 +43,18 @@ function resolveComponentFilter (config) { return config.component || process.env.PLAYWRIGHT_COMPONENT || null; } +function matchesTitleFilter (title, filter) { + const terms = filter.trim().split(/\s+/).filter(Boolean); + + return terms.every(term => { + try { + return new RegExp(term, 'i').test(title); + } catch { + return title.toLowerCase().includes(term.toLowerCase()); + } + }); +} + function shouldIncludeTest (component, testId, title, configComponent) { const componentFilter = resolveComponentFilter({component: configComponent}); if (componentFilter) { @@ -42,7 +70,7 @@ function shouldIncludeTest (component, testId, title, configComponent) { if (testIdFilter != null && testIdFilter !== testId) { return false; } - if (titleFilter && !title.match(new RegExp(titleFilter))) { + if (titleFilter && !matchesTitleFilter(title, titleFilter)) { return false; } return true; diff --git a/tests/screenshot/playwright/utils/shard-registry.js b/tests/screenshot/playwright/utils/shard-registry.js index 9057ef614..e0920e66e 100644 --- a/tests/screenshot/playwright/utils/shard-registry.js +++ b/tests/screenshot/playwright/utils/shard-registry.js @@ -52,8 +52,9 @@ export function validateShardCoverage () { const covering = [...shards].filter(shard => shard <= maxWorkers).sort((a, b) => a - b); const required = [...Array(maxWorkers)].map((_, index) => index + 1); const missing = required.filter(shard => !covering.includes(shard)); + const isShard1OnlyRun = covering.length === 1 && covering[0] === 1; - if (covering.length > 1 && missing.length > 0) { + if (missing.length > 0 && !isShard1OnlyRun) { throw new Error( `Incomplete Playwright shard coverage for ${groupKey}: ` + `PLAYWRIGHT_INSTANCES=${maxWorkers} requires shards ${required.join(', ')}, ` + diff --git a/tests/screenshot/scripts/ensure-screenshot-dist.mjs b/tests/screenshot/scripts/ensure-screenshot-dist.mjs index b6144c050..6128ae957 100644 --- a/tests/screenshot/scripts/ensure-screenshot-dist.mjs +++ b/tests/screenshot/scripts/ensure-screenshot-dist.mjs @@ -3,27 +3,12 @@ import fs from 'fs'; import buildApps from '@enact/ui-test-utils/build-apps'; -import {SCREENSHOT_DIST, SCREENSHOT_VIEW_INDEX} from '../playwright/paths.js'; +import {SCREENSHOT_VIEW_INDEX, assertScreenshotDist} from '../playwright/paths.js'; export function hasScreenshotDist () { return fs.existsSync(SCREENSHOT_VIEW_INDEX); } -function assertBuildOutput () { - if (hasScreenshotDist()) { - return; - } - - const distHint = !fs.existsSync(SCREENSHOT_DIST) ? - `Directory ${SCREENSHOT_DIST} does not exist. ` : - ''; - - throw new Error( - `${distHint}Screenshot build did not produce ${SCREENSHOT_VIEW_INDEX}. ` + - 'Run npm run bootstrap, link @enact/ui-test-utils if build-apps is missing from exports, then retry.' - ); -} - /** * Ensure tests/screenshot/dist exists before Playwright starts its webServer. * Builds when dist is missing (even with --skip-build) so runs do not fail on a cold checkout. @@ -42,7 +27,7 @@ export async function ensureScreenshotDist ({skipBuild = false} = {}) { } await buildApps('screenshot'); - assertBuildOutput(); + assertScreenshotDist(); return {skipBuild: true}; } diff --git a/tests/screenshot/scripts/parse-cli-args.mjs b/tests/screenshot/scripts/parse-cli-args.mjs new file mode 100644 index 000000000..5da7b3442 --- /dev/null +++ b/tests/screenshot/scripts/parse-cli-args.mjs @@ -0,0 +1,91 @@ +const PLAYWRIGHT_VALUE_FLAGS = new Set(['--component', '--spec', '--parallel', '--test-id', '--title']); +const COMPONENT_VALUE_FLAGS = new Set(['--test-id', '--title', '--parallel', '--spec']); + +/** + * Value flags consume all following tokens until the next --flag (multi-word values). + */ +function parseCliArgs (argv = process.argv, {valueFlags, captureFirstPositional = false} = {}) { + const args = argv.slice(2).filter(a => a !== '--'); + const flags = new Set(); + const values = {}; + let positional = null; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (!arg.startsWith('--')) { + if (captureFirstPositional && !positional) { + positional = arg; + } + continue; + } + + if (valueFlags.has(arg)) { + const valueParts = []; + while (++i < args.length && !args[i].startsWith('--')) { + valueParts.push(args[i]); + } + values[arg] = valueParts.length > 0 ? valueParts.join(' ') : null; + i--; + continue; + } + + flags.add(arg); + } + + return {flags, values, positional}; +} + +/** True when argv has a positional component name (test-playwright:component). */ +export function isComponentPlaywrightRun (argv = process.argv) { + const args = argv.slice(2).filter(a => a !== '--'); + + if (args.includes('--component')) { + return false; + } + + return parseComponentArgs(argv).component != null; +} + +/** + * Parse CLI args for npm run test-playwright. + * All flags are optional; default run executes all specs on all components. + */ +export function parsePlaywrightArgs (argv = process.argv) { + const {flags, values} = parseCliArgs(argv, {valueFlags: PLAYWRIGHT_VALUE_FLAGS}); + + return { + component: values['--component'] ?? null, + spec: values['--spec'] ?? null, + update: flags.has('--update'), + skipBuild: flags.has('--skip-build'), + testId: values['--test-id'], + title: values['--title'], + parallel: values['--parallel'] != null ? + Math.max(1, Number.parseInt(values['--parallel']) || 1) : + null + }; +} + +/** + * Parse component CLI args without treating flag values as the component name. + * Component name must be the first positional token (flags may follow). + */ +export function parseComponentArgs (argv = process.argv) { + const {flags, values, positional} = parseCliArgs(argv, { + valueFlags: COMPONENT_VALUE_FLAGS, + captureFirstPositional: true + }); + + return { + component: positional, + update: flags.has('--update'), + // --build is used by benchmark.mjs only; Playwright runners use --skip-build instead. + withBuild: flags.has('--build'), + skipBuild: flags.has('--skip-build'), + testId: values['--test-id'], + title: values['--title'], + spec: values['--spec'], + parallel: Math.max(1, Number.parseInt(values['--parallel'] ?? '1') || 1) + }; +} diff --git a/tests/screenshot/scripts/parse-component-args.mjs b/tests/screenshot/scripts/parse-component-args.mjs deleted file mode 100644 index 6ea6089fb..000000000 --- a/tests/screenshot/scripts/parse-component-args.mjs +++ /dev/null @@ -1,43 +0,0 @@ -const VALUE_FLAGS = new Set(['--test-id', '--title', '--parallel', '--spec']); - -/** - * Parse component CLI args without treating flag values as the component name. - * Component name must be the first positional token (flags may follow). - */ -export function parseComponentArgs (argv = process.argv) { - const args = argv.slice(2).filter(a => a !== '--'); - const flags = new Set(); - const values = {}; - let component = null; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - - if (!arg.startsWith('--')) { - if (!component) { - component = arg; - } - continue; - } - - if (VALUE_FLAGS.has(arg)) { - values[arg] = args[i + 1] ?? null; - i++; - continue; - } - - flags.add(arg); - } - - return { - component, - update: flags.has('--update'), - // --build is used by benchmark.mjs only; Playwright runners use --skip-build instead. - withBuild: flags.has('--build'), - skipBuild: flags.has('--skip-build'), - testId: values['--test-id'], - title: values['--title'], - spec: values['--spec'], - parallel: Math.max(1, Number.parseInt(values['--parallel'] ?? '1') || 1) - }; -} diff --git a/tests/screenshot/scripts/parse-playwright-args.mjs b/tests/screenshot/scripts/parse-playwright-args.mjs deleted file mode 100644 index b986538bb..000000000 --- a/tests/screenshot/scripts/parse-playwright-args.mjs +++ /dev/null @@ -1,39 +0,0 @@ -const VALUE_FLAGS = new Set(['--component', '--spec', '--parallel', '--test-id', '--title']); - -/** - * Parse CLI args for npm run test-playwright. - * All flags are optional; default run executes all specs on all components. - */ -export function parsePlaywrightArgs (argv = process.argv) { - const args = argv.slice(2).filter(a => a !== '--'); - const flags = new Set(); - const values = {}; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - - if (!arg.startsWith('--')) { - continue; - } - - if (VALUE_FLAGS.has(arg)) { - values[arg] = args[i + 1] ?? null; - i++; - continue; - } - - flags.add(arg); - } - - return { - component: values['--component'] ?? null, - spec: values['--spec'] ?? null, - update: flags.has('--update'), - skipBuild: flags.has('--skip-build'), - testId: values['--test-id'], - title: values['--title'], - parallel: values['--parallel'] != null ? - Math.max(1, Number.parseInt(values['--parallel']) || 1) : - null - }; -} diff --git a/tests/screenshot/scripts/run-playwright-component.mjs b/tests/screenshot/scripts/run-playwright-component.mjs deleted file mode 100644 index 11daf5904..000000000 --- a/tests/screenshot/scripts/run-playwright-component.mjs +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env node -/* eslint-disable no-console */ -/** - * Run Playwright screenshot tests for one component (all cases; INSTANCES=1, WORKERS from --parallel). - * - * Usage: - * npm run test-playwright:component -- Sprite - * npm run test-playwright:component -- Sprite --update - * npm run test-playwright:component -- Sprite --skip-build - * npm run test-playwright:component -- Sprite --spec Light - * npm run test-playwright:component -- Sprite --test-id 0 - * npm run test-playwright:component -- Sprite --title <regex> - * npm run test-playwright:component -- Sprite --parallel 5 - */ -import {assertComponentSource} from '../playwright/paths.js'; -import {ensureScreenshotDist} from './ensure-screenshot-dist.mjs'; -import {parseComponentArgs} from './parse-component-args.mjs'; -import {spawnPlaywright} from './spawn-playwright.mjs'; - -const {component, update, skipBuild, spec, testId, title, parallel} = parseComponentArgs(); - -if (!component) { - console.error('Usage: npm run test-playwright:component -- <ComponentName> [options]'); - console.error('Options: --update, --skip-build, --spec <Name>, --test-id <n>, --title <regex>, --parallel <n>'); - console.error('Example: npm run test-playwright:component -- Sprite'); - console.error('Example: npm run test-playwright:component -- Sprite --update'); - console.error('Example: npm run test-playwright:component -- Sprite --skip-build'); - console.error('Example: npm run test-playwright:component -- Sprite --spec Light'); - process.exit(1); -} - -assertComponentSource(component); - -const env = { - PLAYWRIGHT_COMPONENT: component, - PLAYWRIGHT_COMPONENT_EXACT: '1', - PLAYWRIGHT_INSTANCES: '1', - PLAYWRIGHT_WORKERS: String(parallel) -}; - -if (update) { - env.PLAYWRIGHT_FORCE_UPDATE = '1'; -} - -if (spec) { - env.PLAYWRIGHT_SPEC = spec; -} - -if (testId != null) { - env.PLAYWRIGHT_TEST_ID = testId; -} - -if (title != null) { - env.PLAYWRIGHT_TITLE = title; -} - -const filters = [`component ${component}`]; -if (spec) { - filters.push(`spec ~ ${spec}`); -} -console.log(`Playwright: ${filters.join(', ')}`); - -const {skipBuild: skipBuildInSetup} = await ensureScreenshotDist({skipBuild}); - -if (skipBuild || skipBuildInSetup) { - env.PLAYWRIGHT_SKIP_BUILD = '1'; -} - -const exitCode = spawnPlaywright({ - env, - label: `Playwright (${component})` -}); -process.exit(exitCode); diff --git a/tests/screenshot/scripts/run-playwright.mjs b/tests/screenshot/scripts/run-playwright.mjs index 2880090b7..e3cf73e90 100644 --- a/tests/screenshot/scripts/run-playwright.mjs +++ b/tests/screenshot/scripts/run-playwright.mjs @@ -1,9 +1,9 @@ #!/usr/bin/env node /* eslint-disable no-console */ /** - * Run Playwright screenshot tests with optional component/spec filters. + * Run Playwright screenshot tests (full suite or single component). * - * Usage: + * Full suite: * npm run test-playwright * npm run test-playwright -- --component Button * npm run test-playwright -- --spec Light @@ -11,52 +11,134 @@ * npm run test-playwright -- --update * npm run test-playwright -- --skip-build * npm run test-playwright -- --parallel 5 + * + * Single component (positional name; INSTANCES=1, WORKERS from --parallel): + * npm run test-playwright:component -- Sprite + * npm run test-playwright:component -- Sprite --update + * npm run test-playwright:component -- Sprite --skip-build + * npm run test-playwright:component -- Sprite --spec Light + * npm run test-playwright:component -- Sprite --test-id 0 + * npm run test-playwright:component -- Sprite --title <regex> + * npm run test-playwright:component -- Sprite --parallel 5 */ +import {assertComponentSource} from '../playwright/paths.js'; import {ensureScreenshotDist} from './ensure-screenshot-dist.mjs'; -import {parsePlaywrightArgs} from './parse-playwright-args.mjs'; +import {isComponentPlaywrightRun, parseComponentArgs, parsePlaywrightArgs} from './parse-cli-args.mjs'; import {spawnPlaywright} from './spawn-playwright.mjs'; -const {component, spec, update, skipBuild, parallel, testId, title} = parsePlaywrightArgs(); -const env = {}; -const filters = []; - -if (component) { - env.PLAYWRIGHT_COMPONENT = component; - filters.push(`component ~ ${component}`); +function printComponentUsage () { + console.error('Usage: npm run test-playwright:component -- <ComponentName> [options]'); + console.error('Options: --update, --skip-build, --spec <Name>, --test-id <n>, --title <regex>, --parallel <n>'); + console.error('Example: npm run test-playwright:component -- Sprite'); + console.error('Example: npm run test-playwright:component -- Sprite --update'); + console.error('Example: npm run test-playwright:component -- Sprite --skip-build'); + console.error('Example: npm run test-playwright:component -- Sprite --spec Light'); } -if (spec) { - env.PLAYWRIGHT_SPEC = spec; - filters.push(`spec ~ ${spec}`); -} +function buildSuiteEnv () { + const {component, spec, update, skipBuild, parallel, testId, title} = parsePlaywrightArgs(); + const env = {}; + const filters = []; -if (update) { - env.PLAYWRIGHT_FORCE_UPDATE = '1'; -} + let resolvedComponent = component; + let resolvedTitle = title; -if (parallel != null) { - env.PLAYWRIGHT_WORKERS = String(parallel); -} + if (title != null && !component) { + const titleParts = title.trim().split(/\s+/); -if (testId != null) { - env.PLAYWRIGHT_TEST_ID = testId; -} + if (titleParts.length > 1) { + resolvedComponent = titleParts[0]; + resolvedTitle = titleParts.slice(1).join(' '); + } + } + + if (resolvedComponent) { + env.PLAYWRIGHT_COMPONENT = resolvedComponent; + filters.push(`component ~ ${resolvedComponent}`); + } + + if (spec) { + env.PLAYWRIGHT_SPEC = spec; + filters.push(`spec ~ ${spec}`); + } + + if (update) { + env.PLAYWRIGHT_FORCE_UPDATE = '1'; + } + + if (parallel != null) { + env.PLAYWRIGHT_WORKERS = String(parallel); + } + + if (testId != null) { + env.PLAYWRIGHT_TEST_ID = testId; + } + + if (resolvedTitle != null) { + env.PLAYWRIGHT_TITLE = resolvedTitle; + filters.push(`title ~ ${resolvedTitle}`); + } -if (title != null) { - env.PLAYWRIGHT_TITLE = title; + if (filters.length) { + console.log(`Playwright filters: ${filters.join(', ')}`); + } else { + console.log('Playwright: all components, all specs'); + } + + return {env, skipBuild, label: 'Playwright'}; } -if (filters.length) { - console.log(`Playwright filters: ${filters.join(', ')}`); -} else { - console.log('Playwright: all components, all specs'); +function buildComponentEnv () { + const {component, update, skipBuild, spec, testId, title, parallel} = parseComponentArgs(); + + if (!component) { + printComponentUsage(); + process.exit(1); + } + + assertComponentSource(component); + + const env = { + PLAYWRIGHT_COMPONENT: component, + PLAYWRIGHT_COMPONENT_EXACT: '1', + PLAYWRIGHT_INSTANCES: '1', + PLAYWRIGHT_WORKERS: String(parallel) + }; + + if (update) { + env.PLAYWRIGHT_FORCE_UPDATE = '1'; + } + + if (spec) { + env.PLAYWRIGHT_SPEC = spec; + } + + if (testId != null) { + env.PLAYWRIGHT_TEST_ID = testId; + } + + if (title != null) { + env.PLAYWRIGHT_TITLE = title; + } + + const filters = [`component ${component}`]; + if (spec) { + filters.push(`spec ~ ${spec}`); + } + console.log(`Playwright: ${filters.join(', ')}`); + + return {env, skipBuild, label: `Playwright (${component})`}; } +const {env, skipBuild, label} = isComponentPlaywrightRun() ? + buildComponentEnv() : + buildSuiteEnv(); + const {skipBuild: skipBuildInSetup} = await ensureScreenshotDist({skipBuild}); if (skipBuild || skipBuildInSetup) { env.PLAYWRIGHT_SKIP_BUILD = '1'; } -const exitCode = spawnPlaywright({env}); +const exitCode = spawnPlaywright({env, label}); process.exit(exitCode); diff --git a/tests/screenshot/scripts/spawn-playwright.mjs b/tests/screenshot/scripts/spawn-playwright.mjs index 28ebebd93..0875988a2 100644 --- a/tests/screenshot/scripts/spawn-playwright.mjs +++ b/tests/screenshot/scripts/spawn-playwright.mjs @@ -8,6 +8,18 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.join(__dirname, '..', '..', '..'); const config = path.join(__dirname, '..', 'playwright', 'playwright.config.mjs'); +function formatDuration (durationSec) { + const hours = Math.floor(durationSec / 3600); + const minutes = Math.floor((durationSec % 3600) / 60); + const seconds = durationSec % 60; + + const parts = []; + if (hours > 0) parts.push(`${hours}h`); + if (hours > 0 || minutes > 0) parts.push(`${minutes}m`); + parts.push(`${seconds.toFixed(1)}s`); + return parts.join(' '); +} + /** * Run Playwright with optional env overrides and print wall-clock duration. */ @@ -25,7 +37,7 @@ export function spawnPlaywright ({playwrightArgs = [], env = {}, label = 'Playwr ); const durationSec = (Date.now() - start) / 1000; - console.log(`\n${label} finished in ${durationSec.toFixed(1)}s`); + console.log(`\n${label} finished in ${formatDuration(durationSec)}`); return result.status ?? 1; } From 290d8dd885d77aac9f0dd886c5b39921adb83f16 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Thu, 11 Jun 2026 16:11:40 +0300 Subject: [PATCH 14/28] fix lint --- tests/screenshot/playwright/global-setup.js | 2 +- .../playwright/utils/run-screenshot-tests.js | 2 +- tests/screenshot/scripts/run-playwright.mjs | 32 +++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js index ab4140ea6..c719e9cb8 100644 --- a/tests/screenshot/playwright/global-setup.js +++ b/tests/screenshot/playwright/global-setup.js @@ -52,7 +52,7 @@ export default async function globalSetup () { const page = await pwBrowser.newPage(); const requestUrl = `${baseURL}/Limestone-View/?request`; await page.goto(requestUrl, {waitUntil: 'load'}); - await page.waitForFunction(() => window.__TEST_DATA != null, undefined, {timeout: 30000}); + await page.waitForFunction(() => window.__TEST_DATA != null, null, {timeout: 30000}); const testData = await page.evaluate(() => window.__TEST_DATA); if (testData == null) { diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js index ab9692bd9..ff37b6e50 100644 --- a/tests/screenshot/playwright/utils/run-screenshot-tests.js +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -26,7 +26,7 @@ function parseTestIdFilter () { ); } - return Number.parseInt(trimmed, 10); + return Number.parseInt(trimmed); } const testIdFilter = parseTestIdFilter(); diff --git a/tests/screenshot/scripts/run-playwright.mjs b/tests/screenshot/scripts/run-playwright.mjs index e3cf73e90..ae4d90b4d 100644 --- a/tests/screenshot/scripts/run-playwright.mjs +++ b/tests/screenshot/scripts/run-playwright.mjs @@ -36,8 +36,8 @@ function printComponentUsage () { } function buildSuiteEnv () { - const {component, spec, update, skipBuild, parallel, testId, title} = parsePlaywrightArgs(); - const env = {}; + const {component, spec, update, skipBuild: skipBuildFlag, parallel, testId, title} = parsePlaywrightArgs(); + const playwrightEnv = {}; const filters = []; let resolvedComponent = component; @@ -53,29 +53,29 @@ function buildSuiteEnv () { } if (resolvedComponent) { - env.PLAYWRIGHT_COMPONENT = resolvedComponent; + playwrightEnv.PLAYWRIGHT_COMPONENT = resolvedComponent; filters.push(`component ~ ${resolvedComponent}`); } if (spec) { - env.PLAYWRIGHT_SPEC = spec; + playwrightEnv.PLAYWRIGHT_SPEC = spec; filters.push(`spec ~ ${spec}`); } if (update) { - env.PLAYWRIGHT_FORCE_UPDATE = '1'; + playwrightEnv.PLAYWRIGHT_FORCE_UPDATE = '1'; } if (parallel != null) { - env.PLAYWRIGHT_WORKERS = String(parallel); + playwrightEnv.PLAYWRIGHT_WORKERS = String(parallel); } if (testId != null) { - env.PLAYWRIGHT_TEST_ID = testId; + playwrightEnv.PLAYWRIGHT_TEST_ID = testId; } if (resolvedTitle != null) { - env.PLAYWRIGHT_TITLE = resolvedTitle; + playwrightEnv.PLAYWRIGHT_TITLE = resolvedTitle; filters.push(`title ~ ${resolvedTitle}`); } @@ -85,11 +85,11 @@ function buildSuiteEnv () { console.log('Playwright: all components, all specs'); } - return {env, skipBuild, label: 'Playwright'}; + return {env: playwrightEnv, skipBuild: skipBuildFlag, label: 'Playwright'}; } function buildComponentEnv () { - const {component, update, skipBuild, spec, testId, title, parallel} = parseComponentArgs(); + const {component, update, skipBuild: skipBuildFlag, spec, testId, title, parallel} = parseComponentArgs(); if (!component) { printComponentUsage(); @@ -98,7 +98,7 @@ function buildComponentEnv () { assertComponentSource(component); - const env = { + const playwrightEnv = { PLAYWRIGHT_COMPONENT: component, PLAYWRIGHT_COMPONENT_EXACT: '1', PLAYWRIGHT_INSTANCES: '1', @@ -106,19 +106,19 @@ function buildComponentEnv () { }; if (update) { - env.PLAYWRIGHT_FORCE_UPDATE = '1'; + playwrightEnv.PLAYWRIGHT_FORCE_UPDATE = '1'; } if (spec) { - env.PLAYWRIGHT_SPEC = spec; + playwrightEnv.PLAYWRIGHT_SPEC = spec; } if (testId != null) { - env.PLAYWRIGHT_TEST_ID = testId; + playwrightEnv.PLAYWRIGHT_TEST_ID = testId; } if (title != null) { - env.PLAYWRIGHT_TITLE = title; + playwrightEnv.PLAYWRIGHT_TITLE = title; } const filters = [`component ${component}`]; @@ -127,7 +127,7 @@ function buildComponentEnv () { } console.log(`Playwright: ${filters.join(', ')}`); - return {env, skipBuild, label: `Playwright (${component})`}; + return {env: playwrightEnv, skipBuild: skipBuildFlag, label: `Playwright (${component})`}; } const {env, skipBuild, label} = isComponentPlaywrightRun() ? From 68580d1bc45b3a7ddcc58e1ba92508bc773677af Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Fri, 12 Jun 2026 12:15:41 +0300 Subject: [PATCH 15/28] jenkins test 1 --- tests/screenshot/playwright/README.md | 12 +-- .../playwright/playwright.config.mjs | 16 +-- .../playwright/utils/shard-registry.js | 36 ++++--- .../screenshot/playwright/utils/spec-match.js | 99 +++++++++++++++++++ 4 files changed, 131 insertions(+), 32 deletions(-) create mode 100644 tests/screenshot/playwright/utils/spec-match.js diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index b5d291754..a5711c42d 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -351,7 +351,7 @@ npm run test-playwright -- --skip-build | **Workers** | `PLAYWRIGHT_WORKERS` or `--parallel` on component script | How many tests run at once in one Playwright process | | **Component filter** | `PLAYWRIGHT_COMPONENT` | Only one component's cases | -**Shard coverage:** `PLAYWRIGHT_INSTANCES` must match the shard files you actually run. If you set `PLAYWRIGHT_INSTANCES=9` but only invoke shards 1–5, cases with `testId % 9` in `{5,6,7,8}` are silently skipped. `global-teardown` fails the run when it detects partial coverage. The only exemption is a deliberate TV-style run where **only shard 1** ran (`Default-spec` alone). Running a single non-1 shard (e.g. only `Default3-spec`) still fails validation. Shards with `concurrency > PLAYWRIGHT_INSTANCES` are inert (empty) — expected when the default instances is **5** but nine shard files exist per skin. +**Shard coverage:** `global-teardown` checks that every shard **you asked for** actually ran. It derives expected shards from `PLAYWRIGHT_SPEC` (same rules as `testMatch` in `playwright.config.mjs`) — e.g. `PLAYWRIGHT_SPEC=Default3` requires only shard 3; no `PLAYWRIGHT_SPEC` requires all active shards 1…`PLAYWRIGHT_INSTANCES` per skin/HC group. If you set `PLAYWRIGHT_INSTANCES=9` but only invoke shards 1–5, cases with `testId % 9` in `{5,6,7,8}` are silently skipped and teardown fails for the missing expected shards. Shards with `concurrency > PLAYWRIGHT_INSTANCES` are inert (empty) — expected when the default instances is **5** but nine shard files exist per skin. #### `utils/shard-registry.js` — why it exists @@ -363,11 +363,11 @@ Each `*-spec.js` file is independent: it only knows its own `concurrency` value |------|--------|------| | Reset | `global-setup.js` → `clearShardRegistry()` | Deletes `playwright/.shard-registry.jsonl` from any previous run | | Record | `registerScreenshotTests()` → `recordShard()` | Appends one JSON line per spec file: skin, highContrast, `concurrency`, `PLAYWRIGHT_INSTANCES` | -| Validate | `global-teardown.js` → `validateShardCoverage()` | For each skin/HC group, ensures shards `1…PLAYWRIGHT_INSTANCES` all ran (exempt only when shard 1 alone ran) | +| Validate | `global-teardown.js` → `validateShardCoverage()` | For each skin/HC group, ensures every shard in the current `PLAYWRIGHT_SPEC` filter ran (`utils/spec-match.js`) | -**Skipped when** `PLAYWRIGHT_COMPONENT` is set (component script uses `PLAYWRIGHT_INSTANCES=1` and a single spec — no cross-shard risk). +**Skipped when** `PLAYWRIGHT_COMPONENT` is set (component script — no cross-shard risk). -**Example failure:** `PLAYWRIGHT_INSTANCES=9` but only `Default-spec.js` … `Default5-spec.js` are on the command line → teardown throws *Incomplete Playwright shard coverage* listing missing shards. +**Example failure:** no `PLAYWRIGHT_SPEC` (full suite) but only `Default3-spec.js` loads → teardown throws *Incomplete Playwright shard coverage* because shards 1, 2, 4, 5 were expected for that skin/HC group. The registry file is **gitignored** (`.shard-registry.jsonl`); it is a run-time artifact only. @@ -459,7 +459,7 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | `PLAYWRIGHT_TITLE` | Filter cases by title (regex per whitespace-separated term; invalid regex falls back to substring; case-insensitive) | | `PLAYWRIGHT_INSTANCES` | Shard divisor (`testId % instances`); component script sets `1` | | `PLAYWRIGHT_WORKERS` | Parallel workers in `playwright.config.mjs` (default **5**) | -| `PLAYWRIGHT_SPEC` | Limit which `*-spec.js` files run | +| `PLAYWRIGHT_SPEC` | Limit which `*-spec.js` files run; also defines which shards teardown expects | | `PLAYWRIGHT_BASE_URL` | Static server URL (default `http://localhost:4568`) | --- @@ -478,7 +478,7 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | Missing images on build | Place assets under `tests/screenshot/images/`, `videos/` | | Port already in use | Set `PLAYWRIGHT_BASE_URL` to another port; restart any stale `serve` on 4568 | | `build-apps` import error | Link or pin `@enact/ui-test-utils` with export `./build-apps` (see **Link local @enact/ui-test-utils** above) | -| Incomplete shard coverage | Ensure `PLAYWRIGHT_INSTANCES` matches invoked shard files (see **Sharding vs workers**) | +| Incomplete shard coverage | Ensure every spec file matched by `PLAYWRIGHT_SPEC` loaded and ran (see **Sharding vs workers**) | | WDIO `--parallel` ignored | Update `@enact/ui-test-utils` (see ui-test-utils PR for `--parallel` CLI) | --- diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index 58fbd0fa8..1d5a70428 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -3,26 +3,14 @@ import {fileURLToPath} from 'url'; import {defineConfig, devices} from '@playwright/test'; import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_HEALTH_URL} from './paths.js'; +import {resolveTestMatchGlob} from './utils/spec-match.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const distDir = path.join(__dirname, '..', 'dist'); -/** Jenkins: SPEC=Default → Default* shards; PLAYWRIGHT_SPEC=Default-spec → one file (TV). */ -function resolveTestMatch () { - const spec = process.env.PLAYWRIGHT_SPEC?.trim(); - if (!spec) { - return '**/*-spec.js'; - } - const base = spec.replace(/\.js$/, ''); - if (base.endsWith('-spec')) { - return `**/${base}.js`; - } - return `**/*${base}*-spec.js`; -} - export default defineConfig({ testDir: path.join(__dirname, 'specs'), - testMatch: resolveTestMatch(), + testMatch: resolveTestMatchGlob(), testIgnore: ['**/utils/**', '**/scripts/**'], globalSetup: path.join(__dirname, 'global-setup.js'), globalTeardown: path.join(__dirname, 'global-teardown.js'), diff --git a/tests/screenshot/playwright/utils/shard-registry.js b/tests/screenshot/playwright/utils/shard-registry.js index e0920e66e..c4655f91a 100644 --- a/tests/screenshot/playwright/utils/shard-registry.js +++ b/tests/screenshot/playwright/utils/shard-registry.js @@ -2,6 +2,8 @@ import fs from 'fs'; import path from 'path'; import {fileURLToPath} from 'url'; +import {getExpectedShardsByGroup} from './spec-match.js'; + const __dirname = path.dirname(fileURLToPath(import.meta.url)); const registryPath = path.join(__dirname, '..', '.shard-registry.jsonl'); @@ -30,7 +32,7 @@ export function validateShardCoverage () { return; } - const groups = new Map(); + const actualGroups = new Map(); for (const line of fs.readFileSync(registryPath, 'utf8').split('\n')) { if (!line.trim()) { @@ -39,27 +41,37 @@ export function validateShardCoverage () { const {groupKey, concurrency, maxWorkers} = JSON.parse(line); - if (!groups.has(groupKey)) { - groups.set(groupKey, {maxWorkers, shards: new Set()}); + if (!actualGroups.has(groupKey)) { + actualGroups.set(groupKey, {maxWorkers, shards: new Set()}); } - const group = groups.get(groupKey); + const group = actualGroups.get(groupKey); group.maxWorkers = maxWorkers; group.shards.add(concurrency); } - for (const [groupKey, {maxWorkers, shards}] of groups) { - const covering = [...shards].filter(shard => shard <= maxWorkers).sort((a, b) => a - b); - const required = [...Array(maxWorkers)].map((_, index) => index + 1); + const maxWorkers = process.env.PLAYWRIGHT_INSTANCES ? + Number.parseInt(process.env.PLAYWRIGHT_INSTANCES) : + [...actualGroups.values()][0]?.maxWorkers ?? 5; + const expectedGroups = getExpectedShardsByGroup(maxWorkers); + const groupKeys = new Set([...expectedGroups.keys(), ...actualGroups.keys()]); + const specHint = process.env.PLAYWRIGHT_SPEC?.trim() ? + `PLAYWRIGHT_SPEC=${process.env.PLAYWRIGHT_SPEC.trim()}` : + 'all *-spec.js files'; + + for (const groupKey of groupKeys) { + const required = [...(expectedGroups.get(groupKey) || [])].sort((a, b) => a - b); + const covering = [...(actualGroups.get(groupKey)?.shards || [])] + .filter(shard => shard <= maxWorkers) + .sort((a, b) => a - b); const missing = required.filter(shard => !covering.includes(shard)); - const isShard1OnlyRun = covering.length === 1 && covering[0] === 1; - if (missing.length > 0 && !isShard1OnlyRun) { + if (missing.length > 0) { throw new Error( `Incomplete Playwright shard coverage for ${groupKey}: ` + - `PLAYWRIGHT_INSTANCES=${maxWorkers} requires shards ${required.join(', ')}, ` + - `but only ${covering.join(', ')} ran. ` + - `Cases with testId % ${maxWorkers} in {${missing.map(shard => shard - 1).join(', ')}} were never tested.` + `${specHint} requires shards ${required.join(', ')}, ` + + `but only ${covering.length ? covering.join(', ') : '(none)'} ran. ` + + `Missing: ${missing.join(', ')}.` ); } } diff --git a/tests/screenshot/playwright/utils/spec-match.js b/tests/screenshot/playwright/utils/spec-match.js new file mode 100644 index 000000000..26629239b --- /dev/null +++ b/tests/screenshot/playwright/utils/spec-match.js @@ -0,0 +1,99 @@ +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const specsDir = path.join(__dirname, '..', 'specs'); + +/** Same rules as playwright.config.mjs `testMatch`. */ +export function resolveTestMatchGlob () { + const spec = process.env.PLAYWRIGHT_SPEC?.trim(); + + if (!spec) { + return '**/*-spec.js'; + } + + const base = spec.replace(/\.js$/, ''); + + if (base.endsWith('-spec')) { + return `**/${base}.js`; + } + + return `**/*${base}*-spec.js`; +} + +export function specFileMatchesFilter (baseName, specFilter) { + if (!specFilter) { + return baseName.endsWith('-spec'); + } + + const base = specFilter.replace(/\.js$/, ''); + + if (base.endsWith('-spec')) { + return baseName === base; + } + + return baseName.includes(base) && baseName.endsWith('-spec'); +} + +function walkSpecFiles (dir) { + const files = []; + + for (const entry of fs.readdirSync(dir, {withFileTypes: true})) { + const entryPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + files.push(...walkSpecFiles(entryPath)); + } else if (entry.name.endsWith('-spec.js')) { + files.push(entryPath); + } + } + + return files; +} + +function parseSpecMeta (content) { + const concurrency = content.match(/concurrency:\s*(\d+)/)?.[1]; + const skin = content.match(/skin:\s*'([^']+)'/)?.[1]; + const highContrast = content.match(/highContrast:\s*(true|false)/)?.[1]; + + if (!concurrency || !skin || highContrast == null) { + return null; + } + + return { + concurrency: Number.parseInt(concurrency), + skin, + highContrast: highContrast === 'true' + }; +} + +/** Shards that should run per skin/HC group for the current PLAYWRIGHT_SPEC filter. */ +export function getExpectedShardsByGroup (maxWorkers) { + const specFilter = process.env.PLAYWRIGHT_SPEC?.trim() || null; + const groups = new Map(); + + for (const filePath of walkSpecFiles(specsDir)) { + const baseName = path.basename(filePath, '.js'); + + if (!specFileMatchesFilter(baseName, specFilter)) { + continue; + } + + const meta = parseSpecMeta(fs.readFileSync(filePath, 'utf8')); + + if (!meta?.concurrency || meta.concurrency > maxWorkers) { + continue; + } + + const groupKey = `${meta.skin}-${meta.highContrast}`; + + if (!groups.has(groupKey)) { + groups.set(groupKey, new Set()); + } + + groups.get(groupKey).add(meta.concurrency); + } + + return groups; +} From dce3131ed1ac9f35d199d48247371d30431a534c Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Fri, 12 Jun 2026 22:25:50 +0300 Subject: [PATCH 16/28] timeout tests --- tests/screenshot/playwright/playwright.config.mjs | 4 ++++ tests/screenshot/playwright/utils/limestone-page.js | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index 1d5a70428..229bdb17f 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -21,6 +21,8 @@ export default defineConfig({ pathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}') } }, + // WDIO mocha timeout is 1h; default Playwright 30s is too low for CI (goto + hydrate + fonts). + timeout: 120000, fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, @@ -32,6 +34,8 @@ export default defineConfig({ outputDir: path.join(__dirname, 'test-results'), use: { baseURL: PLAYWRIGHT_BASE_URL, + navigationTimeout: 60000, + actionTimeout: 60000, trace: 'on-first-retry', viewport: {width: 1920, height: 1080}, launchOptions: { diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index 5cbaa4b7e..5b32c3e99 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -23,7 +23,7 @@ async function open (page, urlExtra = '?locale=en-US') { export async function openComponent (page, params) { const query = serializeParams(Object.assign({locale: 'en-US'}, params)); await open(page, `?${query}`); - await page.locator('[data-ui-test-id="test"]').waitFor({state: 'visible', timeout: 30000}); + await page.locator('[data-ui-test-id="test"]').waitFor({state: 'visible', timeout: 60000}); // Wait for web fonts to finish loading so glyphs are painted before the screenshot. await page.evaluate(() => document.fonts.ready); } From 09d9c0f4fc4bce1494674a3ad37bf6c564b2e64b Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Sat, 13 Jun 2026 00:02:33 +0300 Subject: [PATCH 17/28] layout --- .../playwright/utils/limestone-page.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index 5b32c3e99..5b78fe18b 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -23,7 +23,21 @@ async function open (page, urlExtra = '?locale=en-US') { export async function openComponent (page, params) { const query = serializeParams(Object.assign({locale: 'en-US'}, params)); await open(page, `?${query}`); - await page.locator('[data-ui-test-id="test"]').waitFor({state: 'visible', timeout: 60000}); + // Match WDIO Page.open: body ready + short settle (no wait on [data-ui-test-id]). + // That node can be attached but not Playwright-"visible" (e.g. Input open in FloatingLayer). + await page.locator('body').waitFor({state: 'visible', timeout: 10000}); + await page.evaluate(() => new Promise((resolve) => setTimeout(resolve, 200))); + await page.waitForFunction(() => { + const root = document.querySelector('#root'); + + if (!root?.children.length) { + return false; + } + + const text = root.textContent || ''; + + return !text.includes('INVALID COMPONENT') && !text.includes('ERROR IN'); + }, {timeout: 60000}); // Wait for web fonts to finish loading so glyphs are painted before the screenshot. await page.evaluate(() => document.fonts.ready); } From 0858c8b63df9585556284dc20f63b020c74b3225 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Mon, 15 Jun 2026 10:06:43 +0300 Subject: [PATCH 18/28] fail tests --- tests/screenshot/playwright/utils/limestone-page.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index 5b78fe18b..a925bd512 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -23,11 +23,14 @@ async function open (page, urlExtra = '?locale=en-US') { export async function openComponent (page, params) { const query = serializeParams(Object.assign({locale: 'en-US'}, params)); await open(page, `?${query}`); - // Match WDIO Page.open: body ready + short settle (no wait on [data-ui-test-id]). - // That node can be attached but not Playwright-"visible" (e.g. Input open in FloatingLayer). - await page.locator('body').waitFor({state: 'visible', timeout: 10000}); + // WDIO Page.open: body.waitForDisplayed + delay(200). Playwright "visible" is stricter + // (body often stays non-visible while #root is ready); wait on app content instead. await page.evaluate(() => new Promise((resolve) => setTimeout(resolve, 200))); await page.waitForFunction(() => { + if (document.readyState !== 'complete') { + return false; + } + const root = document.querySelector('#root'); if (!root?.children.length) { From a69fabd1bb66d119e8ffd79e5bdf3a3cd081212b Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Mon, 15 Jun 2026 11:30:03 +0300 Subject: [PATCH 19/28] fail 1 test --- .../playwright/utils/limestone-page.js | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index a925bd512..45dd636da 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -23,24 +23,9 @@ async function open (page, urlExtra = '?locale=en-US') { export async function openComponent (page, params) { const query = serializeParams(Object.assign({locale: 'en-US'}, params)); await open(page, `?${query}`); - // WDIO Page.open: body.waitForDisplayed + delay(200). Playwright "visible" is stricter - // (body often stays non-visible while #root is ready); wait on app content instead. await page.evaluate(() => new Promise((resolve) => setTimeout(resolve, 200))); - await page.waitForFunction(() => { - if (document.readyState !== 'complete') { - return false; - } - - const root = document.querySelector('#root'); - - if (!root?.children.length) { - return false; - } - - const text = root.textContent || ''; - - return !text.includes('INVALID COMPONENT') && !text.includes('ERROR IN'); - }, {timeout: 60000}); - // Wait for web fonts to finish loading so glyphs are painted before the screenshot. - await page.evaluate(() => document.fonts.ready); + await page.evaluate(() => Promise.race([ + document.fonts.ready, + new Promise((resolve) => setTimeout(resolve, 10000)) + ])); } From c6370a6b3929236c9d9930dfeeadea4b828cf08f Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Mon, 15 Jun 2026 12:32:21 +0300 Subject: [PATCH 20/28] update readme --- tests/screenshot/playwright/README.md | 53 +++++++++++++++++++-------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index a5711c42d..5c5e0a8e9 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -124,9 +124,13 @@ registerScreenshotTests({ /Limestone-View/?locale=en-US&component=Chip&testId=3&skin=neutral&highContrast=false ``` -`Limestone-View` → `prepareTest('Chip', 3)` → renders `[data-ui-test-id="test"]`. +`Limestone-View` → `prepareTest('Chip', 3)` → renders the case under `[data-ui-test-id="test"]` (see **Page ready** below — Playwright does not wait for that node to be visible). -### 7. Screenshot +### 7. Page ready — `utils/limestone-page.js` + +Matches WDIO `Page.open`: navigate, **200 ms** settle, then wait for fonts (capped at **10 s**). Playwright does **not** wait for `body` or `[data-ui-test-id="test"]` to be visible — Enact often leaves `body` non-visible to Playwright while the app is ready; popup content (e.g. Input `open`) lives in `FloatingLayer`. + +### 8. Screenshot `toHaveScreenshot(getScreenshotPathSegments('Chip', 'Limestone', <title>))` writes/compares under `playwright/snapshots/<Component>/<TestName>/<case>.png` — same folder layout as WDIO `dist/screenshots/reference/`. @@ -154,7 +158,9 @@ npm run bootstrap npm run prepare-playwright # optional on machines that already have Chrome (see below) ``` -`prepare-playwright` runs `playwright install chrome`. It is **not** hooked to `postinstall`. Run it on **fresh machines and CI** (`build-scripts/enact-playwright-tests.sh`). On a machine where Chrome is already installed, Playwright prints *"chrome" is already installed on the system!"* — that is success; you can skip this step on later runs. +`prepare-playwright` runs `playwright install chrome`. It is **not** hooked to `postinstall`. Use it on **local machines** without system Chrome. + +**Jenkins / CI** (`build-scripts/enact-playwright-tests.sh`) uses system Chrome from `installChrome` (same as WDIO) and skips `playwright install` to avoid broken apt GPG keys on agents. CI sets `PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS=1`. `global-setup.js` and `ensure-screenshot-dist.mjs` call `assertScreenshotDist()` from `paths.js` when a build is needed. If `tests/screenshot/dist/Limestone-View/index.html` is missing, the run fails immediately with a clear error (not a webServer timeout). @@ -326,17 +332,20 @@ npm run test-playwright -- --component Button --spec Light npm run test-playwright:report ``` -**Parallelism in full suite:** `playwright.config.mjs` defaults to **5 workers** (`PLAYWRIGHT_WORKERS`). Shards split cases by `testId % PLAYWRIGHT_INSTANCES` (default instances **5**). +**Parallelism in full suite:** `playwright.config.mjs` defaults to **5 workers** (`PLAYWRIGHT_WORKERS`) and **120 s** per test (WDIO mocha uses 1 h). Navigation/action timeouts are **60 s**. Shards split cases by `testId % PLAYWRIGHT_INSTANCES` (default instances **5**). -**Limit shards** (same idea as Jenkins `SPEC`): +**Limit shards** (`utils/spec-match.js`): | `PLAYWRIGHT_SPEC` | Runs | |-------------------|------| | *(unset)* | All `*-spec.js` files | -| `Default` | `Default-spec.js` … `Default9-spec.js` | +| `Default` | All neutral default shards: `Default-spec.js` … `Default9-spec.js` | +| `Default2` … `Default5` | Single shard file each (e.g. `Default3-spec.js`) | +| `Default-spec` | Only `specs/neutral/Default-spec.js` (shard 1 — same file as WDIO `Default-specs.js`) | | `HighContrast` | `HighContrast-spec.js` … `HighContrast9-spec.js` | | `Light` | `Light-spec.js` … `Light9-spec.js` | -| `Default-spec` | Only `specs/neutral/Default-spec.js` (TV-style) | + +**Jenkins `SPEC` (WDIO names)** is mapped in `build-scripts/enact-playwright-tests.sh` before export — e.g. `SPEC=Default` → `PLAYWRIGHT_SPEC=Default-spec` (shard 1 only), not the broad `Default` glob above. Matrix desktop: `Default`, `Default2` … `Default5`. ```powershell $env:PLAYWRIGHT_SPEC='Default' @@ -461,6 +470,8 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | `PLAYWRIGHT_WORKERS` | Parallel workers in `playwright.config.mjs` (default **5**) | | `PLAYWRIGHT_SPEC` | Limit which `*-spec.js` files run; also defines which shards teardown expects | | `PLAYWRIGHT_BASE_URL` | Static server URL (default `http://localhost:4568`) | +| `PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS` | Set to `1` on Jenkins agents (build-scripts); skips Playwright host-deps check | +| `PLAYWRIGHT_CI_DEFAULT_SPEC` | Jenkins only: optional subset when `SPEC` is unset (e.g. `Default-spec` for smoke) | --- @@ -471,9 +482,10 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W | Missing dist / build failed | Read the `assertScreenshotDist()` error (fail-fast in global-setup, not a webServer timeout); link `@enact/ui-test-utils` if build-apps fails | | Snapshot diff | Re-baseline with `--update`, or run failing `--test-id` alone | | Snapshot diff on focus cases in parallel | Re-baseline with `--parallel 1`, or run failing `--test-id` alone | -| No tests found | Check `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_SPEC`, `PLAYWRIGHT_TITLE`, `PLAYWRIGHT_TEST_ID` (case titles omit the component name — use `--component Button --title Focused` or `--title Button Focused`); delete stale `.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1` | +| No tests found | Wrong `PLAYWRIGHT_SPEC` (shard 1 = `Default-spec`, not `Default1`); check `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_TITLE`, `PLAYWRIGHT_TEST_ID`; delete stale `.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1` | | Invalid `PLAYWRIGHT_TEST_ID` | Use a non-negative integer for `--test-id` (e.g. `--test-id 3`) | | Stale `.test-data.json` (`undefined` parse error) | Delete `playwright/.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1`; rebuild dist if needed | +| `locator.waitFor` / page timeout on CI | Ensure `playwright.config.mjs` uses `timeout: 120000`; use current `limestone-page.js` (WDIO-style ready, no `visible` on `body` / `data-ui-test-id`) | | Port conflict on CI after setup | Ensure global-setup stopped its temporary `serve` (fresh `.test-data.json` + `--skip-build` should not leave an orphan on :4568) | | Missing images on build | Place assets under `tests/screenshot/images/`, `videos/` | | Port already in use | Set `PLAYWRIGHT_BASE_URL` to another port; restart any stale `serve` on 4568 | @@ -483,16 +495,27 @@ Diminishing returns appear when too many Chrome instances contend for CPU/RAM (W --- -## CI (planned) +## CI + +Jenkins job **`enact-screenshot-tests`** runs WDIO (`npm run test-ss`) then Playwright via `build-scripts/enact-playwright-tests.sh` in the same workspace. Standalone job `enact-playwright-tests` is also supported. See `build-scripts/README.md` for full env var list. + +1. **Chrome:** `installChrome` from build-scripts (same as WDIO); skips `npm run prepare-playwright` on agents +2. **Reference:** optional Nebula `reference/${TARGET_TYPE}-playwright/` (separate from WDIO `reference/${TARGET_TYPE}/`) +3. **`npm run test-playwright`** with Jenkins env (aligned with WDIO `wdio-screenshot-test-opts.sh`): -Jenkins job via `build-scripts/enact-playwright-tests.sh`: +| Jenkins param | Playwright | WDIO (same job) | +|---------------|------------|-----------------| +| *(no `SPEC`, desktop)* | Full suite (`PLAYWRIGHT_SPEC` unset) | `--instances 5 --parallel 5`, all `*-specs.js` | +| `SPEC=Default` | `PLAYWRIGHT_SPEC=Default-spec` | `--spec Default` | +| `SPEC=Default2` … `Default5` | `PLAYWRIGHT_SPEC=Default2-spec`, … | `--spec Default2`, … | +| `COMPONENT` | `PLAYWRIGHT_COMPONENT`, `INSTANCES=1` | `--instances 1 --parallel …` | +| `WDIO_PARALLEL` | `PLAYWRIGHT_WORKERS` (default **5**) | `--parallel` | +| `PLAYWRIGHT_CI_DEFAULT_SPEC` | Optional smoke when no `SPEC` (e.g. `Default-spec`) | — | +| `REFERENCE` | Nebula `limestone-playwright/` | Nebula `limestone/` | -1. `npm run prepare-playwright` -2. Optionally seed `playwright/snapshots/` from Nebula reference -3. `npm run test-playwright` with `PLAYWRIGHT_SPEC`, `PLAYWRIGHT_WORKERS`, `PLAYWRIGHT_COMPONENT` -4. Upload `playwright/reports/html`, `snapshots/`, `test-results/` +4. **Results:** `http://nebula.lge.com/results/playwright-results-<timestamp>/reports/html/index.html` -On TV: typically `PLAYWRIGHT_SPEC=Default-spec` (single shard). +On TV: `PLAYWRIGHT_SPEC=Default-spec`, `INSTANCES=1`, `WORKERS=1`. --- From 6e2ad2158c359b1b5e0292e484ba295dc54fc363 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Mon, 15 Jun 2026 17:01:51 +0300 Subject: [PATCH 21/28] fix lint --- tests/screenshot/playwright/utils/shard-registry.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/screenshot/playwright/utils/shard-registry.js b/tests/screenshot/playwright/utils/shard-registry.js index c4655f91a..30b92b3b6 100644 --- a/tests/screenshot/playwright/utils/shard-registry.js +++ b/tests/screenshot/playwright/utils/shard-registry.js @@ -39,14 +39,14 @@ export function validateShardCoverage () { continue; } - const {groupKey, concurrency, maxWorkers} = JSON.parse(line); + const {groupKey, concurrency, maxWorkers: recordedMaxWorkers} = JSON.parse(line); if (!actualGroups.has(groupKey)) { - actualGroups.set(groupKey, {maxWorkers, shards: new Set()}); + actualGroups.set(groupKey, {maxWorkers: recordedMaxWorkers, shards: new Set()}); } const group = actualGroups.get(groupKey); - group.maxWorkers = maxWorkers; + group.maxWorkers = recordedMaxWorkers; group.shards.add(concurrency); } From b205abf58611fdef1fed54e3c81d0aed1e37496b Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Thu, 18 Jun 2026 15:01:15 +0300 Subject: [PATCH 22/28] update readme --- tests/screenshot/playwright/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index 5c5e0a8e9..94cca5dee 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -345,7 +345,7 @@ npm run test-playwright:report | `HighContrast` | `HighContrast-spec.js` … `HighContrast9-spec.js` | | `Light` | `Light-spec.js` … `Light9-spec.js` | -**Jenkins `SPEC` (WDIO names)** is mapped in `build-scripts/enact-playwright-tests.sh` before export — e.g. `SPEC=Default` → `PLAYWRIGHT_SPEC=Default-spec` (shard 1 only), not the broad `Default` glob above. Matrix desktop: `Default`, `Default2` … `Default5`. +**Jenkins `SPEC` (WDIO names)** is normalized in `build-scripts/enact-playwright-tests.sh` (`Default-specs` → `Default`, etc.) so Playwright uses the same substring filter as WDIO `--spec`. E.g. `SPEC=Default` → `PLAYWRIGHT_SPEC=Default` → all neutral `*Default*-spec.js` shards active for `INSTANCES=5`. Matrix desktop: `Default`, `Default2` … `Default5`. ```powershell $env:PLAYWRIGHT_SPEC='Default' @@ -506,16 +506,16 @@ Jenkins job **`enact-screenshot-tests`** runs WDIO (`npm run test-ss`) then Play | Jenkins param | Playwright | WDIO (same job) | |---------------|------------|-----------------| | *(no `SPEC`, desktop)* | Full suite (`PLAYWRIGHT_SPEC` unset) | `--instances 5 --parallel 5`, all `*-specs.js` | -| `SPEC=Default` | `PLAYWRIGHT_SPEC=Default-spec` | `--spec Default` | -| `SPEC=Default2` … `Default5` | `PLAYWRIGHT_SPEC=Default2-spec`, … | `--spec Default2`, … | +| `SPEC=Default` | `PLAYWRIGHT_SPEC=Default` | `--spec Default` | +| `SPEC=Default2` … `Default5` | `PLAYWRIGHT_SPEC=Default2`, … | `--spec Default2`, … | | `COMPONENT` | `PLAYWRIGHT_COMPONENT`, `INSTANCES=1` | `--instances 1 --parallel …` | | `WDIO_PARALLEL` | `PLAYWRIGHT_WORKERS` (default **5**) | `--parallel` | -| `PLAYWRIGHT_CI_DEFAULT_SPEC` | Optional smoke when no `SPEC` (e.g. `Default-spec`) | — | +| `PLAYWRIGHT_CI_DEFAULT_SPEC` | Optional smoke when no `SPEC` (e.g. `Default-spec` for shard 1 only) | — | | `REFERENCE` | Nebula `limestone-playwright/` | Nebula `limestone/` | 4. **Results:** `http://nebula.lge.com/results/playwright-results-<timestamp>/reports/html/index.html` -On TV: `PLAYWRIGHT_SPEC=Default-spec`, `INSTANCES=1`, `WORKERS=1`. +On TV: `PLAYWRIGHT_SPEC=Default`, `INSTANCES=1`, `WORKERS=1`. --- From 0fa559d241b9987234b09fa90af9649dd91c8322 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Fri, 19 Jun 2026 12:15:51 +0300 Subject: [PATCH 23/28] result tests playwright --- tests/screenshot/playwright/README.md | 2 + .../playwright/playwright.config.mjs | 8 ++-- .../playwright/utils/limestone-page.js | 39 ++++++++++++++++--- .../playwright/utils/run-screenshot-tests.js | 20 ++++------ .../playwright/utils/screenshot-options.js | 25 ++++++++++++ tests/screenshot/scripts/run-playwright.mjs | 4 ++ 6 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 tests/screenshot/playwright/utils/screenshot-options.js diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md index 94cca5dee..969509163 100644 --- a/tests/screenshot/playwright/README.md +++ b/tests/screenshot/playwright/README.md @@ -39,6 +39,8 @@ npm run benchmark-screenshots -- Chip | Baselines | `dist/screenshots/reference/` | `playwright/snapshots/` | | Compare API | `@wdio/visual-service` | `expect(page).toHaveScreenshot()` | | Viewport | 1920×**1167** (WDIO window) | 1920×**1080** (FHD) | +| Page ready | `body` visible + 200 ms + fonts | Same (`limestone-page.js`) | +| Compare tolerance | `ignoreAntialiasing: true` | `threshold: 0.2` (Playwright equivalent) | Playwright does **not** read `apps/components/*.js` at test time. It serves the **built** app from `dist/` and selects cases with URL query parameters (`component`, `testId`, `skin`, …). diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs index 229bdb17f..69fe0aa7f 100644 --- a/tests/screenshot/playwright/playwright.config.mjs +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -4,6 +4,7 @@ import {defineConfig, devices} from '@playwright/test'; import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_HEALTH_URL} from './paths.js'; import {resolveTestMatchGlob} from './utils/spec-match.js'; +import {SCREENSHOT_COMPARE_OPTIONS, SCREENSHOT_VIEWPORT} from './utils/screenshot-options.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const distDir = path.join(__dirname, '..', 'dist'); @@ -18,7 +19,8 @@ export default defineConfig({ snapshotPathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}'), expect: { toHaveScreenshot: { - pathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}') + pathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}'), + ...SCREENSHOT_COMPARE_OPTIONS } }, // WDIO mocha timeout is 1h; default Playwright 30s is too low for CI (goto + hydrate + fonts). @@ -37,7 +39,7 @@ export default defineConfig({ navigationTimeout: 60000, actionTimeout: 60000, trace: 'on-first-retry', - viewport: {width: 1920, height: 1080}, + viewport: SCREENSHOT_VIEWPORT, launchOptions: { args: [ '--disable-infobars', @@ -47,7 +49,7 @@ export default defineConfig({ '--disable-lcd-text', '--force-device-scale-factor=1', '--disable-gpu', - '--window-size=1920,1080' + `--window-size=${SCREENSHOT_VIEWPORT.width},${SCREENSHOT_VIEWPORT.height}` ] } }, diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index 45dd636da..7688459f7 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -1,5 +1,10 @@ import {SCREENSHOT_VIEW} from '../paths.js'; +const BODY_WAIT_MS = 10000; +const SETTLE_MS = 200; +const FONTS_WAIT_MS = 10000; +const READY_STATE_WAIT_MS = 15000; + function serializeParams (params) { const searchParams = new URLSearchParams(); @@ -20,12 +25,36 @@ async function open (page, urlExtra = '?locale=en-US') { await page.goto(buildUrl(urlExtra), {waitUntil: 'load'}); } +async function waitForPageReady (page) { + await page.locator('body').waitFor({state: 'visible', timeout: BODY_WAIT_MS}); + + await page.waitForFunction( + () => document.readyState === 'complete', + {timeout: READY_STATE_WAIT_MS} + ); + + await page.evaluate((ms) => new Promise((resolve) => setTimeout(resolve, ms)), SETTLE_MS); + + await page.evaluate((fontsTimeout) => Promise.race([ + document.fonts.ready, + new Promise((resolve) => setTimeout(resolve, fontsTimeout)) + ]), FONTS_WAIT_MS); + + // MediaOverlay and similar: freeze video on first frame (non-deterministic otherwise). + await page.evaluate(() => { + for (const video of document.querySelectorAll('video')) { + video.pause(); + try { + video.currentTime = 0; + } catch { + // ignore seek errors on unloaded media + } + } + }); +} + export async function openComponent (page, params) { const query = serializeParams(Object.assign({locale: 'en-US'}, params)); await open(page, `?${query}`); - await page.evaluate(() => new Promise((resolve) => setTimeout(resolve, 200))); - await page.evaluate(() => Promise.race([ - document.fonts.ready, - new Promise((resolve) => setTimeout(resolve, 10000)) - ])); + await waitForPageReady(page); } diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js index ff37b6e50..f1f13e0da 100644 --- a/tests/screenshot/playwright/utils/run-screenshot-tests.js +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -7,6 +7,11 @@ import {assertComponentSource, TEST_DATA_FILE} from '../paths.js'; import {getScreenshotPathSegments} from './screenshot-name.js'; import {openComponent} from './limestone-page.js'; import {recordShard} from './shard-registry.js'; +import { + PAGE_SCREENSHOT_OPTIONS, + SCREENSHOT_COMPARE_OPTIONS, + SCREENSHOT_VIEWPORT +} from './screenshot-options.js'; const forceUpdate = process.env.PLAYWRIGHT_FORCE_UPDATE === '1'; const componentExactMatch = process.env.PLAYWRIGHT_COMPONENT_EXACT === '1'; @@ -90,12 +95,7 @@ async function assertPageScreenshot (page, testInfo, segments, options) { if (forceUpdate || !fs.existsSync(filePath)) { fs.mkdirSync(path.dirname(filePath), {recursive: true}); - await page.screenshot({ - animations: 'disabled', - caret: 'hide', - type: 'png', - path: filePath - }); + await page.screenshot({...PAGE_SCREENSHOT_OPTIONS, path: filePath}); return; } @@ -150,7 +150,7 @@ export function registerScreenshotTests (config) { baseTest.describe(suiteName, () => { baseTest.beforeEach(async ({page}) => { - await page.setViewportSize({width: 1920, height: 1080}); + await page.setViewportSize(SCREENSHOT_VIEWPORT); }); for (const screenshotTest of cases) { @@ -164,11 +164,7 @@ export function registerScreenshotTests (config) { config.testName, screenshotTest.params.caseTitle ), - { - animations: 'disabled', - caret: 'hide', - maxDiffPixelRatio: 0 - } + SCREENSHOT_COMPARE_OPTIONS ); }); } diff --git a/tests/screenshot/playwright/utils/screenshot-options.js b/tests/screenshot/playwright/utils/screenshot-options.js new file mode 100644 index 000000000..9f03136d9 --- /dev/null +++ b/tests/screenshot/playwright/utils/screenshot-options.js @@ -0,0 +1,25 @@ +/** + * Shared screenshot settings aligned with WDIO (@enact/ui-test-utils): + * - setWindowSize(1920, 1167) in wdio.conf.js + * - checkScreen({ ignoreAntialiasing: true, ... }) → threshold 0.2 in Playwright + */ +export const SCREENSHOT_VIEWPORT = { + width: 1920, + height: 1080 +}; + +/** Playwright per-pixel color tolerance; mirrors WDIO ignoreAntialiasing. */ +export const SCREENSHOT_THRESHOLD = 0.2; + +export const SCREENSHOT_COMPARE_OPTIONS = { + animations: 'disabled', + caret: 'hide', + threshold: SCREENSHOT_THRESHOLD, + maxDiffPixelRatio: 0 +}; + +export const PAGE_SCREENSHOT_OPTIONS = { + animations: 'disabled', + caret: 'hide', + type: 'png' +}; diff --git a/tests/screenshot/scripts/run-playwright.mjs b/tests/screenshot/scripts/run-playwright.mjs index ae4d90b4d..5fa36e2f0 100644 --- a/tests/screenshot/scripts/run-playwright.mjs +++ b/tests/screenshot/scripts/run-playwright.mjs @@ -64,6 +64,10 @@ function buildSuiteEnv () { if (update) { playwrightEnv.PLAYWRIGHT_FORCE_UPDATE = '1'; + // Serial capture avoids focus/spotlight drift between reference and compare runs. + if (parallel == null) { + playwrightEnv.PLAYWRIGHT_WORKERS = '1'; + } } if (parallel != null) { From fa790eb02049412b286edbd906bfda83ec36cdf0 Mon Sep 17 00:00:00 2001 From: Dan Ichim <dan.ichim@lgepartner.com> Date: Fri, 19 Jun 2026 13:24:00 +0300 Subject: [PATCH 24/28] media fix --- tests/screenshot/playwright/global-setup.js | 2 +- tests/screenshot/playwright/utils/limestone-page.js | 10 +++++++--- tests/screenshot/scripts/ensure-screenshot-dist.mjs | 3 +-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js index c719e9cb8..cfac98f07 100644 --- a/tests/screenshot/playwright/global-setup.js +++ b/tests/screenshot/playwright/global-setup.js @@ -1,6 +1,5 @@ import fs from 'fs'; -import buildApps from '@enact/ui-test-utils/build-apps'; import {chromium} from '@playwright/test'; import {PLAYWRIGHT_BASE_URL, SCREENSHOT_VIEW_INDEX, TEST_DATA_FILE, assertScreenshotDist} from './paths.js'; @@ -28,6 +27,7 @@ export default async function globalSetup () { clearShardRegistry(); if (!process.env.PLAYWRIGHT_SKIP_BUILD) { + const {default: buildApps} = await import('@enact/ui-test-utils/build-apps'); await buildApps('screenshot'); } diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index 7688459f7..87528ff2b 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -26,10 +26,14 @@ async function open (page, urlExtra = '?locale=en-US') { } async function waitForPageReady (page) { - await page.locator('body').waitFor({state: 'visible', timeout: BODY_WAIT_MS}); - + // Enact screenshot app may keep <body> hidden; WDIO waitForDisplayed still passes once #root + // has content. Wait for attached + rendered root instead of body visibility. + await page.locator('body').waitFor({state: 'attached', timeout: BODY_WAIT_MS}); await page.waitForFunction( - () => document.readyState === 'complete', + () => { + const root = document.getElementById('root'); + return document.readyState === 'complete' && root != null && root.childElementCount > 0; + }, {timeout: READY_STATE_WAIT_MS} ); diff --git a/tests/screenshot/scripts/ensure-screenshot-dist.mjs b/tests/screenshot/scripts/ensure-screenshot-dist.mjs index 6128ae957..5f3b6a42d 100644 --- a/tests/screenshot/scripts/ensure-screenshot-dist.mjs +++ b/tests/screenshot/scripts/ensure-screenshot-dist.mjs @@ -1,8 +1,6 @@ /* eslint-disable no-console */ import fs from 'fs'; -import buildApps from '@enact/ui-test-utils/build-apps'; - import {SCREENSHOT_VIEW_INDEX, assertScreenshotDist} from '../playwright/paths.js'; export function hasScreenshotDist () { @@ -26,6 +24,7 @@ export async function ensureScreenshotDist ({skipBuild = false} = {}) { console.log('Building tests/screenshot/dist...'); } + const {default: buildApps} = await import('@enact/ui-test-utils/build-apps'); await buildApps('screenshot'); assertScreenshotDist(); From a911167ff46f6d264d51bf762306f1bda9e43d03 Mon Sep 17 00:00:00 2001 From: daniel-stoian-lgp <daniel.stoian@lgepartner.com> Date: Thu, 25 Jun 2026 15:30:31 +0300 Subject: [PATCH 25/28] added thrshdold 0 --- tests/screenshot/playwright/utils/screenshot-options.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/screenshot/playwright/utils/screenshot-options.js b/tests/screenshot/playwright/utils/screenshot-options.js index 9f03136d9..70d0c95f4 100644 --- a/tests/screenshot/playwright/utils/screenshot-options.js +++ b/tests/screenshot/playwright/utils/screenshot-options.js @@ -9,7 +9,7 @@ export const SCREENSHOT_VIEWPORT = { }; /** Playwright per-pixel color tolerance; mirrors WDIO ignoreAntialiasing. */ -export const SCREENSHOT_THRESHOLD = 0.2; +export const SCREENSHOT_THRESHOLD = 0; export const SCREENSHOT_COMPARE_OPTIONS = { animations: 'disabled', From 45300271da7cbf79fe73060a3999748f195ebdb9 Mon Sep 17 00:00:00 2001 From: daniel-stoian-lgp <daniel.stoian@lgepartner.com> Date: Thu, 25 Jun 2026 17:24:33 +0300 Subject: [PATCH 26/28] removed spellcheck --- .../playwright/utils/limestone-page.js | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index 87528ff2b..e9c8b3ad0 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -21,6 +21,55 @@ function buildUrl (urlExtra = '?locale=en-US') { return `/${SCREENSHOT_VIEW}/${urlExtra}`; } +// Chrome's native spellcheck draws wavy underlines on editable fields with "misspelled" content +// Force spellcheck off on every editable element +// before navigation, so the markers never render regardless of the browser environment. +async function disableSpellcheck (page) { + await page.addInitScript(() => { + const SELECTOR = 'input, textarea, [contenteditable]'; + + const disableWithin = (node) => { + if (node.nodeType !== 1) return; + if (node.matches && node.matches(SELECTOR)) { + node.setAttribute('spellcheck', 'false'); + } + if (node.querySelectorAll) { + for (const el of node.querySelectorAll(SELECTOR)) { + el.setAttribute('spellcheck', 'false'); + } + } + }; + + const apply = () => { + // Set a non-spellchecked default that descendants inherit, then sweep existing fields. + document.documentElement && document.documentElement.setAttribute('spellcheck', 'false'); + disableWithin(document.documentElement || document); + }; + + // React mounts the screenshot app after load, so watch for fields added later. + const observer = new MutationObserver((mutations) => { + for (const mutation of mutations) { + for (const added of mutation.addedNodes) { + disableWithin(added); + } + } + }); + + const start = () => { + apply(); + if (document.body) { + observer.observe(document.body, {childList: true, subtree: true}); + } + }; + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', start, {once: true}); + } else { + start(); + } + }); +} + async function open (page, urlExtra = '?locale=en-US') { await page.goto(buildUrl(urlExtra), {waitUntil: 'load'}); } @@ -58,6 +107,7 @@ async function waitForPageReady (page) { } export async function openComponent (page, params) { + await disableSpellcheck(page); const query = serializeParams(Object.assign({locale: 'en-US'}, params)); await open(page, `?${query}`); await waitForPageReady(page); From d72b392327f8bf9a22f2560df009f2afba35d3fe Mon Sep 17 00:00:00 2001 From: daniel-stoian-lgp <daniel.stoian@lgepartner.com> Date: Thu, 25 Jun 2026 17:42:27 +0300 Subject: [PATCH 27/28] eslint fix --- tests/screenshot/playwright/utils/limestone-page.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js index e9c8b3ad0..d8c28a7a8 100644 --- a/tests/screenshot/playwright/utils/limestone-page.js +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -1,3 +1,5 @@ +/* global MutationObserver */ + import {SCREENSHOT_VIEW} from '../paths.js'; const BODY_WAIT_MS = 10000; @@ -42,7 +44,9 @@ async function disableSpellcheck (page) { const apply = () => { // Set a non-spellchecked default that descendants inherit, then sweep existing fields. - document.documentElement && document.documentElement.setAttribute('spellcheck', 'false'); + if (document.documentElement) { + document.documentElement.setAttribute('spellcheck', 'false'); + } disableWithin(document.documentElement || document); }; From 3a90a934e8cdced2d997d7f2dda8d7ca03909c05 Mon Sep 17 00:00:00 2001 From: daniel-stoian-lgp <daniel.stoian@lgepartner.com> Date: Tue, 30 Jun 2026 11:23:18 +0300 Subject: [PATCH 28/28] fix for serial capture --- tests/screenshot/scripts/run-playwright.mjs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/screenshot/scripts/run-playwright.mjs b/tests/screenshot/scripts/run-playwright.mjs index 5fa36e2f0..ae4d90b4d 100644 --- a/tests/screenshot/scripts/run-playwright.mjs +++ b/tests/screenshot/scripts/run-playwright.mjs @@ -64,10 +64,6 @@ function buildSuiteEnv () { if (update) { playwrightEnv.PLAYWRIGHT_FORCE_UPDATE = '1'; - // Serial capture avoids focus/spotlight drift between reference and compare runs. - if (parallel == null) { - playwrightEnv.PLAYWRIGHT_WORKERS = '1'; - } } if (parallel != null) {