diff --git a/.gitignore b/.gitignore index b92af6988..bcac744df 100644 --- a/.gitignore +++ b/.gitignore @@ -20,5 +20,14 @@ errorShots coverage .eslintcache +# Screenshot tests (tests/screenshot/) — generated artifacts only +# Playwright baselines: snapshots///*.png (same paths as WDIO reference/) +tests/screenshot/playwright/snapshots +tests/screenshot/playwright/test-results +tests/screenshot/playwright/reports +tests/screenshot/playwright/blob-report +tests/screenshot/playwright/.test-data.json +tests/screenshot/playwright/.shard-registry.jsonl + # Mac .DS_Store diff --git a/package-lock.json b/package-lock.json index 308eb4182..40041183e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,8 @@ }, "devDependencies": { "@enact/docs-utils": "^0.4.17", - "@enact/ui-test-utils": "^4.0.2" + "@enact/ui-test-utils": "^4.0.2", + "@playwright/test": "^1.57.0" }, "peerDependencies": { "ilib": "^14.21.0 || ^14.21.0-webos1" @@ -16192,6 +16193,22 @@ "node": ">= 8" } }, + "node_modules/@playwright/test": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.0.tgz", + "integrity": "sha512-cKA5B6lpFEMyMGjxF54QihfYpB4FkEGH+qZhtArDEG+wezQAJY8Pq6C7T1SjWz+FFzt3TbyoXBQYk/0292TdJA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -19599,6 +19616,53 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/playwright": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", + "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", + "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.15", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", diff --git a/package.json b/package.json index 43deda659..edad3fc2e 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,13 @@ "test-ss": "start-tests tests/screenshot/config/wdio.conf.js", "test-ss-docker": "start-tests tests/screenshot/config/wdio.docker.conf.js", "test-ss-tv": "start-tests tests/screenshot/config/wdio.tv.conf.js", + "test-playwright": "node tests/screenshot/scripts/run-playwright.mjs", + "test-playwright:update": "node tests/screenshot/scripts/run-playwright.mjs -- --update", + "test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", + "test-playwright:component": "node tests/screenshot/scripts/run-playwright.mjs", + "test-ss:component": "node tests/screenshot/scripts/run-component-wdio.mjs", + "prepare-playwright": "playwright install chrome", + "benchmark-screenshots": "node tests/screenshot/playwright/benchmark.mjs", "test-json": "enact test --json", "test-watch": "enact test --watch", "transpile": "enact transpile", @@ -64,6 +71,7 @@ }, "devDependencies": { "@enact/docs-utils": "^0.4.17", - "@enact/ui-test-utils": "^4.0.2" + "@enact/ui-test-utils": "^4.0.2", + "@playwright/test": "^1.57.0" } -} \ No newline at end of file +} diff --git a/tests/screenshot/playwright/README.md b/tests/screenshot/playwright/README.md new file mode 100644 index 000000000..969509163 --- /dev/null +++ b/tests/screenshot/playwright/README.md @@ -0,0 +1,549 @@ +# Playwright screenshot tests — developer guide + +Visual regression for Limestone using [Playwright](https://playwright.dev/) `toHaveScreenshot`. This runs **alongside** the existing WebdriverIO suite (`npm run test-ss`). Both share the same scenario source, build output, and case indices — only the runner, port, spec files, and baseline folder differ. + +--- + +## Quick start + +From the **repository root**: + +```bash +npm install +npm run bootstrap # enact CLI — required for the screenshot build +npm link @enact/ui-test-utils # see “Link local ui-test-utils” below until npm publishes ./build-apps +npm run prepare-playwright # optional if Chrome is already available (see below) + +# First time for a component — baselines are created automatically on pass +npm run test-playwright:component -- Sprite + +# Force refresh baselines after UI changes +npm run test-playwright:component -- Sprite --update + +# Compare Playwright vs WDIO runtime on one component +npm run benchmark-screenshots -- Chip +``` + +**Day-to-day:** prefer `test-playwright:component` over the full suite. Button alone has **205** cases; the full suite has **thousands** of screenshots across 27 shards. + +--- + +## How it fits with WDIO + +| | WebdriverIO | Playwright | +|--|-------------|------------| +| Scenario source | `tests/screenshot/apps/components/*.js` | Same | +| Build output | `tests/screenshot/dist/` via `buildApps('screenshot')` | Same | +| Static server | port **4567** | port **4568** (default) | +| Spec files | `tests/screenshot/specs/**/*-specs.js` | `tests/screenshot/playwright/specs/**/*-spec.js` | +| Baselines | `dist/screenshots/reference/` | `playwright/snapshots/` | +| Compare API | `@wdio/visual-service` | `expect(page).toHaveScreenshot()` | +| Viewport | 1920×**1167** (WDIO window) | 1920×**1080** (FHD) | +| Page ready | `body` visible + 200 ms + fonts | Same (`limestone-page.js`) | +| Compare tolerance | `ignoreAntialiasing: true` | `threshold: 0.2` (Playwright equivalent) | + +Playwright does **not** read `apps/components/*.js` at test time. It serves the **built** app from `dist/` and selects cases with URL query parameters (`component`, `testId`, `skin`, …). + +--- + +## Repository layout + +``` +tests/screenshot/ + apps/ + components/ ← scenario JSX (shared with WDIO) + LimestoneComponents.js + Limestone-View.js + images/, videos/ + dist/ ← buildApps('screenshot') output (shared) + specs/ ← WDIO *-specs.js + scripts/ + run-playwright.mjs full suite + component (same entry; mode from argv) + parse-cli-args.mjs shared CLI parsing + ensure-screenshot-dist.mjs + spawn-playwright.mjs + run-component-wdio.mjs + playwright/ ← this folder + specs/neutral|light/ *-spec.js (27 shards) + utils/ registerScreenshotTests, limestone-page, screenshot-name, shard-registry + snapshots/ PNG baselines (gitignored — generate with --update) + .shard-registry.jsonl run-time shard log (gitignored; see shard-registry.js) + global-setup.js build, dist check, .test-data.json (serve :4568 only while fetching) + global-teardown.js shard coverage check + playwright.config.mjs + paths.js SCREENSHOT_VIEW, SCREENSHOT_HEALTH_URL, assertScreenshotDist() + benchmark.mjs +``` + +--- + +## How a component reaches Playwright (example: Chip, testId 3) + +### 1. Author scenarios — `apps/components/Chip.js` + +Each exported array entry is one case; **array index = `testId`**: + +```js +const ChipTests = [ + Default Chip, // testId 0 + Chip with Icon, // testId 1 + Chip with checkmark Icon, // testId 3 + … +]; +export default ChipTests; +``` + +`withConfig({ focus: true }, […])` and `withConfig({ skinVariants: ['largeText'] }, […])` append more cases. Chip has **51** cases total (same as WDIO). + +### 2. Register — `apps/LimestoneComponents.js` + +`Chip` is imported into the `components` map. `generateTestData()` (`@enact/ui-test-utils`) produces `testMetadata.Chip`: one `{ title, … }` per index (titles are derived from JSX props when no explicit `title` is set). + +### 3. Build — `buildApps('screenshot')` + +Triggered from `global-setup.js` unless `PLAYWRIGHT_SKIP_BUILD=1`. Packs `Limestone-View` into `dist/Limestone-View/`. The bundle contains all components and metadata. WDIO uses the same build. + +### 4. Case index — `.test-data.json` + +When `.test-data.json` is missing or older than `dist/Limestone-View/index.html`, `global-setup` briefly starts `serve` on port **4568**, loads `…/Limestone-View/?request` (waits for `window.__TEST_DATA`), writes `playwright/.test-data.json`, then stops that server so Playwright’s `webServer` can bind the same port. Cached runs with `--skip-build` skip the fetch entirely. + +`registerScreenshotTests()` reads this file to register Playwright tests — it never parses `Chip.js` on disk. + +### 5. Spec shard — `specs/neutral/Default-spec.js` + +```js +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 1 // shard 1 of N (see sharding below) +}); +``` + +### 6. Browser URL + +``` +/Limestone-View/?locale=en-US&component=Chip&testId=3&skin=neutral&highContrast=false +``` + +`Limestone-View` → `prepareTest('Chip', 3)` → renders the case under `[data-ui-test-id="test"]` (see **Page ready** below — Playwright does not wait for that node to be visible). + +### 7. Page ready — `utils/limestone-page.js` + +Matches WDIO `Page.open`: navigate, **200 ms** settle, then wait for fonts (capped at **10 s**). Playwright does **not** wait for `body` or `[data-ui-test-id="test"]` to be visible — Enact often leaves `body` non-visible to Playwright while the app is ready; popup content (e.g. Input `open`) lives in `FloatingLayer`. + +### 8. Screenshot + +`toHaveScreenshot(getScreenshotPathSegments('Chip', 'Limestone', ))` writes/compares under `playwright/snapshots/<Component>/<TestName>/<case>.png` — same folder layout as WDIO `dist/screenshots/reference/`. + +**One case only:** + +```bash +npm run test-playwright:component -- Chip --test-id 3 +``` + +--- + +## Setup (detailed) + +### Prerequisites + +- Node.js + npm (same as the rest of Limestone) +- Chrome (Playwright uses `channel: 'chrome'` from `playwright.config.mjs`) +- `@enact/ui-test-utils@^4.0.2` (declared in root `package.json`, same as `test-ss`) + +### Install + +```bash +npm install +npm run bootstrap +npm run prepare-playwright # optional on machines that already have Chrome (see below) +``` + +`prepare-playwright` runs `playwright install chrome`. It is **not** hooked to `postinstall`. Use it on **local machines** without system Chrome. + +**Jenkins / CI** (`build-scripts/enact-playwright-tests.sh`) uses system Chrome from `installChrome` (same as WDIO) and skips `playwright install` to avoid broken apt GPG keys on agents. CI sets `PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS=1`. + +`global-setup.js` and `ensure-screenshot-dist.mjs` call `assertScreenshotDist()` from `paths.js` when a build is needed. If `tests/screenshot/dist/Limestone-View/index.html` is missing, the run fails immediately with a clear error (not a webServer timeout). + +### Build screenshot dist + +The first Playwright run builds `tests/screenshot/dist/` automatically. To reuse an existing build, pass `--skip-build`: + +```bash +npm run test-playwright -- --skip-build +npm run test-playwright:component -- Button --skip-build +``` + +Or set `PLAYWRIGHT_SKIP_BUILD=1` (same effect as `--skip-build`). + +### Link local `@enact/ui-test-utils` (until `./build-apps` is published) + +Playwright `global-setup.js` imports `@enact/ui-test-utils/build-apps`. The npm release **4.0.2** does not export that path yet; the local `ui-test-utils` repo does (see `exports["./build-apps"]` in its `package.json`). Until a new version is published, link the local package: + +```bash +# From your ui-test-utils clone (must include "./build-apps": "./src/build-apps.js" in exports) +cd ../ui-test-utils +npm install +npm link + +# From limestone repo root +cd ../limestone +npm link @enact/ui-test-utils +npm install # refresh lockfile/node_modules if needed +``` + +Alternative without global link — in limestone root `package.json`: + +```json +"@enact/ui-test-utils": "file:../ui-test-utils" +``` + +Then `npm install` from limestone root. + +To undo `npm link`: + +```bash +cd limestone +npm unlink @enact/ui-test-utils +npm install +``` + +When `@enact/ui-test-utils` is published with the `./build-apps` export, remove the link and use the registry version from `package.json` again. + +### Baselines + +Missing PNG baselines under `playwright/snapshots/` are **created automatically** on the first run (same idea as WDIO `autoSaveBaseline`). Use `--update` to **force** overwriting existing baselines. + +```bash +# First run — baselines are written as tests pass +npm run test-playwright:component -- Sprite + +# Force refresh all baselines for one component +npm run test-playwright:component -- Sprite --update + +# Force refresh entire suite +npm run test-playwright:update +``` + +**Tip:** use `--parallel 1` when running `--update` on components with **focus** cases (Chip, Button). Parallel updates can produce slightly different focus pixels and cause flaky compares on later parallel runs. + +--- + +## Running tests + +All commands run from the **repository root**. + +### Command reference + +| Command | What it does | +|---------|----------------| +| `npm run test-playwright` | Full suite — all components, all Playwright shards | +| `npm run test-playwright -- --component Button` | All specs; components whose name contains `Button` | +| `npm run test-playwright -- --spec Light` | `Light*` specs only; all components | +| `npm run test-playwright -- --component Button --spec Light` | Both filters combined | +| `npm run test-playwright -- --title "Focused"` | Filter cases by title (regex per term; case-insensitive fallback) | +| `npm run test-playwright -- --title Button Focused` | Shorthand: `Button` → `--component`, `Focused` → `--title` | +| `npm run test-playwright -- --test-id <n>` | Single case index (invalid values fail fast) | +| `npm run test-playwright -- --skip-build` | Skip `buildApps('screenshot')` when `dist/` already exists | +| `npm run test-playwright:update` | Force-update all Playwright baselines (`--update`) | +| `npm run test-playwright:report` | Open HTML report (`playwright/reports/html/`) | +| `npm run test-playwright:component -- <Name>` | One component, all spec shards | +| `npm run test-playwright:component -- <Name> --update` | Force refresh baselines for that component | +| `npm run test-playwright:component -- <Name> --skip-build` | Skip screenshot build | +| `npm run test-playwright:component -- <Name> --spec Light` | One component, `Light*` specs only | +| `npm run test-playwright:component -- <Name> --test-id <n>` | Single case by index | +| `npm run test-playwright:component -- <Name> --parallel <n>` | Same cases, `n` Playwright workers | +| `npm run test-ss:component -- <Name>` | WDIO equivalent (for comparison) | +| `npm run benchmark-screenshots -- <Name>` | Time Playwright then WDIO for one component | + +### Single component (recommended) + +```bash +# Verify (skip rebuild if dist exists) +npm run test-playwright:component -- Button --skip-build + +# Faster local run +npm run test-playwright:component -- Button --skip-build --parallel 5 + +# Filter by title regex +npm run test-playwright:component -- Button --title "Focused" + +# Light skin only +npm run test-playwright:component -- Button --spec Light +``` + +Each run prints total wall-clock time when finished (e.g. `Playwright (Button) finished in 1m 2.3s`). + +`test-playwright` and `test-playwright:component` both invoke `scripts/run-playwright.mjs`. **Component mode** is selected when the first argument after `--` is a positional name (e.g. `Sprite`); **suite mode** uses flags only (`--component`, `--spec`, …). + +Component mode sets `PLAYWRIGHT_INSTANCES=1` (all `testId`s for that component), `PLAYWRIGHT_WORKERS` from `--parallel` (default **1**), and runs **all** matching spec shards unless `--spec` is set. + +#### `scripts/parse-cli-args.mjs` — why it exists + +npm passes arguments after `--` to the Node script. A naive parser such as `args.find(a => !a.startsWith('--'))` treats **flag values** as the component name — e.g. `… --test-id 0 Sprite` would resolve component `"0"` instead of `"Sprite"`. + +`parse-cli-args.mjs` exports `parsePlaywrightArgs()`, `parseComponentArgs()`, and `isComponentPlaywrightRun()` for: + +- `run-playwright.mjs` — full suite and component entry (same file) +- `benchmark.mjs` — `--build`, `--parallel` + +**Rules:** + +- **Component mode:** the component name is the first positional token (non-flag argument). +- **Suite mode:** use `--component <Name>`; no positional component name. +- Value flags (`--test-id`, `--title`, `--parallel`, `--spec`, `--component`) consume following tokens until the next `--flag` (multi-word values such as `--title Button Focused` are supported). +- Boolean flags: `--update`, `--build`, `--skip-build`. + +**Examples:** + +| Command | Mode / component resolved | +|---------|---------------------------| +| `npm run test-playwright:component -- Button --parallel 5` | component → `Button` | +| `npm run test-playwright:component -- Button --test-id 0` | component → `Button` | +| `npm run test-playwright -- --component Button --spec Light` | suite; component filter `Button` | +| `npm run test-playwright -- --title Button Focused --spec Light` | suite; `Button` + title `Focused` | +| `npm run benchmark-screenshots -- Chip --build` | component → `Chip` | + +Put the component name **first** after `--` in component mode (as in the examples above). `run-component-wdio.mjs` still uses its own parser — use the same name-first order there too. + +### Full suite + +Runs every registered component across **27** spec shards (`Default`…`Default9`, `HighContrast*`, `Light*` under `specs/neutral` and `specs/light`). + +```bash +npm install && npm run bootstrap +npm link @enact/ui-test-utils # until ./build-apps is on npm (see above) +npm run prepare-playwright + +# First run or after UI changes — force refresh all baselines +npm run test-playwright:update + +# Verify (all components, all specs) +npm run test-playwright + +# Skip rebuild when dist/ already exists +npm run test-playwright -- --skip-build + +# Filter examples +npm run test-playwright -- --component Button +npm run test-playwright -- --spec Light +npm run test-playwright -- --component Button --spec Light + +# HTML report +npm run test-playwright:report +``` + +**Parallelism in full suite:** `playwright.config.mjs` defaults to **5 workers** (`PLAYWRIGHT_WORKERS`) and **120 s** per test (WDIO mocha uses 1 h). Navigation/action timeouts are **60 s**. Shards split cases by `testId % PLAYWRIGHT_INSTANCES` (default instances **5**). + +**Limit shards** (`utils/spec-match.js`): + +| `PLAYWRIGHT_SPEC` | Runs | +|-------------------|------| +| *(unset)* | All `*-spec.js` files | +| `Default` | All neutral default shards: `Default-spec.js` … `Default9-spec.js` | +| `Default2` … `Default5` | Single shard file each (e.g. `Default3-spec.js`) | +| `Default-spec` | Only `specs/neutral/Default-spec.js` (shard 1 — same file as WDIO `Default-specs.js`) | +| `HighContrast` | `HighContrast-spec.js` … `HighContrast9-spec.js` | +| `Light` | `Light-spec.js` … `Light9-spec.js` | + +**Jenkins `SPEC` (WDIO names)** is normalized in `build-scripts/enact-playwright-tests.sh` (`Default-specs` → `Default`, etc.) so Playwright uses the same substring filter as WDIO `--spec`. E.g. `SPEC=Default` → `PLAYWRIGHT_SPEC=Default` → all neutral `*Default*-spec.js` shards active for `INSTANCES=5`. Matrix desktop: `Default`, `Default2` … `Default5`. + +```powershell +$env:PLAYWRIGHT_SPEC='Default' +npm run test-playwright -- --skip-build +``` + +### Sharding vs workers + +| Concept | Env / flag | Meaning | +|---------|------------|---------| +| **Shard** | `concurrency` in spec + `PLAYWRIGHT_INSTANCES` | Which `testId`s this spec file owns (`testId % instances === concurrency - 1`) | +| **Workers** | `PLAYWRIGHT_WORKERS` or `--parallel` on component script | How many tests run at once in one Playwright process | +| **Component filter** | `PLAYWRIGHT_COMPONENT` | Only one component's cases | + +**Shard coverage:** `global-teardown` checks that every shard **you asked for** actually ran. It derives expected shards from `PLAYWRIGHT_SPEC` (same rules as `testMatch` in `playwright.config.mjs`) — e.g. `PLAYWRIGHT_SPEC=Default3` requires only shard 3; no `PLAYWRIGHT_SPEC` requires all active shards 1…`PLAYWRIGHT_INSTANCES` per skin/HC group. If you set `PLAYWRIGHT_INSTANCES=9` but only invoke shards 1–5, cases with `testId % 9` in `{5,6,7,8}` are silently skipped and teardown fails for the missing expected shards. Shards with `concurrency > PLAYWRIGHT_INSTANCES` are inert (empty) — expected when the default instances is **5** but nine shard files exist per skin. + +#### `utils/shard-registry.js` — why it exists + +Each `*-spec.js` file is independent: it only knows its own `concurrency` value and cannot see which other spec files ran in the same Playwright process. That makes **partial shard runs look successful** — the suite is green even when whole slices of `testId`s were never rendered. + +`shard-registry.js` collects which shards actually loaded and validates coverage at the end of the run: + +| Step | Where | What | +|------|--------|------| +| Reset | `global-setup.js` → `clearShardRegistry()` | Deletes `playwright/.shard-registry.jsonl` from any previous run | +| Record | `registerScreenshotTests()` → `recordShard()` | Appends one JSON line per spec file: skin, highContrast, `concurrency`, `PLAYWRIGHT_INSTANCES` | +| Validate | `global-teardown.js` → `validateShardCoverage()` | For each skin/HC group, ensures every shard in the current `PLAYWRIGHT_SPEC` filter ran (`utils/spec-match.js`) | + +**Skipped when** `PLAYWRIGHT_COMPONENT` is set (component script — no cross-shard risk). + +**Example failure:** no `PLAYWRIGHT_SPEC` (full suite) but only `Default3-spec.js` loads → teardown throws *Incomplete Playwright shard coverage* because shards 1, 2, 4, 5 were expected for that skin/HC group. + +The registry file is **gitignored** (`.shard-registry.jsonl`); it is a run-time artifact only. + +Full suite: many spec files × 5 workers each. Component run: one spec file, all cases, N workers. + +--- + +## Viewing results + +| Output | Location | +|--------|----------| +| Playwright HTML report | `npm run test-playwright:report` → `playwright/reports/html/index.html` | +| Playwright baselines | `playwright/snapshots/` | +| Playwright failures | `playwright/test-results/` (actual, diff, error-context) | +| WDIO failures | `tests/screenshot/dist/failedTests.html`, `newFiles.html` | +| WDIO baselines | `tests/screenshot/dist/screenshots/reference/` | + +--- + +## Benchmarking + +Compare end-to-end runtime for **one component** (Playwright first, then WDIO, same `dist/`): + +```bash +npm run benchmark-screenshots -- Sprite +npm run benchmark-screenshots -- Chip --parallel 5 +npm run benchmark-screenshots -- Button --parallel 5 + +# Rebuild dist first if missing +npm run benchmark-screenshots -- Chip --build +``` + +- **`--parallel N`** (default `1`): every case for the component, up to `N` at once. +- Playwright: `PLAYWRIGHT_INSTANCES=1`, `PLAYWRIGHT_WORKERS=N`. +- WDIO: `--instances 1` (all cases), `--parallel N` (requires `@enact/ui-test-utils` with `--parallel` support). +- Ports stay separate: Playwright **4568**, WDIO **4567**. + +Benchmark requires **existing baselines** for both runners (Playwright `snapshots/`, WDIO `dist/screenshots/reference/`). Create Playwright baselines with `--update` before benchmarking. + +--- + +## Performance analysis + +Measured **locally on Windows, Chrome 132, June 2026**. Times include browser startup, page load, and screenshot compare. `PLAYWRIGHT_SKIP_BUILD=1` (reused `tests/screenshot/dist`). + +### Results + +| Component | Cases | Parallel | Playwright | WebdriverIO | Faster | PW s/case | WDIO s/case | +|-----------|------:|---------:|-----------:|--------------:|--------|----------:|------------:| +| Sprite | 4 | 1 | 16.2 s | 19.7 s | Playwright (~18%) | 4.1 | 4.9 | +| Sprite | 4 | 5 | 15.7 s | 17.9 s | Playwright (~12%) | 3.9 | 4.5 | +| Chip | 51 | 1 | 103.8 s | 83.9 s | WDIO (~19%) | 2.0 | 1.6 | +| Chip | 51 | 5 | 50.2 s † | 137.7 s | Playwright (~64%) | 1.0 | 2.7 | +| Button | 205 | 1 | 525.4 s | 1427.2 s | Playwright (~63%) | 2.6 | 7.0 | +| Button | 205 | 5 | 96 s † | 979.3 s | Playwright (~89%) | 0.5 | 4.8 | + +† Parallel Playwright runs can **flake on focus cases** (spotlight timing). Wall time is still representative; use `--parallel 1` for `--update` on focus-heavy components, or re-run failed ids with `--test-id`. + +### Takeaways + +1. **Small components (Sprite):** similar cost; Playwright slightly faster even at parallel 1. +2. **Medium (Chip), sequential:** WDIO is ~15–20% faster per case — one browser, mature visual-service path. +3. **Medium/large, parallel 5:** Playwright scales much better — shared server, lighter workers. Chip ~2× faster wall time; Button ~9× faster wall time vs WDIO parallel 5. +4. **Large (Button), sequential:** Playwright ~2.6 s/case vs WDIO ~7.0 s/case — WDIO session overhead dominates at 205 cases. +5. **Full suite:** not benchmarked here; use **component scripts** for daily work. CI should shard by `PLAYWRIGHT_SPEC` / `PLAYWRIGHT_INSTANCES` and set `PLAYWRIGHT_WORKERS` per agent (see `build-scripts/enact-playwright-tests.sh`). + +### Making Playwright faster locally + +```powershell +$env:PLAYWRIGHT_SKIP_BUILD='1' +$env:PLAYWRIGHT_WORKERS='8' # full suite only; component uses --parallel +npm run test-playwright:component -- Button --parallel 8 +``` + +Diminishing returns appear when too many Chrome instances contend for CPU/RAM (WDIO shows this clearly at `--parallel 5` on one machine). + +--- + +## Environment variables + +| Variable | Effect | +|----------|--------| +| `PLAYWRIGHT_SKIP_BUILD=1` | Skip `buildApps('screenshot')` in global-setup (same as `--skip-build`) | +| `PLAYWRIGHT_FORCE_UPDATE=1` | Force overwrite baselines (same as `--update`) | +| `PLAYWRIGHT_COMPONENT_EXACT=1` | Exact component name match (set by component script) | +| `PLAYWRIGHT_COMPONENT` | Filter components (substring match from `--component`; exact when set by component script) | +| `PLAYWRIGHT_REFRESH_TEST_DATA=1` | Force-regenerate `.test-data.json` | +| `PLAYWRIGHT_TEST_ID` | Filter to one case index (must be a non-negative integer; invalid values throw) | +| `PLAYWRIGHT_TITLE` | Filter cases by title (regex per whitespace-separated term; invalid regex falls back to substring; case-insensitive) | +| `PLAYWRIGHT_INSTANCES` | Shard divisor (`testId % instances`); component script sets `1` | +| `PLAYWRIGHT_WORKERS` | Parallel workers in `playwright.config.mjs` (default **5**) | +| `PLAYWRIGHT_SPEC` | Limit which `*-spec.js` files run; also defines which shards teardown expects | +| `PLAYWRIGHT_BASE_URL` | Static server URL (default `http://localhost:4568`) | +| `PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS` | Set to `1` on Jenkins agents (build-scripts); skips Playwright host-deps check | +| `PLAYWRIGHT_CI_DEFAULT_SPEC` | Jenkins only: optional subset when `SPEC` is unset (e.g. `Default-spec` for smoke) | + +--- + +## Troubleshooting + +| Issue | Fix | +|-------|-----| +| Missing dist / build failed | Read the `assertScreenshotDist()` error (fail-fast in global-setup, not a webServer timeout); link `@enact/ui-test-utils` if build-apps fails | +| Snapshot diff | Re-baseline with `--update`, or run failing `--test-id` alone | +| Snapshot diff on focus cases in parallel | Re-baseline with `--parallel 1`, or run failing `--test-id` alone | +| No tests found | Wrong `PLAYWRIGHT_SPEC` (shard 1 = `Default-spec`, not `Default1`); check `PLAYWRIGHT_COMPONENT`, `PLAYWRIGHT_TITLE`, `PLAYWRIGHT_TEST_ID`; delete stale `.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1` | +| Invalid `PLAYWRIGHT_TEST_ID` | Use a non-negative integer for `--test-id` (e.g. `--test-id 3`) | +| Stale `.test-data.json` (`undefined` parse error) | Delete `playwright/.test-data.json` or set `PLAYWRIGHT_REFRESH_TEST_DATA=1`; rebuild dist if needed | +| `locator.waitFor` / page timeout on CI | Ensure `playwright.config.mjs` uses `timeout: 120000`; use current `limestone-page.js` (WDIO-style ready, no `visible` on `body` / `data-ui-test-id`) | +| Port conflict on CI after setup | Ensure global-setup stopped its temporary `serve` (fresh `.test-data.json` + `--skip-build` should not leave an orphan on :4568) | +| Missing images on build | Place assets under `tests/screenshot/images/`, `videos/` | +| Port already in use | Set `PLAYWRIGHT_BASE_URL` to another port; restart any stale `serve` on 4568 | +| `build-apps` import error | Link or pin `@enact/ui-test-utils` with export `./build-apps` (see **Link local @enact/ui-test-utils** above) | +| Incomplete shard coverage | Ensure every spec file matched by `PLAYWRIGHT_SPEC` loaded and ran (see **Sharding vs workers**) | +| WDIO `--parallel` ignored | Update `@enact/ui-test-utils` (see ui-test-utils PR for `--parallel` CLI) | + +--- + +## CI + +Jenkins job **`enact-screenshot-tests`** runs WDIO (`npm run test-ss`) then Playwright via `build-scripts/enact-playwright-tests.sh` in the same workspace. Standalone job `enact-playwright-tests` is also supported. See `build-scripts/README.md` for full env var list. + +1. **Chrome:** `installChrome` from build-scripts (same as WDIO); skips `npm run prepare-playwright` on agents +2. **Reference:** optional Nebula `reference/${TARGET_TYPE}-playwright/` (separate from WDIO `reference/${TARGET_TYPE}/`) +3. **`npm run test-playwright`** with Jenkins env (aligned with WDIO `wdio-screenshot-test-opts.sh`): + +| Jenkins param | Playwright | WDIO (same job) | +|---------------|------------|-----------------| +| *(no `SPEC`, desktop)* | Full suite (`PLAYWRIGHT_SPEC` unset) | `--instances 5 --parallel 5`, all `*-specs.js` | +| `SPEC=Default` | `PLAYWRIGHT_SPEC=Default` | `--spec Default` | +| `SPEC=Default2` … `Default5` | `PLAYWRIGHT_SPEC=Default2`, … | `--spec Default2`, … | +| `COMPONENT` | `PLAYWRIGHT_COMPONENT`, `INSTANCES=1` | `--instances 1 --parallel …` | +| `WDIO_PARALLEL` | `PLAYWRIGHT_WORKERS` (default **5**) | `--parallel` | +| `PLAYWRIGHT_CI_DEFAULT_SPEC` | Optional smoke when no `SPEC` (e.g. `Default-spec` for shard 1 only) | — | +| `REFERENCE` | Nebula `limestone-playwright/` | Nebula `limestone/` | + +4. **Results:** `http://nebula.lge.com/results/playwright-results-<timestamp>/reports/html/index.html` + +On TV: `PLAYWRIGHT_SPEC=Default`, `INSTANCES=1`, `WORKERS=1`. + +--- + +## Maintenance + +### New or changed UI case + +1. Edit `tests/screenshot/apps/components/<Name>.js` +2. Register in `tests/screenshot/apps/LimestoneComponents.js` if new component +3. Run tests (build runs unless `--skip-build`); missing baselines are created automatically +4. `npm run test-playwright:component -- <Name> --update` to force refresh PNGs after UI changes + +### New WDIO shard + +Add matching `playwright/specs/**/<Name>-spec.js` with the same `skin`, `highContrast`, and `concurrency` as the WDIO `*-specs.js` file. + +--- + +## Related npm scripts (root `package.json`) + +```json +"test-playwright": "node tests/screenshot/scripts/run-playwright.mjs", +"test-playwright:update": "node tests/screenshot/scripts/run-playwright.mjs -- --update", +"test-playwright:report": "playwright show-report tests/screenshot/playwright/reports/html", +"test-playwright:component": "node tests/screenshot/scripts/run-playwright.mjs", +"test-ss:component": "node tests/screenshot/scripts/run-component-wdio.mjs", +"prepare-playwright": "playwright install chrome", +"benchmark-screenshots": "node tests/screenshot/playwright/benchmark.mjs" +``` diff --git a/tests/screenshot/playwright/benchmark.mjs b/tests/screenshot/playwright/benchmark.mjs new file mode 100644 index 000000000..7969cdc58 --- /dev/null +++ b/tests/screenshot/playwright/benchmark.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +/** + * Compare Playwright vs WebdriverIO runtime for one component (all its screenshot cases). + * + * Usage: + * npm run benchmark-screenshots -- Sprite + * npm run benchmark-screenshots -- Sprite --build + * npm run benchmark-screenshots -- Chip --parallel 5 + * (reuses tests/screenshot/dist when built; pass --build to rebuild) + */ +import {spawn} from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +import {assertComponentSource, SCREENSHOT_VIEW_INDEX} from './paths.js'; +import {parseComponentArgs} from '../scripts/parse-cli-args.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); + +const {component, withBuild, parallel} = parseComponentArgs(); + +if (!component) { + console.error('Usage: npm run benchmark-screenshots -- <ComponentName> [--build] [--parallel <n>]'); + console.error('Example: npm run benchmark-screenshots -- Sprite'); + console.error('Example: npm run benchmark-screenshots -- Sprite --build'); + console.error('Example: npm run benchmark-screenshots -- Chip --parallel 5'); + process.exit(1); +} + +assertComponentSource(component); + +const distIndex = SCREENSHOT_VIEW_INDEX; +const skipBuild = !withBuild && fs.existsSync(distIndex); + +function runCommand (label, command, commandArgs, env = {}) { + return new Promise((resolve, reject) => { + const start = Date.now(); + const child = spawn(command, commandArgs, { + cwd: repoRoot, + env: {...process.env, ...env}, + shell: true, + stdio: 'inherit' + }); + + child.on('close', code => { + const durationMs = Date.now() - start; + if (code === 0) { + resolve({label, durationMs}); + } else { + reject(new Error(`${label} failed with exit code ${code} after ${(durationMs / 1000).toFixed(1)}s`)); + } + }); + }); +} + +async function main () { + const parallelLabel = parallel === 1 ? + '1 worker / 1 browser (sequential)' : + `parallel ${parallel} (PW: workers=${parallel}, instances=1; WDIO: --parallel ${parallel}, --instances 1)`; + + console.log(`\nBenchmark: ${component} (all cases, ${parallelLabel})`); + if (skipBuild) { + console.log('Using existing tests/screenshot/dist (pass --build to rebuild)\n'); + } else { + console.log('Will build tests/screenshot/dist before run\n'); + } + + const parallelArg = ['--parallel', String(parallel)]; + + const playwright = await runCommand( + 'Playwright', + 'node', + ['tests/screenshot/scripts/run-playwright.mjs', component, ...parallelArg], + skipBuild ? {PLAYWRIGHT_SKIP_BUILD: '1'} : {} + ); + + const wdioArgv = ['tests/screenshot/scripts/run-component-wdio.mjs', component, ...parallelArg]; + if (skipBuild) { + wdioArgv.push('--skip-build'); + } + + const wdio = await runCommand( + 'WebdriverIO', + 'node', + wdioArgv, + {} + ); + + const pwSec = playwright.durationMs / 1000; + const wdioSec = wdio.durationMs / 1000; + const delta = wdioSec - pwSec; + let winner = 'tie'; + + if (delta > 0) { + winner = 'Playwright faster'; + } else if (delta < 0) { + winner = 'WDIO faster'; + } + + console.log('\n--- Results ---'); + console.log(`Component: ${component}`); + console.log(`Parallelism: ${parallel}`); + console.log(`Playwright: ${pwSec.toFixed(1)}s`); + console.log(`WebdriverIO: ${wdioSec.toFixed(1)}s`); + console.log(`Difference: ${Math.abs(delta).toFixed(1)}s (${winner})\n`); +} + +main().catch(err => { + console.error(err.message); + process.exit(1); +}); diff --git a/tests/screenshot/playwright/global-setup.js b/tests/screenshot/playwright/global-setup.js new file mode 100644 index 000000000..cfac98f07 --- /dev/null +++ b/tests/screenshot/playwright/global-setup.js @@ -0,0 +1,72 @@ +import fs from 'fs'; + +import {chromium} from '@playwright/test'; + +import {PLAYWRIGHT_BASE_URL, SCREENSHOT_VIEW_INDEX, TEST_DATA_FILE, assertScreenshotDist} from './paths.js'; +import {ensureStaticServer, stopStaticServer} from './static-server.js'; +import {clearShardRegistry} from './utils/shard-registry.js'; + +const baseURL = PLAYWRIGHT_BASE_URL; +const testDataFile = TEST_DATA_FILE; + +function shouldRefreshTestData () { + if (process.env.PLAYWRIGHT_REFRESH_TEST_DATA === '1') { + return true; + } + if (!fs.existsSync(testDataFile)) { + return true; + } + const distIndex = SCREENSHOT_VIEW_INDEX; + if (!fs.existsSync(distIndex)) { + return true; + } + return fs.statSync(distIndex).mtimeMs > fs.statSync(testDataFile).mtimeMs; +} + +export default async function globalSetup () { + clearShardRegistry(); + + if (!process.env.PLAYWRIGHT_SKIP_BUILD) { + const {default: buildApps} = await import('@enact/ui-test-utils/build-apps'); + await buildApps('screenshot'); + } + + assertScreenshotDist(); + + if (!shouldRefreshTestData()) { + return; + } + + // Playwright lifecycle: globalSetup → webServer (playwright.config.mjs) → tests → teardown. + // We start serve here only to fetch ?request metadata into .test-data.json, then stop our + // process so webServer can own the port for the full run (reuseExistingServer when not CI). + await ensureStaticServer(); + + try { + const pwBrowser = await chromium.launch({ + channel: 'chrome', + headless: true + }); + + try { + const page = await pwBrowser.newPage(); + const requestUrl = `${baseURL}/Limestone-View/?request`; + await page.goto(requestUrl, {waitUntil: 'load'}); + await page.waitForFunction(() => window.__TEST_DATA != null, null, {timeout: 30000}); + const testData = await page.evaluate(() => window.__TEST_DATA); + + if (testData == null) { + throw new Error( + `No window.__TEST_DATA at ${requestUrl}. The screenshot build may be stale or broken — ` + + 'rebuild tests/screenshot/dist (unset PLAYWRIGHT_SKIP_BUILD) and retry.' + ); + } + + fs.writeFileSync(testDataFile, JSON.stringify(testData)); + } finally { + await pwBrowser.close(); + } + } finally { + stopStaticServer(); + } +} diff --git a/tests/screenshot/playwright/global-teardown.js b/tests/screenshot/playwright/global-teardown.js new file mode 100644 index 000000000..19819c835 --- /dev/null +++ b/tests/screenshot/playwright/global-teardown.js @@ -0,0 +1,5 @@ +import {validateShardCoverage} from './utils/shard-registry.js'; + +export default async function globalTeardown () { + validateShardCoverage(); +} diff --git a/tests/screenshot/playwright/package.json b/tests/screenshot/playwright/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/tests/screenshot/playwright/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/tests/screenshot/playwright/paths.js b/tests/screenshot/playwright/paths.js new file mode 100644 index 000000000..d7615428f --- /dev/null +++ b/tests/screenshot/playwright/paths.js @@ -0,0 +1,52 @@ +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const screenshotRoot = path.join(__dirname, '..'); + +/** Default 4568 so WDIO can keep 4567 when both run in parallel. */ +export const PLAYWRIGHT_BASE_URL = process.env.PLAYWRIGHT_BASE_URL || 'http://localhost:4568'; +export const PLAYWRIGHT_PORT = new URL(PLAYWRIGHT_BASE_URL).port || '4568'; + +export const SCREENSHOT_DIST = path.join(screenshotRoot, 'dist'); +export const SCREENSHOT_VIEW = 'Limestone-View'; +export const SCREENSHOT_VIEW_INDEX = path.join(SCREENSHOT_DIST, SCREENSHOT_VIEW, 'index.html'); +export const SCREENSHOT_HEALTH_URL = `${PLAYWRIGHT_BASE_URL}/${SCREENSHOT_VIEW}/index.html`; +export const SCREENSHOT_COMPONENTS = path.join(screenshotRoot, 'apps', 'components'); +export const TEST_DATA_FILE = path.join(__dirname, '.test-data.json'); +export const SNAPSHOTS_DIR = path.join(__dirname, 'snapshots'); + +function componentSourcePath (componentName) { + return path.join(SCREENSHOT_COMPONENTS, `${componentName}.js`); +} + +export function assertScreenshotDist () { + if (fs.existsSync(SCREENSHOT_VIEW_INDEX)) { + return; + } + + const distHint = !fs.existsSync(SCREENSHOT_DIST) ? + `Directory ${SCREENSHOT_DIST} does not exist. ` : + ''; + + if (process.env.PLAYWRIGHT_SKIP_BUILD) { + throw new Error( + `${distHint}Missing ${SCREENSHOT_VIEW_INDEX}. ` + + 'Remove --skip-build (or unset PLAYWRIGHT_SKIP_BUILD) to build automatically, ' + + 'or run npm run test-ss once to populate tests/screenshot/dist/.' + ); + } + + throw new Error( + `${distHint}Screenshot build did not produce ${SCREENSHOT_VIEW_INDEX}. ` + + 'Run npm run bootstrap, link @enact/ui-test-utils if build-apps is missing from exports, then retry.' + ); +} + +export function assertComponentSource (componentName) { + const source = componentSourcePath(componentName); + if (!fs.existsSync(source)) { + throw new Error(`Missing ${source} — add the component under tests/screenshot/apps/components/`); + } +} diff --git a/tests/screenshot/playwright/playwright.config.mjs b/tests/screenshot/playwright/playwright.config.mjs new file mode 100644 index 000000000..69fe0aa7f --- /dev/null +++ b/tests/screenshot/playwright/playwright.config.mjs @@ -0,0 +1,74 @@ +import path from 'path'; +import {fileURLToPath} from 'url'; +import {defineConfig, devices} from '@playwright/test'; + +import {PLAYWRIGHT_BASE_URL, PLAYWRIGHT_PORT, SCREENSHOT_HEALTH_URL} from './paths.js'; +import {resolveTestMatchGlob} from './utils/spec-match.js'; +import {SCREENSHOT_COMPARE_OPTIONS, SCREENSHOT_VIEWPORT} from './utils/screenshot-options.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const distDir = path.join(__dirname, '..', 'dist'); + +export default defineConfig({ + testDir: path.join(__dirname, 'specs'), + testMatch: resolveTestMatchGlob(), + testIgnore: ['**/utils/**', '**/scripts/**'], + globalSetup: path.join(__dirname, 'global-setup.js'), + globalTeardown: path.join(__dirname, 'global-teardown.js'), + updateSnapshots: 'none', + snapshotPathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}'), + expect: { + toHaveScreenshot: { + pathTemplate: path.join(__dirname, 'snapshots/{arg}{ext}'), + ...SCREENSHOT_COMPARE_OPTIONS + } + }, + // WDIO mocha timeout is 1h; default Playwright 30s is too low for CI (goto + hydrate + fonts). + timeout: 120000, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.PLAYWRIGHT_WORKERS ? Number.parseInt(process.env.PLAYWRIGHT_WORKERS) : 5, + reporter: [ + ['list'], + ['html', {outputFolder: path.join(__dirname, 'reports', 'html'), open: 'never'}] + ], + outputDir: path.join(__dirname, 'test-results'), + use: { + baseURL: PLAYWRIGHT_BASE_URL, + navigationTimeout: 60000, + actionTimeout: 60000, + trace: 'on-first-retry', + viewport: SCREENSHOT_VIEWPORT, + launchOptions: { + args: [ + '--disable-infobars', + '--disable-search-engine-choice-screen', + '--disable-notifications', + '--disable-popup-blocking', + '--disable-lcd-text', + '--force-device-scale-factor=1', + '--disable-gpu', + `--window-size=${SCREENSHOT_VIEWPORT.width},${SCREENSHOT_VIEWPORT.height}` + ] + } + }, + webServer: { + command: process.platform === 'win32' ? + `npx.cmd --yes serve "${distDir}" -l ${PLAYWRIGHT_PORT}` : + `npx --yes serve ${distDir} -l ${PLAYWRIGHT_PORT}`, + url: SCREENSHOT_HEALTH_URL, + reuseExistingServer: !process.env.CI, + timeout: 120000, + shell: process.platform === 'win32' + }, + projects: [ + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + channel: 'chrome' + } + } + ] +}); diff --git a/tests/screenshot/playwright/specs/light/Light-spec.js b/tests/screenshot/playwright/specs/light/Light-spec.js new file mode 100644 index 000000000..e4f4e9758 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 1 +}); diff --git a/tests/screenshot/playwright/specs/light/Light2-spec.js b/tests/screenshot/playwright/specs/light/Light2-spec.js new file mode 100644 index 000000000..b43afb1cc --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light2-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 2 +}); diff --git a/tests/screenshot/playwright/specs/light/Light3-spec.js b/tests/screenshot/playwright/specs/light/Light3-spec.js new file mode 100644 index 000000000..9a752cd96 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light3-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 3 +}); diff --git a/tests/screenshot/playwright/specs/light/Light4-spec.js b/tests/screenshot/playwright/specs/light/Light4-spec.js new file mode 100644 index 000000000..7173e2054 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light4-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 4 +}); diff --git a/tests/screenshot/playwright/specs/light/Light5-spec.js b/tests/screenshot/playwright/specs/light/Light5-spec.js new file mode 100644 index 000000000..0e47df980 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light5-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 5 +}); diff --git a/tests/screenshot/playwright/specs/light/Light6-spec.js b/tests/screenshot/playwright/specs/light/Light6-spec.js new file mode 100644 index 000000000..6c31e0d00 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light6-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 6 +}); diff --git a/tests/screenshot/playwright/specs/light/Light7-spec.js b/tests/screenshot/playwright/specs/light/Light7-spec.js new file mode 100644 index 000000000..87bbe08b8 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light7-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 7 +}); diff --git a/tests/screenshot/playwright/specs/light/Light8-spec.js b/tests/screenshot/playwright/specs/light/Light8-spec.js new file mode 100644 index 000000000..28fbefab0 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light8-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 8 +}); diff --git a/tests/screenshot/playwright/specs/light/Light9-spec.js b/tests/screenshot/playwright/specs/light/Light9-spec.js new file mode 100644 index 000000000..215d84363 --- /dev/null +++ b/tests/screenshot/playwright/specs/light/Light9-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone Light', + skin: 'light', + highContrast: false, + concurrency: 9 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default-spec.js b/tests/screenshot/playwright/specs/neutral/Default-spec.js new file mode 100644 index 000000000..4d9c64902 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 1 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default2-spec.js b/tests/screenshot/playwright/specs/neutral/Default2-spec.js new file mode 100644 index 000000000..89e00d887 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default2-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 2 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default3-spec.js b/tests/screenshot/playwright/specs/neutral/Default3-spec.js new file mode 100644 index 000000000..82162dad7 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default3-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 3 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default4-spec.js b/tests/screenshot/playwright/specs/neutral/Default4-spec.js new file mode 100644 index 000000000..733ecdaff --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default4-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 4 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default5-spec.js b/tests/screenshot/playwright/specs/neutral/Default5-spec.js new file mode 100644 index 000000000..3ea5b09f6 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default5-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 5 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default6-spec.js b/tests/screenshot/playwright/specs/neutral/Default6-spec.js new file mode 100644 index 000000000..081f8ea3d --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default6-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 6 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default7-spec.js b/tests/screenshot/playwright/specs/neutral/Default7-spec.js new file mode 100644 index 000000000..134b7c7c0 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default7-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 7 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default8-spec.js b/tests/screenshot/playwright/specs/neutral/Default8-spec.js new file mode 100644 index 000000000..3de7684c6 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default8-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 8 +}); diff --git a/tests/screenshot/playwright/specs/neutral/Default9-spec.js b/tests/screenshot/playwright/specs/neutral/Default9-spec.js new file mode 100644 index 000000000..a1485f337 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/Default9-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone', + skin: 'neutral', + highContrast: false, + concurrency: 9 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast-spec.js new file mode 100644 index 000000000..f607c25a4 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 1 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast2-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast2-spec.js new file mode 100644 index 000000000..724696a6b --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast2-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 2 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast3-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast3-spec.js new file mode 100644 index 000000000..c42c328d7 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast3-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 3 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast4-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast4-spec.js new file mode 100644 index 000000000..ca4ff6a8d --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast4-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 4 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast5-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast5-spec.js new file mode 100644 index 000000000..c4bfaeba8 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast5-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 5 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast6-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast6-spec.js new file mode 100644 index 000000000..1f257a5ec --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast6-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 6 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast7-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast7-spec.js new file mode 100644 index 000000000..a3433b018 --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast7-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 7 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast8-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast8-spec.js new file mode 100644 index 000000000..1b5e901ba --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast8-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 8 +}); diff --git a/tests/screenshot/playwright/specs/neutral/HighContrast9-spec.js b/tests/screenshot/playwright/specs/neutral/HighContrast9-spec.js new file mode 100644 index 000000000..cded4447a --- /dev/null +++ b/tests/screenshot/playwright/specs/neutral/HighContrast9-spec.js @@ -0,0 +1,8 @@ +import {registerScreenshotTests} from '../../utils/run-screenshot-tests.js'; + +registerScreenshotTests({ + testName: 'Limestone High Contrast', + skin: 'neutral', + highContrast: true, + concurrency: 9 +}); diff --git a/tests/screenshot/playwright/static-server.js b/tests/screenshot/playwright/static-server.js new file mode 100644 index 000000000..21484beb4 --- /dev/null +++ b/tests/screenshot/playwright/static-server.js @@ -0,0 +1,127 @@ +import {spawn, spawnSync} from 'child_process'; + +import {PLAYWRIGHT_PORT, SCREENSHOT_DIST, SCREENSHOT_HEALTH_URL} from './paths.js'; + +const distPath = SCREENSHOT_DIST; +const port = PLAYWRIGHT_PORT; + +/** Set when global-setup starts `serve` for ?request metadata (stopped before tests). */ +export const staticServer = { + process: null, + started: false +}; + +function killProcessTree (childProcess) { + if (process.platform === 'win32') { + spawnSync('taskkill', ['/T', '/F', '/PID', String(childProcess.pid)], { + stdio: 'ignore', + shell: true + }); + return; + } + + childProcess.kill(); +} + +/** Free the Playwright port when a previous run left `serve` listening or wedged. */ +export function killProcessOnPort (listenPort = port) { + if (process.platform === 'win32') { + const netstat = spawnSync('netstat', ['-ano'], {encoding: 'utf8', shell: true}); + if (netstat.status !== 0) { + return; + } + + const portPattern = new RegExp(`:${String(listenPort)}(?!\\d)`); + + for (const line of netstat.stdout.split('\n')) { + if (!portPattern.test(line) || !line.includes('LISTENING')) { + continue; + } + + const pid = line.trim().split(/\s+/).at(-1); + if (pid && /^\d+$/.test(pid)) { + spawnSync('taskkill', ['/T', '/F', '/PID', pid], {stdio: 'ignore', shell: true}); + } + } + return; + } + + const lsof = spawnSync('lsof', ['-ti', `:${listenPort}`], {encoding: 'utf8'}); + if (lsof.status !== 0 || !lsof.stdout.trim()) { + return; + } + + for (const pid of lsof.stdout.trim().split('\n')) { + if (/^\d+$/.test(pid)) { + spawnSync('kill', ['-9', pid], {stdio: 'ignore'}); + } + } +} + +export function stopStaticServer () { + if (!staticServer.started || !staticServer.process) { + return; + } + + killProcessTree(staticServer.process); + staticServer.process = null; + staticServer.started = false; + killProcessOnPort(port); +} + +async function fetchWithTimeout (url, timeoutMs = 5000) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + return await fetch(url, {signal: controller.signal}); + } finally { + clearTimeout(timeout); + } +} + +async function isServerHealthy () { + try { + const response = await fetchWithTimeout(SCREENSHOT_HEALTH_URL); + return response.ok; + } catch { + return false; + } +} + +function startStaticServer () { + const command = process.platform === 'win32' ? 'npx.cmd' : 'npx'; + staticServer.process = spawn(command, ['--yes', 'serve', distPath, '-l', port], { + stdio: 'pipe', + shell: process.platform === 'win32' + }); + staticServer.started = true; +} + +async function waitForServer (url, timeout = 120000) { + const start = Date.now(); + + while (Date.now() - start < timeout) { + try { + const response = await fetchWithTimeout(url, 5000); + if (response.ok) { + return; + } + } catch { + // retry until the static server is ready + } + await new Promise(resolve => setTimeout(resolve, 500)); + } + + throw new Error(`Static server at ${url} did not start within ${timeout}ms`); +} + +export async function ensureStaticServer () { + if (await isServerHealthy()) { + return; + } + + stopStaticServer(); + startStaticServer(); + await waitForServer(SCREENSHOT_HEALTH_URL); +} diff --git a/tests/screenshot/playwright/utils/limestone-page.js b/tests/screenshot/playwright/utils/limestone-page.js new file mode 100644 index 000000000..d8c28a7a8 --- /dev/null +++ b/tests/screenshot/playwright/utils/limestone-page.js @@ -0,0 +1,118 @@ +/* global MutationObserver */ + +import {SCREENSHOT_VIEW} from '../paths.js'; + +const BODY_WAIT_MS = 10000; +const SETTLE_MS = 200; +const FONTS_WAIT_MS = 10000; +const READY_STATE_WAIT_MS = 15000; + +function serializeParams (params) { + const searchParams = new URLSearchParams(); + + for (const [key, value] of Object.entries(params)) { + if (value != null) { + searchParams.set(key, String(value)); + } + } + + return searchParams.toString(); +} + +function buildUrl (urlExtra = '?locale=en-US') { + return `/${SCREENSHOT_VIEW}/${urlExtra}`; +} + +// Chrome's native spellcheck draws wavy underlines on editable fields with "misspelled" content +// Force spellcheck off on every editable element +// before navigation, so the markers never render regardless of the browser environment. +async function disableSpellcheck (page) { + await page.addInitScript(() => { + const SELECTOR = 'input, textarea, [contenteditable]'; + + const disableWithin = (node) => { + if (node.nodeType !== 1) return; + if (node.matches && node.matches(SELECTOR)) { + node.setAttribute('spellcheck', 'false'); + } + if (node.querySelectorAll) { + for (const el of node.querySelectorAll(SELECTOR)) { + el.setAttribute('spellcheck', 'false'); + } + } + }; + + const apply = () => { + // Set a non-spellchecked default that descendants inherit, then sweep existing fields. + if (document.documentElement) { + document.documentElement.setAttribute('spellcheck', 'false'); + } + disableWithin(document.documentElement || document); + }; + + // React mounts the screenshot app after load, so watch for fields added later. + const observer = new MutationObserver((mutations) => { + for (const mutation of mutations) { + for (const added of mutation.addedNodes) { + disableWithin(added); + } + } + }); + + const start = () => { + apply(); + if (document.body) { + observer.observe(document.body, {childList: true, subtree: true}); + } + }; + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', start, {once: true}); + } else { + start(); + } + }); +} + +async function open (page, urlExtra = '?locale=en-US') { + await page.goto(buildUrl(urlExtra), {waitUntil: 'load'}); +} + +async function waitForPageReady (page) { + // Enact screenshot app may keep <body> hidden; WDIO waitForDisplayed still passes once #root + // has content. Wait for attached + rendered root instead of body visibility. + await page.locator('body').waitFor({state: 'attached', timeout: BODY_WAIT_MS}); + await page.waitForFunction( + () => { + const root = document.getElementById('root'); + return document.readyState === 'complete' && root != null && root.childElementCount > 0; + }, + {timeout: READY_STATE_WAIT_MS} + ); + + await page.evaluate((ms) => new Promise((resolve) => setTimeout(resolve, ms)), SETTLE_MS); + + await page.evaluate((fontsTimeout) => Promise.race([ + document.fonts.ready, + new Promise((resolve) => setTimeout(resolve, fontsTimeout)) + ]), FONTS_WAIT_MS); + + // MediaOverlay and similar: freeze video on first frame (non-deterministic otherwise). + await page.evaluate(() => { + for (const video of document.querySelectorAll('video')) { + video.pause(); + try { + video.currentTime = 0; + } catch { + // ignore seek errors on unloaded media + } + } + }); +} + +export async function openComponent (page, params) { + await disableSpellcheck(page); + const query = serializeParams(Object.assign({locale: 'en-US'}, params)); + await open(page, `?${query}`); + await waitForPageReady(page); +} diff --git a/tests/screenshot/playwright/utils/run-screenshot-tests.js b/tests/screenshot/playwright/utils/run-screenshot-tests.js new file mode 100644 index 000000000..f1f13e0da --- /dev/null +++ b/tests/screenshot/playwright/utils/run-screenshot-tests.js @@ -0,0 +1,172 @@ +import fs from 'fs'; +import path from 'path'; + +import {test as baseTest, expect as baseExpect} from '@playwright/test'; + +import {assertComponentSource, TEST_DATA_FILE} from '../paths.js'; +import {getScreenshotPathSegments} from './screenshot-name.js'; +import {openComponent} from './limestone-page.js'; +import {recordShard} from './shard-registry.js'; +import { + PAGE_SCREENSHOT_OPTIONS, + SCREENSHOT_COMPARE_OPTIONS, + SCREENSHOT_VIEWPORT +} from './screenshot-options.js'; + +const forceUpdate = process.env.PLAYWRIGHT_FORCE_UPDATE === '1'; +const componentExactMatch = process.env.PLAYWRIGHT_COMPONENT_EXACT === '1'; + +function parseTestIdFilter () { + const raw = process.env.PLAYWRIGHT_TEST_ID; + + if (raw == null) { + return null; + } + + const trimmed = raw.trim(); + + if (!/^\d+$/.test(trimmed)) { + throw new Error( + `Invalid PLAYWRIGHT_TEST_ID "${raw}": must be a non-negative integer (e.g. --test-id 3).` + ); + } + + return Number.parseInt(trimmed); +} + +const testIdFilter = parseTestIdFilter(); +const titleFilter = process.env.PLAYWRIGHT_TITLE; +const maxInstances = process.env.PLAYWRIGHT_INSTANCES ? + Number.parseInt(process.env.PLAYWRIGHT_INSTANCES) : + 5; + +function loadTestData () { + return JSON.parse(fs.readFileSync(TEST_DATA_FILE, 'utf8')); +} + +function resolveComponentFilter (config) { + return config.component || process.env.PLAYWRIGHT_COMPONENT || null; +} + +function matchesTitleFilter (title, filter) { + const terms = filter.trim().split(/\s+/).filter(Boolean); + + return terms.every(term => { + try { + return new RegExp(term, 'i').test(title); + } catch { + return title.toLowerCase().includes(term.toLowerCase()); + } + }); +} + +function shouldIncludeTest (component, testId, title, configComponent) { + const componentFilter = resolveComponentFilter({component: configComponent}); + if (componentFilter) { + if (componentExactMatch) { + const pattern = componentFilter.startsWith('^') ? componentFilter : `^${componentFilter}$`; + if (!component.match(new RegExp(pattern))) { + return false; + } + } else if (!component.includes(componentFilter)) { + return false; + } + } + if (testIdFilter != null && testIdFilter !== testId) { + return false; + } + if (titleFilter && !matchesTitleFilter(title, titleFilter)) { + return false; + } + return true; +} + +function resolveSnapshotPath (testInfo, segments) { + return testInfo.snapshotPath(...segments, {kind: 'screenshot'}); +} + +/** + * WDIO autoSaveBaseline: create missing baselines without --update. + * PLAYWRIGHT_FORCE_UPDATE (--update) always overwrites existing baselines. + * Snapshot path: snapshots/<Component>/<TestName>/<case>.png (same as WDIO). + */ +async function assertPageScreenshot (page, testInfo, segments, options) { + const filePath = resolveSnapshotPath(testInfo, segments); + + if (forceUpdate || !fs.existsSync(filePath)) { + fs.mkdirSync(path.dirname(filePath), {recursive: true}); + await page.screenshot({...PAGE_SCREENSHOT_OPTIONS, path: filePath}); + return; + } + + await baseExpect(page).toHaveScreenshot(segments, options); +} + +/** + * Cases from tests/screenshot/apps/components/*.js (built into tests/screenshot/dist). + */ +function getScreenshotTests ({testName, skin, highContrast, concurrency, component: configComponent, maxWorkers = maxInstances}) { + if (concurrency && concurrency > maxWorkers) { + return []; + } + + if (configComponent) { + assertComponentSource(configComponent); + } + + const testCases = loadTestData(); + const tests = []; + + for (const component in testCases) { + testCases[component].forEach((testCase, testId) => { + if (concurrency && testId % maxWorkers !== concurrency - 1) { + return; + } + if (!shouldIncludeTest(component, testId, testCase.title, configComponent)) { + return; + } + + tests.push({ + title: `${component}~/${testName}~/${testCase.title}~/${testId}`, + params: {component, testId, skin, highContrast, caseTitle: testCase.title} + }); + }); + } + + return tests; +} + +export function registerScreenshotTests (config) { + const component = resolveComponentFilter(config); + let suiteName = config.testName; + + if (component) { + suiteName = `${config.testName} — ${component}`; + } else if (config.concurrency) { + suiteName = `${config.testName} (shard ${config.concurrency})`; + } + const cases = getScreenshotTests(config); + recordShard(config, maxInstances); + + baseTest.describe(suiteName, () => { + baseTest.beforeEach(async ({page}) => { + await page.setViewportSize(SCREENSHOT_VIEWPORT); + }); + + for (const screenshotTest of cases) { + baseTest(screenshotTest.title, async ({page}, testInfo) => { + await openComponent(page, screenshotTest.params); + await assertPageScreenshot( + page, + testInfo, + getScreenshotPathSegments( + screenshotTest.params.component, + config.testName, + screenshotTest.params.caseTitle + ), + SCREENSHOT_COMPARE_OPTIONS + ); + }); + } + }); +} diff --git a/tests/screenshot/playwright/utils/screenshot-name.js b/tests/screenshot/playwright/utils/screenshot-name.js new file mode 100644 index 000000000..e1518e561 --- /dev/null +++ b/tests/screenshot/playwright/utils/screenshot-name.js @@ -0,0 +1,24 @@ +import {createHash} from 'node:crypto'; +import path from 'path'; + +/** + * Case file name only (WDIO visual-service convention). + * @see ui-test-utils/utils/runTest.js, screenshot/utils/confHelpers.js + */ +export function getScreenshotBasename (title) { + let testCaseName = title.replace(/[/\\:?*"|<>]/g, '_'); + testCaseName = testCaseName.substring(0, 128) + '-' + createHash('md5').update(testCaseName).digest('hex'); + + return `${testCaseName}.png`.replace(/ /g, '_'); +} + +/** Relative path: <Component>/<TestName>/<case>.png (matches WDIO reference layout). */ +export function getScreenshotName (component, testName, title) { + return path.posix.join(component, testName, getScreenshotBasename(title)); +} + +/** Playwright snapshot segments — array form keeps WDIO folder layout (not flat dashes). */ +export function getScreenshotPathSegments (component, testName, title) { + return getScreenshotName(component, testName, title).split('/'); +} + diff --git a/tests/screenshot/playwright/utils/screenshot-options.js b/tests/screenshot/playwright/utils/screenshot-options.js new file mode 100644 index 000000000..70d0c95f4 --- /dev/null +++ b/tests/screenshot/playwright/utils/screenshot-options.js @@ -0,0 +1,25 @@ +/** + * Shared screenshot settings aligned with WDIO (@enact/ui-test-utils): + * - setWindowSize(1920, 1167) in wdio.conf.js + * - checkScreen({ ignoreAntialiasing: true, ... }) → threshold 0.2 in Playwright + */ +export const SCREENSHOT_VIEWPORT = { + width: 1920, + height: 1080 +}; + +/** Playwright per-pixel color tolerance; mirrors WDIO ignoreAntialiasing. */ +export const SCREENSHOT_THRESHOLD = 0; + +export const SCREENSHOT_COMPARE_OPTIONS = { + animations: 'disabled', + caret: 'hide', + threshold: SCREENSHOT_THRESHOLD, + maxDiffPixelRatio: 0 +}; + +export const PAGE_SCREENSHOT_OPTIONS = { + animations: 'disabled', + caret: 'hide', + type: 'png' +}; diff --git a/tests/screenshot/playwright/utils/shard-registry.js b/tests/screenshot/playwright/utils/shard-registry.js new file mode 100644 index 000000000..30b92b3b6 --- /dev/null +++ b/tests/screenshot/playwright/utils/shard-registry.js @@ -0,0 +1,78 @@ +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +import {getExpectedShardsByGroup} from './spec-match.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const registryPath = path.join(__dirname, '..', '.shard-registry.jsonl'); + +export function clearShardRegistry () { + if (fs.existsSync(registryPath)) { + fs.unlinkSync(registryPath); + } +} + +export function recordShard ({skin, highContrast, concurrency}, maxWorkers) { + if (process.env.PLAYWRIGHT_COMPONENT || !concurrency) { + return; + } + + const entry = JSON.stringify({ + groupKey: `${skin}-${highContrast}`, + concurrency, + maxWorkers + }); + + fs.appendFileSync(registryPath, `${entry}\n`); +} + +export function validateShardCoverage () { + if (process.env.PLAYWRIGHT_COMPONENT || !fs.existsSync(registryPath)) { + return; + } + + const actualGroups = new Map(); + + for (const line of fs.readFileSync(registryPath, 'utf8').split('\n')) { + if (!line.trim()) { + continue; + } + + const {groupKey, concurrency, maxWorkers: recordedMaxWorkers} = JSON.parse(line); + + if (!actualGroups.has(groupKey)) { + actualGroups.set(groupKey, {maxWorkers: recordedMaxWorkers, shards: new Set()}); + } + + const group = actualGroups.get(groupKey); + group.maxWorkers = recordedMaxWorkers; + group.shards.add(concurrency); + } + + const maxWorkers = process.env.PLAYWRIGHT_INSTANCES ? + Number.parseInt(process.env.PLAYWRIGHT_INSTANCES) : + [...actualGroups.values()][0]?.maxWorkers ?? 5; + const expectedGroups = getExpectedShardsByGroup(maxWorkers); + const groupKeys = new Set([...expectedGroups.keys(), ...actualGroups.keys()]); + const specHint = process.env.PLAYWRIGHT_SPEC?.trim() ? + `PLAYWRIGHT_SPEC=${process.env.PLAYWRIGHT_SPEC.trim()}` : + 'all *-spec.js files'; + + for (const groupKey of groupKeys) { + const required = [...(expectedGroups.get(groupKey) || [])].sort((a, b) => a - b); + const covering = [...(actualGroups.get(groupKey)?.shards || [])] + .filter(shard => shard <= maxWorkers) + .sort((a, b) => a - b); + const missing = required.filter(shard => !covering.includes(shard)); + + if (missing.length > 0) { + throw new Error( + `Incomplete Playwright shard coverage for ${groupKey}: ` + + `${specHint} requires shards ${required.join(', ')}, ` + + `but only ${covering.length ? covering.join(', ') : '(none)'} ran. ` + + `Missing: ${missing.join(', ')}.` + ); + } + } +} diff --git a/tests/screenshot/playwright/utils/spec-match.js b/tests/screenshot/playwright/utils/spec-match.js new file mode 100644 index 000000000..26629239b --- /dev/null +++ b/tests/screenshot/playwright/utils/spec-match.js @@ -0,0 +1,99 @@ +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const specsDir = path.join(__dirname, '..', 'specs'); + +/** Same rules as playwright.config.mjs `testMatch`. */ +export function resolveTestMatchGlob () { + const spec = process.env.PLAYWRIGHT_SPEC?.trim(); + + if (!spec) { + return '**/*-spec.js'; + } + + const base = spec.replace(/\.js$/, ''); + + if (base.endsWith('-spec')) { + return `**/${base}.js`; + } + + return `**/*${base}*-spec.js`; +} + +export function specFileMatchesFilter (baseName, specFilter) { + if (!specFilter) { + return baseName.endsWith('-spec'); + } + + const base = specFilter.replace(/\.js$/, ''); + + if (base.endsWith('-spec')) { + return baseName === base; + } + + return baseName.includes(base) && baseName.endsWith('-spec'); +} + +function walkSpecFiles (dir) { + const files = []; + + for (const entry of fs.readdirSync(dir, {withFileTypes: true})) { + const entryPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + files.push(...walkSpecFiles(entryPath)); + } else if (entry.name.endsWith('-spec.js')) { + files.push(entryPath); + } + } + + return files; +} + +function parseSpecMeta (content) { + const concurrency = content.match(/concurrency:\s*(\d+)/)?.[1]; + const skin = content.match(/skin:\s*'([^']+)'/)?.[1]; + const highContrast = content.match(/highContrast:\s*(true|false)/)?.[1]; + + if (!concurrency || !skin || highContrast == null) { + return null; + } + + return { + concurrency: Number.parseInt(concurrency), + skin, + highContrast: highContrast === 'true' + }; +} + +/** Shards that should run per skin/HC group for the current PLAYWRIGHT_SPEC filter. */ +export function getExpectedShardsByGroup (maxWorkers) { + const specFilter = process.env.PLAYWRIGHT_SPEC?.trim() || null; + const groups = new Map(); + + for (const filePath of walkSpecFiles(specsDir)) { + const baseName = path.basename(filePath, '.js'); + + if (!specFileMatchesFilter(baseName, specFilter)) { + continue; + } + + const meta = parseSpecMeta(fs.readFileSync(filePath, 'utf8')); + + if (!meta?.concurrency || meta.concurrency > maxWorkers) { + continue; + } + + const groupKey = `${meta.skin}-${meta.highContrast}`; + + if (!groups.has(groupKey)) { + groups.set(groupKey, new Set()); + } + + groups.get(groupKey).add(meta.concurrency); + } + + return groups; +} diff --git a/tests/screenshot/scripts/ensure-screenshot-dist.mjs b/tests/screenshot/scripts/ensure-screenshot-dist.mjs new file mode 100644 index 000000000..5f3b6a42d --- /dev/null +++ b/tests/screenshot/scripts/ensure-screenshot-dist.mjs @@ -0,0 +1,32 @@ +/* eslint-disable no-console */ +import fs from 'fs'; + +import {SCREENSHOT_VIEW_INDEX, assertScreenshotDist} from '../playwright/paths.js'; + +export function hasScreenshotDist () { + return fs.existsSync(SCREENSHOT_VIEW_INDEX); +} + +/** + * Ensure tests/screenshot/dist exists before Playwright starts its webServer. + * Builds when dist is missing (even with --skip-build) so runs do not fail on a cold checkout. + * + * @returns {Promise<{skipBuild: boolean}>} skipBuild=true → set PLAYWRIGHT_SKIP_BUILD for global-setup + */ +export async function ensureScreenshotDist ({skipBuild = false} = {}) { + if (hasScreenshotDist()) { + return {skipBuild}; + } + + if (skipBuild) { + console.log('tests/screenshot/dist is missing; building before Playwright (ignoring --skip-build)'); + } else { + console.log('Building tests/screenshot/dist...'); + } + + const {default: buildApps} = await import('@enact/ui-test-utils/build-apps'); + await buildApps('screenshot'); + assertScreenshotDist(); + + return {skipBuild: true}; +} diff --git a/tests/screenshot/scripts/parse-cli-args.mjs b/tests/screenshot/scripts/parse-cli-args.mjs new file mode 100644 index 000000000..5da7b3442 --- /dev/null +++ b/tests/screenshot/scripts/parse-cli-args.mjs @@ -0,0 +1,91 @@ +const PLAYWRIGHT_VALUE_FLAGS = new Set(['--component', '--spec', '--parallel', '--test-id', '--title']); +const COMPONENT_VALUE_FLAGS = new Set(['--test-id', '--title', '--parallel', '--spec']); + +/** + * Value flags consume all following tokens until the next --flag (multi-word values). + */ +function parseCliArgs (argv = process.argv, {valueFlags, captureFirstPositional = false} = {}) { + const args = argv.slice(2).filter(a => a !== '--'); + const flags = new Set(); + const values = {}; + let positional = null; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + + if (!arg.startsWith('--')) { + if (captureFirstPositional && !positional) { + positional = arg; + } + continue; + } + + if (valueFlags.has(arg)) { + const valueParts = []; + while (++i < args.length && !args[i].startsWith('--')) { + valueParts.push(args[i]); + } + values[arg] = valueParts.length > 0 ? valueParts.join(' ') : null; + i--; + continue; + } + + flags.add(arg); + } + + return {flags, values, positional}; +} + +/** True when argv has a positional component name (test-playwright:component). */ +export function isComponentPlaywrightRun (argv = process.argv) { + const args = argv.slice(2).filter(a => a !== '--'); + + if (args.includes('--component')) { + return false; + } + + return parseComponentArgs(argv).component != null; +} + +/** + * Parse CLI args for npm run test-playwright. + * All flags are optional; default run executes all specs on all components. + */ +export function parsePlaywrightArgs (argv = process.argv) { + const {flags, values} = parseCliArgs(argv, {valueFlags: PLAYWRIGHT_VALUE_FLAGS}); + + return { + component: values['--component'] ?? null, + spec: values['--spec'] ?? null, + update: flags.has('--update'), + skipBuild: flags.has('--skip-build'), + testId: values['--test-id'], + title: values['--title'], + parallel: values['--parallel'] != null ? + Math.max(1, Number.parseInt(values['--parallel']) || 1) : + null + }; +} + +/** + * Parse component CLI args without treating flag values as the component name. + * Component name must be the first positional token (flags may follow). + */ +export function parseComponentArgs (argv = process.argv) { + const {flags, values, positional} = parseCliArgs(argv, { + valueFlags: COMPONENT_VALUE_FLAGS, + captureFirstPositional: true + }); + + return { + component: positional, + update: flags.has('--update'), + // --build is used by benchmark.mjs only; Playwright runners use --skip-build instead. + withBuild: flags.has('--build'), + skipBuild: flags.has('--skip-build'), + testId: values['--test-id'], + title: values['--title'], + spec: values['--spec'], + parallel: Math.max(1, Number.parseInt(values['--parallel'] ?? '1') || 1) + }; +} diff --git a/tests/screenshot/scripts/run-component-wdio.mjs b/tests/screenshot/scripts/run-component-wdio.mjs new file mode 100644 index 000000000..7be5b3787 --- /dev/null +++ b/tests/screenshot/scripts/run-component-wdio.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +/** + * Run WDIO screenshot tests for one component (Default shard, instances 1 = all cases). + * + * Usage: + * npm run test-ss:component -- Sprite + * npm run test-ss:component -- Sprite --skip-build + * npm run test-ss:component -- Sprite --parallel 5 + */ +import {spawnSync} from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); + +const args = process.argv.slice(2).filter(a => a !== '--'); + +function flagValue (name) { + const index = args.indexOf(name); + return index === -1 ? null : args[index + 1]; +} + +const component = args.find(a => !a.startsWith('--')); +const skipBuild = args.includes('--skip-build'); +const parallel = Math.max(1, Number.parseInt(flagValue('--parallel') ?? '1') || 1); + +if (!component) { + console.error('Usage: npm run test-ss:component -- <ComponentName> [--skip-build] [--parallel <n>]'); + console.error('Example: npm run test-ss:component -- Sprite'); + console.error('Example: npm run test-ss:component -- Sprite --skip-build'); + console.error('Example: npm run test-ss:component -- Sprite --parallel 5'); + process.exit(1); +} + +const screenshotSource = path.join(repoRoot, 'tests', 'screenshot', 'apps', 'components', `${component}.js`); +if (!fs.existsSync(screenshotSource)) { + console.error(`Missing ${screenshotSource}`); + process.exit(1); +} + +const spec = path.join(repoRoot, 'tests', 'screenshot', 'specs', 'neutral', 'Default-specs.js'); + +console.log(`WDIO screenshots: ${component} (${spec}, parallel=${parallel})`); + +const wdioArgs = [ + 'run', 'test-ss', + '--', + '--component', `^${component}$`, + '--instances', '1', + '--spec', spec +]; + +wdioArgs.push('--parallel', String(parallel)); + +if (skipBuild) { + wdioArgs.push('--skip-build'); +} + +const result = spawnSync( + 'npm', + wdioArgs, + { + cwd: repoRoot, + stdio: 'inherit', + shell: true + } +); + +process.exit(result.status ?? 1); diff --git a/tests/screenshot/scripts/run-playwright.mjs b/tests/screenshot/scripts/run-playwright.mjs new file mode 100644 index 000000000..ae4d90b4d --- /dev/null +++ b/tests/screenshot/scripts/run-playwright.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node +/* eslint-disable no-console */ +/** + * Run Playwright screenshot tests (full suite or single component). + * + * Full suite: + * npm run test-playwright + * npm run test-playwright -- --component Button + * npm run test-playwright -- --spec Light + * npm run test-playwright -- --component Button --spec Light + * npm run test-playwright -- --update + * npm run test-playwright -- --skip-build + * npm run test-playwright -- --parallel 5 + * + * Single component (positional name; INSTANCES=1, WORKERS from --parallel): + * npm run test-playwright:component -- Sprite + * npm run test-playwright:component -- Sprite --update + * npm run test-playwright:component -- Sprite --skip-build + * npm run test-playwright:component -- Sprite --spec Light + * npm run test-playwright:component -- Sprite --test-id 0 + * npm run test-playwright:component -- Sprite --title <regex> + * npm run test-playwright:component -- Sprite --parallel 5 + */ +import {assertComponentSource} from '../playwright/paths.js'; +import {ensureScreenshotDist} from './ensure-screenshot-dist.mjs'; +import {isComponentPlaywrightRun, parseComponentArgs, parsePlaywrightArgs} from './parse-cli-args.mjs'; +import {spawnPlaywright} from './spawn-playwright.mjs'; + +function printComponentUsage () { + console.error('Usage: npm run test-playwright:component -- <ComponentName> [options]'); + console.error('Options: --update, --skip-build, --spec <Name>, --test-id <n>, --title <regex>, --parallel <n>'); + console.error('Example: npm run test-playwright:component -- Sprite'); + console.error('Example: npm run test-playwright:component -- Sprite --update'); + console.error('Example: npm run test-playwright:component -- Sprite --skip-build'); + console.error('Example: npm run test-playwright:component -- Sprite --spec Light'); +} + +function buildSuiteEnv () { + const {component, spec, update, skipBuild: skipBuildFlag, parallel, testId, title} = parsePlaywrightArgs(); + const playwrightEnv = {}; + const filters = []; + + let resolvedComponent = component; + let resolvedTitle = title; + + if (title != null && !component) { + const titleParts = title.trim().split(/\s+/); + + if (titleParts.length > 1) { + resolvedComponent = titleParts[0]; + resolvedTitle = titleParts.slice(1).join(' '); + } + } + + if (resolvedComponent) { + playwrightEnv.PLAYWRIGHT_COMPONENT = resolvedComponent; + filters.push(`component ~ ${resolvedComponent}`); + } + + if (spec) { + playwrightEnv.PLAYWRIGHT_SPEC = spec; + filters.push(`spec ~ ${spec}`); + } + + if (update) { + playwrightEnv.PLAYWRIGHT_FORCE_UPDATE = '1'; + } + + if (parallel != null) { + playwrightEnv.PLAYWRIGHT_WORKERS = String(parallel); + } + + if (testId != null) { + playwrightEnv.PLAYWRIGHT_TEST_ID = testId; + } + + if (resolvedTitle != null) { + playwrightEnv.PLAYWRIGHT_TITLE = resolvedTitle; + filters.push(`title ~ ${resolvedTitle}`); + } + + if (filters.length) { + console.log(`Playwright filters: ${filters.join(', ')}`); + } else { + console.log('Playwright: all components, all specs'); + } + + return {env: playwrightEnv, skipBuild: skipBuildFlag, label: 'Playwright'}; +} + +function buildComponentEnv () { + const {component, update, skipBuild: skipBuildFlag, spec, testId, title, parallel} = parseComponentArgs(); + + if (!component) { + printComponentUsage(); + process.exit(1); + } + + assertComponentSource(component); + + const playwrightEnv = { + PLAYWRIGHT_COMPONENT: component, + PLAYWRIGHT_COMPONENT_EXACT: '1', + PLAYWRIGHT_INSTANCES: '1', + PLAYWRIGHT_WORKERS: String(parallel) + }; + + if (update) { + playwrightEnv.PLAYWRIGHT_FORCE_UPDATE = '1'; + } + + if (spec) { + playwrightEnv.PLAYWRIGHT_SPEC = spec; + } + + if (testId != null) { + playwrightEnv.PLAYWRIGHT_TEST_ID = testId; + } + + if (title != null) { + playwrightEnv.PLAYWRIGHT_TITLE = title; + } + + const filters = [`component ${component}`]; + if (spec) { + filters.push(`spec ~ ${spec}`); + } + console.log(`Playwright: ${filters.join(', ')}`); + + return {env: playwrightEnv, skipBuild: skipBuildFlag, label: `Playwright (${component})`}; +} + +const {env, skipBuild, label} = isComponentPlaywrightRun() ? + buildComponentEnv() : + buildSuiteEnv(); + +const {skipBuild: skipBuildInSetup} = await ensureScreenshotDist({skipBuild}); + +if (skipBuild || skipBuildInSetup) { + env.PLAYWRIGHT_SKIP_BUILD = '1'; +} + +const exitCode = spawnPlaywright({env, label}); +process.exit(exitCode); diff --git a/tests/screenshot/scripts/spawn-playwright.mjs b/tests/screenshot/scripts/spawn-playwright.mjs new file mode 100644 index 000000000..0875988a2 --- /dev/null +++ b/tests/screenshot/scripts/spawn-playwright.mjs @@ -0,0 +1,43 @@ +import {spawnSync} from 'child_process'; +import path from 'path'; +import {fileURLToPath} from 'url'; + +/* eslint-disable no-console */ + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(__dirname, '..', '..', '..'); +const config = path.join(__dirname, '..', 'playwright', 'playwright.config.mjs'); + +function formatDuration (durationSec) { + const hours = Math.floor(durationSec / 3600); + const minutes = Math.floor((durationSec % 3600) / 60); + const seconds = durationSec % 60; + + const parts = []; + if (hours > 0) parts.push(`${hours}h`); + if (hours > 0 || minutes > 0) parts.push(`${minutes}m`); + parts.push(`${seconds.toFixed(1)}s`); + return parts.join(' '); +} + +/** + * Run Playwright with optional env overrides and print wall-clock duration. + */ +export function spawnPlaywright ({playwrightArgs = [], env = {}, label = 'Playwright'}) { + const start = Date.now(); + const result = spawnSync( + process.platform === 'win32' ? 'npx.cmd' : 'npx', + ['playwright', 'test', '--config', config, ...playwrightArgs], + { + cwd: repoRoot, + stdio: 'inherit', + env: {...process.env, ...env}, + shell: process.platform === 'win32' + } + ); + const durationSec = (Date.now() - start) / 1000; + + console.log(`\n${label} finished in ${formatDuration(durationSec)}`); + + return result.status ?? 1; +}