From 72807166dfa98f6116c0f5a283e9072ce68fca60 Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Thu, 30 Jul 2026 15:45:45 +0200 Subject: [PATCH 1/2] Add packages/qa: script-based runner for the CLI Pre-release QA flow Automates the QA doc steps that need no human interaction by driving the CLI directly: execa for non-interactive commands, node-pty for interactive ones (app dev, app config link, hydrogen init). Emits a summary that mirrors the QA doc structure, with manual/delegated items reported as skipped. Verified locally so far: Preparation (version), Hydrogen init + build (hydrogen dev blocked locally by Santa denying workerd; fine on CI runners). Assisted-By: devx/2410058d-4b68-49a7-ba23-7233a4497d19 --- packages/qa/.gitignore | 2 + packages/qa/README.md | 67 ++++ packages/qa/package.json | 22 ++ packages/qa/src/context.ts | 116 +++++++ packages/qa/src/proc.ts | 250 +++++++++++++++ packages/qa/src/report.ts | 104 +++++++ packages/qa/src/run.ts | 102 +++++++ packages/qa/src/steps/apps.ts | 488 ++++++++++++++++++++++++++++++ packages/qa/src/steps/hydrogen.ts | 124 ++++++++ packages/qa/src/steps/prep.ts | 51 ++++ packages/qa/src/steps/theme.ts | 16 + packages/qa/src/types.ts | 59 ++++ packages/qa/tsconfig.json | 15 + pnpm-lock.yaml | 68 +++++ 14 files changed, 1484 insertions(+) create mode 100644 packages/qa/.gitignore create mode 100644 packages/qa/README.md create mode 100644 packages/qa/package.json create mode 100644 packages/qa/src/context.ts create mode 100644 packages/qa/src/proc.ts create mode 100644 packages/qa/src/report.ts create mode 100644 packages/qa/src/run.ts create mode 100644 packages/qa/src/steps/apps.ts create mode 100644 packages/qa/src/steps/hydrogen.ts create mode 100644 packages/qa/src/steps/prep.ts create mode 100644 packages/qa/src/steps/theme.ts create mode 100644 packages/qa/src/types.ts create mode 100644 packages/qa/tsconfig.json diff --git a/packages/qa/.gitignore b/packages/qa/.gitignore new file mode 100644 index 00000000000..5dda80d9c8e --- /dev/null +++ b/packages/qa/.gitignore @@ -0,0 +1,2 @@ +qa-report/ +node_modules/ diff --git a/packages/qa/README.md b/packages/qa/README.md new file mode 100644 index 00000000000..2cb82e85cd6 --- /dev/null +++ b/packages/qa/README.md @@ -0,0 +1,67 @@ +# @shopify/qa — CLI Pre-release QA flow runner + +Automates the steps of the [CLI Pre-release QA flow doc](https://docs.google.com/document/d/1XX6QnS6kKZTT1shcCZVcso74VWn-Ui4V2IASenHHi1E/edit) +that need **no human interaction**, by driving the CLI directly (no Playwright, +no test framework): non-interactive commands run via `execa`, interactive ones +(`app dev`, `app config link`, `hydrogen init`) run in a pseudo-terminal via +`node-pty` and are driven with the same keypresses the doc describes (`g`, `q`, +Enter, CTRL+C). + +Every checklist item of the doc is represented, in order: + +- **auto** — executed by the runner. +- **manual** — needs a browser/human (dev console, theme editor, visual + confirmations). Reported as `⏭️ manual` and listed in the + "Remaining manual checklist" section of the summary — never silently dropped. +- **delegated** — the Theme section, owned by the themes team per the doc. + +The summary (`qa-summary.md`, also appended to `$GITHUB_STEP_SUMMARY` on CI) +mirrors the QA doc structure section by section. + +## Running locally + +```sh +# Build the CLI first (the runner targets the repo build by default) +pnpm nx run-many --all --target=build + +cd packages/qa +pnpm install + +# Requires an authenticated CLI session (shopify auth login) or +# SHOPIFY_APP_AUTOMATION_TOKEN, plus an org and a dev store: +QA_ORG_ID= QA_STORE_FQDN=.myshopify.com pnpm qa + +# Run a subset while iterating: +QA_ONLY=hydrogen pnpm qa +QA_ONLY=prep,apps QA_ORG_ID=… QA_STORE_FQDN=… pnpm qa +``` + +## Environment variables + +| Variable | Purpose | +| --- | --- | +| `QA_CLI_BIN` | Path to the CLI under test. Default: `packages/cli/bin/run.js` (repo build). Point it at an installed `shopify` binary to QA a published version. | +| `QA_EXPECTED_VERSION` | Assert `shopify version` equals this (doc: "the version should match the nightly you just created"). | +| `QA_ORG_ID` / `E2E_ORG_ID` | Organization used by `app init`. | +| `QA_STORE_FQDN` / `E2E_STORE_FQDN` | Dev store passed to `app dev`. | +| `QA_PACKAGE_MANAGER` | Package manager for `app init` (default `pnpm`). | +| `QA_ISOLATE` | `1` = fresh XDG dirs (don't reuse the local CLI session). Set on CI. | +| `QA_WORK_DIR` | Scratch dir (default: fresh temp dir). | +| `QA_ONLY` | Comma-separated section filter (`prep`, `general`, `apps`, `theme`, `hydrogen`). | +| `QA_REPORT_DIR` | Where `qa-summary.md` / `qa-report.json` are written (default `./qa-report`). | +| `SHOPIFY_APP_AUTOMATION_TOKEN` | Headless auth for app-platform commands on CI. | + +## Auth + +- **Locally**: the runner reuses your ambient `shopify auth login` session. +- **CI**: `QA_ISOLATE=1` plus `SHOPIFY_APP_AUTOMATION_TOKEN` (Genghis e2e org). + Steps that turn out to require a browser user session are expected to fail + loudly in the report rather than hang: every interactive prompt has a + timeout that dumps the captured output. + +## Relationship to packages/e2e + +`packages/e2e` is the Playwright-based PR test suite. This package is +intentionally independent (per-team decision): it duplicates the few markers it +needs (e.g. the dev "Ready, watching for changes" string) instead of importing +from e2e, so the QA flow stays a faithful, standalone replica of the doc. diff --git a/packages/qa/package.json b/packages/qa/package.json new file mode 100644 index 00000000000..2c73f4fb8fd --- /dev/null +++ b/packages/qa/package.json @@ -0,0 +1,22 @@ +{ + "name": "@shopify/qa", + "version": "1.0.0", + "packageManager": "pnpm@10.11.1", + "private": true, + "type": "module", + "scripts": { + "qa": "tsx src/run.ts", + "type-check": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "18.19.130", + "execa": "^7.2.0", + "node-pty": "^1.0.0", + "strip-ansi": "^7.2.0", + "tsx": "^4.23.1", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=22.12.0" + } +} diff --git a/packages/qa/src/context.ts b/packages/qa/src/context.ts new file mode 100644 index 00000000000..f1db1e6aa32 --- /dev/null +++ b/packages/qa/src/context.ts @@ -0,0 +1,116 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {fileURLToPath} from 'url' +import type {PtyProcess} from './proc.js' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +export interface CtxState { + /** Directory of the app created by `app init`. */ + appDir?: string + /** Name given to the app created by `app init`. */ + appName?: string + /** GraphiQL server coordinates passed to `app dev`. */ + graphiql?: {port: number; key: string} + /** The long-running `app dev` pty process. */ + devProc?: PtyProcess + /** Directory of the hydrogen app. */ + hydrogenDir?: string + /** Name of the secondary config created by `app config link` (e.g. "staging"). */ + secondaryConfig?: string + /** Any pty processes to kill during teardown. */ + ptyProcs: PtyProcess[] +} + +export interface Ctx { + /** argv prefix used to invoke the CLI under test, e.g. ['node', '/path/to/run.js'] or ['shopify']. */ + cliInvoke: string[] + /** Human description of what is being tested (repo build vs installed package). */ + cliTarget: string + /** Root scratch directory for this run. */ + workDir: string + /** Environment passed to every CLI invocation. */ + env: {[key: string]: string} + orgId?: string + storeFqdn?: string + /** When set, the version step asserts `shopify version` equals this. */ + expectedVersion?: string + state: CtxState + log: (msg: string) => void +} + +function repoRoot(): string { + // packages/qa/src -> repo root + return path.resolve(__dirname, '..', '..', '..') +} + +/** + * Build the run context from environment variables. + * + * QA_CLI_BIN Path to the CLI entrypoint (JS file or executable). + * Defaults to the repo build at packages/cli/bin/run.js. + * QA_EXPECTED_VERSION Assert `shopify version` equals this value. + * QA_ORG_ID Organization id used for `app init` (falls back to E2E_ORG_ID). + * QA_STORE_FQDN Dev store passed to `app dev` (falls back to E2E_STORE_FQDN). + * QA_WORK_DIR Scratch dir (defaults to a fresh mkdtemp). + * QA_ISOLATE When "1", run with fresh XDG dirs so the ambient CLI session + * is not used. CI sets this; local runs default to ambient auth. + */ +export function createContext(log: (msg: string) => void = defaultLog): Ctx { + const binEnv = process.env.QA_CLI_BIN + const defaultBin = path.join(repoRoot(), 'packages', 'cli', 'bin', 'run.js') + const bin = binEnv ?? defaultBin + + let cliInvoke: string[] + if (bin.endsWith('.js') || bin.endsWith('.mjs') || bin.endsWith('.cjs')) { + cliInvoke = [process.execPath, bin] + } else { + cliInvoke = [bin] + } + const cliTarget = binEnv ? `installed CLI at ${bin}` : 'repo build (packages/cli/bin/run.js)' + + const workDir = + process.env.QA_WORK_DIR ?? fs.mkdtempSync(path.join(os.tmpdir(), 'shopify-qa-')) + fs.mkdirSync(workDir, {recursive: true}) + + const env: {[key: string]: string} = {} + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) env[key] = value + } + // Deterministic, machine-readable CLI output. + env.FORCE_COLOR = '0' + env.SHOPIFY_CLI_NO_ANALYTICS = '1' + env.SHOPIFY_FLAG_VERBOSE = '' + delete env.NODE_OPTIONS + // pnpm sets INIT_CWD to where `pnpm qa` was invoked; cli-kit's cwd() prefers + // it over the real process cwd, which would break every --path resolution. + // proc.ts re-points INIT_CWD/PWD at the effective cwd per invocation. + delete env.INIT_CWD + delete env.PWD + + if (process.env.QA_ISOLATE === '1') { + const authDir = path.join(workDir, 'xdg') + for (const name of ['XDG_DATA_HOME', 'XDG_CONFIG_HOME', 'XDG_STATE_HOME', 'XDG_CACHE_HOME']) { + const dir = path.join(authDir, name) + fs.mkdirSync(dir, {recursive: true}) + env[name] = dir + } + } + + return { + cliInvoke, + cliTarget, + workDir, + env, + orgId: process.env.QA_ORG_ID ?? process.env.E2E_ORG_ID, + storeFqdn: process.env.QA_STORE_FQDN ?? process.env.E2E_STORE_FQDN, + expectedVersion: process.env.QA_EXPECTED_VERSION, + state: {ptyProcs: []}, + log, + } +} + +function defaultLog(msg: string): void { + process.stdout.write(`[qa] ${msg}\n`) +} diff --git a/packages/qa/src/proc.ts b/packages/qa/src/proc.ts new file mode 100644 index 00000000000..46fa646b34e --- /dev/null +++ b/packages/qa/src/proc.ts @@ -0,0 +1,250 @@ +/** + * Process helpers for driving the CLI under test. + * + * `exec` runs non-interactive commands via execa. + * `spawnPty` runs interactive commands (app dev, config link, hydrogen init) + * in a pseudo-terminal so the CLI renders its interactive UI and accepts + * keypresses — no browser or test framework involved. + */ +import * as fs from 'fs' +import * as path from 'path' +import {createRequire} from 'module' +import stripAnsi from 'strip-ansi' +import {execa} from 'execa' +import * as pty from 'node-pty' +import type {Ctx} from './context.js' + +/** + * pnpm extracts node-pty prebuilds without the executable bit on macOS/Linux, + * which makes every pty.spawn fail with "posix_spawnp failed.". Fix it once. + */ +export function ensurePtySpawnHelperExecutable(): void { + if (process.platform === 'win32') return + try { + const require = createRequire(import.meta.url) + const packageDir = path.dirname(require.resolve('node-pty/package.json')) + const prebuilds = path.join(packageDir, 'prebuilds') + if (!fs.existsSync(prebuilds)) return + for (const platformDir of fs.readdirSync(prebuilds)) { + const helper = path.join(prebuilds, platformDir, 'spawn-helper') + if (fs.existsSync(helper)) fs.chmodSync(helper, 0o755) + } + } catch { + // Best effort — a real failure will surface as a spawn error with context. + } +} + +export interface ExecResult { + stdout: string + stderr: string + exitCode: number +} + +export interface ExecOptions { + cwd?: string + env?: {[key: string]: string} + timeoutMs?: number + /** Piped to stdin (used by `app function run`). */ + input?: string +} + +const DEFAULT_EXEC_TIMEOUT = 5 * 60_000 + +export async function exec(ctx: Ctx, args: string[], opts: ExecOptions = {}): Promise { + const [command, ...prefix] = ctx.cliInvoke + ctx.log(`$ shopify ${args.join(' ')}`) + const cwd = opts.cwd ?? ctx.workDir + const result = await execa(command as string, [...prefix, ...args], { + cwd, + env: {...ctx.env, ...opts.env, INIT_CWD: cwd, PWD: cwd}, + extendEnv: false, + timeout: opts.timeoutMs ?? DEFAULT_EXEC_TIMEOUT, + input: opts.input, + reject: false, + all: true, + }) + return { + stdout: stripAnsi(result.stdout ?? ''), + stderr: stripAnsi(result.stderr ?? ''), + exitCode: result.exitCode ?? 1, + } +} + +/** Throw with a readable error when a command exits non-zero. */ +export function expectSuccess(result: ExecResult, what: string): ExecResult { + if (result.exitCode !== 0) { + throw new Error(`${what} exited with ${result.exitCode}\n--- output ---\n${tail(result.stdout + result.stderr)}`) + } + return result +} + +export function tail(text: string, lines = 40): string { + const all = text.split('\n') + return all.slice(Math.max(0, all.length - lines)).join('\n') +} + +export interface WaitOptions { + timeoutMs?: number +} + +export interface PtyProcess { + /** Wait until the (ANSI-stripped) output matches. */ + waitFor(match: string | RegExp, opts?: WaitOptions): Promise + /** Send raw input, e.g. 'q', '\r', '\x03' (Ctrl+C). */ + write(data: string): void + /** Send a line followed by Enter. */ + sendLine(line: string): void + waitForExit(timeoutMs?: number): Promise + kill(): void + /** All output captured so far, ANSI-stripped. */ + output(): string + exited: boolean +} + +const DEFAULT_WAIT_TIMEOUT = 3 * 60_000 + +export function spawnPty( + ctx: Ctx, + args: string[], + opts: {cwd?: string; env?: {[key: string]: string}} = {}, +): PtyProcess { + const [command, ...prefix] = ctx.cliInvoke + ctx.log(`$ shopify ${args.join(' ')} (pty)`) + + const cwd = opts.cwd ?? ctx.workDir + const proc = pty.spawn(command as string, [...prefix, ...args], { + name: 'xterm-color', + cols: 120, + rows: 40, + cwd, + env: {...ctx.env, ...opts.env, INIT_CWD: cwd, PWD: cwd}, + }) + + let buffer = '' + let exited = false + let exitCode: number | undefined + const exitWaiters: ((code: number) => void)[] = [] + interface Waiter { + match: string | RegExp + resolve: () => void + from: number + } + const waiters: Waiter[] = [] + + const matches = (match: string | RegExp, from: number): boolean => { + const haystack = buffer.slice(from) + return typeof match === 'string' ? haystack.includes(match) : match.test(haystack) + } + + proc.onData((data) => { + buffer += stripAnsi(data) + for (let i = waiters.length - 1; i >= 0; i--) { + const waiter = waiters[i] as Waiter + if (matches(waiter.match, waiter.from)) { + waiters.splice(i, 1) + waiter.resolve() + } + } + }) + + proc.onExit(({exitCode: code}) => { + exited = true + exitCode = code + while (exitWaiters.length) exitWaiters.shift()?.(code) + }) + + const handle: PtyProcess = { + get exited() { + return exited + }, + output: () => buffer, + write: (data) => proc.write(data), + sendLine: (line) => proc.write(`${line}\r`), + kill: () => { + if (!exited) proc.kill() + }, + waitFor: (match, waitOpts = {}) => { + const from = 0 + if (matches(match, from)) return Promise.resolve() + if (exited) { + return Promise.reject( + new Error(`Process exited (code ${exitCode}) before matching ${String(match)}\n--- output ---\n${tail(buffer)}`), + ) + } + return new Promise((resolve, reject) => { + const timeoutMs = waitOpts.timeoutMs ?? DEFAULT_WAIT_TIMEOUT + const timer = setTimeout(() => { + const index = waiters.findIndex((waiter) => waiter.resolve === wrapped) + if (index >= 0) waiters.splice(index, 1) + reject(new Error(`Timed out (${timeoutMs}ms) waiting for ${String(match)}\n--- output ---\n${tail(buffer)}`)) + }, timeoutMs) + const wrapped = () => { + clearTimeout(timer) + resolve() + } + waiters.push({match, resolve: wrapped, from}) + exitWaiters.push(() => { + const index = waiters.findIndex((waiter) => waiter.resolve === wrapped) + if (index >= 0) { + waiters.splice(index, 1) + clearTimeout(timer) + reject( + new Error( + `Process exited (code ${exitCode}) before matching ${String(match)}\n--- output ---\n${tail(buffer)}`, + ), + ) + } + }) + }) + }, + waitForExit: (timeoutMs = DEFAULT_WAIT_TIMEOUT) => { + if (exited) return Promise.resolve(exitCode ?? 0) + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`Timed out (${timeoutMs}ms) waiting for exit`)), timeoutMs) + exitWaiters.push((code) => { + clearTimeout(timer) + resolve(code) + }) + }) + }, + } + + ctx.state.ptyProcs.push(handle) + return handle +} + +/** Simple HTTP fetch helper with timeout; avoids extra deps (Node >=22 has fetch). */ +export async function httpRequest( + url: string, + init: {method?: string; headers?: {[key: string]: string}; body?: string; timeoutMs?: number} = {}, +): Promise<{status: number; body: string}> { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), init.timeoutMs ?? 15_000) + try { + const response = await fetch(url, { + method: init.method ?? 'GET', + headers: init.headers, + body: init.body, + signal: controller.signal, + }) + const body = await response.text() + return {status: response.status, body} + } finally { + clearTimeout(timer) + } +} + +export async function retry(fn: () => Promise, attempts: number, delayMs: number): Promise { + let lastError: unknown + for (let attempt = 0; attempt < attempts; attempt++) { + try { + // eslint-disable-next-line no-await-in-loop + return await fn() + } catch (error) { + lastError = error + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, delayMs)) + } + } + throw lastError +} diff --git a/packages/qa/src/report.ts b/packages/qa/src/report.ts new file mode 100644 index 00000000000..b78dcbbf517 --- /dev/null +++ b/packages/qa/src/report.ts @@ -0,0 +1,104 @@ +import * as fs from 'fs' +import type {QAReport, StepResult, StepStatus} from './types.js' + +const STATUS_ICON: {[key in StepStatus]: string} = { + pass: '✅', + fail: '❌', + skipped: '⏭️', + blocked: '🚫', +} + +function statusLabel(step: StepResult): string { + if (step.status === 'skipped' && step.kind === 'manual') return '⏭️ manual' + if (step.status === 'skipped' && step.kind === 'delegated') return '➡️ delegated' + return `${STATUS_ICON[step.status]} ${step.status}` +} + +function duration(ms: number): string { + if (ms === 0) return '' + if (ms < 1000) return `${ms}ms` + const seconds = Math.round(ms / 1000) + if (seconds < 60) return `${seconds}s` + return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, '0')}s` +} + +export function renderMarkdown(report: QAReport): string { + const lines: string[] = [] + lines.push('# CLI Pre-release QA flow — automated run') + lines.push('') + lines.push(`- **CLI under test:** ${report.cliTarget}`) + lines.push(`- **Version:** ${report.cliVersion || 'unknown'}`) + lines.push(`- **OS:** ${report.os}`) + lines.push(`- **Started:** ${report.startedAt} — **finished:** ${report.finishedAt}`) + lines.push('') + + const all = report.sections.flatMap((section) => section.steps) + const counts = { + pass: all.filter((step) => step.status === 'pass').length, + fail: all.filter((step) => step.status === 'fail').length, + blocked: all.filter((step) => step.status === 'blocked').length, + manual: all.filter((step) => step.kind === 'manual').length, + delegated: all.filter((step) => step.kind === 'delegated').length, + } + lines.push( + `**${counts.pass} passed** · ${counts.fail} failed · ${counts.blocked} blocked · ` + + `${counts.manual} left to manual QA · ${counts.delegated} delegated`, + ) + lines.push('') + + for (const section of report.sections) { + lines.push(`## ${section.title}`) + lines.push('') + lines.push('| QA doc step | Result | Time | Notes |') + lines.push('|---|---|---|---|') + for (const step of section.steps) { + const note = step.status === 'fail' ? 'see error details below' : step.note ?? '' + lines.push( + `| ${escapeCell(step.doc)} | ${statusLabel(step)} | ${duration(step.durationMs)} | ${escapeCell(note)} |`, + ) + } + lines.push('') + const failures = section.steps.filter((step) => step.status === 'fail') + for (const failure of failures) { + lines.push(`
❌ ${failure.id} — error details`) + lines.push('') + lines.push('```') + lines.push(failure.error ?? 'no error captured') + lines.push('```') + lines.push('
') + lines.push('') + } + } + + const manualSteps = all.filter((step) => step.kind === 'manual') + if (manualSteps.length > 0) { + lines.push('## Remaining manual checklist') + lines.push('') + lines.push('These QA-doc items still need a human (browser/visual checks):') + lines.push('') + for (const step of manualSteps) { + lines.push(`- [ ] ${step.doc}${step.note ? ` _(${step.note})_` : ''}`) + } + lines.push('') + } + + return lines.join('\n') +} + +function escapeCell(text: string): string { + return text.replace(/\|/g, '\\|').replace(/\n/g, ' ') +} + +export function writeReports(report: QAReport, outDir: string): {markdownPath: string; jsonPath: string} { + fs.mkdirSync(outDir, {recursive: true}) + const markdown = renderMarkdown(report) + const markdownPath = `${outDir}/qa-summary.md` + const jsonPath = `${outDir}/qa-report.json` + fs.writeFileSync(markdownPath, markdown) + fs.writeFileSync(jsonPath, JSON.stringify(report, null, 2)) + + const stepSummary = process.env.GITHUB_STEP_SUMMARY + if (stepSummary) fs.appendFileSync(stepSummary, `${markdown}\n`) + + return {markdownPath, jsonPath} +} diff --git a/packages/qa/src/run.ts b/packages/qa/src/run.ts new file mode 100644 index 00000000000..e04c6401994 --- /dev/null +++ b/packages/qa/src/run.ts @@ -0,0 +1,102 @@ +/** + * CLI Pre-release QA flow runner. + * + * Executes the QA doc steps that need no human interaction directly against + * the CLI under test (repo build by default, or QA_CLI_BIN), records one + * result per QA-doc checklist item and emits a summary that mirrors the doc. + * + * Usage: pnpm --filter @shopify/qa qa (all sections) + * QA_ONLY=hydrogen pnpm --filter @shopify/qa qa + */ +import * as path from 'path' +import {createContext} from './context.js' +import {ensurePtySpawnHelperExecutable, exec} from './proc.js' +import {extractVersion} from './steps/prep.js' +import {writeReports} from './report.js' +import {appsSection} from './steps/apps.js' +import {crossOSSection, hydrogenSection} from './steps/hydrogen.js' +import {generalSection, prepSection} from './steps/prep.js' +import {themeSection} from './steps/theme.js' +import type {QAReport, SectionDef, SectionResult, StepResult} from './types.js' + +const ALL_SECTIONS: SectionDef[] = [prepSection, generalSection, appsSection, themeSection, hydrogenSection, crossOSSection] + +function selectedSections(): SectionDef[] { + const only = process.env.QA_ONLY + if (!only) return ALL_SECTIONS + const wanted = only.split(',').map((part) => part.trim().toLowerCase()) + return ALL_SECTIONS.filter((section) => wanted.some((want) => section.title.toLowerCase().includes(want))) +} + +async function main(): Promise { + ensurePtySpawnHelperExecutable() + const ctx = createContext() + const startedAt = new Date().toISOString() + ctx.log(`CLI under test: ${ctx.cliTarget}`) + ctx.log(`work dir: ${ctx.workDir}`) + + const sections: SectionResult[] = [] + for (const section of selectedSections()) { + ctx.log(`── ${section.title} ──`) + const results: StepResult[] = [] + let chainBroken = false + + for (const step of section.steps) { + const base = {id: step.id, doc: step.doc, kind: step.kind} + if (step.kind !== 'auto' || !step.run) { + results.push({...base, status: 'skipped', durationMs: 0, note: step.reason}) + continue + } + if (chainBroken && !step.independent) { + results.push({...base, status: 'blocked', durationMs: 0, note: 'blocked by an earlier failure'}) + continue + } + const start = Date.now() + try { + const note = await step.run(ctx) + results.push({...base, status: 'pass', durationMs: Date.now() - start, note: note ?? undefined}) + ctx.log(`✅ ${step.id}`) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + results.push({...base, status: 'fail', durationMs: Date.now() - start, error: message}) + ctx.log(`❌ ${step.id}: ${message.split('\n')[0]}`) + if (!step.independent) chainBroken = true + } + } + sections.push({title: section.title, steps: results}) + } + + // Teardown: make sure no interactive process outlives the run. + for (const proc of ctx.state.ptyProcs) { + if (!proc.exited) proc.kill() + } + + let cliVersion = '' + try { + const result = await exec(ctx, ['version'], {timeoutMs: 60_000}) + cliVersion = extractVersion(result.stdout) + } catch { + // version already reported in the prep section; ignore here. + } + + const report: QAReport = { + startedAt, + finishedAt: new Date().toISOString(), + cliVersion, + cliTarget: ctx.cliTarget, + os: `${process.platform} ${process.arch}`, + sections, + } + + const outDir = process.env.QA_REPORT_DIR ?? path.join(process.cwd(), 'qa-report') + const {markdownPath, jsonPath} = writeReports(report, outDir) + ctx.log(`report: ${markdownPath} / ${jsonPath}`) + + const failed = sections.some((section) => section.steps.some((step) => step.status === 'fail' || step.status === 'blocked')) + process.exit(failed ? 1 : 0) +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/packages/qa/src/steps/apps.ts b/packages/qa/src/steps/apps.ts new file mode 100644 index 00000000000..1e4fb090e31 --- /dev/null +++ b/packages/qa/src/steps/apps.ts @@ -0,0 +1,488 @@ +/** + * "Apps" section of the CLI Pre-release QA flow. + * + * Each step mirrors one checklist item of the QA doc, in order. Items that + * require a human (browser/visual checks) are declared kind: 'manual' and + * reported as skipped — never silently dropped. + */ +import * as fs from 'fs' +import * as net from 'net' +import * as path from 'path' +import {exec, expectSuccess, httpRequest, retry, spawnPty, tail} from '../proc.js' +import type {Ctx} from '../context.js' +import type {SectionDef} from '../types.js' + +const READY_MESSAGE = 'Ready, watching for changes in your app' +const UPDATED_MESSAGE = 'Updated dev preview' + +const FUNCTION_INPUT = + '{"cart":{"lines":[{"id":"gid://shopify/CartLine/0","cost":{"subtotalAmount":{"amount":"10.0"}}}]},"discount":{"discountClasses":["PRODUCT","ORDER","SHIPPING"]}}' + +const EXTENSIONS = { + adminAction: 'qa-admin-action', + theme: 'qa-theme-ext', + discount: 'qa-discount', + flowAction: 'qa-flow-action', + choice: 'qa-admin-block', + midDev: 'qa-admin-link', +} + +function appDirOrThrow(ctx: Ctx): string { + const dir = ctx.state.appDir + if (!dir) throw new Error('No app directory (app init did not succeed)') + return dir +} + +function devProcOrThrow(ctx: Ctx) { + const proc = ctx.state.devProc + if (!proc || proc.exited) throw new Error('`app dev` is not running') + return proc +} + +async function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer() + server.listen(0, '127.0.0.1', () => { + const address = server.address() + if (address && typeof address === 'object') { + const port = address.port + server.close(() => resolve(port)) + } else { + server.close(() => reject(new Error('Could not allocate a free port'))) + } + }) + server.on('error', reject) + }) +} + +async function generateExtension(ctx: Ctx, template: string, name: string, flavor?: string): Promise { + const appDir = appDirOrThrow(ctx) + const args = ['app', 'generate', 'extension', `--template=${template}`, `--name=${name}`] + if (flavor) args.push(`--flavor=${flavor}`) + expectSuccess( + await exec(ctx, args, {cwd: appDir, timeoutMs: 5 * 60_000}), + `app generate extension --template=${template}`, + ) + const extDir = path.join(appDir, 'extensions', name) + if (!fs.existsSync(extDir)) throw new Error(`Extension directory not created: ${extDir}`) + return `created extensions/${name}` +} + +/** Edit a file and return a note; throws when the expected content is missing. */ +function editFile(filePath: string, mutate: (content: string) => string): void { + const content = fs.readFileSync(filePath, 'utf8') + fs.writeFileSync(filePath, mutate(content)) +} + +function findFile(dir: string, candidates: string[]): string { + for (const candidate of candidates) { + const filePath = path.join(dir, candidate) + if (fs.existsSync(filePath)) return filePath + } + throw new Error(`None of ${candidates.join(', ')} found under ${dir}`) +} + +export const appsSection: SectionDef = { + title: 'Apps', + steps: [ + { + id: 'apps.init', + doc: 'Create a new dev platform app: `shopify app init --template reactRouter`', + kind: 'auto', + run: async (ctx) => { + const name = `qa-ci-${new Date().toISOString().slice(0, 10)}-${process.pid}` + const args = [ + 'app', + 'init', + '--template', + 'reactRouter', + // Flags below only replace interactive prompts; the doc answers them by hand. + '--name', + name, + '--flavor', + 'javascript', + '--package-manager', + process.env.QA_PACKAGE_MANAGER ?? 'pnpm', + '--path', + ctx.workDir, + ] + if (ctx.orgId) args.push('--organization-id', ctx.orgId) + const result = expectSuccess(await exec(ctx, args, {timeoutMs: 10 * 60_000}), 'app init') + + const output = `${result.stdout}\n${result.stderr}` + const match = output.match(/([\w-]+) is ready for you to build!/) + let appDir: string | undefined + if (match?.[1]) { + appDir = path.join(ctx.workDir, match[1]) + } else { + const entries = fs.readdirSync(ctx.workDir, {withFileTypes: true}) + const appEntry = entries.find( + (entry) => entry.isDirectory() && fs.existsSync(path.join(ctx.workDir, entry.name, 'shopify.app.toml')), + ) + if (appEntry) appDir = path.join(ctx.workDir, appEntry.name) + } + if (!appDir || !fs.existsSync(appDir)) { + throw new Error(`Could not locate created app in ${ctx.workDir}\n${tail(output)}`) + } + ctx.state.appDir = appDir + ctx.state.appName = name + return `created ${path.basename(appDir)}` + }, + }, + { + id: 'apps.ext.admin-action', + doc: '`shopify app generate extension --template=admin_action`', + kind: 'auto', + independent: true, + run: (ctx) => generateExtension(ctx, 'admin_action', EXTENSIONS.adminAction), + }, + { + id: 'apps.ext.theme', + doc: '`shopify app generate extension --template=theme_app_extension`', + kind: 'auto', + independent: true, + run: (ctx) => generateExtension(ctx, 'theme_app_extension', EXTENSIONS.theme), + }, + { + id: 'apps.ext.discount', + doc: '`shopify app generate extension --template=discount --flavor=typescript`', + kind: 'auto', + independent: true, + run: (ctx) => generateExtension(ctx, 'discount', EXTENSIONS.discount, 'typescript'), + }, + { + id: 'apps.ext.flow-action', + doc: '`shopify app generate extension --template=flow_action`', + kind: 'auto', + independent: true, + run: (ctx) => generateExtension(ctx, 'flow_action', EXTENSIONS.flowAction), + }, + { + id: 'apps.ext.choice', + doc: '`shopify app generate extension` (a random extension of your choice — admin_block)', + kind: 'auto', + independent: true, + run: (ctx) => generateExtension(ctx, 'admin_block', EXTENSIONS.choice), + }, + { + id: 'apps.dev.start', + doc: 'Run `shopify app dev`', + kind: 'auto', + run: async (ctx) => { + const appDir = appDirOrThrow(ctx) + const port = await freePort() + const key = 'qa-graphiql-key' + ctx.state.graphiql = {port, key} + const args = ['app', 'dev', '--graphiql-port', String(port), '--graphiql-key', key] + if (ctx.storeFqdn) args.push('--store', ctx.storeFqdn) + const proc = spawnPty(ctx, args, {cwd: appDir}) + ctx.state.devProc = proc + await proc.waitFor(READY_MESSAGE, {timeoutMs: 5 * 60_000}) + return 'dev is ready and watching for changes' + }, + }, + { + id: 'apps.dev.console.open', + doc: 'Dev Console: open the shop and see the dev console', + kind: 'manual', + reason: 'browser check', + }, + { + id: 'apps.dev.console.connected', + doc: 'The app you ran dev on should be the first dev preview and show as connected (green icon)', + kind: 'manual', + reason: 'browser check', + }, + { + id: 'apps.dev.admin-action.product', + doc: 'If your shop has no products, create one now (needed to test the extensions)', + kind: 'manual', + reason: 'store admin browser step — the QA store is expected to already have a product', + }, + { + id: 'apps.dev.admin-action.modal', + doc: 'From dev-console, open the admin-action link: product admin page opens the action modal', + kind: 'manual', + reason: 'browser check', + }, + { + id: 'apps.dev.admin-action.hot-reload', + doc: 'Change the message inside `src/ActionExtension.js` — you should see it hot reload', + kind: 'auto', + run: async (ctx) => { + const proc = devProcOrThrow(ctx) + const extDir = path.join(appDirOrThrow(ctx), 'extensions', EXTENSIONS.adminAction, 'src') + const file = findFile(extDir, ['ActionExtension.jsx', 'ActionExtension.js', 'ActionExtension.tsx']) + editFile(file, (content) => content.replace(/current product/i, 'QA hot reload message')) + await proc.waitFor(UPDATED_MESSAGE, {timeoutMs: 3 * 60_000}) + return `edited ${path.basename(file)}; dev reported "${UPDATED_MESSAGE}" (visual confirmation stays manual)` + }, + }, + { + id: 'apps.dev.extension-mid-dev', + doc: 'Add another extension and see it show up in the dev console', + kind: 'auto', + run: async (ctx) => { + const proc = devProcOrThrow(ctx) + await generateExtension(ctx, 'admin_link', EXTENSIONS.midDev) + await proc.waitFor(/Extension created|Updated dev preview/, {timeoutMs: 3 * 60_000}) + return 'new extension picked up by the running dev session (dev-console visual stays manual)' + }, + }, + { + id: 'apps.dev.graphiql', + doc: 'Press `g` to open GraphiQL; test that it works with `query { shop { name } }`', + kind: 'auto', + run: async (ctx) => { + devProcOrThrow(ctx) + const graphiql = ctx.state.graphiql + if (!graphiql) throw new Error('GraphiQL port/key were not configured') + const base = `http://localhost:${graphiql.port}` + // Equivalent of pressing `g`: the key only opens this URL in a browser. + await retry(async () => { + const ping = await httpRequest(`${base}/graphiql/ping`) + if (ping.status !== 200) throw new Error(`GraphiQL ping returned ${ping.status}`) + }, 10, 3000) + const response = await httpRequest(`${base}/graphiql/graphql.json?key=${graphiql.key}&api_version=unstable`, { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({query: 'query { shop { name } }'}), + timeoutMs: 30_000, + }) + if (response.status !== 200) { + throw new Error(`GraphiQL query returned HTTP ${response.status}: ${tail(response.body, 10)}`) + } + const parsed = JSON.parse(response.body) as {data?: {shop?: {name?: string}}; errors?: unknown} + if (!parsed.data?.shop?.name) { + throw new Error(`GraphiQL query did not return shop name: ${tail(response.body, 10)}`) + } + return `GraphiQL answered: shop name "${parsed.data.shop.name}"` + }, + }, + { + id: 'apps.dev.execute', + doc: "Test the same query via command: `shopify app execute --query 'query { shop { name } }'`", + kind: 'auto', + run: async (ctx) => { + const result = expectSuccess( + await exec(ctx, ['app', 'execute', '--query', 'query { shop { name } }'], { + cwd: appDirOrThrow(ctx), + timeoutMs: 2 * 60_000, + }), + 'app execute', + ) + const output = result.stdout + result.stderr + if (!/"name"\s*:/.test(output)) throw new Error(`app execute output has no shop name:\n${tail(output, 15)}`) + return 'app execute returned shop data' + }, + }, + { + id: 'apps.dev.theme-ext.uploaded', + doc: 'Theme extension files should have been uploaded by now (if not, wait for it)', + kind: 'auto', + run: async (ctx) => { + const proc = devProcOrThrow(ctx) + await proc.waitFor(/theme app extension|host theme|9292/i, {timeoutMs: 4 * 60_000}) + return 'dev output shows the theme app extension is being served' + }, + }, + { + id: 'apps.dev.theme-ext.setup-link', + doc: 'Click on the "Setup your theme app extension in the host theme" link from the CLI output', + kind: 'manual', + reason: 'browser check (theme editor)', + }, + { + id: 'apps.dev.theme-ext.add-section', + doc: '"Add section", choose your app and "Save"', + kind: 'manual', + reason: 'browser check (theme editor)', + }, + { + id: 'apps.dev.theme-ext.preview', + doc: 'Open the theme app extension local preview (e.g. http://127.0.0.1:9292)', + kind: 'auto', + run: async (ctx) => { + const proc = devProcOrThrow(ctx) + const match = proc.output().match(/https?:\/\/(?:127\.0\.0\.1|localhost):(\d{4,5})/g) + const themeUrl = + match?.find((url) => url.includes('9292')) ?? 'http://127.0.0.1:9292' + const response = await retry(() => httpRequest(themeUrl, {timeoutMs: 20_000}), 6, 5000) + return `theme preview at ${themeUrl} responded with HTTP ${response.status} (visual confirmation stays manual)` + }, + }, + { + id: 'apps.dev.theme-ext.hot-reload', + doc: "Add `Hello` inside the span in the theme's `blocks/star_rating.liquid` — it should hot reload", + kind: 'auto', + run: async (ctx) => { + const proc = devProcOrThrow(ctx) + const blocksDir = path.join(appDirOrThrow(ctx), 'extensions', EXTENSIONS.theme, 'blocks') + const file = findFile(blocksDir, ['star_rating.liquid']) + editFile(file, (content) => { + if (!content.includes(' found in ${file}`) + return content.replace(/(]*>)/, '$1Hello ') + }) + await proc.waitFor(/star_rating\.liquid|Updated dev preview|hot reload/i, {timeoutMs: 3 * 60_000}) + return 'edited star_rating.liquid; dev picked up the change (visual confirmation stays manual)' + }, + }, + { + id: 'apps.dev.quit', + doc: 'Press `q` to stop dev', + kind: 'auto', + run: async (ctx) => { + const proc = devProcOrThrow(ctx) + proc.write('q') + const code = await proc.waitForExit(60_000) + ctx.state.devProc = undefined + if (code !== 0) throw new Error(`app dev exited with code ${code} after pressing q\n${tail(proc.output())}`) + return 'dev stopped cleanly' + }, + }, + { + id: 'apps.dev.console.disconnected', + doc: 'See that the dev console still reports a dev preview, but shown as disconnected', + kind: 'manual', + reason: 'browser check', + }, + { + id: 'apps.dev.clean', + doc: 'Run `shopify app dev clean` to end the preview; the dev preview is now hidden', + kind: 'auto', + run: async (ctx) => { + expectSuccess( + await exec(ctx, ['app', 'dev', 'clean'], {cwd: appDirOrThrow(ctx), timeoutMs: 2 * 60_000}), + 'app dev clean', + ) + return 'dev preview cleaned (dev-console visual stays manual)' + }, + }, + { + id: 'apps.function.build', + doc: 'Move to the function directory and run `shopify app function build`', + kind: 'auto', + run: async (ctx) => { + const functionDir = path.join(appDirOrThrow(ctx), 'extensions', EXTENSIONS.discount) + expectSuccess( + await exec(ctx, ['app', 'function', 'build'], {cwd: functionDir, timeoutMs: 5 * 60_000}), + 'app function build', + ) + return 'function built' + }, + }, + { + id: 'apps.function.run', + doc: "echo '' | shopify app function run — should complete without an error", + kind: 'auto', + run: async (ctx) => { + const functionDir = path.join(appDirOrThrow(ctx), 'extensions', EXTENSIONS.discount) + const result = expectSuccess( + await exec(ctx, ['app', 'function', 'run'], { + cwd: functionDir, + input: FUNCTION_INPUT, + timeoutMs: 3 * 60_000, + }), + 'app function run', + ) + return `function ran (output tail: ${tail(result.stdout, 3).trim().slice(0, 120)})` + }, + }, + { + id: 'apps.deploy.v1', + doc: 'Run `shopify app deploy --version v1` and release the version — should complete without error', + kind: 'auto', + run: async (ctx) => { + // --allow-updates replaces the interactive confirmation (the flag the CLI + // documents as the CI/CD equivalent). Deploy releases by default. + expectSuccess( + await exec(ctx, ['app', 'deploy', '--version', 'v1', '--allow-updates'], { + cwd: appDirOrThrow(ctx), + timeoutMs: 10 * 60_000, + }), + 'app deploy --version v1', + ) + return 'v1 deployed and released' + }, + }, + { + id: 'apps.versions.list', + doc: 'Run `shopify app versions list` and validate that your version is there', + kind: 'auto', + run: async (ctx) => { + const result = expectSuccess( + await exec(ctx, ['app', 'versions', 'list'], {cwd: appDirOrThrow(ctx), timeoutMs: 2 * 60_000}), + 'app versions list', + ) + const output = result.stdout + result.stderr + if (!output.includes('v1')) throw new Error(`versions list does not contain v1:\n${tail(output, 20)}`) + return 'v1 present in versions list' + }, + }, + { + id: 'apps.config.link', + doc: 'Run `shopify app config link` and create a new app', + kind: 'auto', + run: async (ctx) => { + const appDir = appDirOrThrow(ctx) + const configName = 'staging' + const newAppName = `${ctx.state.appName ?? 'qa-ci-app'}-staging` + const proc = spawnPty(ctx, ['app', 'config', 'link'], {cwd: appDir}) + + // Prompt flow: configuration file name → (org) → app selection + // ("Create this project as a new app" is the first choice) → app name. + await proc.waitFor(/configuration file name/i, {timeoutMs: 2 * 60_000}) + proc.sendLine(configName) + + await proc.waitFor(/create this project as a new app|which existing app|organization/i, { + timeoutMs: 3 * 60_000, + }) + if (/organization/i.test(proc.output()) && !/create this project as a new app/i.test(proc.output())) { + // Organization select — accept the highlighted option. + proc.write('\r') + await proc.waitFor(/create this project as a new app|which existing app/i, {timeoutMs: 2 * 60_000}) + } + // "Create this project as a new app on Shopify?" (yes is default) or app select with create-new first. + proc.write('\r') + + await proc.waitFor(/app name/i, {timeoutMs: 2 * 60_000}) + proc.sendLine(newAppName) + + const code = await proc.waitForExit(3 * 60_000) + if (code !== 0) throw new Error(`app config link exited with ${code}\n${tail(proc.output())}`) + const tomlPath = path.join(appDir, `shopify.app.${configName}.toml`) + if (!fs.existsSync(tomlPath)) { + throw new Error(`Expected ${tomlPath} to exist after config link\n${tail(proc.output())}`) + } + ctx.state.secondaryConfig = configName + return `created shopify.app.${configName}.toml linked to new app "${newAppName}"` + }, + }, + { + id: 'apps.deploy.second', + doc: 'Run `shopify app deploy` again and validate that it is deployed to the new app', + kind: 'auto', + run: async (ctx) => { + const appDir = appDirOrThrow(ctx) + const config = ctx.state.secondaryConfig + if (!config) throw new Error('No secondary config from `app config link`') + expectSuccess( + await exec(ctx, ['app', 'deploy', '--config', config, '--allow-updates'], { + cwd: appDir, + timeoutMs: 10 * 60_000, + }), + `app deploy --config ${config}`, + ) + const versions = expectSuccess( + await exec(ctx, ['app', 'versions', 'list', '--config', config], {cwd: appDir, timeoutMs: 2 * 60_000}), + 'app versions list --config', + ) + const output = versions.stdout + versions.stderr + if (!/1 app version|app versions?/i.test(output)) { + throw new Error(`Could not validate deploy on the new app:\n${tail(output, 20)}`) + } + return 'second deploy landed on the newly created app' + }, + }, + ], +} diff --git a/packages/qa/src/steps/hydrogen.ts b/packages/qa/src/steps/hydrogen.ts new file mode 100644 index 00000000000..67a57ebc9c1 --- /dev/null +++ b/packages/qa/src/steps/hydrogen.ts @@ -0,0 +1,124 @@ +import * as fs from 'fs' +import * as path from 'path' +import {exec, expectSuccess, httpRequest, retry, spawnPty, tail} from '../proc.js' +import type {PtyProcess} from '../proc.js' +import type {SectionDef} from '../types.js' + +/** + * Press Enter on every prompt until `until` matches the output (the QA doc says + * "choose default answers to all questions"). A prompt is considered pending + * when new output containing a question indicator appeared and stayed quiet. + */ +async function answerDefaultsUntil(proc: PtyProcess, until: RegExp, timeoutMs: number): Promise { + const startedAt = Date.now() + let lastLength = 0 + let quietSince = Date.now() + let lastAnsweredLength = -1 + + while (Date.now() - startedAt < timeoutMs) { + if (until.test(proc.output())) return + if (proc.exited) { + if (until.test(proc.output())) return + throw new Error(`hydrogen init exited before completing\n${tail(proc.output())}`) + } + + const length = proc.output().length + if (length !== lastLength) { + lastLength = length + quietSince = Date.now() + } else if (Date.now() - quietSince > 2000 && length !== lastAnsweredLength) { + // Output has been quiet for 2s — if a prompt is on screen, accept the default. + const recent = proc.output().slice(-2000) + if (/\?|❯|›|Yes|No|\(y\/n\)/i.test(recent)) { + proc.write('\r') + lastAnsweredLength = length + } + } + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 250)) + } + throw new Error(`Timed out driving hydrogen init prompts\n${tail(proc.output())}`) +} + +function hydrogenDirOrThrow(ctx: {state: {hydrogenDir?: string}}): string { + const dir = ctx.state.hydrogenDir + if (!dir) throw new Error('No hydrogen project (hydrogen init did not succeed)') + return dir +} + +export const hydrogenSection: SectionDef = { + title: 'Hydrogen', + steps: [ + { + id: 'hydrogen.init', + doc: 'Create a Hydrogen app: `shopify hydrogen init` and choose default answers to all questions', + kind: 'auto', + run: async (ctx) => { + const dir = path.join(ctx.workDir, 'qa-hydrogen') + const proc = spawnPty(ctx, ['hydrogen', 'init', '--path', dir], {cwd: ctx.workDir}) + // "Storefront setup complete!" is the definitive end-of-init marker; the + // deps install between the last prompt and this box can take many minutes. + await answerDefaultsUntil(proc, /Storefront setup complete!|Next steps/, 20 * 60_000) + await proc.waitForExit(5 * 60_000) + if (!fs.existsSync(dir)) throw new Error(`Hydrogen project not created at ${dir}\n${tail(proc.output())}`) + ctx.state.hydrogenDir = dir + return 'hydrogen project scaffolded with default answers' + }, + }, + { + id: 'hydrogen.build', + doc: 'cd into the hydrogen project and run `shopify hydrogen build`', + kind: 'auto', + run: async (ctx) => { + expectSuccess( + await exec(ctx, ['hydrogen', 'build'], {cwd: hydrogenDirOrThrow(ctx), timeoutMs: 10 * 60_000}), + 'hydrogen build', + ) + return 'hydrogen build succeeded' + }, + }, + { + id: 'hydrogen.dev', + doc: 'Run `shopify hydrogen dev`; follow the "View Hydrogen app" link and confirm you can see a storefront', + kind: 'auto', + run: async (ctx) => { + const proc = spawnPty(ctx, ['hydrogen', 'dev'], {cwd: hydrogenDirOrThrow(ctx)}) + await proc.waitFor(/View [\w ]*app:|localhost:\d+/i, {timeoutMs: 5 * 60_000}) + const announced = + proc.output().match(/https?:\/\/(?:127\.0\.0\.1|localhost):\d{4,5}\/?/)?.[0] ?? 'http://localhost:3000' + // Node's fetch resolves `localhost` to ::1 first on macOS while the dev + // server listens on IPv4 — probe 127.0.0.1 explicitly. + const url = announced.replace('localhost', '127.0.0.1') + const response = await retry(async () => { + try { + const res = await httpRequest(url, {timeoutMs: 20_000}) + if (res.status >= 500) throw new Error(`storefront returned HTTP ${res.status}`) + return res + } catch (error) { + const message = error instanceof Error ? (error.cause instanceof Error ? error.cause.message : error.message) : String(error) + throw new Error(`probe of ${url} failed: ${message}`) + } + }, 10, 5000) + if (!/ proc.kill()) + return `storefront responded at ${url} (HTTP ${response.status}); stopped with CTRL+C` + }, + }, + ], +} + +export const crossOSSection: SectionDef = { + title: 'Linux & Windows', + steps: [ + { + id: 'cross-os.matrix', + doc: 'Run the same QA flow above for linux and windows platforms', + kind: 'manual', + reason: 'covered by the workflow OS matrix — rollout starts with macOS, linux/windows follow', + }, + ], +} diff --git a/packages/qa/src/steps/prep.ts b/packages/qa/src/steps/prep.ts new file mode 100644 index 00000000000..de63637422c --- /dev/null +++ b/packages/qa/src/steps/prep.ts @@ -0,0 +1,51 @@ +import {exec, expectSuccess} from '../proc.js' +import type {SectionDef} from '../types.js' + +/** Pull the semver out of `shopify version` output (it may be followed by an upgrade-notice box). */ +export function extractVersion(stdout: string): string { + const match = stdout.match(/^\s*(\d+\.\d+\.\d+(?:-[\w.]+)?)\s*$/m) + return match?.[1] ?? stdout.trim().split('\n')[0]?.trim() ?? '' +} + +export const prepSection: SectionDef = { + title: 'Preparation', + steps: [ + { + id: 'prep.release', + doc: 'Make a nightly release before starting the QA flow', + kind: 'manual', + reason: 'CI provides the artifact under test (repo build on CI runs, published nightly on manual runs)', + }, + { + id: 'prep.version', + doc: 'Run `shopify version`, the version should match the nightly you just created', + kind: 'auto', + run: async (ctx) => { + const result = expectSuccess(await exec(ctx, ['version'], {timeoutMs: 60_000}), 'shopify version') + const observed = extractVersion(result.stdout) + if (ctx.expectedVersion && observed !== ctx.expectedVersion) { + throw new Error(`Expected version ${ctx.expectedVersion}, got ${observed}`) + } + return `version ${observed}${ctx.expectedVersion ? ' (matches expected)' : ''}` + }, + }, + { + id: 'prep.review-changes', + doc: 'Review the changes in the new release and take detours in the QA steps to exercise them', + kind: 'manual', + reason: 'human judgement — review the Version Packages PR', + }, + ], +} + +export const generalSection: SectionDef = { + title: 'General', + steps: [ + { + id: 'general.lockdown', + doc: 'Turn off Shopify Lockdown (blocks adhoc-signed binaries via AMFI, used by functions)', + kind: 'manual', + reason: 'not applicable on CI runners (no Shopify Lockdown installed); required only on Shopify-issued Macs', + }, + ], +} diff --git a/packages/qa/src/steps/theme.ts b/packages/qa/src/steps/theme.ts new file mode 100644 index 00000000000..17829722958 --- /dev/null +++ b/packages/qa/src/steps/theme.ts @@ -0,0 +1,16 @@ +import type {SectionDef} from '../types.js' + +const REASON = + 'QA doc: "No need to run these anymore, themes have their own checklist" — owned by the themes team (#online-store-developer-platforms)' + +export const themeSection: SectionDef = { + title: 'Theme', + steps: [ + {id: 'theme.init', doc: 'Create a theme: `shopify theme init`', kind: 'delegated', reason: REASON}, + {id: 'theme.check', doc: 'Run `shopify theme check --fail-level crash`', kind: 'delegated', reason: REASON}, + {id: 'theme.package', doc: 'Run `shopify theme package`', kind: 'delegated', reason: REASON}, + {id: 'theme.dev', doc: 'Run `shopify theme dev --store ` (+ hot reload checks)', kind: 'delegated', reason: REASON}, + {id: 'theme.push', doc: 'Run `shopify theme push`', kind: 'delegated', reason: REASON}, + {id: 'theme.list', doc: 'Run `shopify theme list`', kind: 'delegated', reason: REASON}, + ], +} diff --git a/packages/qa/src/types.ts b/packages/qa/src/types.ts new file mode 100644 index 00000000000..c4804585a6b --- /dev/null +++ b/packages/qa/src/types.ts @@ -0,0 +1,59 @@ +import type {Ctx} from './context.js' + +/** + * How a QA-doc item is handled by the runner: + * - auto: executed by the runner. + * - manual: requires a human (browser/visual checks) — reported as skipped. + * - delegated: owned by another team per the QA doc (Theme section). + */ +export type StepKind = 'auto' | 'manual' | 'delegated' + +export type StepStatus = 'pass' | 'fail' | 'skipped' | 'blocked' + +export interface StepDef { + /** Stable id, e.g. "apps.init" */ + id: string + /** The QA-doc wording for this item (shown verbatim in the summary). */ + doc: string + kind: StepKind + /** For manual/delegated steps: why the runner skips it. */ + reason?: string + /** + * When true, this step runs even if a previous step in the section failed. + * Defaults to false: a failure blocks the remaining auto steps in the section. + */ + independent?: boolean + run?: (ctx: Ctx) => Promise +} + +export interface SectionDef { + /** QA-doc section title, e.g. "Apps". */ + title: string + steps: StepDef[] +} + +export interface StepResult { + id: string + doc: string + kind: StepKind + status: StepStatus + durationMs: number + /** Short human note (e.g. observed version, skip reason). */ + note?: string + /** Error tail for failures. */ + error?: string +} + +export interface SectionResult { + title: string + steps: StepResult[] +} + +export interface QAReport { + startedAt: string + finishedAt: string + cliVersion: string + cliTarget: string + os: string + sections: SectionResult[] +} diff --git a/packages/qa/tsconfig.json b/packages/qa/tsconfig.json new file mode 100644 index 00000000000..bb57fcfe385 --- /dev/null +++ b/packages/qa/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "types": ["node"], + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 90357c9c908..6fbb68e4d86 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -665,6 +665,27 @@ importers: specifier: ^3.2.7 version: 3.2.7(vitest@4.1.10) + packages/qa: + devDependencies: + '@types/node': + specifier: 18.19.130 + version: 18.19.130 + execa: + specifier: ^7.2.0 + version: 7.2.0 + node-pty: + specifier: ^1.0.0 + version: 1.1.0 + strip-ansi: + specifier: ^7.2.0 + version: 7.2.0 + tsx: + specifier: ^4.23.1 + version: 4.23.1 + typescript: + specifier: 5.9.3 + version: 5.9.3 + packages/store: dependencies: '@graphql-typed-document-node/core': @@ -862,48 +883,56 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@ast-grep/napi-linux-arm64-gnu@0.43.0': resolution: {integrity: sha512-yJSRPxwwrvVW94J2rtaatcixSAGWcSaHNbAh6soXD6HXgq6I7uMc+cyMnJstFL789yd6Pu3QIhTlpD9VY0oYhw==, tarball: https://registry.npmjs.org/@ast-grep/napi-linux-arm64-gnu/-/napi-linux-arm64-gnu-0.43.0.tgz} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@ast-grep/napi-linux-arm64-musl@0.34.1': resolution: {integrity: sha512-IXdqwTbkdqHrcuQb448Qzd82QdTqVFe/f0sSkFYQTic8P2qNzmiHsVnxgEFsQPPbe09BVAoZ885j3OnaNfcDYA==, tarball: https://registry.npmjs.org/@ast-grep/napi-linux-arm64-musl/-/napi-linux-arm64-musl-0.34.1.tgz} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@ast-grep/napi-linux-arm64-musl@0.43.0': resolution: {integrity: sha512-mknXLDsf66HvT/JEl18ZQSvR7/qWgWfqh3eHuVRqD2lE6cKBDXRnAzx9ZNZkUnL2Z5ph54Yk8dKVu09k45cegA==, tarball: https://registry.npmjs.org/@ast-grep/napi-linux-arm64-musl/-/napi-linux-arm64-musl-0.43.0.tgz} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@ast-grep/napi-linux-x64-gnu@0.34.1': resolution: {integrity: sha512-on4LyIeN/zN7SIh8zr5v+NTzVu3kXm2mG28ib1Qe9GVcf35dz52ckf7bilulayKSa2MHZWAXMjuc6NYMiNEw+w==, tarball: https://registry.npmjs.org/@ast-grep/napi-linux-x64-gnu/-/napi-linux-x64-gnu-0.34.1.tgz} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@ast-grep/napi-linux-x64-gnu@0.43.0': resolution: {integrity: sha512-KM6M5KKFsHG9Y7VKCKnMsWQQ1sYwj/SPdyr91SKp66AeFJ5xMtXb13WVQ3Joe9NEsi84dzzOBIJgMddz+UMvQw==, tarball: https://registry.npmjs.org/@ast-grep/napi-linux-x64-gnu/-/napi-linux-x64-gnu-0.43.0.tgz} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@ast-grep/napi-linux-x64-musl@0.34.1': resolution: {integrity: sha512-l1R5L9LOp0jTPjs8C+LUndZOA8cRw7PFlvoVxVbi2jCfcns00dqatSYc4yA/ke6ng2K0LSxjoV/jS8tefve0sA==, tarball: https://registry.npmjs.org/@ast-grep/napi-linux-x64-musl/-/napi-linux-x64-musl-0.34.1.tgz} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@ast-grep/napi-linux-x64-musl@0.43.0': resolution: {integrity: sha512-NfjI74m7CEEOsLi7ZkYwcxY10CfKIHPHRrr9aqMulmnrC4FGvxQMk1qpDrTqkjmEd8LdZt3PsPrmBa8AZCErew==, tarball: https://registry.npmjs.org/@ast-grep/napi-linux-x64-musl/-/napi-linux-x64-musl-0.43.0.tgz} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@ast-grep/napi-win32-arm64-msvc@0.34.1': resolution: {integrity: sha512-eVsdMtnY7jmN2xQjYY9gaqIxRHA44+QYivlP1uLbg8w3P4YlZWTFgOJ7aa357Hg/257mjeQCpodCkr0lGRsSYQ==, tarball: https://registry.npmjs.org/@ast-grep/napi-win32-arm64-msvc/-/napi-win32-arm64-msvc-0.34.1.tgz} @@ -3007,21 +3036,25 @@ packages: resolution: {integrity: sha512-CfzWaS+b32lI/inlYPv1ZAK9CWTWFHzT7TPThvOHML65nrgFl8cQ4Z4FeGdyCbHu2NoG3vR1E36O2tPI2/DLGg==, tarball: https://registry.npmjs.org/@nx/nx-linux-arm64-gnu/-/nx-linux-arm64-gnu-22.7.7.tgz} cpu: [arm64] os: [linux] + libc: [glibc] '@nx/nx-linux-arm64-musl@22.7.7': resolution: {integrity: sha512-53QFuODMEHc1gobCT2WOmYz89DZtEIuLphirdIgnXkkPBTdrP77LPbJUXMRXCMKr90BQaSWhTX+J/ubANRE8og==, tarball: https://registry.npmjs.org/@nx/nx-linux-arm64-musl/-/nx-linux-arm64-musl-22.7.7.tgz} cpu: [arm64] os: [linux] + libc: [musl] '@nx/nx-linux-x64-gnu@22.7.7': resolution: {integrity: sha512-unzmqGIDXGzujo1ZtbrMj2e5D5ldQ1feZmqDPhvO4casK2d96jpT8LtgIILpVDUfhLjl+By185gb0ArrWTsyKw==, tarball: https://registry.npmjs.org/@nx/nx-linux-x64-gnu/-/nx-linux-x64-gnu-22.7.7.tgz} cpu: [x64] os: [linux] + libc: [glibc] '@nx/nx-linux-x64-musl@22.7.7': resolution: {integrity: sha512-oTjMGuM105ok8aH5rlg2YP0Kgkjg0VfUj1exwp49S70gGBJ0r8v5rEtJwOSdCf1WTHuEh0xi66Y/b1S0khF8dg==, tarball: https://registry.npmjs.org/@nx/nx-linux-x64-musl/-/nx-linux-x64-musl-22.7.7.tgz} cpu: [x64] os: [linux] + libc: [musl] '@nx/nx-win32-arm64-msvc@22.7.7': resolution: {integrity: sha512-K741meG/l48TPPeDqnhYTV7pP+1ljjZ+0DRJBxMrPFIu8qKE3Abko/7NKWmWRjcSXJvtxvYkgG3wZds4N2Bhow==, tarball: https://registry.npmjs.org/@nx/nx-win32-arm64-msvc/-/nx-win32-arm64-msvc-22.7.7.tgz} @@ -3327,41 +3360,49 @@ packages: resolution: {integrity: sha512-0bJnmYFp62JdZ4nVMDUZ/C58BCZOCcqgKtnUlp7L9Ojf/czIN+3j72YlLPeWLkzlr6SlYvIQA4SGV/HyO0d+qg==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-11.20.0.tgz} cpu: [arm64] os: [linux] + libc: [glibc] '@oxc-resolver/binding-linux-arm64-musl@11.20.0': resolution: {integrity: sha512-wKHHzPKZo7Ufhv/Bt6yxT7FOgnIgW4gwXcJUipkShGp68W3wGVqvr1Sr0fY65lN0Oy6y41+g2kIDvkgZaMMUkw==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-linux-arm64-musl/-/binding-linux-arm64-musl-11.20.0.tgz} cpu: [arm64] os: [linux] + libc: [musl] '@oxc-resolver/binding-linux-ppc64-gnu@11.20.0': resolution: {integrity: sha512-RN8goF7Ie0B79L4i4G6OeBocTgSC56vJbQ65VJje+oXnldVpLnOU7j/AQ/dP94TcCS+Yh6WG8u3Qt4ETteXFNQ==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-11.20.0.tgz} cpu: [ppc64] os: [linux] + libc: [glibc] '@oxc-resolver/binding-linux-riscv64-gnu@11.20.0': resolution: {integrity: sha512-5l1yU6/xQEqLZRzxqmMxJfWPslpwCmBsdDGaBvABPehxquCXDC7dd7oraNdKSJUMDXSM7VvVj8H2D2FTjU7oWw==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-11.20.0.tgz} cpu: [riscv64] os: [linux] + libc: [glibc] '@oxc-resolver/binding-linux-riscv64-musl@11.20.0': resolution: {integrity: sha512-xHEvkbgz6UC+A3JOyDQy76LkUaxsNSfIr3/GV8slwZsnuooJiIB34gzJfsyvR4JdCYNUUPsRJc/w/oWkODu+hg==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-11.20.0.tgz} cpu: [riscv64] os: [linux] + libc: [musl] '@oxc-resolver/binding-linux-s390x-gnu@11.20.0': resolution: {integrity: sha512-aWPDUUmSeyHvlW+SoEUd+JIJsQhVhu6a5tBpDRMu058naPAchTgAVGCFy35zjbnFlt0i8hLWziff6HX0D3LU4g==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-11.20.0.tgz} cpu: [s390x] os: [linux] + libc: [glibc] '@oxc-resolver/binding-linux-x64-gnu@11.20.0': resolution: {integrity: sha512-x2YeSimvhJjKLVD8KSu8f/rqU1potcdEMkApIPJqjZWN7c2Fpt4g2X32WDg1p+XDAmyT7nuQGe0vnhvXeLbH+g==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-linux-x64-gnu/-/binding-linux-x64-gnu-11.20.0.tgz} cpu: [x64] os: [linux] + libc: [glibc] '@oxc-resolver/binding-linux-x64-musl@11.20.0': resolution: {integrity: sha512-kcRLEIxpZefeYfLChjpgFf3ilBzRDZ+yobMrpRsQlSrxuFGtm3U6PMU7AaEpMqo3NfDGVyJJseAjnRLzMFHjwQ==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-linux-x64-musl/-/binding-linux-x64-musl-11.20.0.tgz} cpu: [x64] os: [linux] + libc: [musl] '@oxc-resolver/binding-openharmony-arm64@11.20.0': resolution: {integrity: sha512-HHcfnApSZGtKhTiHqe8OZruOZe5XuFQH5/E0Yhj3u8fnFvzkM4/k6WjacUf4SvA0SPEAbfbgYmVPuo0VX/fIBQ==, tarball: https://registry.npmjs.org/@oxc-resolver/binding-openharmony-arm64/-/binding-openharmony-arm64-11.20.0.tgz} @@ -3412,36 +3453,42 @@ packages: engines: {node: '>= 10.0.0'} cpu: [arm] os: [linux] + libc: [glibc] '@parcel/watcher-linux-arm-musl@2.5.6': resolution: {integrity: sha512-Ve3gUCG57nuUUSyjBq/MAM0CzArtuIOxsBdQ+ftz6ho8n7s1i9E1Nmk/xmP323r2YL0SONs1EuwqBp2u1k5fxg==, tarball: https://registry.npmjs.org/@parcel/watcher-linux-arm-musl/-/watcher-linux-arm-musl-2.5.6.tgz} engines: {node: '>= 10.0.0'} cpu: [arm] os: [linux] + libc: [musl] '@parcel/watcher-linux-arm64-glibc@2.5.6': resolution: {integrity: sha512-f2g/DT3NhGPdBmMWYoxixqYr3v/UXcmLOYy16Bx0TM20Tchduwr4EaCbmxh1321TABqPGDpS8D/ggOTaljijOA==, tarball: https://registry.npmjs.org/@parcel/watcher-linux-arm64-glibc/-/watcher-linux-arm64-glibc-2.5.6.tgz} engines: {node: '>= 10.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] '@parcel/watcher-linux-arm64-musl@2.5.6': resolution: {integrity: sha512-qb6naMDGlbCwdhLj6hgoVKJl2odL34z2sqkC7Z6kzir8b5W65WYDpLB6R06KabvZdgoHI/zxke4b3zR0wAbDTA==, tarball: https://registry.npmjs.org/@parcel/watcher-linux-arm64-musl/-/watcher-linux-arm64-musl-2.5.6.tgz} engines: {node: '>= 10.0.0'} cpu: [arm64] os: [linux] + libc: [musl] '@parcel/watcher-linux-x64-glibc@2.5.6': resolution: {integrity: sha512-kbT5wvNQlx7NaGjzPFu8nVIW1rWqV780O7ZtkjuWaPUgpv2NMFpjYERVi0UYj1msZNyCzGlaCWEtzc+exjMGbQ==, tarball: https://registry.npmjs.org/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.6.tgz} engines: {node: '>= 10.0.0'} cpu: [x64] os: [linux] + libc: [glibc] '@parcel/watcher-linux-x64-musl@2.5.6': resolution: {integrity: sha512-1JRFeC+h7RdXwldHzTsmdtYR/Ku8SylLgTU/reMuqdVD7CtLwf0VR1FqeprZ0eHQkO0vqsbvFLXUmYm/uNKJBg==, tarball: https://registry.npmjs.org/@parcel/watcher-linux-x64-musl/-/watcher-linux-x64-musl-2.5.6.tgz} engines: {node: '>= 10.0.0'} cpu: [x64] os: [linux] + libc: [musl] '@parcel/watcher-win32-arm64@2.5.6': resolution: {integrity: sha512-3ukyebjc6eGlw9yRt678DxVF7rjXatWiHvTXqphZLvo7aC5NdEgFufVwjFfY51ijYEWpXbqF5jtrK275z52D4Q==, tarball: https://registry.npmjs.org/@parcel/watcher-win32-arm64/-/watcher-win32-arm64-2.5.6.tgz} @@ -3565,66 +3612,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==, tarball: https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==, tarball: https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz} @@ -4138,41 +4198,49 @@ packages: resolution: {integrity: sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-gnu/-/resolver-binding-linux-arm64-gnu-1.11.1.tgz} cpu: [arm64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.11.1': resolution: {integrity: sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-linux-arm64-musl/-/resolver-binding-linux-arm64-musl-1.11.1.tgz} cpu: [arm64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': resolution: {integrity: sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-linux-ppc64-gnu/-/resolver-binding-linux-ppc64-gnu-1.11.1.tgz} cpu: [ppc64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': resolution: {integrity: sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-gnu/-/resolver-binding-linux-riscv64-gnu-1.11.1.tgz} cpu: [riscv64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': resolution: {integrity: sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-linux-riscv64-musl/-/resolver-binding-linux-riscv64-musl-1.11.1.tgz} cpu: [riscv64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': resolution: {integrity: sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-linux-s390x-gnu/-/resolver-binding-linux-s390x-gnu-1.11.1.tgz} cpu: [s390x] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.11.1': resolution: {integrity: sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-gnu/-/resolver-binding-linux-x64-gnu-1.11.1.tgz} cpu: [x64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.11.1': resolution: {integrity: sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-linux-x64-musl/-/resolver-binding-linux-x64-musl-1.11.1.tgz} cpu: [x64] os: [linux] + libc: [musl] '@unrs/resolver-binding-wasm32-wasi@1.11.1': resolution: {integrity: sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==, tarball: https://registry.npmjs.org/@unrs/resolver-binding-wasm32-wasi/-/resolver-binding-wasm32-wasi-1.11.1.tgz} From 3e3b8ebcb07910a311d22a0fe877513237122f4e Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Thu, 30 Jul 2026 18:26:01 +0200 Subject: [PATCH 2/2] QA flow: manual app dev block, prompt fixes, workflow - app dev block reclassified as manual (interactive session, team decision): no store/password needed by the runner at all - config link driven like the e2e helper: --config + --organization-id skip prompts; create-new-app + app name answered over the pty - exec runs with CI=1 (fail fast on unexpected prompts) and app init writes frozen-lockfile=false, matching GitHub Actions behavior - pty kill() now kills the process group (workerd/vite orphans kept ports) - hydrogen dev pinned to a free port; storefront probe tries IPv4 and IPv6 - qa-flow.yml: non-blocking PR/merge_group job (repo build) + dispatch (published version, default nightly), macOS first Assisted-By: devx/2410058d-4b68-49a7-ba23-7233a4497d19 --- .github/workflows/qa-flow.yml | 92 +++++++++++ packages/qa/README.md | 14 +- packages/qa/src/context.ts | 12 +- packages/qa/src/proc.ts | 43 ++++- packages/qa/src/steps/apps.ts | 263 +++++------------------------- packages/qa/src/steps/hydrogen.ts | 47 ++++-- 6 files changed, 217 insertions(+), 254 deletions(-) create mode 100644 .github/workflows/qa-flow.yml diff --git a/.github/workflows/qa-flow.yml b/.github/workflows/qa-flow.yml new file mode 100644 index 00000000000..43eafccea8a --- /dev/null +++ b/.github/workflows/qa-flow.yml @@ -0,0 +1,92 @@ +name: QA flow + +# Automated CLI Pre-release QA flow (packages/qa): runs the QA-doc steps that +# need no human interaction, non-blocking, and publishes a summary that mirrors +# the doc structure (with the remaining manual checklist for gardeners). +# +# - CI runs (PRs / merge queue): tests the CLI built from the current branch. +# - Manual runs (workflow_dispatch): tests a published version, default nightly. + +on: + pull_request: + merge_group: + workflow_dispatch: + inputs: + cli-version: + type: string + required: true + default: 'nightly' + description: 'npm dist-tag or version of @shopify/cli to QA (or "repo" for the branch build)' + +concurrency: + group: qa-flow-${{ github.event_name }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + DEBUG: '1' + PNPM_VERSION: '10.11.1' + QA_NODE_VERSION: '24.1.0' + +jobs: + qa-flow: + name: 'QA flow (${{ matrix.os }})' + # Non-blocking: failures show up in the summary, not as a red required check. + continue-on-error: true + # Secrets are unavailable to forks; run only for internal PRs, the merge + # queue and manual dispatches. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + # Rollout starts with macOS (the QA doc's default OS); + # ubuntu-latest and windows-latest follow once stable. + os: ['macos-latest'] + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Setup deps + uses: ./.github/actions/setup-cli-deps + with: + node-version: ${{ env.QA_NODE_VERSION }} + + - name: Build CLI from branch + if: github.event_name != 'workflow_dispatch' || inputs.cli-version == 'repo' + run: pnpm nx run-many --all --skip-nx-cache --target=build --output-style=stream + + - name: Install @shopify/cli@${{ inputs.cli-version }} + if: github.event_name == 'workflow_dispatch' && inputs.cli-version != 'repo' + id: install-cli + run: | + mkdir -p "$RUNNER_TEMP/qa-cli" + npm install --prefix "$RUNNER_TEMP/qa-cli" "@shopify/cli@${{ inputs.cli-version }}" --registry=https://registry.npmjs.org + BIN="$RUNNER_TEMP/qa-cli/node_modules/@shopify/cli/bin/run.js" + VERSION=$(node -p "require('$RUNNER_TEMP/qa-cli/node_modules/@shopify/cli/package.json').version") + echo "bin=$BIN" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Installed @shopify/cli@$VERSION" + + - name: Rebuild node-pty + run: pnpm rebuild node-pty + + - name: Run QA flow + working-directory: packages/qa + env: + QA_CLI_BIN: ${{ steps.install-cli.outputs.bin }} + QA_EXPECTED_VERSION: ${{ steps.install-cli.outputs.version }} + QA_ISOLATE: '1' + QA_ORG_ID: ${{ secrets.QA_ORG_ID }} + QA_REPORT_DIR: ${{ github.workspace }}/qa-report + SHOPIFY_APP_AUTOMATION_TOKEN: ${{ secrets.QA_APP_AUTOMATION_TOKEN }} + run: pnpm qa + + - name: Upload QA report + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: qa-report-${{ matrix.os }} + path: qa-report/ + retention-days: 30 diff --git a/packages/qa/README.md b/packages/qa/README.md index 2cb82e85cd6..a6894a9be1f 100644 --- a/packages/qa/README.md +++ b/packages/qa/README.md @@ -10,9 +10,10 @@ Enter, CTRL+C). Every checklist item of the doc is represented, in order: - **auto** — executed by the runner. -- **manual** — needs a browser/human (dev console, theme editor, visual - confirmations). Reported as `⏭️ manual` and listed in the - "Remaining manual checklist" section of the summary — never silently dropped. +- **manual** — needs a human. This covers browser/visual checks and the whole + `shopify app dev` block (kept manual by team decision — it is an interactive + session). Reported as `⏭️ manual` and listed in the "Remaining manual + checklist" section of the summary — never silently dropped. - **delegated** — the Theme section, owned by the themes team per the doc. The summary (`qa-summary.md`, also appended to `$GITHUB_STEP_SUMMARY` on CI) @@ -28,12 +29,12 @@ cd packages/qa pnpm install # Requires an authenticated CLI session (shopify auth login) or -# SHOPIFY_APP_AUTOMATION_TOKEN, plus an org and a dev store: -QA_ORG_ID= QA_STORE_FQDN=.myshopify.com pnpm qa +# SHOPIFY_APP_AUTOMATION_TOKEN, plus an org: +QA_ORG_ID= pnpm qa # Run a subset while iterating: QA_ONLY=hydrogen pnpm qa -QA_ONLY=prep,apps QA_ORG_ID=… QA_STORE_FQDN=… pnpm qa +QA_ONLY=prep,apps QA_ORG_ID=… pnpm qa ``` ## Environment variables @@ -43,7 +44,6 @@ QA_ONLY=prep,apps QA_ORG_ID=… QA_STORE_FQDN=… pnpm qa | `QA_CLI_BIN` | Path to the CLI under test. Default: `packages/cli/bin/run.js` (repo build). Point it at an installed `shopify` binary to QA a published version. | | `QA_EXPECTED_VERSION` | Assert `shopify version` equals this (doc: "the version should match the nightly you just created"). | | `QA_ORG_ID` / `E2E_ORG_ID` | Organization used by `app init`. | -| `QA_STORE_FQDN` / `E2E_STORE_FQDN` | Dev store passed to `app dev`. | | `QA_PACKAGE_MANAGER` | Package manager for `app init` (default `pnpm`). | | `QA_ISOLATE` | `1` = fresh XDG dirs (don't reuse the local CLI session). Set on CI. | | `QA_WORK_DIR` | Scratch dir (default: fresh temp dir). | diff --git a/packages/qa/src/context.ts b/packages/qa/src/context.ts index f1db1e6aa32..2d25f3c84a1 100644 --- a/packages/qa/src/context.ts +++ b/packages/qa/src/context.ts @@ -11,10 +11,6 @@ export interface CtxState { appDir?: string /** Name given to the app created by `app init`. */ appName?: string - /** GraphiQL server coordinates passed to `app dev`. */ - graphiql?: {port: number; key: string} - /** The long-running `app dev` pty process. */ - devProc?: PtyProcess /** Directory of the hydrogen app. */ hydrogenDir?: string /** Name of the secondary config created by `app config link` (e.g. "staging"). */ @@ -33,7 +29,6 @@ export interface Ctx { /** Environment passed to every CLI invocation. */ env: {[key: string]: string} orgId?: string - storeFqdn?: string /** When set, the version step asserts `shopify version` equals this. */ expectedVersion?: string state: CtxState @@ -52,7 +47,6 @@ function repoRoot(): string { * Defaults to the repo build at packages/cli/bin/run.js. * QA_EXPECTED_VERSION Assert `shopify version` equals this value. * QA_ORG_ID Organization id used for `app init` (falls back to E2E_ORG_ID). - * QA_STORE_FQDN Dev store passed to `app dev` (falls back to E2E_STORE_FQDN). * QA_WORK_DIR Scratch dir (defaults to a fresh mkdtemp). * QA_ISOLATE When "1", run with fresh XDG dirs so the ambient CLI session * is not used. CI sets this; local runs default to ambient auth. @@ -88,6 +82,11 @@ export function createContext(log: (msg: string) => void = defaultLog): Ctx { // proc.ts re-points INIT_CWD/PWD at the effective cwd per invocation. delete env.INIT_CWD delete env.PWD + // Some environments (nix shells, agent harnesses) export NPM_CONFIG_NODE_VERSION; + // pnpm trusts it over the real node version and fails template engine checks. + for (const key of Object.keys(env)) { + if (key.toLowerCase() === 'npm_config_node_version') delete env[key] + } if (process.env.QA_ISOLATE === '1') { const authDir = path.join(workDir, 'xdg') @@ -104,7 +103,6 @@ export function createContext(log: (msg: string) => void = defaultLog): Ctx { workDir, env, orgId: process.env.QA_ORG_ID ?? process.env.E2E_ORG_ID, - storeFqdn: process.env.QA_STORE_FQDN ?? process.env.E2E_STORE_FQDN, expectedVersion: process.env.QA_EXPECTED_VERSION, state: {ptyProcs: []}, log, diff --git a/packages/qa/src/proc.ts b/packages/qa/src/proc.ts index 46fa646b34e..67f449b7209 100644 --- a/packages/qa/src/proc.ts +++ b/packages/qa/src/proc.ts @@ -54,9 +54,10 @@ export async function exec(ctx: Ctx, args: string[], opts: ExecOptions = {}): Pr const [command, ...prefix] = ctx.cliInvoke ctx.log(`$ shopify ${args.join(' ')}`) const cwd = opts.cwd ?? ctx.workDir + // CI=1 makes the CLI fail fast on unexpected prompts instead of hanging. const result = await execa(command as string, [...prefix, ...args], { cwd, - env: {...ctx.env, ...opts.env, INIT_CWD: cwd, PWD: cwd}, + env: {...ctx.env, ...opts.env, INIT_CWD: cwd, PWD: cwd, CI: '1'}, extendEnv: false, timeout: opts.timeoutMs ?? DEFAULT_EXEC_TIMEOUT, input: opts.input, @@ -87,6 +88,28 @@ export interface WaitOptions { timeoutMs?: number } +export function settle(ms = 250): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +/** Allocate a free localhost port. */ +export async function freePort(): Promise { + const net = await import('net') + return new Promise((resolve, reject) => { + const server = net.createServer() + server.listen(0, '127.0.0.1', () => { + const address = server.address() + if (address && typeof address === 'object') { + const port = address.port + server.close(() => resolve(port)) + } else { + server.close(() => reject(new Error('Could not allocate a free port'))) + } + }) + server.on('error', reject) + }) +} + export interface PtyProcess { /** Wait until the (ANSI-stripped) output matches. */ waitFor(match: string | RegExp, opts?: WaitOptions): Promise @@ -112,12 +135,15 @@ export function spawnPty( ctx.log(`$ shopify ${args.join(' ')} (pty)`) const cwd = opts.cwd ?? ctx.workDir + // Interactive prompts only render when the CLI does not think it's on CI. + const env: {[key: string]: string} = {...ctx.env, ...opts.env, INIT_CWD: cwd, PWD: cwd} + delete env.CI const proc = pty.spawn(command as string, [...prefix, ...args], { name: 'xterm-color', cols: 120, rows: 40, cwd, - env: {...ctx.env, ...opts.env, INIT_CWD: cwd, PWD: cwd}, + env, }) let buffer = '' @@ -161,7 +187,18 @@ export function spawnPty( write: (data) => proc.write(data), sendLine: (line) => proc.write(`${line}\r`), kill: () => { - if (!exited) proc.kill() + if (exited) return + // Kill the whole process group: dev servers spawn children (workerd, + // vite) that would otherwise survive and keep ports bound. + if (process.platform !== 'win32') { + try { + process.kill(-proc.pid, 'SIGTERM') + return + } catch { + // Fall through to killing the direct child only. + } + } + proc.kill() }, waitFor: (match, waitOpts = {}) => { const from = 0 diff --git a/packages/qa/src/steps/apps.ts b/packages/qa/src/steps/apps.ts index 1e4fb090e31..14a00cc1707 100644 --- a/packages/qa/src/steps/apps.ts +++ b/packages/qa/src/steps/apps.ts @@ -2,18 +2,17 @@ * "Apps" section of the CLI Pre-release QA flow. * * Each step mirrors one checklist item of the QA doc, in order. Items that - * require a human (browser/visual checks) are declared kind: 'manual' and - * reported as skipped — never silently dropped. + * require a human are declared kind: 'manual' and reported as skipped — never + * silently dropped. The whole `shopify app dev` block is manual by team + * decision: it is an interactive session with browser/visual checks. */ import * as fs from 'fs' -import * as net from 'net' import * as path from 'path' -import {exec, expectSuccess, httpRequest, retry, spawnPty, tail} from '../proc.js' +import {exec, expectSuccess, settle, spawnPty, tail} from '../proc.js' import type {Ctx} from '../context.js' import type {SectionDef} from '../types.js' -const READY_MESSAGE = 'Ready, watching for changes in your app' -const UPDATED_MESSAGE = 'Updated dev preview' +const DEV_MANUAL_REASON = 'part of the interactive `shopify app dev` session — validated by hand during release QA' const FUNCTION_INPUT = '{"cart":{"lines":[{"id":"gid://shopify/CartLine/0","cost":{"subtotalAmount":{"amount":"10.0"}}}]},"discount":{"discountClasses":["PRODUCT","ORDER","SHIPPING"]}}' @@ -24,7 +23,6 @@ const EXTENSIONS = { discount: 'qa-discount', flowAction: 'qa-flow-action', choice: 'qa-admin-block', - midDev: 'qa-admin-link', } function appDirOrThrow(ctx: Ctx): string { @@ -33,28 +31,6 @@ function appDirOrThrow(ctx: Ctx): string { return dir } -function devProcOrThrow(ctx: Ctx) { - const proc = ctx.state.devProc - if (!proc || proc.exited) throw new Error('`app dev` is not running') - return proc -} - -async function freePort(): Promise { - return new Promise((resolve, reject) => { - const server = net.createServer() - server.listen(0, '127.0.0.1', () => { - const address = server.address() - if (address && typeof address === 'object') { - const port = address.port - server.close(() => resolve(port)) - } else { - server.close(() => reject(new Error('Could not allocate a free port'))) - } - }) - server.on('error', reject) - }) -} - async function generateExtension(ctx: Ctx, template: string, name: string, flavor?: string): Promise { const appDir = appDirOrThrow(ctx) const args = ['app', 'generate', 'extension', `--template=${template}`, `--name=${name}`] @@ -68,20 +44,6 @@ async function generateExtension(ctx: Ctx, template: string, name: string, flavo return `created extensions/${name}` } -/** Edit a file and return a note; throws when the expected content is missing. */ -function editFile(filePath: string, mutate: (content: string) => string): void { - const content = fs.readFileSync(filePath, 'utf8') - fs.writeFileSync(filePath, mutate(content)) -} - -function findFile(dir: string, candidates: string[]): string { - for (const candidate of candidates) { - const filePath = path.join(dir, candidate) - if (fs.existsSync(filePath)) return filePath - } - throw new Error(`None of ${candidates.join(', ')} found under ${dir}`) -} - export const appsSection: SectionDef = { title: 'Apps', steps: [ @@ -124,6 +86,9 @@ export const appsSection: SectionDef = { if (!appDir || !fs.existsSync(appDir)) { throw new Error(`Could not locate created app in ${ctx.workDir}\n${tail(output)}`) } + // On CI, pnpm defaults to frozen-lockfile and `generate extension` adds + // workspace packages that are not in the template lockfile yet. + fs.appendFileSync(path.join(appDir, '.npmrc'), 'frozen-lockfile=false\n') ctx.state.appDir = appDir ctx.state.appName = name return `created ${path.basename(appDir)}` @@ -167,196 +132,56 @@ export const appsSection: SectionDef = { { id: 'apps.dev.start', doc: 'Run `shopify app dev`', - kind: 'auto', - run: async (ctx) => { - const appDir = appDirOrThrow(ctx) - const port = await freePort() - const key = 'qa-graphiql-key' - ctx.state.graphiql = {port, key} - const args = ['app', 'dev', '--graphiql-port', String(port), '--graphiql-key', key] - if (ctx.storeFqdn) args.push('--store', ctx.storeFqdn) - const proc = spawnPty(ctx, args, {cwd: appDir}) - ctx.state.devProc = proc - await proc.waitFor(READY_MESSAGE, {timeoutMs: 5 * 60_000}) - return 'dev is ready and watching for changes' - }, - }, - { - id: 'apps.dev.console.open', - doc: 'Dev Console: open the shop and see the dev console', kind: 'manual', - reason: 'browser check', + reason: DEV_MANUAL_REASON, }, { - id: 'apps.dev.console.connected', - doc: 'The app you ran dev on should be the first dev preview and show as connected (green icon)', + id: 'apps.dev.console', + doc: 'Dev Console: open the shop, see the dev console, app shows as connected (green icon)', kind: 'manual', - reason: 'browser check', + reason: DEV_MANUAL_REASON, }, { - id: 'apps.dev.admin-action.product', - doc: 'If your shop has no products, create one now (needed to test the extensions)', + id: 'apps.dev.admin-action', + doc: 'Test the admin-action: product admin opens the action modal; editing `src/ActionExtension.js` hot reloads', kind: 'manual', - reason: 'store admin browser step — the QA store is expected to already have a product', - }, - { - id: 'apps.dev.admin-action.modal', - doc: 'From dev-console, open the admin-action link: product admin page opens the action modal', - kind: 'manual', - reason: 'browser check', - }, - { - id: 'apps.dev.admin-action.hot-reload', - doc: 'Change the message inside `src/ActionExtension.js` — you should see it hot reload', - kind: 'auto', - run: async (ctx) => { - const proc = devProcOrThrow(ctx) - const extDir = path.join(appDirOrThrow(ctx), 'extensions', EXTENSIONS.adminAction, 'src') - const file = findFile(extDir, ['ActionExtension.jsx', 'ActionExtension.js', 'ActionExtension.tsx']) - editFile(file, (content) => content.replace(/current product/i, 'QA hot reload message')) - await proc.waitFor(UPDATED_MESSAGE, {timeoutMs: 3 * 60_000}) - return `edited ${path.basename(file)}; dev reported "${UPDATED_MESSAGE}" (visual confirmation stays manual)` - }, + reason: DEV_MANUAL_REASON, }, { id: 'apps.dev.extension-mid-dev', doc: 'Add another extension and see it show up in the dev console', - kind: 'auto', - run: async (ctx) => { - const proc = devProcOrThrow(ctx) - await generateExtension(ctx, 'admin_link', EXTENSIONS.midDev) - await proc.waitFor(/Extension created|Updated dev preview/, {timeoutMs: 3 * 60_000}) - return 'new extension picked up by the running dev session (dev-console visual stays manual)' - }, + kind: 'manual', + reason: DEV_MANUAL_REASON, }, { id: 'apps.dev.graphiql', - doc: 'Press `g` to open GraphiQL; test that it works with `query { shop { name } }`', - kind: 'auto', - run: async (ctx) => { - devProcOrThrow(ctx) - const graphiql = ctx.state.graphiql - if (!graphiql) throw new Error('GraphiQL port/key were not configured') - const base = `http://localhost:${graphiql.port}` - // Equivalent of pressing `g`: the key only opens this URL in a browser. - await retry(async () => { - const ping = await httpRequest(`${base}/graphiql/ping`) - if (ping.status !== 200) throw new Error(`GraphiQL ping returned ${ping.status}`) - }, 10, 3000) - const response = await httpRequest(`${base}/graphiql/graphql.json?key=${graphiql.key}&api_version=unstable`, { - method: 'POST', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({query: 'query { shop { name } }'}), - timeoutMs: 30_000, - }) - if (response.status !== 200) { - throw new Error(`GraphiQL query returned HTTP ${response.status}: ${tail(response.body, 10)}`) - } - const parsed = JSON.parse(response.body) as {data?: {shop?: {name?: string}}; errors?: unknown} - if (!parsed.data?.shop?.name) { - throw new Error(`GraphiQL query did not return shop name: ${tail(response.body, 10)}`) - } - return `GraphiQL answered: shop name "${parsed.data.shop.name}"` - }, + doc: 'Press `g` to open GraphiQL and test `query { shop { name } }`', + kind: 'manual', + reason: DEV_MANUAL_REASON, }, { id: 'apps.dev.execute', doc: "Test the same query via command: `shopify app execute --query 'query { shop { name } }'`", - kind: 'auto', - run: async (ctx) => { - const result = expectSuccess( - await exec(ctx, ['app', 'execute', '--query', 'query { shop { name } }'], { - cwd: appDirOrThrow(ctx), - timeoutMs: 2 * 60_000, - }), - 'app execute', - ) - const output = result.stdout + result.stderr - if (!/"name"\s*:/.test(output)) throw new Error(`app execute output has no shop name:\n${tail(output, 15)}`) - return 'app execute returned shop data' - }, - }, - { - id: 'apps.dev.theme-ext.uploaded', - doc: 'Theme extension files should have been uploaded by now (if not, wait for it)', - kind: 'auto', - run: async (ctx) => { - const proc = devProcOrThrow(ctx) - await proc.waitFor(/theme app extension|host theme|9292/i, {timeoutMs: 4 * 60_000}) - return 'dev output shows the theme app extension is being served' - }, - }, - { - id: 'apps.dev.theme-ext.setup-link', - doc: 'Click on the "Setup your theme app extension in the host theme" link from the CLI output', kind: 'manual', - reason: 'browser check (theme editor)', + reason: DEV_MANUAL_REASON, }, { - id: 'apps.dev.theme-ext.add-section', - doc: '"Add section", choose your app and "Save"', + id: 'apps.dev.theme-ext', + doc: 'Theme app extension: setup link, "Add section" + Save, local preview at 127.0.0.1:9292, `star_rating.liquid` hot reload', kind: 'manual', - reason: 'browser check (theme editor)', - }, - { - id: 'apps.dev.theme-ext.preview', - doc: 'Open the theme app extension local preview (e.g. http://127.0.0.1:9292)', - kind: 'auto', - run: async (ctx) => { - const proc = devProcOrThrow(ctx) - const match = proc.output().match(/https?:\/\/(?:127\.0\.0\.1|localhost):(\d{4,5})/g) - const themeUrl = - match?.find((url) => url.includes('9292')) ?? 'http://127.0.0.1:9292' - const response = await retry(() => httpRequest(themeUrl, {timeoutMs: 20_000}), 6, 5000) - return `theme preview at ${themeUrl} responded with HTTP ${response.status} (visual confirmation stays manual)` - }, - }, - { - id: 'apps.dev.theme-ext.hot-reload', - doc: "Add `Hello` inside the span in the theme's `blocks/star_rating.liquid` — it should hot reload", - kind: 'auto', - run: async (ctx) => { - const proc = devProcOrThrow(ctx) - const blocksDir = path.join(appDirOrThrow(ctx), 'extensions', EXTENSIONS.theme, 'blocks') - const file = findFile(blocksDir, ['star_rating.liquid']) - editFile(file, (content) => { - if (!content.includes(' found in ${file}`) - return content.replace(/(]*>)/, '$1Hello ') - }) - await proc.waitFor(/star_rating\.liquid|Updated dev preview|hot reload/i, {timeoutMs: 3 * 60_000}) - return 'edited star_rating.liquid; dev picked up the change (visual confirmation stays manual)' - }, + reason: DEV_MANUAL_REASON, }, { id: 'apps.dev.quit', - doc: 'Press `q` to stop dev', - kind: 'auto', - run: async (ctx) => { - const proc = devProcOrThrow(ctx) - proc.write('q') - const code = await proc.waitForExit(60_000) - ctx.state.devProc = undefined - if (code !== 0) throw new Error(`app dev exited with code ${code} after pressing q\n${tail(proc.output())}`) - return 'dev stopped cleanly' - }, - }, - { - id: 'apps.dev.console.disconnected', - doc: 'See that the dev console still reports a dev preview, but shown as disconnected', + doc: 'Press `q` to stop dev; dev console shows the preview as disconnected', kind: 'manual', - reason: 'browser check', + reason: DEV_MANUAL_REASON, }, { id: 'apps.dev.clean', doc: 'Run `shopify app dev clean` to end the preview; the dev preview is now hidden', - kind: 'auto', - run: async (ctx) => { - expectSuccess( - await exec(ctx, ['app', 'dev', 'clean'], {cwd: appDirOrThrow(ctx), timeoutMs: 2 * 60_000}), - 'app dev clean', - ) - return 'dev preview cleaned (dev-console visual stays manual)' - }, + kind: 'manual', + reason: DEV_MANUAL_REASON, }, { id: 'apps.function.build', @@ -427,27 +252,27 @@ export const appsSection: SectionDef = { const appDir = appDirOrThrow(ctx) const configName = 'staging' const newAppName = `${ctx.state.appName ?? 'qa-ci-app'}-staging` - const proc = spawnPty(ctx, ['app', 'config', 'link'], {cwd: appDir}) - - // Prompt flow: configuration file name → (org) → app selection - // ("Create this project as a new app" is the first choice) → app name. - await proc.waitFor(/configuration file name/i, {timeoutMs: 2 * 60_000}) - proc.sendLine(configName) + // --organization-id and --config replace prompts the doc answers by hand; + // the "create new app" + "app name" prompts are driven over the pty. + const args = ['app', 'config', 'link', '--config', configName] + if (ctx.orgId) args.push('--organization-id', ctx.orgId) + const proc = spawnPty(ctx, args, {cwd: appDir}) - await proc.waitFor(/create this project as a new app|which existing app|organization/i, { - timeoutMs: 3 * 60_000, - }) - if (/organization/i.test(proc.output()) && !/create this project as a new app/i.test(proc.output())) { - // Organization select — accept the highlighted option. + // First prompt is "Create this project as a new app" when the org has + // existing apps, or "App name" straight away when it does not. + await proc.waitFor(/Create this project as a new app|App name/, {timeoutMs: 3 * 60_000}) + if (proc.output().includes('Create this project as a new app')) { + await settle(100) proc.write('\r') - await proc.waitFor(/create this project as a new app|which existing app/i, {timeoutMs: 2 * 60_000}) + await proc.waitFor(/App name/, {timeoutMs: 2 * 60_000}) } - // "Create this project as a new app on Shopify?" (yes is default) or app select with create-new first. + // Write the name as a single data event, then Enter separately so Ink + // treats it as submission (mirrors how a human types + confirms). + await settle(250) + proc.write(newAppName) + await settle(250) proc.write('\r') - await proc.waitFor(/app name/i, {timeoutMs: 2 * 60_000}) - proc.sendLine(newAppName) - const code = await proc.waitForExit(3 * 60_000) if (code !== 0) throw new Error(`app config link exited with ${code}\n${tail(proc.output())}`) const tomlPath = path.join(appDir, `shopify.app.${configName}.toml`) diff --git a/packages/qa/src/steps/hydrogen.ts b/packages/qa/src/steps/hydrogen.ts index 67a57ebc9c1..50d42ca7483 100644 --- a/packages/qa/src/steps/hydrogen.ts +++ b/packages/qa/src/steps/hydrogen.ts @@ -1,9 +1,27 @@ import * as fs from 'fs' import * as path from 'path' -import {exec, expectSuccess, httpRequest, retry, spawnPty, tail} from '../proc.js' +import {exec, expectSuccess, freePort, httpRequest, retry, spawnPty, tail} from '../proc.js' import type {PtyProcess} from '../proc.js' import type {SectionDef} from '../types.js' +/** Probe both address families — MiniOxygen may bind IPv6 ::1 or IPv4 127.0.0.1. */ +async function probeLocalhost(port: number): Promise<{status: number; body: string; url: string}> { + const urls = [`http://127.0.0.1:${port}/`, `http://[::1]:${port}/`] + let lastError: unknown + for (const url of urls) { + try { + // eslint-disable-next-line no-await-in-loop + const response = await httpRequest(url, {timeoutMs: 20_000}) + return {...response, url} + } catch (error) { + lastError = error + } + } + const message = + lastError instanceof Error ? (lastError.cause instanceof Error ? lastError.cause.message : lastError.message) : String(lastError) + throw new Error(`probe of port ${port} failed on IPv4 and IPv6: ${message}`) +} + /** * Press Enter on every prompt until `until` matches the output (the QA doc says * "choose default answers to all questions"). A prompt is considered pending @@ -82,30 +100,23 @@ export const hydrogenSection: SectionDef = { doc: 'Run `shopify hydrogen dev`; follow the "View Hydrogen app" link and confirm you can see a storefront', kind: 'auto', run: async (ctx) => { - const proc = spawnPty(ctx, ['hydrogen', 'dev'], {cwd: hydrogenDirOrThrow(ctx)}) - await proc.waitFor(/View [\w ]*app:|localhost:\d+/i, {timeoutMs: 5 * 60_000}) - const announced = - proc.output().match(/https?:\/\/(?:127\.0\.0\.1|localhost):\d{4,5}\/?/)?.[0] ?? 'http://localhost:3000' - // Node's fetch resolves `localhost` to ::1 first on macOS while the dev - // server listens on IPv4 — probe 127.0.0.1 explicitly. - const url = announced.replace('localhost', '127.0.0.1') + // Pin the port so the probe is deterministic even if something else + // occupies hydrogen's default 3000. + const port = await freePort() + const proc = spawnPty(ctx, ['hydrogen', 'dev', '--port', String(port)], {cwd: hydrogenDirOrThrow(ctx)}) + await proc.waitFor(new RegExp(`View [\\w ]*app:|localhost:${port}`, 'i'), {timeoutMs: 5 * 60_000}) const response = await retry(async () => { - try { - const res = await httpRequest(url, {timeoutMs: 20_000}) - if (res.status >= 500) throw new Error(`storefront returned HTTP ${res.status}`) - return res - } catch (error) { - const message = error instanceof Error ? (error.cause instanceof Error ? error.cause.message : error.message) : String(error) - throw new Error(`probe of ${url} failed: ${message}`) - } + const res = await probeLocalhost(port) + if (res.status >= 500) throw new Error(`storefront returned HTTP ${res.status}`) + return res }, 10, 5000) if (!/ proc.kill()) - return `storefront responded at ${url} (HTTP ${response.status}); stopped with CTRL+C` + return `storefront responded at ${response.url} (HTTP ${response.status}); stopped with CTRL+C` }, }, ],