Add automated QA flow: packages/qa runner + qa-flow workflow

.github/workflows/qa-flow.yml

@@ -0,0 +1,92 @@
+name: QA flow
+
+# Automated CLI Pre-release QA flow (packages/qa): runs the QA-doc steps that
+# need no human interaction, non-blocking, and publishes a summary that mirrors
+# the doc structure (with the remaining manual checklist for gardeners).
+#
+# - CI runs (PRs / merge queue): tests the CLI built from the current branch.
+# - Manual runs (workflow_dispatch): tests a published version, default nightly.
+
+on:
+  pull_request:
+  merge_group:
+  workflow_dispatch:
+    inputs:
+      cli-version:
+        type: string
+        required: true
+        default: 'nightly'
+        description: 'npm dist-tag or version of @shopify/cli to QA (or "repo" for the branch build)'
+
+concurrency:
+  group: qa-flow-${{ github.event_name }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  DEBUG: '1'
+  PNPM_VERSION: '10.11.1'
+  QA_NODE_VERSION: '24.1.0'
+
+jobs:
+  qa-flow:
+    name: 'QA flow (${{ matrix.os }})'
+    # Non-blocking: failures show up in the summary, not as a red required check.
+    continue-on-error: true
+    # Secrets are unavailable to forks; run only for internal PRs, the merge
+    # queue and manual dispatches.
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        # Rollout starts with macOS (the QA doc's default OS);
+        # ubuntu-latest and windows-latest follow once stable.
+        os: ['macos-latest']
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+
+      - name: Setup deps
+        uses: ./.github/actions/setup-cli-deps
+        with:
+          node-version: ${{ env.QA_NODE_VERSION }}
+
+      - name: Build CLI from branch
+        if: github.event_name != 'workflow_dispatch' || inputs.cli-version == 'repo'
+        run: pnpm nx run-many --all --skip-nx-cache --target=build --output-style=stream
+
+      - name: Install @shopify/cli@${{ inputs.cli-version }}
+        if: github.event_name == 'workflow_dispatch' && inputs.cli-version != 'repo'
+        id: install-cli
+        run: |
+          mkdir -p "$RUNNER_TEMP/qa-cli"
+          npm install --prefix "$RUNNER_TEMP/qa-cli" "@shopify/cli@${{ inputs.cli-version }}" --registry=https://registry.npmjs.org
+          BIN="$RUNNER_TEMP/qa-cli/node_modules/@shopify/cli/bin/run.js"
+          VERSION=$(node -p "require('$RUNNER_TEMP/qa-cli/node_modules/@shopify/cli/package.json').version")
+          echo "bin=$BIN" >> "$GITHUB_OUTPUT"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Installed @shopify/cli@$VERSION"
+
+      - name: Rebuild node-pty
+        run: pnpm rebuild node-pty
+
+      - name: Run QA flow
+        working-directory: packages/qa
+        env:
+          QA_CLI_BIN: ${{ steps.install-cli.outputs.bin }}
+          QA_EXPECTED_VERSION: ${{ steps.install-cli.outputs.version }}
+          QA_ISOLATE: '1'
+          QA_ORG_ID: ${{ secrets.QA_ORG_ID }}
+          QA_REPORT_DIR: ${{ github.workspace }}/qa-report
+          SHOPIFY_APP_AUTOMATION_TOKEN: ${{ secrets.QA_APP_AUTOMATION_TOKEN }}
+        run: pnpm qa
+
+      - name: Upload QA report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: qa-report-${{ matrix.os }}
+          path: qa-report/
+          retention-days: 30

packages/qa/.gitignore

@@ -0,0 +1,2 @@
+qa-report/
+node_modules/

packages/qa/README.md

@@ -0,0 +1,67 @@
+# @shopify/qa — CLI Pre-release QA flow runner
+
+Automates the steps of the [CLI Pre-release QA flow doc](https://docs.google.com/document/d/1XX6QnS6kKZTT1shcCZVcso74VWn-Ui4V2IASenHHi1E/edit)
+that need **no human interaction**, by driving the CLI directly (no Playwright,
+no test framework): non-interactive commands run via `execa`, interactive ones
+(`app dev`, `app config link`, `hydrogen init`) run in a pseudo-terminal via
+`node-pty` and are driven with the same keypresses the doc describes (`g`, `q`,
+Enter, CTRL+C).
+
+Every checklist item of the doc is represented, in order:
+
+- **auto** — executed by the runner.
+- **manual** — needs a human. This covers browser/visual checks and the whole
+  `shopify app dev` block (kept manual by team decision — it is an interactive
+  session). Reported as `⏭️ manual` and listed in the "Remaining manual
+  checklist" section of the summary — never silently dropped.
+- **delegated** — the Theme section, owned by the themes team per the doc.
+
+The summary (`qa-summary.md`, also appended to `$GITHUB_STEP_SUMMARY` on CI)
+mirrors the QA doc structure section by section.
+
+## Running locally
+
+```sh
+# Build the CLI first (the runner targets the repo build by default)
+pnpm nx run-many --all --target=build
+
+cd packages/qa
+pnpm install
+
+# Requires an authenticated CLI session (shopify auth login) or
+# SHOPIFY_APP_AUTOMATION_TOKEN, plus an org:
+QA_ORG_ID=<org-id> pnpm qa
+
+# Run a subset while iterating:
+QA_ONLY=hydrogen pnpm qa
+QA_ONLY=prep,apps QA_ORG_ID=… pnpm qa
+```
+
+## Environment variables
+
+| Variable | Purpose |
+| --- | --- |
+| `QA_CLI_BIN` | Path to the CLI under test. Default: `packages/cli/bin/run.js` (repo build). Point it at an installed `shopify` binary to QA a published version. |
+| `QA_EXPECTED_VERSION` | Assert `shopify version` equals this (doc: "the version should match the nightly you just created"). |
+| `QA_ORG_ID` / `E2E_ORG_ID` | Organization used by `app init`. |
+| `QA_PACKAGE_MANAGER` | Package manager for `app init` (default `pnpm`). |
+| `QA_ISOLATE` | `1` = fresh XDG dirs (don't reuse the local CLI session). Set on CI. |
+| `QA_WORK_DIR` | Scratch dir (default: fresh temp dir). |
+| `QA_ONLY` | Comma-separated section filter (`prep`, `general`, `apps`, `theme`, `hydrogen`). |
+| `QA_REPORT_DIR` | Where `qa-summary.md` / `qa-report.json` are written (default `./qa-report`). |
+| `SHOPIFY_APP_AUTOMATION_TOKEN` | Headless auth for app-platform commands on CI. |
+
+## Auth
+
+- **Locally**: the runner reuses your ambient `shopify auth login` session.
+- **CI**: `QA_ISOLATE=1` plus `SHOPIFY_APP_AUTOMATION_TOKEN` (Genghis e2e org).
+  Steps that turn out to require a browser user session are expected to fail
+  loudly in the report rather than hang: every interactive prompt has a
+  timeout that dumps the captured output.
+
+## Relationship to packages/e2e
+
+`packages/e2e` is the Playwright-based PR test suite. This package is
+intentionally independent (per-team decision): it duplicates the few markers it
+needs (e.g. the dev "Ready, watching for changes" string) instead of importing
+from e2e, so the QA flow stays a faithful, standalone replica of the doc.

packages/qa/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@shopify/qa",
+  "version": "1.0.0",
+  "packageManager": "pnpm@10.11.1",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "qa": "tsx src/run.ts",
+    "type-check": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "@types/node": "18.19.130",
+    "execa": "^7.2.0",
+    "node-pty": "^1.0.0",
+    "strip-ansi": "^7.2.0",
+    "tsx": "^4.23.1",
+    "typescript": "5.9.3"
+  },
+  "engines": {
+    "node": ">=22.12.0"
+  }
+}

packages/qa/src/context.ts

@@ -0,0 +1,114 @@
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
+import {fileURLToPath} from 'url'
+import type {PtyProcess} from './proc.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export interface CtxState {
+  /** Directory of the app created by `app init`. */
+  appDir?: string
+  /** Name given to the app created by `app init`. */
+  appName?: string
+  /** Directory of the hydrogen app. */
+  hydrogenDir?: string
+  /** Name of the secondary config created by `app config link` (e.g. "staging"). */
+  secondaryConfig?: string
+  /** Any pty processes to kill during teardown. */
+  ptyProcs: PtyProcess[]
+}
+
+export interface Ctx {
+  /** argv prefix used to invoke the CLI under test, e.g. ['node', '/path/to/run.js'] or ['shopify']. */
+  cliInvoke: string[]
+  /** Human description of what is being tested (repo build vs installed package). */
+  cliTarget: string
+  /** Root scratch directory for this run. */
+  workDir: string
+  /** Environment passed to every CLI invocation. */
+  env: {[key: string]: string}
+  orgId?: string
+  /** When set, the version step asserts `shopify version` equals this. */
+  expectedVersion?: string
+  state: CtxState
+  log: (msg: string) => void
+}
+
+function repoRoot(): string {
+  // packages/qa/src -> repo root
+  return path.resolve(__dirname, '..', '..', '..')
+}
+
+/**
+ * Build the run context from environment variables.
+ *
+ * QA_CLI_BIN            Path to the CLI entrypoint (JS file or executable).
+ *                       Defaults to the repo build at packages/cli/bin/run.js.
+ * QA_EXPECTED_VERSION   Assert `shopify version` equals this value.
+ * QA_ORG_ID             Organization id used for `app init` (falls back to E2E_ORG_ID).
+ * QA_WORK_DIR           Scratch dir (defaults to a fresh mkdtemp).
+ * QA_ISOLATE            When "1", run with fresh XDG dirs so the ambient CLI session
+ *                       is not used. CI sets this; local runs default to ambient auth.
+ */
+export function createContext(log: (msg: string) => void = defaultLog): Ctx {
+  const binEnv = process.env.QA_CLI_BIN
+  const defaultBin = path.join(repoRoot(), 'packages', 'cli', 'bin', 'run.js')
+  const bin = binEnv ?? defaultBin
+
+  let cliInvoke: string[]
+  if (bin.endsWith('.js') || bin.endsWith('.mjs') || bin.endsWith('.cjs')) {
+    cliInvoke = [process.execPath, bin]
+  } else {
+    cliInvoke = [bin]
+  }
+  const cliTarget = binEnv ? `installed CLI at ${bin}` : 'repo build (packages/cli/bin/run.js)'
+
+  const workDir =
+    process.env.QA_WORK_DIR ?? fs.mkdtempSync(path.join(os.tmpdir(), 'shopify-qa-'))
+  fs.mkdirSync(workDir, {recursive: true})
+
+  const env: {[key: string]: string} = {}
+  for (const [key, value] of Object.entries(process.env)) {
+    if (value !== undefined) env[key] = value
+  }
+  // Deterministic, machine-readable CLI output.
+  env.FORCE_COLOR = '0'
+  env.SHOPIFY_CLI_NO_ANALYTICS = '1'
+  env.SHOPIFY_FLAG_VERBOSE = ''
+  delete env.NODE_OPTIONS
+  // pnpm sets INIT_CWD to where `pnpm qa` was invoked; cli-kit's cwd() prefers
+  // it over the real process cwd, which would break every --path resolution.
+  // proc.ts re-points INIT_CWD/PWD at the effective cwd per invocation.
+  delete env.INIT_CWD
+  delete env.PWD
+  // Some environments (nix shells, agent harnesses) export NPM_CONFIG_NODE_VERSION;
+  // pnpm trusts it over the real node version and fails template engine checks.
+  for (const key of Object.keys(env)) {
+    if (key.toLowerCase() === 'npm_config_node_version') delete env[key]
+  }
+
+  if (process.env.QA_ISOLATE === '1') {
+    const authDir = path.join(workDir, 'xdg')
+    for (const name of ['XDG_DATA_HOME', 'XDG_CONFIG_HOME', 'XDG_STATE_HOME', 'XDG_CACHE_HOME']) {
+      const dir = path.join(authDir, name)
+      fs.mkdirSync(dir, {recursive: true})
+      env[name] = dir
+    }
+  }
+
+  return {
+    cliInvoke,
+    cliTarget,
+    workDir,
+    env,
+    orgId: process.env.QA_ORG_ID ?? process.env.E2E_ORG_ID,
+    expectedVersion: process.env.QA_EXPECTED_VERSION,
+    state: {ptyProcs: []},
+    log,
+  }
+}
+
+function defaultLog(msg: string): void {
+  process.stdout.write(`[qa] ${msg}\n`)
+}