From c433bb5ccd5db0c16ce0a307466ab52f292ddb4b Mon Sep 17 00:00:00 2001 From: Ryan Hughes Date: Sun, 20 Sep 2026 15:43:32 -0400 Subject: [PATCH] Reuse matching daily builder images in package jobs --- .github/workflows/build-pr.yml | 1 + .github/workflows/publish.yml | 21 ++++-- README.md | 28 +++++++- bin/build | 12 ++++ bin/builder-image | 41 +++++++++++- tests/builder-image.cjs | 114 ++++++++++++++++++++++++++++++++- 6 files changed, 206 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index 7adbfcc30..021ef838b 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -161,6 +161,7 @@ jobs: - name: Build ${{ matrix.package }} (${{ matrix.arch }}, ships to ${{ matrix.channels }}) env: CONTAINER_ENGINE: docker + OMARCHY_PREBUILT_IMAGES: '1' run: bin/build --mirror edge --arch ${{ matrix.arch }} --package ${{ matrix.package }} # The artifact label carries the package directory's git tree hash so # the publish step can find the build for exactly the tree that merged. diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 75f236fa0..5db0c4e2d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -67,6 +67,8 @@ jobs: runs-on: [self-hosted, omarchy-builder] environment: publish timeout-minutes: 240 + env: + OMARCHY_PREBUILT_IMAGES: '1' steps: - uses: actions/checkout@v4 with: @@ -141,8 +143,20 @@ jobs: exit 1 fi + - name: Prepare publishing image + id: publish-image + env: + CONTAINER_ENGINE: docker + run: | + set -euo pipefail + if compgen -G "build-output/edge/*/*.pkg.tar.zst" >/dev/null; then + image=$(bin/builder-image prepare --arch x86_64 --mirror edge) + echo "image=$image" >> "$GITHUB_OUTPUT" + fi + - name: Publish env: + PUBLISH_BUILDER_IMAGE: ${{ steps.publish-image.outputs.image }} GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} RCLONE_CONFIG_R2_TYPE: s3 @@ -168,9 +182,6 @@ jobs: cat publish-record.json exit 0 fi - docker image inspect omarchy-pkg-builder:latest-x86_64-edge >/dev/null 2>&1 \ - || docker buildx build --load -t omarchy-pkg-builder:latest-x86_64-edge --build-arg MIRROR=edge build - # Group the merge's files by the (channel, architecture) slot each # belongs to. A package's files live under build-output/edge// and are named --.pkg.tar.zst; a @@ -178,7 +189,7 @@ jobs: # on the artifact list rather than the name. # pkgbase is read inside the builder image: the Ubuntu host has no # bsdtar. One container call maps every file to its pkgbase. - docker run --rm -v "$PWD:/w:ro" -w /w omarchy-pkg-builder:latest-x86_64-edge bash -c ' + docker run --rm -v "$PWD:/w:ro" -w /w "$PUBLISH_BUILDER_IMAGE" bash -c ' for f in build-output/edge/*/*.pkg.tar.zst; do printf "%s %s\n" "$f" "$(bsdtar -xOf "$f" .PKGINFO | awk -F" = " "\$1==\"pkgbase\"{print \$2}")" done' > pkgbase.txt @@ -215,7 +226,7 @@ jobs: -e RCLONE_CONFIG_R2_TYPE -e RCLONE_CONFIG_R2_PROVIDER -e RCLONE_CONFIG_R2_ENDPOINT -e RCLONE_CONFIG_R2_NO_CHECK_BUCKET \ -e RCLONE_CONFIG_R2_ACCESS_KEY_ID -e RCLONE_CONFIG_R2_SECRET_ACCESS_KEY \ -v "$PWD:/w:ro" -w /w \ - omarchy-pkg-builder:latest-x86_64-edge \ + "$PUBLISH_BUILDER_IMAGE" \ bin/publish-artifact --remote R2:omarchy-pkgs --mirror "$mirror" --arch "$parch" $files; then record_slot "$mirror" "$parch" published "$files" else diff --git a/README.md b/README.md index 8823595e3..3849a156c 100644 --- a/README.md +++ b/README.md @@ -861,8 +861,32 @@ creates the package private. In the `omacom/omarchy-pkg-builder` package settings, change visibility to **Public**, then rerun the failed refresh job. The workflow checks anonymous registry access before advancing the compatible tag, so fork PRs will not be directed to an image they cannot pull. Subsequent -refreshes preserve that package visibility. This change only produces images; -package jobs keep their existing behavior until image consumption is enabled. +refreshes preserve that package visibility. + +PR builds and publishing set `OMARCHY_PREBUILT_IMAGES=1`. `bin/build` then +pulls the compatible tag, verifies its input key and architecture, and uses +its immutable registry digest throughout that invocation. Missing, private, +unavailable, or mismatched images fall back to building from the checked-out +inputs and pinning the resulting local image ID. Publishing uses the same +selection for its Arch utilities, before signing credentials are supplied. +The selected reference is printed in the job log. No default-branch checkout +is substituted for the job's own build definition. + +Local builds retain their existing behavior unless this flag is set. An +explicit `OMARCHY_SKIP_BUILDER_IMAGE=1` still takes precedence, including in +isolation tests. `bin/builder-image prepare --arch x86_64 --mirror edge` can +also be used directly; stdout is the immutable reference and diagnostics go +to stderr. `OMARCHY_BUILDER_IMAGE_REPOSITORY` overrides the registry repository +for local testing. Package containers still run `pacman -Syu` before installing +dependencies, and installed package state remains isolated between builds. + +Roll out image production first, make the GHCR package public, and rerun its +initial refresh. Then enable consumption and compare x86_64 and ARM job times. +During a registry outage or before the first matching publication, source +builds continue to work. Reverting the consumer integration restores the +previous image construction in package builds and publishing; the daily image +producer can remain enabled. Neither stage requires a DO controller deployment +or a VM snapshot change. ## Version Management diff --git a/bin/build b/bin/build index 765e0713d..fd29e2bec 100755 --- a/bin/build +++ b/bin/build @@ -26,12 +26,16 @@ DRY_RUN=false # that builds the image once with an external # BuildKit cache can then fan out over many # package jobs without each one rebuilding it. +# OMARCHY_PREBUILT_IMAGES=1 prefer a tested registry image matching this +# checkout's build inputs; otherwise build +# locally. SKIP_BUILDER_IMAGE takes precedence. # OMARCHY_DEFER_RUNTIME_DEPS=true build the omarchy/omarchy-settings pair # with --nodeps (see build/build.sh); only # for a pipeline that verifies the install # transaction afterwards. KEEP_BUILD_WORKSPACE=${OMARCHY_KEEP_BUILD_WORKSPACE:-0} SKIP_BUILDER_IMAGE=${OMARCHY_SKIP_BUILDER_IMAGE:-0} +PREBUILT_IMAGES=${OMARCHY_PREBUILT_IMAGES:-0} DEFER_RUNTIME_DEPS=${OMARCHY_DEFER_RUNTIME_DEPS:-false} # Parse command line arguments @@ -91,6 +95,7 @@ while [[ $# -gt 0 ]]; do echo "Environment (for CI and resumed builds; defaults keep today's behaviour):" echo " OMARCHY_KEEP_BUILD_WORKSPACE=1 keep build-output and reuse packages already there" echo " OMARCHY_SKIP_BUILDER_IMAGE=1 use the existing builder image instead of building it" + echo " OMARCHY_PREBUILT_IMAGES=1 pull a matching tested image, or build locally if unavailable" echo " OMARCHY_DEFER_RUNTIME_DEPS=true build the omarchy pair with --nodeps (transaction verified later)" echo " OMARCHY_PUBLISHED_REPO_URL= channel to plan and resolve against when no local tree exists" echo " (default https://pkgs.omarchy.org; empty disables the fallback)" @@ -106,6 +111,11 @@ done require_valid_arch "$ARCH" +if [[ $PREBUILT_IMAGES != 0 && $PREBUILT_IMAGES != 1 ]]; then + print_error "OMARCHY_PREBUILT_IMAGES must be 0 or 1" + exit 1 +fi + if [[ $DEFER_RUNTIME_DEPS != "false" && $DEFER_RUNTIME_DEPS != "true" ]]; then print_error "OMARCHY_DEFER_RUNTIME_DEPS must be true or false" exit 1 @@ -224,6 +234,8 @@ if [[ $SKIP_BUILDER_IMAGE == "1" ]]; then exit 1 fi print_info "Using prepared builder image: $IMAGE_TAG" +elif [[ $PREBUILT_IMAGES == 1 ]]; then + IMAGE_TAG=$("$BUILD_ROOT/bin/builder-image" prepare --arch "$ARCH" --mirror "$MIRROR") else build_docker_image "$BUILD_DIR" "$ARCH" "$MIRROR" fi diff --git a/bin/builder-image b/bin/builder-image index f23f64bf0..bdf575f8b 100755 --- a/bin/builder-image +++ b/bin/builder-image @@ -6,7 +6,7 @@ BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") source "$BUILD_ROOT/helpers/paths.sh" usage() { - echo "Usage: bin/builder-image {key|build} [--arch x86_64|aarch64] [--mirror edge|rc|stable] [--tag IMAGE] [--fresh]" + echo "Usage: bin/builder-image {key|build|prepare} [--arch x86_64|aarch64] [--mirror edge|rc|stable] [--tag IMAGE] [--fresh]" } command=${1:-} @@ -24,8 +24,8 @@ while (( $# )); do done require_valid_arch "$ARCH" validate_mirror "$MIRROR" || { echo "Invalid mirror: $MIRROR" >&2; exit 1; } -case "$command" in key|build) ;; *) usage >&2; exit 1 ;; esac -if [[ $command == key && ( -n $tag || $fresh == true ) ]]; then +case "$command" in key|build|prepare) ;; *) usage >&2; exit 1 ;; esac +if [[ $command != build && ( -n $tag || $fresh == true ) ]]; then usage >&2 exit 1 fi @@ -47,6 +47,41 @@ source "$BUILD_ROOT/helpers/docker-helpers.sh" check_engine platform=$(get_platform_arg "$ARCH") tag=${tag:-omarchy-pkg-builder:latest-$ARCH-$MIRROR} + +if [[ $command == prepare ]]; then + # All diagnostics go to stderr; stdout is one immutable reference, suitable + # for every container in this build. Never use a registry's unqualified + # "latest", and never trust a local tag left over from a failed pull. + repository=${OMARCHY_BUILDER_IMAGE_REPOSITORY:-ghcr.io/omacom/omarchy-pkg-builder} + reference="$repository:$key" + echo "Looking for prepared builder: $reference" >&2 + if "$CONTAINER_ENGINE" pull "$platform" "$reference" >&2; then + if metadata=$("$CONTAINER_ENGINE" image inspect "$reference"); then + expected_arch=${platform##*/} + if pinned=$(jq -er --arg key "$key" --arg arch "$expected_arch" --arg repository "$repository" ' + .[0] | select(.Config.Labels["org.omarchy.builder.key"] == $key and + .Architecture == $arch and .Os == "linux") | + first(.RepoDigests[] | select(startswith($repository + "@sha256:")) | + select(split("@sha256:")[1] | test("^[a-f0-9]{64}$"))) + ' <<<"$metadata"); then + echo "Using prepared builder: $pinned" >&2 + echo "$pinned" + exit 0 + fi + fi + echo "Prepared builder metadata does not match this checkout; building locally." >&2 + else + echo "No accessible matching builder; building this checkout's image locally." >&2 + fi + build_docker_image "$BUILD_DIR" "$ARCH" "$MIRROR" >&2 + # Pin the local fallback too, rather than running a mutable convenience tag. + image_id=$("$CONTAINER_ENGINE" image inspect "$tag" --format '{{.Id}}') + [[ $image_id =~ ^sha256:[a-f0-9]{64}$ ]] || { echo "Invalid local image ID" >&2; exit 1; } + echo "Using locally built environment: $image_id" >&2 + echo "$image_id" + exit 0 +fi + revision=$(git -C "$BUILD_ROOT" rev-parse HEAD 2>/dev/null || echo unknown) args=("$platform" --build-arg "MIRROR=$MIRROR" --label "org.omarchy.builder.key=$key" diff --git a/tests/builder-image.cjs b/tests/builder-image.cjs index c20f306a8..9a3816b7d 100644 --- a/tests/builder-image.cjs +++ b/tests/builder-image.cjs @@ -15,6 +15,7 @@ function fixture(t) { mkdirSync(join(directory, 'build')); cpSync(join(root, 'helpers'), join(directory, 'helpers'), { recursive: true }); cpSync(join(root, 'bin/builder-image'), join(directory, 'bin/builder-image')); + cpSync(join(root, 'bin/build'), join(directory, 'bin/build')); writeFileSync(join(directory, 'build/Dockerfile'), 'FROM scratch\nCOPY input /input\n'); writeFileSync(join(directory, 'build/input'), 'original input\n'); const engine = join(directory, 'engine'); @@ -26,7 +27,20 @@ const args = process.argv.slice(2); fs.appendFileSync(process.env.ENGINE_LOG, JSON.stringify(args) + '\\n'); if (args[0] === 'manifest' && process.env.PRIVATE_IMAGE === '1') process.exit(1); if (args[0] === 'push' && process.env.PUSH_FAIL === '1') process.exit(1); -if (args[0] === 'image' && args[1] === 'inspect') console.log('ghcr.io/omacom/omarchy-pkg-builder@sha256:' + 'a'.repeat(64)); +if (args[0] === 'pull' && process.env.PULL_FAIL === '1') process.exit(1); +if (args[0] === 'buildx' && process.env.BUILD_FAIL === '1') process.exit(1); +if (args[0] === 'image' && args[1] === 'inspect') { + if (!args.includes('--format') && process.env.INSPECT_FAIL === '1') process.exit(1); + if (args.includes('{{.Id}}')) console.log('sha256:' + 'b'.repeat(64)); + else if (args.includes('--format')) console.log('ghcr.io/omacom/omarchy-pkg-builder@sha256:' + 'a'.repeat(64)); + else console.log(process.env.IMAGE_METADATA || '[]'); +} +if (args[0] === 'run' && process.env.FAKE_PACKAGE_RUN === '1' && args.includes('DRY_RUN=true')) { + const mount = args.find(arg => arg.endsWith(':/build-plan')).split(':')[0]; + fs.writeFileSync(mount + '/packages', 'example\\n'); + fs.writeFileSync(mount + '/skipped', ''); + fs.writeFileSync(mount + '/dependencies', ''); +} `); chmodSync(join(engine, 'docker'), 0o755); const env = { @@ -128,3 +142,101 @@ test('a private image or failed push never replaces the previous compatible-imag assert.equal(f.calls().some(args => args[0] === 'push' && args[1] === `ghcr.io/omacom/omarchy-pkg-builder:${key}`), false); } }); + +const repository = 'ghcr.io/omacom/omarchy-pkg-builder'; +const digest = `${repository}@sha256:${'a'.repeat(64)}`; +const localId = `sha256:${'b'.repeat(64)}`; +function metadata(f, overrides = {}) { + return JSON.stringify([{ + Config: { Labels: { 'org.omarchy.builder.key': f.key() } }, + Architecture: 'amd64', Os: 'linux', RepoDigests: [digest], ...overrides, + }]); +} + +test('a matching image is selected by input key and pinned to its registry digest', t => { + const f = fixture(t); + const result = f.run(['prepare'], { IMAGE_METADATA: metadata(f) }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout.trim(), digest); + assert.deepEqual(f.calls().find(args => args[0] === 'pull'), ['pull', '--platform=linux/amd64', `${repository}:${f.key()}`]); + assert.equal(f.calls().some(args => args[0] === 'buildx'), false); +}); + +test('mismatched or incomplete registry metadata falls back to this checkout and pins the local image', t => { + for (const overrides of [ + { Config: { Labels: { 'org.omarchy.builder.key': 'different-inputs' } } }, + { Config: {} }, { Architecture: 'arm64' }, { Os: 'windows' }, + { RepoDigests: [] }, { RepoDigests: [`${repository}@sha256:invalid`] }, + { RepoDigests: ['unrelated.example/builder@sha256:' + 'a'.repeat(64)] }, + ]) { + const f = fixture(t); + const result = f.run(['prepare'], { IMAGE_METADATA: metadata(f, overrides) }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout.trim(), localId); + const build = f.calls().find(args => args[0] === 'buildx'); + assert.equal(build.at(-1), join(f.directory, 'build')); + assert.ok(build.includes('MIRROR=edge')); + } +}); + +test('missing or inaccessible images and inspection failures preserve local builds', t => { + for (const extraEnv of [{ PULL_FAIL: '1' }, { INSPECT_FAIL: '1' }]) { + const f = fixture(t); + const result = f.run(['prepare'], { IMAGE_METADATA: metadata(f), ...extraEnv }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout.trim(), localId); + assert.ok(f.calls().some(args => args[0] === 'buildx')); + if (extraEnv.PULL_FAIL) { + assert.ok(f.calls().findIndex(args => args[0] === 'buildx') < f.calls().findIndex(args => args[0] === 'image'), 'do not reuse a stale local tag after a failed pull'); + } + } +}); + +test('a failed fallback build fails the job instead of reusing a stale image', t => { + const f = fixture(t); + const result = f.run(['prepare'], { PULL_FAIL: '1', BUILD_FAIL: '1' }); + assert.notEqual(result.status, 0); + assert.equal(result.stdout, ''); + assert.equal(f.calls().some(args => args[0] === 'image'), false); +}); + +test('all package containers use the selected immutable image; local and explicit-image modes still work', t => { + for (const mode of ['registry', 'fallback', 'local', 'explicit']) { + const f = fixture(t); + const result = spawnSync(join(f.directory, 'bin/build'), ['--package', 'example'], { + cwd: f.directory, encoding: 'utf8', env: { + ...f.env, IMAGE_METADATA: metadata(f), FAKE_PACKAGE_RUN: '1', + OMARCHY_PREBUILT_IMAGES: mode === 'local' ? '0' : '1', + OMARCHY_SKIP_BUILDER_IMAGE: mode === 'explicit' ? '1' : '0', + PULL_FAIL: mode === 'fallback' ? '1' : '0', + }, + }); + assert.equal(result.status, 0, `${mode}: ${result.stderr}\n${result.stdout}`); + const calls = f.calls(); + const containers = calls.filter(args => args[0] === 'run'); + assert.equal(containers.length, 2, 'one planning container and one package container'); + const expected = mode === 'registry' ? digest : mode === 'fallback' ? localId : 'omarchy-pkg-builder:latest-x86_64-edge'; + for (const args of containers) assert.equal(args.at(-2), expected); + if (mode === 'local' || mode === 'explicit') assert.equal(calls.some(args => args[0] === 'pull'), false); + if (mode === 'explicit') assert.equal(calls.some(args => args[0] === 'buildx'), false); + } +}); + +test('publishing prepares the same matching image only when there are artifacts', t => { + const publishWorkflow = readFileSync(join(root, '.github/workflows/publish.yml'), 'utf8'); + const script = publishWorkflow.split(' - name: Prepare publishing image\n')[1].split(' run: |\n')[1].split(' - name: Publish\n')[0]; + for (const artifacts of [false, true]) { + const f = fixture(t); + if (artifacts) { + mkdirSync(join(f.directory, 'build-output/edge/x86_64'), { recursive: true }); + writeFileSync(join(f.directory, 'build-output/edge/x86_64/example.pkg.tar.zst'), 'fixture'); + } + const output = join(f.directory, 'outputs'); + writeFileSync(output, ''); + const result = spawnSync('bash', ['-e', '-o', 'pipefail', '-c', script], { + cwd: f.directory, encoding: 'utf8', env: { ...f.env, IMAGE_METADATA: metadata(f), GITHUB_OUTPUT: output }, + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(output, 'utf8'), artifacts ? `image=${digest}\n` : ''); + } +});