Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 139 additions & 45 deletions .github/workflows/build-images.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,27 @@
name: Build & push md-workflows images

# Builds the three-stage image chain on the self-hosted diffuse builder and pushes to Harbor.
# Builds the three-stage image chain on the self-hosted diffuse builder and publishes it:
# - base + gromacs -> ghcr.io/prism-science/md-workflows (public; tags prefixed base-/gromacs-),
# pushed with the workflow's own GITHUB_TOKEN;
# - actl (the Astera ACTL overlay) -> Astera's internal registry, library/md-workflows. The
# registry host is the ASTERA_REGISTRY secret, never written in this public repo; GitHub masks
# it wherever it is echoed, including inside the longer image refs built from it.
#
# Security: this workflow has NO pull_request trigger, so fork PRs can never reach the
# self-hosted runner or the Harbor credentials. Heavy builds run only on push to `astera`
# (path-filtered) and manual dispatch. Credentials live in the `harbor` GitHub Environment,
# which must be restricted to the `astera` deployment branch; the `diffuse-sh-builder` runner
# should sit in a runner group scoped to this repo. See the runbook in the PR description.
# self-hosted runner or any credentials. Heavy builds run only on push to `astera`
# (path-filtered) and manual dispatch. All three publishing jobs run in the `harbor` GitHub
# Environment, which must be restricted to the `astera` deployment branch: that restriction is
# what limits publishing -- including the public GHCR images -- to `astera`, and the environment
# holds the Astera registry secrets (ASTERA_REGISTRY, HARBOR_USERNAME, HARBOR_PASSWORD). The
# `diffuse-sh-builder` runner should sit in a runner group scoped to this repo. See the runbook
# in the PR description.
#
# Manual dispatch runs only the `test-build` job unless `publish` is ticked: all three stages
# built on the runner's local Docker daemon, smoke-tested, then untagged. It pushes nothing and
# uses no registry credentials, so it needs no `harbor` environment and can be dispatched from
# any branch to try Dockerfile changes before they reach `astera`:
# gh workflow run build-images.yml --ref <branch> # build + smoke-test
# gh workflow run build-images.yml --ref astera -f publish=true # build + publish
#
# NOTE: runs-on matches only the unique custom label `diffuse-sh-builder` (not `self-hosted`),
# because that runner was registered with --no-default-labels and so lacks the implicit
Expand All @@ -22,6 +37,11 @@ on:
- md_workflows/**
- .github/workflows/build-images.yml
workflow_dispatch:
inputs:
publish:
description: "Publish the images (GHCR + Astera registry). Leave unticked to build + smoke-test only."
type: boolean
default: false

permissions:
contents: read
Expand All @@ -31,15 +51,15 @@ concurrency:
cancel-in-progress: true

env:
REGISTRY: harbor.astera.sh
# base + gromacs (bake the non-redistributable ChimeraX) live in the proprietary project,
# distinguished by a stage tag prefix. The final consumable actl image lands in library.
PROPRIETARY_IMAGE: harbor.astera.sh/diffuse-proprietary/md-workflows
LIBRARY_IMAGE: harbor.astera.sh/library/md-workflows
# Public stages, distinguished by a stage tag prefix.
PUBLIC_IMAGE: ghcr.io/prism-science/md-workflows
# Path of the actl overlay on the Astera registry; the host comes from secrets.ASTERA_REGISTRY.
ASTERA_IMAGE_PATH: library/md-workflows

jobs:
version:
name: Compute version
if: github.event_name == 'push' || inputs.publish
runs-on: ubuntu-latest
outputs:
semver: ${{ steps.v.outputs.semver }}
Expand All @@ -61,20 +81,25 @@ jobs:

base:
name: Build base
if: github.event_name == 'push' || inputs.publish
needs: version
runs-on: [diffuse-sh-builder]
# Uses no environment secrets; the environment's branch restriction gates publishing.
environment: harbor
permissions:
contents: read
packages: write
outputs:
digest: ${{ steps.build.outputs.digest }}
steps:
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- name: Login to Harbor
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.HARBOR_USERNAME }}
password: ${{ secrets.HARBOR_PASSWORD }}
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- id: build
uses: docker/build-push-action@v6
with:
Expand All @@ -84,29 +109,34 @@ jobs:
push: true
provenance: false
tags: |
${{ env.PROPRIETARY_IMAGE }}:base-${{ needs.version.outputs.semver }}
${{ env.PROPRIETARY_IMAGE }}:base-${{ needs.version.outputs.build }}
${{ env.PROPRIETARY_IMAGE }}:base-${{ needs.version.outputs.sha }}
${{ env.PROPRIETARY_IMAGE }}:base
cache-from: type=registry,ref=${{ env.PROPRIETARY_IMAGE }}:base-buildcache
cache-to: type=registry,ref=${{ env.PROPRIETARY_IMAGE }}:base-buildcache,mode=max
${{ env.PUBLIC_IMAGE }}:base-${{ needs.version.outputs.semver }}
${{ env.PUBLIC_IMAGE }}:base-${{ needs.version.outputs.build }}
${{ env.PUBLIC_IMAGE }}:base-${{ needs.version.outputs.sha }}
${{ env.PUBLIC_IMAGE }}:base
cache-from: type=registry,ref=${{ env.PUBLIC_IMAGE }}:base-buildcache
cache-to: type=registry,ref=${{ env.PUBLIC_IMAGE }}:base-buildcache,mode=max

gromacs:
name: Build gromacs
if: github.event_name == 'push' || inputs.publish
needs: [version, base]
runs-on: [diffuse-sh-builder]
# Uses no environment secrets; the environment's branch restriction gates publishing.
environment: harbor
permissions:
contents: read
packages: write
outputs:
digest: ${{ steps.build.outputs.digest }}
steps:
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- name: Login to Harbor
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.HARBOR_USERNAME }}
password: ${{ secrets.HARBOR_PASSWORD }}
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- id: build
uses: docker/build-push-action@v6
with:
Expand All @@ -116,29 +146,48 @@ jobs:
push: true
provenance: false
build-args: |
BASE_IMAGE=${{ env.PROPRIETARY_IMAGE }}@${{ needs.base.outputs.digest }}
BASE_IMAGE=${{ env.PUBLIC_IMAGE }}@${{ needs.base.outputs.digest }}
tags: |
${{ env.PROPRIETARY_IMAGE }}:gromacs-${{ needs.version.outputs.semver }}
${{ env.PROPRIETARY_IMAGE }}:gromacs-${{ needs.version.outputs.build }}
${{ env.PROPRIETARY_IMAGE }}:gromacs-${{ needs.version.outputs.sha }}
${{ env.PROPRIETARY_IMAGE }}:gromacs
cache-from: type=registry,ref=${{ env.PROPRIETARY_IMAGE }}:gromacs-buildcache
cache-to: type=registry,ref=${{ env.PROPRIETARY_IMAGE }}:gromacs-buildcache,mode=max
${{ env.PUBLIC_IMAGE }}:gromacs-${{ needs.version.outputs.semver }}
${{ env.PUBLIC_IMAGE }}:gromacs-${{ needs.version.outputs.build }}
${{ env.PUBLIC_IMAGE }}:gromacs-${{ needs.version.outputs.sha }}
${{ env.PUBLIC_IMAGE }}:gromacs
cache-from: type=registry,ref=${{ env.PUBLIC_IMAGE }}:gromacs-buildcache
cache-to: type=registry,ref=${{ env.PUBLIC_IMAGE }}:gromacs-buildcache,mode=max

actl:
name: Build actl (consumable)
needs: [version, gromacs]
name: Build actl (Astera overlay)
if: github.event_name == 'push' || inputs.publish
needs: [version, base, gromacs]
runs-on: [diffuse-sh-builder]
environment: harbor
permissions:
contents: read
packages: read # pull the gromacs parent from GHCR, even while the package is private
env:
# Bare registry host only; the image path stays in the workflow env above.
ASTERA_REGISTRY: ${{ secrets.ASTERA_REGISTRY }}
outputs:
digest: ${{ steps.build.outputs.digest }}
steps:
- name: Check the registry secret is set
run: |
if [ -z "${ASTERA_REGISTRY}" ]; then
echo "::error::Set ASTERA_REGISTRY (the bare Astera registry host) in the harbor environment."
exit 1
fi
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- name: Login to Harbor
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to the Astera registry
uses: docker/login-action@v3
with:
registry: ${{ env.ASTERA_REGISTRY }}
username: ${{ secrets.HARBOR_USERNAME }}
password: ${{ secrets.HARBOR_PASSWORD }}
- id: build
Expand All @@ -150,18 +199,63 @@ jobs:
push: true
provenance: false
build-args: |
GROMACS_IMAGE=${{ env.PROPRIETARY_IMAGE }}@${{ needs.gromacs.outputs.digest }}
GROMACS_IMAGE=${{ env.PUBLIC_IMAGE }}@${{ needs.gromacs.outputs.digest }}
tags: |
${{ env.LIBRARY_IMAGE }}:${{ needs.version.outputs.semver }}
${{ env.LIBRARY_IMAGE }}:${{ needs.version.outputs.build }}
${{ env.LIBRARY_IMAGE }}:${{ needs.version.outputs.sha }}
${{ env.LIBRARY_IMAGE }}:actl
${{ env.LIBRARY_IMAGE }}:latest
cache-from: type=registry,ref=${{ env.LIBRARY_IMAGE }}:actl-buildcache
cache-to: type=registry,ref=${{ env.LIBRARY_IMAGE }}:actl-buildcache,mode=max
${{ env.ASTERA_REGISTRY }}/${{ env.ASTERA_IMAGE_PATH }}:${{ needs.version.outputs.semver }}
${{ env.ASTERA_REGISTRY }}/${{ env.ASTERA_IMAGE_PATH }}:${{ needs.version.outputs.build }}
${{ env.ASTERA_REGISTRY }}/${{ env.ASTERA_IMAGE_PATH }}:${{ needs.version.outputs.sha }}
${{ env.ASTERA_REGISTRY }}/${{ env.ASTERA_IMAGE_PATH }}:actl
${{ env.ASTERA_REGISTRY }}/${{ env.ASTERA_IMAGE_PATH }}:latest
cache-from: type=registry,ref=${{ env.ASTERA_REGISTRY }}/${{ env.ASTERA_IMAGE_PATH }}:actl-buildcache
cache-to: type=registry,ref=${{ env.ASTERA_REGISTRY }}/${{ env.ASTERA_IMAGE_PATH }}:actl-buildcache,mode=max
- name: Report digests
env:
BUILD: ${{ needs.version.outputs.build }}
run: |
echo "base digest: ${{ needs.base.outputs.digest }}" || true
echo "base digest: ${{ needs.base.outputs.digest }}"
echo "gromacs digest: ${{ needs.gromacs.outputs.digest }}"
echo "actl digest: ${{ steps.build.outputs.digest }}"
echo "Published ${{ env.LIBRARY_IMAGE }}:${{ needs.version.outputs.build }}"
echo "Published ${PUBLIC_IMAGE}:gromacs-${BUILD}"
echo "Published ${ASTERA_REGISTRY}/${ASTERA_IMAGE_PATH}:${BUILD}"

# publish unticked: the chain above pulls each parent stage from a registry by digest, so
# without a push the stages must instead be built in one job and chained through local tags,
# as in the README. `--builder default` pins the daemon's own BuildKit (docker driver), which
# can see locally built images in FROM, regardless of any buildx builder left selected on the
# runner.
test-build:
name: Build all stages (no push)
if: github.event_name == 'workflow_dispatch' && !inputs.publish
runs-on: [diffuse-sh-builder]
env:
TAG: test-${{ github.run_id }}
steps:
- uses: actions/checkout@v4
- name: Build base
run: >-
docker buildx build --builder default --platform linux/amd64
-f Dockerfile.base -t "md-base:${TAG}" .
- name: Build gromacs
run: >-
docker buildx build --builder default --platform linux/amd64
-f Dockerfile.gromacs --build-arg BASE_IMAGE="md-base:${TAG}" -t "md-gromacs:${TAG}" .
- name: Build actl
run: >-
docker buildx build --builder default --platform linux/amd64
-f Dockerfile.actl --build-arg GROMACS_IMAGE="md-gromacs:${TAG}" -t "md-actl:${TAG}" .
# GPU-free: tools the pipeline shells out to, the gemmi crystal code, and no ChimeraX.
- name: Smoke test
run: |
docker run --rm "md-actl:${TAG}" bash -c '
set -euo pipefail
md-workflows --help > /dev/null
python -c "import gemmi, md_workflows.core.crystal; print(gemmi.__version__)"
for cmd in gmx pdb4amber; do command -v "${cmd}"; done
if command -v chimerax ChimeraX > /dev/null 2>&1; then
echo "ChimeraX is still installed" >&2
exit 1
fi
'
- name: Remove test tags
if: always()
run: docker image rm -f "md-actl:${TAG}" "md-gromacs:${TAG}" "md-base:${TAG}" || true
27 changes: 27 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,33 @@ jobs:
# Contract + CLI tests are gmx-free, so they run on the hosted runner.
run: pytest -q

# The image's lunus env pins Python 3.10 (Dockerfile.base) while `lint` runs on 3.12, so
# 3.11+-only code -- an unconditional `import tomllib` -- once passed CI and broke the image's
# CLI. A separate job rather than a matrix on `lint` leaves that job's check name unchanged for
# any branch-protection rule that requires it.
test-py310:
name: Tests (Python 3.10, as in the image)
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'

- name: Install package
run: |
python -m pip install --upgrade pip
pip install '.[dev]'

- name: Run tests
run: pytest -q

- name: Check the CLI starts
run: md-workflows --help > /dev/null

build-check:
name: Dockerfile build checks (no push)
runs-on: ubuntu-latest
Expand Down
21 changes: 21 additions & 0 deletions .gitleaksignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Accepted findings for the org secret scan (prism-science/github-workflows, secret-scan.yml).
#
# Rule astera-internal-hostname: historical references to Astera's internal container registry
# host, in the build workflows, Dockerfiles and README of these commits (on main, astera and
# dev). The repo is public, so they are already disclosed and git history keeps them; the host
# was removed from the working tree in PR #15, where build-images.yml now reads it from the
# ASTERA_REGISTRY secret. The host is a registry endpoint, not a credential; registry access
# stays behind the Harbor credentials in the `harbor` environment.
119f1f5ad21e6a60a6d599e2ddee5ebd2ea73d30:.github/workflows/build-images.yml:astera-internal-hostname:36
f91aa04f88471ec7b71fc3e94c95821e05ed7057:Dockerfile.actl:astera-internal-hostname:11
f91aa04f88471ec7b71fc3e94c95821e05ed7057:.github/workflows/build-images.yml:astera-internal-hostname:30
f91aa04f88471ec7b71fc3e94c95821e05ed7057:.github/workflows/build-images.yml:astera-internal-hostname:33
f91aa04f88471ec7b71fc3e94c95821e05ed7057:.github/workflows/build-images.yml:astera-internal-hostname:34
7578642a282b3ce46bd55d0293459c008c9a2884:Dockerfile.astera:astera-internal-hostname:14
425a67755927313d8a065654fd7935c2c6e6ab27:Dockerfile.astera:astera-internal-hostname:13
425a67755927313d8a065654fd7935c2c6e6ab27:README.md:astera-internal-hostname:10
425a67755927313d8a065654fd7935c2c6e6ab27:README.md:astera-internal-hostname:40
9f2ddc1b545a90e828dd97c9ca359d60d522f7ac:Dockerfile.astera:astera-internal-hostname:13
9f2ddc1b545a90e828dd97c9ca359d60d522f7ac:.github/workflows/astera-docker.yml:astera-internal-hostname:12
9f2ddc1b545a90e828dd97c9ca359d60d522f7ac:README.md:astera-internal-hostname:10
9f2ddc1b545a90e828dd97c9ca359d60d522f7ac:README.md:astera-internal-hostname:40
2 changes: 1 addition & 1 deletion Dockerfile.actl
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
# Build locally (after building md-gromacs:local):
# docker buildx build --platform linux/amd64 \
# -f Dockerfile.actl --build-arg GROMACS_IMAGE=md-gromacs:local \
# -t harbor.astera.sh/library/md-workflows:local-actl .
# -t md-actl:local .
ARG GROMACS_IMAGE=md-gromacs:local
FROM ${GROMACS_IMAGE} AS actl

Expand Down
28 changes: 2 additions & 26 deletions Dockerfile.base
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# ==================== md-workflows base (stage 1 of 3) ====================
# Architecture-neutral foundation: CUDA 12.6 *devel* toolchain + the full scientific
# conda stack (AmberTools / DIALS / cctbx / mdtraj / …), lunus, and ChimeraX. This layer
# conda stack (AmberTools / DIALS / cctbx / mdtraj / …) and lunus. This layer
# carries everything that is portable across compute; the non-portable GROMACS compile and
# the md-workflows app are added downstream in Dockerfile.gromacs, and Astera workspace
# conventions in Dockerfile.actl.
Expand Down Expand Up @@ -106,33 +106,9 @@ RUN $MAMBA_EXE remove -n lunus -y scons \
rm -rf /opt/packages/lunus/.git; \
true

# ---------- ChimeraX ----------
ARG CHIMERAX_URL="https://www.cgl.ucsf.edu/chimerax/cgi-bin/secure/chimerax-get.py?file=current/ubuntu-22.04/chimerax-daily.deb"
# The download is a two-step token dance behind www.cgl.ucsf.edu (POST choice=Accept ->
# meta-refresh URL carrying a session ident -> follow with the cookie). Run it under bash with
# `set -euo pipefail` and `curl -fsS` so a failed request aborts loudly instead of feeding an
# empty body into grep -- a silent `curl -s | grep` masks both curl's exit code (pipe takes the
# last command's status) and the real error (grep just exits 1 on no match), which is exactly how
# this step failed in CI with a bare "exit code 1" and zero diagnostics.
RUN apt-get update \
&& bash -euo pipefail -c '\
url="$1"; \
echo "ChimeraX: requesting download token..."; \
resp="$(curl -fsS --retry 5 --retry-all-errors --retry-delay 5 --connect-timeout 30 -c /tmp/cx_cookies -d "choice=Accept" "$url")"; \
redirect="$(printf "%s" "$resp" | grep -oP "url=\K[^\"]*" | head -n1 || true)"; \
if [ -z "$redirect" ]; then echo "ChimeraX ERROR: no download URL in response (network block or page change). First 2KB:"; printf "%s" "$resp" | head -c 2048; echo; exit 1; fi; \
echo "ChimeraX: downloading $redirect"; \
curl -fsS --retry 5 --retry-all-errors --retry-delay 5 --connect-timeout 30 -b /tmp/cx_cookies -o /tmp/chimerax.deb "https://www.cgl.ucsf.edu$redirect"; \
dpkg-deb --info /tmp/chimerax.deb >/dev/null 2>&1 || { echo "ChimeraX ERROR: downloaded file is not a valid .deb. First 512B:"; head -c 512 /tmp/chimerax.deb; echo; exit 1; }; \
apt-get install -y /tmp/chimerax.deb \
' _ "${CHIMERAX_URL}" \
&& rm -f /tmp/chimerax.deb /tmp/cx_cookies \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir -p /home/dev/.config/ChimeraX

# ---------- interactive shells & pager ----------
# Kept out of the build-time list at the top so runtime additions don't invalidate the unpinned
# conda solve and ChimeraX fetch above. tcsh also provides /bin/csh (via update-alternatives).
# conda solve above. tcsh also provides /bin/csh (via update-alternatives).
RUN apt-get update && apt-get install -y --no-install-recommends \
less \
tcsh \
Expand Down
Loading
Loading