From f58a5447d77a9a2032b1523735a0b4c55347f645 Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Mon, 27 Jul 2026 09:34:56 -0300 Subject: [PATCH 1/5] fix ci --- .github/workflows/bench-abba.yml | 92 ++++++++++++++++++---- .github/workflows/benchmark-gpu.yml | 112 +++++++++++++++++++++------ bench_vs/lambda/recursion/Cargo.toml | 5 +- bin/cli/Cargo.toml | 3 +- crypto/crypto/Cargo.toml | 2 + crypto/math/Cargo.toml | 2 + crypto/stark/Cargo.toml | 3 +- prover/Cargo.toml | 5 +- 8 files changed, 183 insertions(+), 41 deletions(-) diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml index 8e8863c30..2f6e02b87 100644 --- a/.github/workflows/bench-abba.yml +++ b/.github/workflows/bench-abba.yml @@ -2,8 +2,18 @@ name: Bench ABBA tiebreaker # Drift-free paired (A/B/B/A) prover benchmark for resolving small (~1%) deltas the # cheap PR benchmark can't confirm. It builds both binaries and runs ~20 interleaved -# pairs, so it OCCUPIES THE SINGLE BENCH SERVER FOR ~30-40 MIN. For that reason it +# pairs — at the default workload that OCCUPIES THE SINGLE BENCH SERVER FOR SEVERAL +# HOURS (a 100tx continuations prove is ~7 min on a 16-core box). For that reason it # NEVER auto-triggers -- it runs only on an explicit `/bench-abba` comment on a PR. +# +# Syntax: "/bench-abba [N] [cont[TX]|mono[TX]]" (N = pair count, default 20). The +# workload defaults to ethrex 100tx with --continuations — the production proving +# mode at a size where per-epoch effects amortize (63 epochs). "cont10" gives the +# small continuation workload (~45s/prove — good when a coarse answer fast matters +# more than resolution); "mono[TX]", e.g. "mono5", the legacy monolithic prove. The +# ~3.1 GiB cont100 proof requires rkyv pointer_width_64 on BOTH sides (see +# prover/Cargo.toml), so PR branches older than that fix must rebase before a cont100 +# bench. on: issue_comment: types: [created] @@ -26,9 +36,12 @@ jobs: startsWith(github.event.comment.body, '/bench-abba') && contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) runs-on: [self-hosted, bench] - # Generous ceiling so a hang/OOM can't strand the single bench runner; the - # workload itself is ~30-40 min at the default 20 pairs (clamped to <=40). - timeout-minutes: 120 + # Generous ceiling so a hang/OOM can't strand the single bench runner — not the + # expected duration. Measured on a 16-core box: a 100tx-continuations CPU prove + # is ~7 min, so the default 20 pairs (40 proves) runs ~4.5-5 hr and the 40-pair + # clamp ceiling ~9.5 hr; builds add ~10-20 min. (The old 5tx-monolithic workload + # was ~30-40 min total at 20 pairs.) + timeout-minutes: 600 steps: - name: Acknowledge (react + occupancy notice) uses: actions/github-script@v7 @@ -41,10 +54,10 @@ jobs: await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, - body: '⏳ **ABBA tiebreaker started** on the bench server (~30–40 min). The bench server is occupied until it finishes.' + body: '⏳ **ABBA tiebreaker started** on the bench server (~4-5 hr at the default 20 pairs of ethrex 100tx continuations; pass a smaller pair count or `cont10` for a quicker, coarser run). The bench server is occupied until it finishes.' }); - - name: Resolve PR head + pair count + - name: Resolve PR head + bench config id: cfg env: GH_TOKEN: ${{ github.token }} @@ -56,15 +69,59 @@ jobs: # force-push race mid-run. HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" - # Optional pair count, e.g. "/bench-abba 32"; default 20. Clamp to [2,40] - # so a "/bench-abba 10000" can't monopolize the single bench server. - N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-abba[[:space:]]*\([0-9]\+\).*|\1|p') - N=${N:-20} - if [ "$N" -lt 2 ] 2>/dev/null || [ "$N" -gt 40 ] 2>/dev/null; then - echo "::warning::pair count $N out of range [2,40]; using 20" - N=20 + # Everything after "/bench-abba" on its line, tokens in any order: a number = + # pair count (default 20); cont[TX]/mono[TX] = workload. Default: ethrex 100tx + # with --continuations — the production proving mode, where the epoch-level + # optimizations (pipeline, K provers, cross-epoch caches) actually execute, + # at a size (63 epochs) where per-epoch effects amortize. The old + # 5tx-monolithic default never entered prove_continuation and under-reported + # continuation PRs ~8x. "mono[TX]" keeps the legacy monolithic prove + # reachable for historical comparison (TX defaults to 5). + ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-abba||p' | head -n1) + PAIRS=20; CONTINUATIONS=1; TX_COUNT=100 + for tok in $ARGS; do + case "$tok" in + cont) CONTINUATIONS=1; TX_COUNT=100 ;; + mono) CONTINUATIONS=0; TX_COUNT=5 ;; + cont[0-9]*) CONTINUATIONS=1; TX_COUNT="${tok#cont}" ;; + mono[0-9]*) CONTINUATIONS=0; TX_COUNT="${tok#mono}" ;; + [0-9]*) PAIRS="$tok" ;; + *) echo "::warning::ignoring unrecognized token '$tok'" ;; + esac + done + # Digits-only + clamps: TX_COUNT sizes the fixture (the 10-transfer one is + # committed; others are generated on the runner); PAIRS is clamped to [2,40] + # so a "/bench-abba 10000" can't monopolize the single bench server. The TX + # ceiling is mode-aware: monolithic peak heap grows with the trace (20tx was + # ~78 GB), so mono is capped at the classic 5; continuations are flat-memory, + # capped at the validated 100. + if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi + case "$TX_COUNT" in + ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;; + esac + if [ "$TX_COUNT" -lt 1 ] || [ "$TX_COUNT" -gt "$TX_MAX" ]; then + echo "::warning::tx count $TX_COUNT out of range [1,$TX_MAX] for this mode; using $TX_DEFAULT" + TX_COUNT=$TX_DEFAULT fi - echo "pairs=$N" >> "$GITHUB_OUTPUT" + case "$PAIRS" in + ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 20"; PAIRS=20 ;; + esac + if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 40 ]; then + echo "::warning::pair count $PAIRS out of range [2,40]; using 20" + PAIRS=20 + fi + if [ "$CONTINUATIONS" = "1" ]; then + WORKLOAD="ethrex ${TX_COUNT}tx continuations" + else + WORKLOAD="ethrex ${TX_COUNT}tx monolithic" + fi + { + echo "pairs=$PAIRS" + echo "continuations=$CONTINUATIONS" + echo "tx_count=$TX_COUNT" + echo "workload=$WORKLOAD" + } >> "$GITHUB_OUTPUT" + echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD" - name: Checkout (full history for ref resolution) uses: actions/checkout@v4 @@ -84,6 +141,9 @@ jobs: env: HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} PAIRS: ${{ steps.cfg.outputs.pairs }} + # Workload knobs read directly by bench_abba.sh from the environment. + CONTINUATIONS: ${{ steps.cfg.outputs.continuations }} + TX_COUNT: ${{ steps.cfg.outputs.tx_count }} run: | export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" set -o pipefail @@ -100,12 +160,14 @@ jobs: HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} PAIRS: ${{ steps.cfg.outputs.pairs }} OUTCOME: ${{ steps.run.outcome }} + WORKLOAD: ${{ steps.cfg.outputs.workload }} with: script: | const fs = require('fs'); const read = (p) => { try { return fs.readFileSync(p, 'utf8').trim(); } catch { return ''; } }; const head = (process.env.HEAD_SHA || '').slice(0, 10), pairs = process.env.PAIRS; - let body = `## ABBA tiebreaker — \`${head}\` vs \`main\` (${pairs} pairs)\n\n`; + const workload = process.env.WORKLOAD || 'ethrex'; + let body = `## ABBA tiebreaker — \`${head}\` vs \`main\` (${pairs} pairs, ${workload})\n\n`; if (process.env.OUTCOME === 'success') { const res = read('/tmp/abba_result.txt') || read('/tmp/abba_out.txt'); body += '```\n' + res + '\n```\n'; diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml index 6928255d9..7e3a5f567 100644 --- a/.github/workflows/benchmark-gpu.yml +++ b/.github/workflows/benchmark-gpu.yml @@ -6,9 +6,15 @@ name: Benchmark GPU (PR) # It builds the cli at the PR head and at main, runs N interleaved pairs on the GPU, # posts the paired-t + Wilcoxon verdict back to the PR, then ALWAYS destroys the box. # -# Triggered by a "/bench-gpu [N]" comment on a PR (N = pair count, default 14) or via -# workflow_dispatch. Orchestration runs on a GitHub-hosted runner; all GPU work happens -# on the rented Vast box (provisioned by the template onstart). +# Triggered by a "/bench-gpu [N] [cont[TX]|mono[TX]]" comment on a PR (N = pair count, +# default 14) or via workflow_dispatch. The workload defaults to ethrex 100tx with +# --continuations — the production proving mode at a size where per-epoch effects +# amortize (63 epochs; the proof is ~3.1 GiB, which requires rkyv pointer_width_64 on +# BOTH sides — see prover/Cargo.toml — so old PR branches must rebase past that fix +# before a cont bench). "cont10" gives the small continuation workload; "mono[TX]" +# (e.g. "mono5") the legacy monolithic prove, for historical comparison. Orchestration +# runs on a GitHub-hosted runner; all GPU work happens on the rented Vast box +# (provisioned by the template onstart). # # Requires repo secrets: # VAST_API_KEY — https://cloud.vast.ai/manage-keys/ @@ -20,6 +26,9 @@ on: pairs: description: "Number of A/B/B/A pairs" default: "14" + mode: + description: "Workload: cont[TX] (--continuations, TX defaults to 100) or mono[TX] (monolithic, TX defaults to 5)" + default: "cont100" issue_comment: types: [created] @@ -57,12 +66,14 @@ jobs: github.event.issue.pull_request && startsWith(github.event.comment.body, '/bench-gpu') && contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)) - # ABBA on the GPU: provisioning + dual cuda build (~30 min) + 2*pairs proves - # (~95s each). At the max 32 pairs (64 proves) a slow-provision box runs ~3 hr, - # so allow headroom over that; teardown still always destroys the box. - timeout-minutes: 210 + # ABBA on the GPU: provisioning + dual cuda build (~30 min) + 2*pairs proves. + # The default 100tx-continuations prove is ~3.5 min on a 16-core/5090 box + # (10tx cont ~25s, legacy 5tx monolithic ~95s). Worst case — 32 pairs (64 + # proves) at 100tx on a slow-provision box — is ~4.5 hr, so allow headroom + # over that; teardown still always destroys the box. + timeout-minutes: 330 steps: - - name: Resolve PR ref + pair count + - name: Resolve PR ref + bench config id: config env: GH_TOKEN: ${{ github.token }} @@ -70,24 +81,62 @@ jobs: COMMENT_BODY: ${{ github.event.comment.body }} PR_NUM: ${{ github.event.issue.number }} DISPATCH_PAIRS: ${{ github.event.inputs.pairs }} + DISPATCH_MODE: ${{ github.event.inputs.mode }} DISPATCH_REF: ${{ github.ref_name }} run: | if [ "$EVENT_NAME" = "issue_comment" ]; then # Pin the head SHA (works for fork PRs; avoids a force-push race mid-run). HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) OUT_PR_NUM="$PR_NUM"; OUT_HEAD_SHA="$HEAD_SHA"; OUT_BRANCH="" - # "/bench-gpu 20" -> 20 pairs; otherwise default. - N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-gpu[[:space:]]*\([0-9]\+\).*|\1|p') - PAIRS=${N:-14} + # Everything after "/bench-gpu" on its line, tokens in any order: + # a number = pair count; cont[TX]/mono[TX] = workload (see loop below). + ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-gpu||p' | head -n1) + PAIRS=14 else # workflow_dispatch: compare this branch vs main. OUT_PR_NUM=""; OUT_HEAD_SHA=""; OUT_BRANCH="$DISPATCH_REF" + ARGS="$DISPATCH_MODE" PAIRS=${DISPATCH_PAIRS:-14} fi + # Workload default: ethrex 100tx with --continuations — the production proving + # mode, where the epoch-level optimizations (pipeline, K provers, global-prove + # overlap, cross-epoch caches) actually execute, at a size (63 epochs) where + # per-epoch effects amortize. The old 5tx-monolithic default never entered + # prove_continuation and under-reported continuation PRs ~8x (PR #863 measured + # -4.7% here vs -38% real). "mono[TX]" keeps the legacy monolithic prove + # reachable for historical comparison (TX defaults to 5). + CONTINUATIONS=1; TX_COUNT=100 + for tok in $ARGS; do + case "$tok" in + cont) CONTINUATIONS=1; TX_COUNT=100 ;; + mono) CONTINUATIONS=0; TX_COUNT=5 ;; + cont[0-9]*) CONTINUATIONS=1; TX_COUNT="${tok#cont}" ;; + mono[0-9]*) CONTINUATIONS=0; TX_COUNT="${tok#mono}" ;; + [0-9]*) PAIRS="$tok" ;; + *) echo "::warning::ignoring unrecognized token '$tok'" ;; + esac + done + # TX_COUNT and PAIRS are interpolated into the remote bash -lc below — enforce + # digits-only before anything else. TX_COUNT also sizes the fixture (the + # 10-transfer one is committed; others are generated on the box). The ceiling + # is mode-aware: monolithic peak heap grows with the trace (20tx was ~78 GB, + # over the 48 GB box floor), so mono is capped at the classic 5; continuations + # are flat-memory, capped at the validated 100. + if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi + case "$TX_COUNT" in + ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;; + esac + if [ "$TX_COUNT" -lt 1 ] || [ "$TX_COUNT" -gt "$TX_MAX" ]; then + echo "::warning::tx count $TX_COUNT out of range [1,$TX_MAX] for this mode; using $TX_DEFAULT" + TX_COUNT=$TX_DEFAULT + fi + case "$PAIRS" in + ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 14"; PAIRS=14 ;; + esac # Clamp to [2,32]; out-of-range -> default. 14 ~ resolves a 2% delta. The ceiling # keeps the worst-case run (64 proves + provisioning + dual build) under the job # timeout above. - if [ "$PAIRS" -lt 2 ] 2>/dev/null || [ "$PAIRS" -gt 32 ] 2>/dev/null; then + if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 32 ]; then echo "::warning::pair count out of range [2,32], defaulting to 14" PAIRS=14 fi @@ -96,19 +145,28 @@ jobs: PAIRS=$((PAIRS + 1)) echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance" fi + if [ "$CONTINUATIONS" = "1" ]; then + WORKLOAD="ethrex ${TX_COUNT}tx continuations" + else + WORKLOAD="ethrex ${TX_COUNT}tx monolithic" + fi { echo "pr_num=$OUT_PR_NUM" echo "head_sha=$OUT_HEAD_SHA" echo "branch=$OUT_BRANCH" echo "pairs=$PAIRS" + echo "continuations=$CONTINUATIONS" + echo "tx_count=$TX_COUNT" + echo "workload=$WORKLOAD" } >> "$GITHUB_OUTPUT" - echo "Using $PAIRS A/B/B/A pairs" + echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD" - name: Acknowledge (react + occupancy notice) if: github.event_name == 'issue_comment' uses: actions/github-script@v7 env: PAIRS: ${{ steps.config.outputs.pairs }} + WORKLOAD: ${{ steps.config.outputs.workload }} with: script: | await github.rest.reactions.createForIssueComment({ @@ -118,7 +176,7 @@ jobs: // Post the "started" notice under the SAME marker the result step uses, so the // result updates this comment in place (and re-runs reuse it rather than stacking). const marker = 'GPU Benchmark (ABBA)'; - const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) on the CUDA prover path. This takes ~1 hr; the result will replace this comment.`; + const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) of ${process.env.WORKLOAD} on the CUDA prover path. This takes ~2.5 hr at the default workload; the result will replace this comment.`; const comments = await github.paginate(github.rest.issues.listComments, { owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, per_page: 100, @@ -170,10 +228,12 @@ jobs: # because vast can't numerically compare the driver_version string server-side. MIN_DRIVER: "580" run: | - # cpu_ram filter is in GB. Floor 48 GB: the bench workload moved to the - # 5-transfer ethrex fixture (executor/tests/ethrex_5_transfers.bin), far smaller - # than the old 20-transfer prove (~78 GB heap) that set the previous 96 GB floor. - # 48 GB widens the dedicated pool (~15 vs ~11 offers). + # cpu_ram filter is in GB. Floor 48 GB: the default workload (ethrex 100tx with + # --continuations) proves in epochs with flat peak memory (~10 GB at + # epoch_size_log2=20, plus a ~3 GiB proof buffer at write time), and even the + # legacy 5tx monolithic prove fits — both far below the old 20-transfer + # monolithic prove (~78 GB heap) that set the previous 96 GB floor. 48 GB + # widens the dedicated pool (~15 vs ~11 offers). # gpu_frac=1 requires a WHOLE-MACHINE offer (you rent every GPU on the host), so # Vast places no other tenant on the box: CPU cores, RAM/memory bandwidth, PCIe, # and NVMe are fully dedicated. Without it the "most expensive" sort below lands on @@ -331,6 +391,8 @@ jobs: HEAD_SHA: ${{ steps.config.outputs.head_sha }} BRANCH: ${{ steps.config.outputs.branch }} PAIRS: ${{ steps.config.outputs.pairs }} + CONTINUATIONS: ${{ steps.config.outputs.continuations }} + TX_COUNT: ${{ steps.config.outputs.tx_count }} run: | SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes -i $KEY -p $PORT root@$HOST" @@ -355,8 +417,10 @@ jobs: # explicit and robust to the template default changing.) The harness still builds the # cli at REF_A (the PR) and origin/main in isolated worktrees, runs PAIRS interleaved # A/B/B/A proves, and prints the paired-t CI + Wilcoxon verdict. BENCH_FEATURES routes - # the build through the CUDA prover path. NOTE: requires this PR's bench_abba.sh change - # (the BENCH_FEATURES env) to be on main — i.e. it only takes effect after merge. + # the build through the CUDA prover path; CONTINUATIONS/TX_COUNT pick the workload + # (validated digits-only in the config step — they're interpolated into this remote + # command). NOTE: since the harness runs from main, workflow/script changes only take + # effect after they merge. # REBUILD=1: each Vast box is fresh, GPU-specific hardware — always rebuild both # binaries (cubin is compiled for the detected arch); never trust a cached binary. # CUDARC_PIN: compat shim for pre-pin baseline shas. cudarc's CUDA version is now pinned @@ -371,6 +435,7 @@ jobs: git fetch --force origin main; $FETCH; \ git checkout -f origin/main; \ REBUILD=1 CUDARC_PIN=cuda-12080 SYSROOT_DIR=/opt/lambda-vm-sysroot BENCH_FEATURES='$BENCH_FEATURES' \ + CONTINUATIONS=$CONTINUATIONS TX_COUNT=$TX_COUNT \ scripts/bench_abba.sh $REF_A origin/main $PAIRS" # pipefail so a failed remote bench (e.g. a prove that dies) propagates through the @@ -386,9 +451,10 @@ jobs: if: always() && (steps.bench.outcome == 'success' || steps.bench.outcome == 'failure') env: OUTCOME: ${{ steps.bench.outcome }} + WORKLOAD: ${{ steps.config.outputs.workload }} run: | { - echo "## GPU ABBA — ethrex 20 transfers (vs main)" + echo "## GPU ABBA — ${WORKLOAD:-ethrex} (vs main)" if [ "$OUTCOME" = "success" ] && [ -s "$RUNNER_TEMP/abba_result.txt" ]; then echo '```' cat "$RUNNER_TEMP/abba_result.txt" @@ -410,6 +476,7 @@ jobs: OUTCOME: ${{ steps.bench.outcome }} GPU_NAME: ${{ env.GPU_NAME }} OFFER_PRICE: ${{ steps.offer.outputs.price }} + WORKLOAD: ${{ steps.config.outputs.workload }} with: script: | const fs = require('fs'); @@ -419,9 +486,10 @@ jobs: const pairs = process.env.PAIRS; const gpu = (process.env.GPU_NAME || '').replace('_', ' '); const price = process.env.OFFER_PRICE; + const workload = process.env.WORKLOAD || 'ethrex'; let body = `## GPU Benchmark (ABBA) — \`${head}\` vs \`main\` (${pairs} pairs)\n\n`; - body += `${gpu} · Vast.ai datacenter${price ? ` @ \$${price}/hr` : ''} · \`prover/cuda\` · drift-free A/B/B/A\n\n`; + body += `${gpu} · Vast.ai datacenter${price ? ` @ \$${price}/hr` : ''} · \`prover/cuda\` · ${workload} · drift-free A/B/B/A\n\n`; if (process.env.OUTCOME === 'success') { const res = read(`${tmp}/abba_result.txt`) || read(`${tmp}/abba_out.txt`); body += '```\n' + res + '\n```\n'; diff --git a/bench_vs/lambda/recursion/Cargo.toml b/bench_vs/lambda/recursion/Cargo.toml index f612949a8..50af81359 100644 --- a/bench_vs/lambda/recursion/Cargo.toml +++ b/bench_vs/lambda/recursion/Cargo.toml @@ -65,7 +65,10 @@ lambda-vm-prover = { path = "../../../prover", default-features = false, feature "profile-markers", ] } lambda-vm-syscalls = { path = "../../../syscalls" } -rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] } +# pointer_width_64: proof-format pointer width — must match the host serializer +# (see prover/Cargo.toml). Feature unification with the prover dep already +# implies it; declared here so the in-VM zero-copy reads can't silently drift. +rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } [profile.release] debug = 2 diff --git a/bin/cli/Cargo.toml b/bin/cli/Cargo.toml index e4fcdb7fd..71e89beef 100644 --- a/bin/cli/Cargo.toml +++ b/bin/cli/Cargo.toml @@ -9,7 +9,8 @@ executor = { path = "../../executor" } prover = { path = "../../prover", package = "lambda-vm-prover" } stark = { path = "../../crypto/stark" } clap = { version = "4.3.10", features = ["derive"] } -rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] } +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. +rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } tempfile = "3" tikv-jemallocator = "0.6" tikv-jemalloc-ctl = { version = "0.6", features = ["stats"], optional = true } diff --git a/crypto/crypto/Cargo.toml b/crypto/crypto/Cargo.toml index 6b78f81e7..9a36b2614 100644 --- a/crypto/crypto/Cargo.toml +++ b/crypto/crypto/Cargo.toml @@ -22,10 +22,12 @@ rand_chacha = { version = "0.3.1", default-features = false } memmap2 = { version = "0.9", optional = true } tempfile = { version = "3", optional = true } libc = { version = "0.2", optional = true } +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. rkyv = { version = "0.8.10", default-features = false, features = [ "alloc", "bytecheck", "aligned", + "pointer_width_64", ], optional = true } [target.'cfg(target_arch = "riscv64")'.dependencies] diff --git a/crypto/math/Cargo.toml b/crypto/math/Cargo.toml index df43ea975..a161056ca 100644 --- a/crypto/math/Cargo.toml +++ b/crypto/math/Cargo.toml @@ -25,10 +25,12 @@ num-traits = { version = "0.2.19", default-features = false } # rkyv zero-copy (de)serialization. Optional; used by the recursion verifier to # read a proof straight from its byte buffer with no deserialization pass. +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. rkyv = { version = "0.8.10", default-features = false, features = [ "alloc", "bytecheck", "aligned", + "pointer_width_64", ], optional = true } [dev-dependencies] diff --git a/crypto/stark/Cargo.toml b/crypto/stark/Cargo.toml index 78d95be67..09a5c1d9d 100644 --- a/crypto/stark/Cargo.toml +++ b/crypto/stark/Cargo.toml @@ -20,7 +20,8 @@ log = "0.4.17" digest = "0.10.7" serde = { version = "1.0", features = ["derive"] } itertools = "0.11.0" -rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] } +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. +rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } # Parallelization crates rayon = { version = "1.8.0", optional = true } diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 15344d138..83a02b137 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -24,7 +24,10 @@ rayon = { version = "1.8.0", optional = true } sysinfo = { version = "0.31", default-features = false, features = ["system"] } log = "0.4" digest = "0.10.7" -rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned"] } +# pointer_width_64: rkyv's default 32-bit relative pointers cap an archive at +# ~2 GiB, and a continuation proof crosses that near ~40 epochs (~54 MB/epoch on +# ethrex). Must match on every crate that reads or writes the proof format. +rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } [dev-dependencies] env_logger = "*" From aac66421d037f72252b3ec68b8c53a0e48af26d8 Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Mon, 27 Jul 2026 09:44:14 -0300 Subject: [PATCH 2/5] fix comments --- .github/workflows/bench-abba.yml | 44 +++++++---------------- .github/workflows/benchmark-gpu.yml | 52 +++++++++------------------- bench_vs/lambda/recursion/Cargo.toml | 4 +-- prover/Cargo.toml | 5 ++- 4 files changed, 33 insertions(+), 72 deletions(-) diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml index 2f6e02b87..9d7d1be40 100644 --- a/.github/workflows/bench-abba.yml +++ b/.github/workflows/bench-abba.yml @@ -1,19 +1,14 @@ name: Bench ABBA tiebreaker # Drift-free paired (A/B/B/A) prover benchmark for resolving small (~1%) deltas the -# cheap PR benchmark can't confirm. It builds both binaries and runs ~20 interleaved -# pairs — at the default workload that OCCUPIES THE SINGLE BENCH SERVER FOR SEVERAL -# HOURS (a 100tx continuations prove is ~7 min on a 16-core box). For that reason it -# NEVER auto-triggers -- it runs only on an explicit `/bench-abba` comment on a PR. +# cheap PR benchmark can't confirm. At the default workload it OCCUPIES THE SINGLE +# BENCH SERVER FOR SEVERAL HOURS, so it NEVER auto-triggers -- it runs only on an +# explicit `/bench-abba` comment on a PR. # -# Syntax: "/bench-abba [N] [cont[TX]|mono[TX]]" (N = pair count, default 20). The -# workload defaults to ethrex 100tx with --continuations — the production proving -# mode at a size where per-epoch effects amortize (63 epochs). "cont10" gives the -# small continuation workload (~45s/prove — good when a coarse answer fast matters -# more than resolution); "mono[TX]", e.g. "mono5", the legacy monolithic prove. The -# ~3.1 GiB cont100 proof requires rkyv pointer_width_64 on BOTH sides (see -# prover/Cargo.toml), so PR branches older than that fix must rebase before a cont100 -# bench. +# Syntax: "/bench-abba [N] [cont[TX]|mono[TX]]" (default: 20 pairs, ethrex 100tx +# --continuations; "cont10" is the quick coarse option, "mono[TX]" the legacy +# monolithic prove). Cont proofs need rkyv pointer_width_64 on both sides, so PR +# branches older than that fix must rebase before a cont bench. on: issue_comment: types: [created] @@ -36,11 +31,8 @@ jobs: startsWith(github.event.comment.body, '/bench-abba') && contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) runs-on: [self-hosted, bench] - # Generous ceiling so a hang/OOM can't strand the single bench runner — not the - # expected duration. Measured on a 16-core box: a 100tx-continuations CPU prove - # is ~7 min, so the default 20 pairs (40 proves) runs ~4.5-5 hr and the 40-pair - # clamp ceiling ~9.5 hr; builds add ~10-20 min. (The old 5tx-monolithic workload - # was ~30-40 min total at 20 pairs.) + # Hang guardrail, not expected duration: a cont100 CPU prove is ~7 min, so the + # default 20 pairs runs ~4.5-5 hr and the 40-pair clamp ~9.5 hr, plus builds. timeout-minutes: 600 steps: - name: Acknowledge (react + occupancy notice) @@ -70,13 +62,7 @@ jobs: HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" # Everything after "/bench-abba" on its line, tokens in any order: a number = - # pair count (default 20); cont[TX]/mono[TX] = workload. Default: ethrex 100tx - # with --continuations — the production proving mode, where the epoch-level - # optimizations (pipeline, K provers, cross-epoch caches) actually execute, - # at a size (63 epochs) where per-epoch effects amortize. The old - # 5tx-monolithic default never entered prove_continuation and under-reported - # continuation PRs ~8x. "mono[TX]" keeps the legacy monolithic prove - # reachable for historical comparison (TX defaults to 5). + # pair count; cont[TX]/mono[TX] = workload. ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-abba||p' | head -n1) PAIRS=20; CONTINUATIONS=1; TX_COUNT=100 for tok in $ARGS; do @@ -89,12 +75,9 @@ jobs: *) echo "::warning::ignoring unrecognized token '$tok'" ;; esac done - # Digits-only + clamps: TX_COUNT sizes the fixture (the 10-transfer one is - # committed; others are generated on the runner); PAIRS is clamped to [2,40] - # so a "/bench-abba 10000" can't monopolize the single bench server. The TX - # ceiling is mode-aware: monolithic peak heap grows with the trace (20tx was - # ~78 GB), so mono is capped at the classic 5; continuations are flat-memory, - # capped at the validated 100. + # Digits-only + clamps: PAIRS capped so one comment can't monopolize the + # single bench server; mono capped at 5tx (monolithic peak heap grows with + # the trace; 20tx needs ~78 GB). if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi case "$TX_COUNT" in ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;; @@ -141,7 +124,6 @@ jobs: env: HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} PAIRS: ${{ steps.cfg.outputs.pairs }} - # Workload knobs read directly by bench_abba.sh from the environment. CONTINUATIONS: ${{ steps.cfg.outputs.continuations }} TX_COUNT: ${{ steps.cfg.outputs.tx_count }} run: | diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml index 7e3a5f567..7e47eb427 100644 --- a/.github/workflows/benchmark-gpu.yml +++ b/.github/workflows/benchmark-gpu.yml @@ -7,14 +7,11 @@ name: Benchmark GPU (PR) # posts the paired-t + Wilcoxon verdict back to the PR, then ALWAYS destroys the box. # # Triggered by a "/bench-gpu [N] [cont[TX]|mono[TX]]" comment on a PR (N = pair count, -# default 14) or via workflow_dispatch. The workload defaults to ethrex 100tx with -# --continuations — the production proving mode at a size where per-epoch effects -# amortize (63 epochs; the proof is ~3.1 GiB, which requires rkyv pointer_width_64 on -# BOTH sides — see prover/Cargo.toml — so old PR branches must rebase past that fix -# before a cont bench). "cont10" gives the small continuation workload; "mono[TX]" -# (e.g. "mono5") the legacy monolithic prove, for historical comparison. Orchestration -# runs on a GitHub-hosted runner; all GPU work happens on the rented Vast box -# (provisioned by the template onstart). +# default 14) or via workflow_dispatch. Workload default: ethrex 100tx --continuations; +# "mono[TX]" = legacy monolithic prove. Cont proofs need rkyv pointer_width_64 on both +# sides, so PR branches older than that fix must rebase before a cont bench. +# Orchestration runs on a GitHub-hosted runner; all GPU work happens on the rented +# Vast box (provisioned by the template onstart). # # Requires repo secrets: # VAST_API_KEY — https://cloud.vast.ai/manage-keys/ @@ -66,11 +63,9 @@ jobs: github.event.issue.pull_request && startsWith(github.event.comment.body, '/bench-gpu') && contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)) - # ABBA on the GPU: provisioning + dual cuda build (~30 min) + 2*pairs proves. - # The default 100tx-continuations prove is ~3.5 min on a 16-core/5090 box - # (10tx cont ~25s, legacy 5tx monolithic ~95s). Worst case — 32 pairs (64 - # proves) at 100tx on a slow-provision box — is ~4.5 hr, so allow headroom - # over that; teardown still always destroys the box. + # Provisioning + dual cuda build (~30 min) + 2*pairs proves (~3.5 min each at + # the default cont100). Sized for the 32-pair worst case (~4.5 hr) with headroom; + # teardown still always destroys the box. timeout-minutes: 330 steps: - name: Resolve PR ref + bench config @@ -98,13 +93,7 @@ jobs: ARGS="$DISPATCH_MODE" PAIRS=${DISPATCH_PAIRS:-14} fi - # Workload default: ethrex 100tx with --continuations — the production proving - # mode, where the epoch-level optimizations (pipeline, K provers, global-prove - # overlap, cross-epoch caches) actually execute, at a size (63 epochs) where - # per-epoch effects amortize. The old 5tx-monolithic default never entered - # prove_continuation and under-reported continuation PRs ~8x (PR #863 measured - # -4.7% here vs -38% real). "mono[TX]" keeps the legacy monolithic prove - # reachable for historical comparison (TX defaults to 5). + # Defaults: continuations with the 100-transfer fixture (production mode). CONTINUATIONS=1; TX_COUNT=100 for tok in $ARGS; do case "$tok" in @@ -116,12 +105,9 @@ jobs: *) echo "::warning::ignoring unrecognized token '$tok'" ;; esac done - # TX_COUNT and PAIRS are interpolated into the remote bash -lc below — enforce - # digits-only before anything else. TX_COUNT also sizes the fixture (the - # 10-transfer one is committed; others are generated on the box). The ceiling - # is mode-aware: monolithic peak heap grows with the trace (20tx was ~78 GB, - # over the 48 GB box floor), so mono is capped at the classic 5; continuations - # are flat-memory, capped at the validated 100. + # TX_COUNT and PAIRS are interpolated into the remote bash -lc below: enforce + # digits-only. Mono is capped at 5tx (monolithic peak heap grows with the + # trace; 20tx needs ~78 GB, over the 48 GB box floor). if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi case "$TX_COUNT" in ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;; @@ -228,11 +214,9 @@ jobs: # because vast can't numerically compare the driver_version string server-side. MIN_DRIVER: "580" run: | - # cpu_ram filter is in GB. Floor 48 GB: the default workload (ethrex 100tx with - # --continuations) proves in epochs with flat peak memory (~10 GB at - # epoch_size_log2=20, plus a ~3 GiB proof buffer at write time), and even the - # legacy 5tx monolithic prove fits — both far below the old 20-transfer - # monolithic prove (~78 GB heap) that set the previous 96 GB floor. 48 GB + # cpu_ram filter is in GB. Floor 48 GB: continuation proves are flat-memory + # (~10 GB) and the legacy 5tx monolithic prove also fits — far below the old + # 20-transfer prove (~78 GB heap) that set the previous 96 GB floor. 48 GB # widens the dedicated pool (~15 vs ~11 offers). # gpu_frac=1 requires a WHOLE-MACHINE offer (you rent every GPU on the host), so # Vast places no other tenant on the box: CPU cores, RAM/memory bandwidth, PCIe, @@ -417,10 +401,8 @@ jobs: # explicit and robust to the template default changing.) The harness still builds the # cli at REF_A (the PR) and origin/main in isolated worktrees, runs PAIRS interleaved # A/B/B/A proves, and prints the paired-t CI + Wilcoxon verdict. BENCH_FEATURES routes - # the build through the CUDA prover path; CONTINUATIONS/TX_COUNT pick the workload - # (validated digits-only in the config step — they're interpolated into this remote - # command). NOTE: since the harness runs from main, workflow/script changes only take - # effect after they merge. + # the build through the CUDA prover path; CONTINUATIONS/TX_COUNT pick the workload. + # The harness runs from main, so workflow/script changes take effect post-merge. # REBUILD=1: each Vast box is fresh, GPU-specific hardware — always rebuild both # binaries (cubin is compiled for the detected arch); never trust a cached binary. # CUDARC_PIN: compat shim for pre-pin baseline shas. cudarc's CUDA version is now pinned diff --git a/bench_vs/lambda/recursion/Cargo.toml b/bench_vs/lambda/recursion/Cargo.toml index 50af81359..cc4d00a70 100644 --- a/bench_vs/lambda/recursion/Cargo.toml +++ b/bench_vs/lambda/recursion/Cargo.toml @@ -65,9 +65,7 @@ lambda-vm-prover = { path = "../../../prover", default-features = false, feature "profile-markers", ] } lambda-vm-syscalls = { path = "../../../syscalls" } -# pointer_width_64: proof-format pointer width — must match the host serializer -# (see prover/Cargo.toml). Feature unification with the prover dep already -# implies it; declared here so the in-VM zero-copy reads can't silently drift. +# pointer_width_64: proof-format pointer width — see prover/Cargo.toml. rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } [profile.release] diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 83a02b137..748ddd9a8 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -24,9 +24,8 @@ rayon = { version = "1.8.0", optional = true } sysinfo = { version = "0.31", default-features = false, features = ["system"] } log = "0.4" digest = "0.10.7" -# pointer_width_64: rkyv's default 32-bit relative pointers cap an archive at -# ~2 GiB, and a continuation proof crosses that near ~40 epochs (~54 MB/epoch on -# ethrex). Must match on every crate that reads or writes the proof format. +# pointer_width_64: 32-bit rel-ptrs cap an archive at ~2 GiB, which large +# continuation proofs exceed. Keep in sync across all proof-format crates. rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } [dev-dependencies] From 323b3fc030ded1b0b67713a9ce9f51d313b2b8ea Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Mon, 27 Jul 2026 09:57:02 -0300 Subject: [PATCH 3/5] resolve coments --- .github/workflows/bench-abba.yml | 14 +++++++++++++- .github/workflows/benchmark-gpu.yml | 13 +++++++++++++ prover/src/lib.rs | 5 +++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml index 9d7d1be40..3852987bb 100644 --- a/.github/workflows/bench-abba.yml +++ b/.github/workflows/bench-abba.yml @@ -33,7 +33,7 @@ jobs: runs-on: [self-hosted, bench] # Hang guardrail, not expected duration: a cont100 CPU prove is ~7 min, so the # default 20 pairs runs ~4.5-5 hr and the 40-pair clamp ~9.5 hr, plus builds. - timeout-minutes: 600 + timeout-minutes: 720 steps: - name: Acknowledge (react + occupancy notice) uses: actions/github-script@v7 @@ -65,6 +65,7 @@ jobs: # pair count; cont[TX]/mono[TX] = workload. ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-abba||p' | head -n1) PAIRS=20; CONTINUATIONS=1; TX_COUNT=100 + set -f # tokens must not glob-expand against the runner's CWD for tok in $ARGS; do case "$tok" in cont) CONTINUATIONS=1; TX_COUNT=100 ;; @@ -93,6 +94,17 @@ jobs: echo "::warning::pair count $PAIRS out of range [2,40]; using 20" PAIRS=20 fi + # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx + # continuation proof exceeds rkyv's old 2 GiB cap and only dies after + # blocking the single bench server for hours. Skip if the fetch fails. + if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then + SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$HEAD_SHA" \ + -H "Accept: application/vnd.github.raw" 2>/dev/null || true) + if [ -n "$SIDE" ] && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then + echo "::error::PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." + exit 1 + fi + fi if [ "$CONTINUATIONS" = "1" ]; then WORKLOAD="ethrex ${TX_COUNT}tx continuations" else diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml index 7e47eb427..2bb94cf2f 100644 --- a/.github/workflows/benchmark-gpu.yml +++ b/.github/workflows/benchmark-gpu.yml @@ -95,6 +95,7 @@ jobs: fi # Defaults: continuations with the 100-transfer fixture (production mode). CONTINUATIONS=1; TX_COUNT=100 + set -f # tokens must not glob-expand against the runner's CWD for tok in $ARGS; do case "$tok" in cont) CONTINUATIONS=1; TX_COUNT=100 ;; @@ -131,6 +132,18 @@ jobs: PAIRS=$((PAIRS + 1)) echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance" fi + # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx + # continuation proof exceeds rkyv's old 2 GiB cap and only dies after the + # full dual build (~1 hr of GPU rental). Skip the check if the fetch fails. + if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then + REF="${OUT_HEAD_SHA:-$OUT_BRANCH}" + SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$REF" \ + -H "Accept: application/vnd.github.raw" 2>/dev/null || true) + if [ -n "$SIDE" ] && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then + echo "::error::PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." + exit 1 + fi + fi if [ "$CONTINUATIONS" = "1" ]; then WORKLOAD="ethrex ${TX_COUNT}tx continuations" else diff --git a/prover/src/lib.rs b/prover/src/lib.rs index ff9601bb4..a88e9f81c 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -209,8 +209,9 @@ pub struct GuestInput { /// 4-byte magic identifying a lambda-vm recursion input blob ("LVMR"). pub const RECURSION_INPUT_MAGIC: [u8; 4] = *b"LVMR"; -/// Wire-format version of the recursion input blob. -pub const RECURSION_INPUT_VERSION: u32 = 1; +/// Wire-format version of the recursion input blob. v2: rkyv pointer_width_64 +/// (64-bit rel-ptrs) — v1 archives use 32-bit offsets and are incompatible. +pub const RECURSION_INPUT_VERSION: u32 = 2; /// Required alignment (bytes) of the archive's first byte in guest memory. pub const RECURSION_INPUT_ALIGN: usize = 16; From c4cc52f8b22360fe1051abb8c4648d494c2286f4 Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Mon, 27 Jul 2026 10:10:42 -0300 Subject: [PATCH 4/5] fix --- .github/workflows/bench-abba.yml | 5 +++++ .github/workflows/benchmark-gpu.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml index 3852987bb..cb341ac9f 100644 --- a/.github/workflows/bench-abba.yml +++ b/.github/workflows/bench-abba.yml @@ -94,6 +94,11 @@ jobs: echo "::warning::pair count $PAIRS out of range [2,40]; using 20" PAIRS=20 fi + # Even is ideal so the AB/BA orders balance; round an odd request up by one. + if [ "$((PAIRS % 2))" -ne 0 ]; then + PAIRS=$((PAIRS + 1)) + echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance" + fi # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx # continuation proof exceeds rkyv's old 2 GiB cap and only dies after # blocking the single bench server for hours. Skip if the fetch fails. diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml index 2bb94cf2f..6525c51c3 100644 --- a/.github/workflows/benchmark-gpu.yml +++ b/.github/workflows/benchmark-gpu.yml @@ -260,7 +260,7 @@ jobs: sleep "$OFFER_INTERVAL" done if [ -z "$OFFER_ID" ]; then - echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (>=16 cores, >=96GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, <= \$${PRICE_CAP}/hr)" + echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (>=16 cores, >=48GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, <= \$${PRICE_CAP}/hr)" exit 1 fi echo "id=$OFFER_ID" >> "$GITHUB_OUTPUT" From 578327f4cd1313d583e0542e2db73fcfb1d60632 Mon Sep 17 00:00:00 2001 From: Joaquin Carletti Date: Mon, 27 Jul 2026 18:11:03 -0300 Subject: [PATCH 5/5] fix --- .github/workflows/bench-abba.yml | 65 +++++++++++++++--------- .github/workflows/benchmark-gpu.yml | 31 ++++++----- crypto/stark/src/proof/stark.rs | 11 ++++ prover/src/lib.rs | 17 ++++++- prover/src/tests/recursion_smoke_test.rs | 48 +++++++++++++++++ 5 files changed, 133 insertions(+), 39 deletions(-) diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml index cb341ac9f..aa066bbd5 100644 --- a/.github/workflows/bench-abba.yml +++ b/.github/workflows/bench-abba.yml @@ -35,20 +35,6 @@ jobs: # default 20 pairs runs ~4.5-5 hr and the 40-pair clamp ~9.5 hr, plus builds. timeout-minutes: 720 steps: - - name: Acknowledge (react + occupancy notice) - uses: actions/github-script@v7 - with: - script: | - await github.rest.reactions.createForIssueComment({ - owner: context.repo.owner, repo: context.repo.repo, - comment_id: context.payload.comment.id, content: 'eyes' - }); - await github.rest.issues.createComment({ - owner: context.repo.owner, repo: context.repo.repo, - issue_number: context.issue.number, - body: '⏳ **ABBA tiebreaker started** on the bench server (~4-5 hr at the default 20 pairs of ethrex 100tx continuations; pass a smaller pair count or `cont10` for a quicker, coarser run). The bench server is occupied until it finishes.' - }); - - name: Resolve PR head + bench config id: cfg env: @@ -56,6 +42,9 @@ jobs: PR_NUM: ${{ github.event.issue.number }} COMMENT_BODY: ${{ github.event.comment.body }} run: | + # The runner is persistent self-hosted: drop the previous run's logs so + # the always() result comment never tails a stale /tmp/abba_out.txt. + rm -f /tmp/abba_out.txt /tmp/abba_result.txt # Resolve the head SHA (not the branch name): pinning the commit works for # fork PRs too (the branch lives in the fork, not origin/) and avoids a # force-push race mid-run. @@ -99,30 +88,56 @@ jobs: PAIRS=$((PAIRS + 1)) echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance" fi - # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx - # continuation proof exceeds rkyv's old 2 GiB cap and only dies after - # blocking the single bench server for hours. Skip if the fetch fails. - if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then - SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$HEAD_SHA" \ - -H "Accept: application/vnd.github.raw" 2>/dev/null || true) - if [ -n "$SIDE" ] && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then - echo "::error::PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." - exit 1 - fi - fi if [ "$CONTINUATIONS" = "1" ]; then WORKLOAD="ethrex ${TX_COUNT}tx continuations" else WORKLOAD="ethrex ${TX_COUNT}tx monolithic" fi + # Outputs land before the fail-fast below so the always() result + # comment is fully labeled even when this step exits early. { echo "pairs=$PAIRS" echo "continuations=$CONTINUATIONS" echo "tx_count=$TX_COUNT" echo "workload=$WORKLOAD" } >> "$GITHUB_OUTPUT" + # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx + # continuation proof exceeds rkyv's old 2 GiB cap and only dies after + # blocking the single bench server for hours. Skip the check when the + # fetch fails: gh api prints HTTP error bodies to stdout, so gate on + # its exit status AND on the payload looking like the manifest (it + # declares rkyv) — never treat an error blob as a missing feature. + # Purely textual; deletable once every open branch postdates the fix. + if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then + if SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$HEAD_SHA" \ + -H "Accept: application/vnd.github.raw" 2>/dev/null) \ + && printf '%s' "$SIDE" | grep -q '^rkyv' \ + && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then + MSG="PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." + echo "$MSG" > /tmp/abba_out.txt # surfaces in the result comment + echo "::error::$MSG" + exit 1 + fi + fi echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD" + - name: Acknowledge (react + occupancy notice) + uses: actions/github-script@v7 + env: + PAIRS: ${{ steps.cfg.outputs.pairs }} + WORKLOAD: ${{ steps.cfg.outputs.workload }} + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, repo: context.repo.repo, + comment_id: context.payload.comment.id, content: 'eyes' + }); + await github.rest.issues.createComment({ + owner: context.repo.owner, repo: context.repo.repo, + issue_number: context.issue.number, + body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} (a cont100 pair is ~15 min, so the default 20 pairs runs ~4.5-5 hr; pass a smaller pair count or \`cont10\` for a quicker, coarser run). The bench server is occupied until it finishes.` + }); + - name: Checkout (full history for ref resolution) uses: actions/checkout@v4 with: diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml index 6525c51c3..040b51c89 100644 --- a/.github/workflows/benchmark-gpu.yml +++ b/.github/workflows/benchmark-gpu.yml @@ -132,23 +132,13 @@ jobs: PAIRS=$((PAIRS + 1)) echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance" fi - # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx - # continuation proof exceeds rkyv's old 2 GiB cap and only dies after the - # full dual build (~1 hr of GPU rental). Skip the check if the fetch fails. - if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then - REF="${OUT_HEAD_SHA:-$OUT_BRANCH}" - SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$REF" \ - -H "Accept: application/vnd.github.raw" 2>/dev/null || true) - if [ -n "$SIDE" ] && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then - echo "::error::PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." - exit 1 - fi - fi if [ "$CONTINUATIONS" = "1" ]; then WORKLOAD="ethrex ${TX_COUNT}tx continuations" else WORKLOAD="ethrex ${TX_COUNT}tx monolithic" fi + # Outputs land before the fail-fast below so the always() result + # comment is fully labeled even when this step exits early. { echo "pr_num=$OUT_PR_NUM" echo "head_sha=$OUT_HEAD_SHA" @@ -158,6 +148,23 @@ jobs: echo "tx_count=$TX_COUNT" echo "workload=$WORKLOAD" } >> "$GITHUB_OUTPUT" + # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx + # continuation proof exceeds rkyv's old 2 GiB cap and only dies after the + # full dual build (~1 hr of GPU rental). Skip the check when the fetch + # fails: gh api prints HTTP error bodies to stdout, so gate on its exit + # status AND on the payload looking like the manifest (it declares + # rkyv) — never treat an error blob as a missing feature. Purely + # textual; deletable once every open branch postdates the fix. + if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then + REF="${OUT_HEAD_SHA:-$OUT_BRANCH}" + if SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$REF" \ + -H "Accept: application/vnd.github.raw" 2>/dev/null) \ + && printf '%s' "$SIDE" | grep -q '^rkyv' \ + && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then + echo "::error::PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." + exit 1 + fi + fi echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD" - name: Acknowledge (react + occupancy notice) diff --git a/crypto/stark/src/proof/stark.rs b/crypto/stark/src/proof/stark.rs index ba4aca2dc..9ce3ed32f 100644 --- a/crypto/stark/src/proof/stark.rs +++ b/crypto/stark/src/proof/stark.rs @@ -16,6 +16,17 @@ use crate::{ // `tests/bus_tests/completeness_tests.rs`. Do not add a production serde // dependency on these types. +// With no pointer-width feature enabled rkyv silently falls back to 32-bit +// rel-ptrs, capping an archive at ~2 GiB — which large continuation proofs +// exceed, and which CI round-trips (all under 2 GiB) can't catch. Pinned here, +// where the archived proof types live, so standalone builds of this crate fail +// if a Cargo.toml loses `pointer_width_64`; lambda-vm-prover repeats the +// assert to cover the host + riscv64 guest graphs. +const _: () = assert!( + size_of::() == 8, + "proof wire format requires rkyv's pointer_width_64 feature on every proof-format crate", +); + #[derive( Debug, Clone, diff --git a/prover/src/lib.rs b/prover/src/lib.rs index a88e9f81c..a8e89f989 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -235,6 +235,18 @@ const _: () = { ); }; +// With no pointer-width feature enabled rkyv silently falls back to 32-bit +// rel-ptrs, capping an archive at ~2 GiB — which large continuation proofs +// exceed, and which nothing else catches: every CI round-trip fits 32-bit +// offsets, and RECURSION_INPUT_VERSION can't flag it since host and guest +// compile the constant from this same file. This compiles into both the host +// and the riscv64 guest graph (the recursion guests path-depend on this +// crate), so a Cargo.toml losing the feature fails the build here. +const _: () = assert!( + size_of::() == 8, + "proof wire format v2 requires rkyv's pointer_width_64 feature on every proof-format crate", +); + /// Encode a [`GuestInput`] into the on-wire blob: a 12-byte /// `magic + version + reserved` prefix followed by the rkyv archive. The prefix /// both aligns the archive in guest memory (so in-place reads don't trap) and @@ -252,8 +264,9 @@ pub fn encode_recursion_input(input: &GuestInput) -> Result, Error> { } /// Validate the wire prefix and return the archive bytes (zero-copy slice). -/// Returns `None` if the magic or version doesn't match — the caller should -/// halt cleanly rather than proceed into an `access_unchecked`. +/// Returns `None` if the blob is too short or the magic or version doesn't +/// match — callers halt with a legible wrong-format error instead of +/// surfacing whatever bytecheck makes of old-format bytes. pub fn recursion_archive_bytes(blob: &[u8]) -> Option<&[u8]> { if blob.len() < RECURSION_INPUT_PREFIX_LEN { return None; diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index 1f4800011..90482a3a4 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -594,6 +594,54 @@ fn run_recursion_pipeline( ); } +/// The wire-prefix rejection path: a stale version (v1 blobs predate rkyv +/// pointer_width_64), a corrupted magic, and a blob shorter than the prefix +/// must all yield `None` — the clean "bad magic or version" error the +/// breaking-change story leans on, rather than a bytecheck error over +/// old-format bytes. Pure function, no proving needed. +#[test] +fn test_recursion_prefix_rejects_wrong_magic_version_and_short_blobs() { + let archive = [0xAAu8; 16]; + let mut blob = Vec::with_capacity(crate::RECURSION_INPUT_PREFIX_LEN + archive.len()); + blob.extend_from_slice(&crate::RECURSION_INPUT_MAGIC); + blob.extend_from_slice(&crate::RECURSION_INPUT_VERSION.to_le_bytes()); + blob.extend_from_slice(&[0u8; 4]); // reserved + blob.extend_from_slice(&archive); + + // Baseline: a well-formed prefix passes and returns exactly the archive. + assert_eq!( + crate::recursion_archive_bytes(&blob), + Some(&archive[..]), + "well-formed prefix must expose the archive bytes" + ); + + // (a) Stale wire version: a v1 blob (32-bit rel-ptrs) must be rejected. + let mut stale = blob.clone(); + stale[4..8].copy_from_slice(&1u32.to_le_bytes()); + assert_eq!( + crate::recursion_archive_bytes(&stale), + None, + "v1 blob must be rejected by the version check" + ); + + // (b) Corrupted magic. + let mut bad_magic = blob.clone(); + bad_magic[0] ^= 0xFF; + assert_eq!( + crate::recursion_archive_bytes(&bad_magic), + None, + "flipped magic byte must be rejected" + ); + + // (c) Shorter than the 12-byte prefix (including empty). + assert_eq!( + crate::recursion_archive_bytes(&blob[..crate::RECURSION_INPUT_PREFIX_LEN - 1]), + None, + "blob shorter than the prefix must be rejected" + ); + assert_eq!(crate::recursion_archive_bytes(&[]), None); +} + /// Decode the blob on the host and mirror the guest's verify+attest, then run /// the consumer check — a cheap guard on the encode/decode/attest contract /// without running the VM.