tooling(recursion): count keccak hashes to verify a proof (excl. grinding)

A host-only diagnostic, behind the `hash-metrics` cargo feature, that counts the
keccak-256 hashes done to VERIFY a proof — a fast, deterministic proxy for the
recursion guest's dominant cost (Merkle path hashing + Fiat-Shamir).

A default build is provably unchanged: PlatformKeccak256 stays = sha3::Keccak256
and every counter compiles to nothing. With the feature on, the host keccak
wrapper bumps a global counter on every finalize (inlined, byte-identical digest);
the Merkle backends and the grinding PoW check tag their own sub-counters. So
test_count_recursion_hashes reports total(excl. grinding), merkle (nodes/leaves),
transcript+other, and grinding. Nothing is added to the verifier and there is no
enable/disable toggle — no cross-thread race under a parallel verify. Compiled out
on the riscv64 guest.

  RECURSION_DUMP_PRESET=<preset> cargo test --release --features hash-metrics \
    -p lambda-vm-prover --lib test_count_recursion_hashes -- --ignored --nocapture

Author: ColoCarletti

crypto/crypto/Cargo.toml

@@ -44,4 +44,7 @@ serde = ["dep:serde"]
 parallel = ["dep:rayon"]
 disk-spill = ["std", "dep:memmap2", "dep:tempfile", "dep:libc"]
 alloc = []
-rkyv = ["dep:rkyv", "math/rkyv"]
+rkyv = ["dep:rkyv", "math/rkyv"]
+# Host-only diagnostic: count keccak finalizes during verify (see `hash_metrics`).
+# Off by default → `PlatformKeccak256 = sha3::Keccak256`, provably unchanged.
+hash-metrics = []

crypto/crypto/src/hash/platform_keccak.rs

@@ -58,7 +58,59 @@ mod imp {
     }
 }
 
-#[cfg(not(target_arch = "riscv64"))]
+// Host, `hash-metrics` feature ON: `sha3::Keccak256` plus a finalize counter for
+// [`crate::hash_metrics`]. The counter is a PURE SIDE EFFECT — every method
+// forwards to the inner hasher (byte-identical digest) and is `#[inline(always)]`,
+// so no cross-crate call is added over the bare alias.
+#[cfg(all(not(target_arch = "riscv64"), feature = "hash-metrics"))]
+mod imp {
+    use digest::{
+        FixedOutput, FixedOutputReset, HashMarker, Output, OutputSizeUser, Reset, Update,
+    };
+
+    #[derive(Clone, Default)]
+    pub struct PlatformKeccak256(sha3::Keccak256);
+
+    impl HashMarker for PlatformKeccak256 {}
+
+    impl OutputSizeUser for PlatformKeccak256 {
+        type OutputSize = digest::typenum::U32;
+    }
+
+    impl Update for PlatformKeccak256 {
+        #[inline(always)]
+        fn update(&mut self, data: &[u8]) {
+            Update::update(&mut self.0, data);
+        }
+    }
+
+    impl FixedOutput for PlatformKeccak256 {
+        #[inline(always)]
+        fn finalize_into(self, out: &mut Output<Self>) {
+            crate::hash_metrics::count_total();
+            FixedOutput::finalize_into(self.0, out);
+        }
+    }
+
+    impl Reset for PlatformKeccak256 {
+        #[inline(always)]
+        fn reset(&mut self) {
+            Reset::reset(&mut self.0);
+        }
+    }
+
+    impl FixedOutputReset for PlatformKeccak256 {
+        #[inline(always)]
+        fn finalize_into_reset(&mut self, out: &mut Output<Self>) {
+            crate::hash_metrics::count_total();
+            FixedOutputReset::finalize_into_reset(&mut self.0, out);
+        }
+    }
+}
+
+// Default host build (no `hash-metrics` feature): the plain alias, provably
+// unchanged from upstream.
+#[cfg(all(not(target_arch = "riscv64"), not(feature = "hash-metrics")))]
 mod imp {
     pub type PlatformKeccak256 = sha3::Keccak256;
 }

crypto/crypto/src/hash_metrics.rs

@@ -0,0 +1,102 @@
+//! Host-only keccak-hash counters for measuring the cost of VERIFYING a proof
+//! (a proxy for the recursion guest's dominant work: keccak hashing).
+//!
+//! Behind the `hash-metrics` cargo feature: a normal build keeps
+//! `PlatformKeccak256 = sha3::Keccak256` and every counter call compiles to
+//! nothing, so the prover is provably unchanged. With the feature on (host only),
+//! the host `PlatformKeccak256` wrapper bumps [`count_total`] on EVERY keccak-256
+//! finalize — Merkle trees, the Fiat-Shamir transcript, the program-id/ELF fold.
+//! The Merkle backends split their share ([`count_merkle`] / [`count_merkle_node`])
+//! and grinding tags its own ([`count_grinding`]), so a caller can report
+//! `total`, `merkle` (nodes/leaves), `grinding`, and everything else.
+//!
+//! No enable/disable toggle and nothing in the verifier: counting is always on
+//! under the feature, and a measuring caller just [`reset`]s before the verify
+//! and reads [`snapshot`] after. Grinding is separated by counter, not excluded
+//! at a call site, so there is no cross-thread race under a parallel verify.
+
+#[cfg(all(not(target_arch = "riscv64"), feature = "hash-metrics"))]
+mod imp {
+    use core::sync::atomic::{AtomicU64, Ordering};
+
+    static TOTAL: AtomicU64 = AtomicU64::new(0);
+    static MERKLE: AtomicU64 = AtomicU64::new(0);
+    static MERKLE_NODES: AtomicU64 = AtomicU64::new(0);
+    static GRINDING: AtomicU64 = AtomicU64::new(0);
+
+    /// Every keccak-256 finalize, from any site (host `PlatformKeccak256`).
+    #[inline(always)]
+    pub fn count_total() {
+        TOTAL.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// A Merkle finalize (leaf or node), counted ONLY when the backend digest is
+    /// the platform keccak wrapper — the one whose `finalize` also bumps
+    /// [`count_total`]. This keeps `merkle` a strict subset of `total` for ANY
+    /// `D` (a non-keccak backend, as in the crypto tests, does not go through the
+    /// counted wrapper, so counting it here would let `merkle` exceed `total`).
+    #[inline(always)]
+    pub fn count_merkle<D: 'static>() {
+        if core::any::TypeId::of::<D>()
+            == core::any::TypeId::of::<crate::hash::platform_keccak::PlatformKeccak256>()
+        {
+            MERKLE.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+
+    /// A Merkle parent (auth-path) compression. Subset of [`count_merkle`];
+    /// same keccak-only guard.
+    #[inline(always)]
+    pub fn count_merkle_node<D: 'static>() {
+        if core::any::TypeId::of::<D>()
+            == core::any::TypeId::of::<crate::hash::platform_keccak::PlatformKeccak256>()
+        {
+            MERKLE_NODES.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+
+    /// A grinding (proof-of-work) finalize. Subset of [`count_total`]; a caller
+    /// reports `total - grinding` to exclude the PoW check.
+    #[inline(always)]
+    pub fn count_grinding() {
+        GRINDING.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// Zero all counters.
+    pub fn reset() {
+        TOTAL.store(0, Ordering::Relaxed);
+        MERKLE.store(0, Ordering::Relaxed);
+        MERKLE_NODES.store(0, Ordering::Relaxed);
+        GRINDING.store(0, Ordering::Relaxed);
+    }
+
+    /// `(total, merkle, merkle_nodes, grinding)`. leaves = merkle − nodes;
+    /// transcript+other = total − merkle − grinding; excl. grinding = total − grinding.
+    pub fn snapshot() -> (u64, u64, u64, u64) {
+        (
+            TOTAL.load(Ordering::Relaxed),
+            MERKLE.load(Ordering::Relaxed),
+            MERKLE_NODES.load(Ordering::Relaxed),
+            GRINDING.load(Ordering::Relaxed),
+        )
+    }
+}
+
+// Feature off, or the riscv64 guest: every entry compiles to nothing.
+#[cfg(any(target_arch = "riscv64", not(feature = "hash-metrics")))]
+mod imp {
+    #[inline(always)]
+    pub fn count_total() {}
+    #[inline(always)]
+    pub fn count_merkle<D: 'static>() {}
+    #[inline(always)]
+    pub fn count_merkle_node<D: 'static>() {}
+    #[inline(always)]
+    pub fn count_grinding() {}
+    pub fn reset() {}
+    pub fn snapshot() -> (u64, u64, u64, u64) {
+        (0, 0, 0, 0)
+    }
+}
+
+pub use imp::{count_grinding, count_merkle, count_merkle_node, count_total, reset, snapshot};

crypto/crypto/src/lib.rs

@@ -9,6 +9,7 @@ extern crate alloc;
 
 pub mod fiat_shamir;
 pub mod hash;
+pub mod hash_metrics;
 pub mod merkle_tree;
 #[cfg(feature = "disk-spill")]
 pub mod mmap_util;

crypto/crypto/src/merkle_tree/backends/field_element.rs

@@ -22,7 +22,7 @@ impl<F, D: Digest, const NUM_BYTES: usize> Default for FieldElementBackend<F, D,
     }
 }
 
-impl<F, D: Digest, const NUM_BYTES: usize> IsMerkleTreeBackend
+impl<F, D: Digest + 'static, const NUM_BYTES: usize> IsMerkleTreeBackend
     for FieldElementBackend<F, D, NUM_BYTES>
 where
     F: IsField,
@@ -33,12 +33,18 @@ where
     type Data = FieldElement<F>;
 
     fn hash_data(input: &FieldElement<F>) -> [u8; NUM_BYTES] {
+        // Merkle leaf finalize (see `crate::hash_metrics`); counts only when `D`
+        // is the platform keccak (so `merkle ⊆ total`), no-op without the feature.
+        crate::hash_metrics::count_merkle::<D>();
         let mut hasher = D::new();
         input.stream_bytes(&mut |b| hasher.update(b));
         hasher.finalize().into()
     }
 
     fn hash_new_parent(left: &[u8; NUM_BYTES], right: &[u8; NUM_BYTES]) -> [u8; NUM_BYTES] {
+        // Merkle auth-path (node) compression; keccak-only guard, no-op without
+        // the feature.
+        crate::hash_metrics::count_merkle_node::<D>();
         let mut hasher = D::new();
         hasher.update(left);
         hasher.update(right);

crypto/crypto/src/merkle_tree/backends/field_element_vector.rs

@@ -43,6 +43,9 @@ use lambda_vm_syscalls::keccak::Keccak256 as SyscallKeccak256;
 fn hash_streamed<D: Digest + 'static, const NUM_BYTES: usize>(
     feed: impl Fn(&mut dyn FnMut(&[u8])),
 ) -> [u8; NUM_BYTES] {
+    // Metric: a Merkle finalize (leaf or node). Counts only when `D` is the
+    // platform keccak (so `merkle ⊆ total`); no-op on guest / without the feature.
+    crate::hash_metrics::count_merkle::<D>();
     #[cfg(target_arch = "riscv64")]
     if NUM_BYTES == 32 && TypeId::of::<D>() == TypeId::of::<PlatformKeccak256>() {
         let mut hasher = SyscallKeccak256::new();
@@ -75,6 +78,10 @@ fn hash_new_parent_bytes<D: Digest + 'static, const NUM_BYTES: usize>(
     left: &[u8; NUM_BYTES],
     right: &[u8; NUM_BYTES],
 ) -> [u8; NUM_BYTES] {
+    // Metric: a Merkle parent (auth-path) compression. On the host this also
+    // flows through `hash_streamed` (one `count_merkle`), so merkle − nodes =
+    // leaves. Keccak-only guard; no-op on guest / without the feature.
+    crate::hash_metrics::count_merkle_node::<D>();
     #[cfg(target_arch = "riscv64")]
     if NUM_BYTES == 32 && TypeId::of::<D>() == TypeId::of::<PlatformKeccak256>() {
         let l: &[u8; 32] = left[..].try_into().unwrap();

crypto/stark/src/grinding.rs

@@ -65,6 +65,9 @@ pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option<u64> {
 /// when interpreted as `u64`.
 #[inline(always)]
 fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool {
+    // Tag this finalize as grinding so a verify-hash metric can report it apart
+    // (see `crypto::hash_metrics`); no-op unless the `hash-metrics` feature is on.
+    crypto::hash_metrics::count_grinding();
     let mut data = [0; 40];
     data[..32].copy_from_slice(inner_hash);
     data[32..].copy_from_slice(&candidate_nonce.to_be_bytes());
@@ -79,6 +82,8 @@ fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, li
 /// Hash(prefix || seed || grinding_factor)
 /// `prefix` is the bit-string `0x123456789abcded`
 fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] {
+    // Grinding finalize (see `crypto::hash_metrics`); no-op unless enabled.
+    crypto::hash_metrics::count_grinding();
     let mut inner_data = [0u8; 41];
     inner_data[0..8].copy_from_slice(&PREFIX);
     inner_data[8..40].copy_from_slice(seed);

prover/Cargo.toml

@@ -15,6 +15,8 @@ instruments = ["stark/instruments"]
 nvtx = ["cuda", "instruments", "stark/nvtx"]
 profile-markers = ["stark/profile-markers"]
 disk-spill = ["stark/disk-spill"]
+# Host-only verify-hash counter for `test_count_recursion_hashes` (see `hash_metrics`).
+hash-metrics = ["crypto/hash-metrics"]
 
 [dependencies]
 stark = { path = "../crypto/stark" }

prover/src/tests/recursion_smoke_test.rs

@@ -1052,6 +1052,78 @@ fn test_dump_recursion_input() {
     }
 }
 
+/// Count the keccak hashes done to VERIFY the dumped recursion blob — a
+/// prover-change metric: fewer hashes ⇒ a cheaper recursion guest. Runs the exact
+/// guest verify (`verify_continuation_and_attest`) on `/tmp/recursion_input.bin`
+/// (override with `RECURSION_INPUT_PATH`) with `crypto::hash_metrics` counting
+/// every keccak-256 finalize, and reports the grinding proof-of-work hashes apart
+/// so the headline `total` excludes them.
+///
+/// Requires:
+/// * the `hash-metrics` cargo feature — without it counting is a no-op (all zeros
+///   → this test asserts and tells you to add the feature);
+/// * a CONTINUATION dump: `test_dump_recursion_input` with `RECURSION_DUMP_EPOCH_LOG2`
+///   set (this path verifies via `verify_continuation_and_attest`);
+/// * `RECURSION_DUMP_PRESET` matching the dump (default `min`), else the verify fails.
+///
+/// Loop: change the prover → re-run `test_dump_recursion_input` (re-proves + dumps
+/// the new blob) → run this (fast, verify-only) → compare `total`.
+///
+///   RECURSION_DUMP_PRESET=blowup4 cargo test --release --features hash-metrics \
+///     -p lambda-vm-prover --lib test_count_recursion_hashes -- --ignored --nocapture
+#[test]
+#[ignore = "diagnostic: counts keccak hashes verifying the dumped recursion blob"]
+fn test_count_recursion_hashes() {
+    let preset_name = std::env::var("RECURSION_DUMP_PRESET").unwrap_or_else(|_| "min".to_string());
+    let preset = Preset::ALL
+        .into_iter()
+        .find(|p| p.name() == preset_name)
+        .unwrap_or_else(|| panic!("unknown RECURSION_DUMP_PRESET '{preset_name}'"));
+    let path = std::env::var("RECURSION_INPUT_PATH")
+        .unwrap_or_else(|_| "/tmp/recursion_input.bin".to_string());
+    let blob = std::fs::read(&path)
+        .unwrap_or_else(|e| panic!("read {path} (run test_dump_recursion_input first): {e}"));
+
+    // Counting is always-on under the `hash-metrics` feature, so just zero the
+    // counters, verify, and read — no enable/disable, nothing in the verifier.
+    crypto::hash_metrics::reset();
+    let attestation = recursion::verify_continuation_and_attest(&blob, &preset.options()).expect(
+        "verify_continuation_and_attest errored — needs a CONTINUATION dump \
+         (RECURSION_DUMP_EPOCH_LOG2 set) under a matching RECURSION_DUMP_PRESET",
+    );
+    let (total, merkle, nodes, grinding) = crypto::hash_metrics::snapshot();
+
+    assert!(
+        attestation.is_some(),
+        "the blob must verify under preset '{}' — does it match the dump's RECURSION_DUMP_PRESET?",
+        preset.name()
+    );
+    assert!(
+        total > 0,
+        "hash counters are zero — build with `--features hash-metrics`"
+    );
+    // `merkle` and `grinding` are disjoint subsets of `total`; `nodes` ⊆ `merkle`.
+    // (Holds for the keccak recursion verify; flags a backend/counter mismatch.)
+    assert!(
+        nodes <= merkle && merkle + grinding <= total,
+        "inconsistent counters: total={total} merkle={merkle} nodes={nodes} grinding={grinding}"
+    );
+
+    println!(
+        "[hash-count] preset={} blob={}B fri_queries={} | total(excl. grinding)={} | \
+         merkle={} (nodes={} leaves={}) | transcript+other={} | grinding={}",
+        preset.name(),
+        blob.len(),
+        preset.options().fri_number_of_queries,
+        total - grinding,
+        merkle,
+        nodes,
+        merkle - nodes,
+        total - merkle - grinding,
+        grinding,
+    );
+}
+
 /// Cycle count only of the recursion guest verifying a 1-query inner proof.
 #[test]
 #[ignore = "diagnostic: fast; recursion guest cycle count (1 query)"]