merge #1010's head 88b0d31 into noepoch/whir

The one-binary whole-block A/B compares #1010's epoch tree and the block
tree on the same code: Gruen GKR rounds, no lift, the global-after-last
latch.

# Conflicts:
#	crypto/multilinear/src/constraint_argument.rs

Author: MauroToscano

crypto/math-cuda/kernels/sumcheck.cu

@@ -17,7 +17,7 @@ use std::sync::Arc;
 
 use crate::Result;
 use crate::device::{DeviceReservation, alloc_or_trim, alloc_zeros_or_trim, backend, htod_or_trim};
-use crate::sumcheck::{DeviceFactors, eq_table_ext3, launch_shape};
+use crate::sumcheck::{FactorView, eq_table_ext3, launch_shape};
 
 const BLOCK_DIM: u32 = 256;
 const MAX_GRID: u32 = 4096;
@@ -50,7 +50,7 @@ pub struct FusedBus<'a> {
 /// One table's fused zerocheck on the card.
 pub struct FusedZerocheck<'f> {
     stream: Arc<CudaStream>,
-    source: &'f DeviceFactors,
+    source: FactorView<'f>,
     width: usize,
     /// The cube the lifted factors span.
     rows: usize,
@@ -151,16 +151,16 @@ impl<'f> FusedZerocheck<'f> {
     /// `r_tail` and `rho_tail` are `r[2..]` and `ρ[2..]` (three u64 a
     /// coordinate): the grid pass's weights.
     pub fn new(
-        source: &'f DeviceFactors,
+        source: FactorView<'f>,
         program: FusedProgram<'_>,
         bus: FusedBus<'_>,
         r_tail: &[u64],
         rho_tail: &[u64],
         grid_rows: usize,
         gruen_rows: usize,
     ) -> Result<Option<Self>> {
-        let rows = source.len();
-        let width = source.width();
+        let rows = source.rows;
+        let width = source.width;
         assert!(
             rows >= 8 && rows.is_power_of_two(),
             "the grid needs four rows a group"
@@ -194,7 +194,7 @@ impl<'f> FusedZerocheck<'f> {
         )) else {
             return Ok(None);
         };
-        let stream = source.stream().clone();
+        let stream = source.stream.clone();
         let q = rows / 4;
         let nonempty = |v: &[u64]| -> Vec<u64> { if v.is_empty() { vec![0] } else { v.to_vec() } };
         let nodes = htod_or_trim(&stream, &nonempty(program.nodes))?;
@@ -316,7 +316,7 @@ impl<'f> FusedZerocheck<'f> {
             q.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32,
             BLOCK_DIM,
         );
-        let factors = self.source.factor_ptrs();
+        let factors = self.source.ptrs;
         unsafe {
             self.stream
                 .launch_builder(&be.zc_bus_u)
@@ -328,6 +328,7 @@ impl<'f> FusedZerocheck<'f> {
                 .arg(&self.constant)
                 .arg(&self.eq_rho)
                 .arg(&mut self.partials)
+                .arg(&self.source.stride)
                 .launch(LaunchConfig {
                     grid_dim: (grid, 1, 1),
                     block_dim: (block, 1, 1),
@@ -357,7 +358,7 @@ impl<'f> FusedZerocheck<'f> {
         }
         self.stream.memcpy_htod(&[u64::MAX], &mut self.violation)?;
         let (grid, block) = self.shape(q, points.len(), self.num_slots as u64)?;
-        let factors = self.source.factor_ptrs();
+        let factors = self.source.ptrs;
         unsafe {
             self.stream
                 .launch_builder(&be.zc_grid01)
@@ -373,6 +374,7 @@ impl<'f> FusedZerocheck<'f> {
                 .arg(&mut self.partials)
                 .arg(&mut self.violation)
                 .arg(&u32::from(keep_corners))
+                .arg(&self.source.stride)
                 .launch(LaunchConfig {
                     grid_dim: (grid, points.len() as u32, 1),
                     block_dim: (block, 1, 1),
@@ -396,7 +398,7 @@ impl<'f> FusedZerocheck<'f> {
         let width = self.width as u64;
         let total = width * q;
         let grid = total.div_ceil(BLOCK_DIM as u64).clamp(1, MAX_GRID as u64) as u32;
-        let factors = self.source.factor_ptrs();
+        let factors = self.source.ptrs;
         unsafe {
             self.stream
                 .launch_builder(&be.zc_fold2)
@@ -405,6 +407,7 @@ impl<'f> FusedZerocheck<'f> {
                 .arg(&width)
                 .arg(&self.scratch)
                 .arg(&mut self.folded)
+                .arg(&self.source.stride)
                 .launch(LaunchConfig {
                     grid_dim: (grid, 1, 1),
                     block_dim: (BLOCK_DIM, 1, 1),

crypto/math-cuda/src/argue_fused.rs

@@ -366,6 +366,12 @@ pub struct Backend {
     pub zc_bus_column: CudaFunction,
     pub zc_halve: CudaFunction,
     pub zc_round_gruen: CudaFunction,
+    // D-ARGUE S1-3's GKR layer rounds (`crate::gkr::GruenLayer`).
+    pub gkr_eq_levels_ext3: CudaFunction,
+    pub gkr_round_gruen: CudaFunction,
+    pub gkr_gruen_finish: CudaFunction,
+    // D-BATCH M1-2's input layer from the base columns (`crate::gkr::input_from_columns`).
+    pub gkr_input_from_columns: CudaFunction,
     pub sumcheck_fold_ext3: CudaFunction,
     pub mle_fold_base_ext3: CudaFunction,
     pub eq_expand_level_ext3: CudaFunction,
@@ -1192,6 +1198,10 @@ impl Backend {
             zc_bus_column: sumcheck.load_function("zc_bus_column")?,
             zc_halve: sumcheck.load_function("zc_halve")?,
             zc_round_gruen: sumcheck.load_function("zc_round_gruen")?,
+            gkr_eq_levels_ext3: sumcheck.load_function("gkr_eq_levels_ext3")?,
+            gkr_round_gruen: sumcheck.load_function("gkr_round_gruen")?,
+            gkr_gruen_finish: sumcheck.load_function("gkr_gruen_finish")?,
+            gkr_input_from_columns: sumcheck.load_function("gkr_input_from_columns")?,
             sumcheck_fold_ext3: sumcheck.load_function("sumcheck_fold_ext3")?,
             mle_fold_base_ext3: sumcheck.load_function("mle_fold_base_ext3")?,
             eq_expand_level_ext3: sumcheck.load_function("eq_expand_level_ext3")?,