lfm: the driver's sampler and node builder never panic

HostSampler::stop expected its thread to be there and not to have
panicked, and the node builder expected a slot it had just filled: both
are on the production path now that the whole-block driver runs in the
CLI. A sampler that could not run reports no peak; a node whose program
never arrived is the builder's error.

Author: MauroToscano

prover/src/lfm/block_tree_pipeline.rs

@@ -429,7 +429,9 @@ impl Pipe {
                                 emit += secs;
                                 ready[l][jj] = Some(program);
                             }
-                            ready[lv][j].take().expect("filled above")?
+                            ready[lv][j].take().ok_or_else(|| {
+                                format!("level {} node {j}: its program never arrived", lv + 1)
+                            })??
                         }
                     };
                     let (artifacts, derived) = built(&program)?;

prover/src/lfm/tree_run.rs

@@ -389,11 +389,11 @@ impl HostSampler {
     /// `(peak GiB, argmax UNIX seconds)` over this sampler's window.
     pub(crate) fn stop(mut self) -> (f64, f64) {
         self.stop.store(true, std::sync::atomic::Ordering::Relaxed);
+        // A sampler that could not run reports no peak, at the time it stopped.
         self.handle
             .take()
-            .expect("a sampler is stopped once")
-            .join()
-            .expect("the sampler thread must not panic")
+            .and_then(|h| h.join().ok())
+            .unwrap_or_else(|| (0.0, unix_now()))
     }
 }