From 1ee78246dd8e12f69d775765c0c0dcfa6bc0e553 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:03:07 -0400 Subject: [PATCH 001/127] fix(ci): include audio quality in JavaScript gates Cover every Bun workspace with the scope regression check. Fixes #48. Co-Authored-By: Codex --- js/justfile | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/js/justfile b/js/justfile index bc8cf3fdcf..7d999cd98e 100644 --- a/js/justfile +++ b/js/justfile @@ -12,7 +12,7 @@ set working-directory := '.' # # The playout corpus lives in rs/moq-audio because that crate has to include_str! it, but it is # generated and freshness-checked from js/hang, so editing it has to run these suites too. -scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|rs/moq-audio/tests/playout-01\.json$|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' +scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/audio-quality/|test/wasm/|rs/moq-audio/tests/playout-01\.json$|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' # Run all checks by default. default: @@ -57,6 +57,7 @@ check $FILES="": fi # doc/ is in this scope, so the binding pages are checked here. python3 ../rs/scripts/stats-docs.py + just _scope-test bun install --frozen-lockfile bun common/deps.ts if tty -s; then @@ -67,6 +68,20 @@ check $FILES="": bun biome check just build +# Every workspace must trigger the JavaScript gates. +[private] +_scope-test: + #!/usr/bin/env bash + set -euo pipefail + missing=() + while IFS= read -r workspace; do + grep -qE '{{ scope }}' <<< "$workspace/package.json" || missing+=("$workspace") + done < <(jq -r '.workspaces[]' ../package.json) + if ((${#missing[@]})); then + echo "js: workspaces outside the JS scope: ${missing[*]}" >&2 + exit 1 + fi + # Auto-fix lint and formatting issues. Skips on the same terms as `check`. fix $FILES="": #!/usr/bin/env bash From 1dfa1971949b754e807d64fa84b7f7354ce6c709 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:03:07 -0400 Subject: [PATCH 002/127] test(audio-quality): prove output and sampling safeguards Exercise real replay output paths and probe sampling, and test large recording floors. Each regression fails when its corresponding safeguard is reverted. Covers the harness portion of #50. Co-Authored-By: Codex --- test/audio-quality/clients/js/replay.ts | 14 ++---- .../clients/js/src/probe.test.ts | 37 ++++++++++++-- test/audio-quality/clients/js/src/run.test.ts | 48 +++++++++++++++++++ .../clients/js/src/schema.test.ts | 12 ++++- test/audio-quality/clients/js/src/schema.ts | 11 +++++ 5 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 test/audio-quality/clients/js/src/run.test.ts diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts index e1eb01d53a..6ee84af598 100644 --- a/test/audio-quality/clients/js/replay.ts +++ b/test/audio-quality/clients/js/replay.ts @@ -32,6 +32,7 @@ import { BUCKET_MS, convergence, episodes as episodesOf, + frameFloor, type Point, type Ring, type Row, @@ -136,20 +137,11 @@ for (const { recording, row } of matrix) { const tag = rowKey(row); const arrivals = recorded(recording.fixture); - // The catalog floor `Sync` holds the target above is the codec's frame duration, which is - // the trace's own nominal spacing. Reduced rather than spread, because a recording is tens of - // thousands of frames and `Math.min(...)` of that many arguments blows the stack. - const media = arrivals.map((a) => a.media).sort((a, b) => a - b); - let smallest = Number.POSITIVE_INFINITY; - for (let i = 1; i < media.length; i++) { - const gap = media[i] - media[i - 1]; - if (gap > 0 && gap < smallest) smallest = gap; - } - if (!Number.isFinite(smallest)) { + const floorMs = frameFloor(arrivals.map((a) => a.media)); + if (floorMs === undefined) { console.error(`error: ${tag} has no two distinct media timestamps, so it has no frame duration`); process.exit(2); } - const floorMs = Math.ceil(smallest); const build = row.ring === "isolated" ? shared(recording.rate) : post(recording.rate); const result = replay(build, arrivals, { diff --git a/test/audio-quality/clients/js/src/probe.test.ts b/test/audio-quality/clients/js/src/probe.test.ts index 95b9e22347..efa3aa6e6e 100644 --- a/test/audio-quality/clients/js/src/probe.test.ts +++ b/test/audio-quality/clients/js/src/probe.test.ts @@ -1,9 +1,9 @@ -import { expect, test } from "bun:test"; +import { expect, jest, test } from "bun:test"; import type MoqWatch from "@moq/watch/element"; -import { threadOf } from "./probe.ts"; +import { probe, threadOf } from "./probe.ts"; // Only what `threadOf` reads: `audio.out`, with or without the signal. -const watch = (out: Record) => ({ audio: { out } }) as unknown as MoqWatch; +const watch = (out: Record) => ({ audio: { out }, sync: { out: {} } }) as unknown as MoqWatch; const signal = (value: unknown) => ({ peek: () => value }); test("a build with the signal says which thread feeds the ring", () => { @@ -21,3 +21,34 @@ test("a worker still starting is pending, not a build that cannot say", () => { expect(threadOf(watch({ thread: signal(undefined) }))).toEqual({ kind: "pending" }); expect(threadOf(watch({}))).toBeUndefined(); }); + +test("the probe sums buffered ranges and tolerates builds without a range list", () => { + jest.useFakeTimers(); + try { + for (const [buffered, expected] of [ + [ + [ + { start: 0, end: 40 }, + { start: 60, end: 100 }, + ], + 80, + ], + [12, undefined], + [{}, undefined], + [undefined, undefined], + [[], undefined], + ] as const) { + const running = probe(watch({ buffered: signal(buffered) })); + try { + jest.advanceTimersByTime(250); + const samples = running.drain(); + expect(samples).toHaveLength(1); + expect(samples[0].buffered).toBe(expected); + } finally { + running.stop(); + } + } + } finally { + jest.useRealTimers(); + } +}); diff --git a/test/audio-quality/clients/js/src/run.test.ts b/test/audio-quality/clients/js/src/run.test.ts new file mode 100644 index 0000000000..a2ad23a9a5 --- /dev/null +++ b/test/audio-quality/clients/js/src/run.test.ts @@ -0,0 +1,48 @@ +import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const script = fileURLToPath(new URL("../../../run.sh", import.meta.url)); + +test("replay refuses an occupied output directory without changing its files", () => { + const out = mkdtempSync(join(tmpdir(), "moq-audio-output-")); + try { + writeFileSync(join(out, "marker"), "keep this run"); + const result = spawnSync("bash", [script, "--runtime", "replay", "--out", out], { encoding: "utf8" }); + expect(result.status).toBe(2); + expect(result.stderr).toContain("is not empty; remove it or name a new directory"); + expect(readFileSync(join(out, "marker"), "utf8")).toBe("keep this run"); + expect(readdirSync(out)).toEqual(["marker"]); + const listed = spawnSync("bash", [script, "--runtime", "replay", "--list", "--out", out], { encoding: "utf8" }); + expect(listed.status).toBe(0); + expect(listed.stdout).toContain("replay-aac-44100-lan-bbb-isolated"); + expect(readdirSync(out)).toEqual(["marker"]); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}); + +test("replay saves results into empty and missing output directories", () => { + const root = mkdtempSync(join(tmpdir(), "moq-audio-output-")); + try { + mkdirSync(join(root, "empty")); + for (const name of ["empty", "missing"]) { + const out = join(root, name); + const result = spawnSync( + "bash", + [script, "--runtime", "replay", "--profiles", "mic-local", "--rings", "plain", "--out", out], + { + encoding: "utf8", + env: { ...process.env, MOQ_TEST_RUNS: join(root, "runs") }, + }, + ); + expect(result.status).toBe(0); + expect(readdirSync(out)).toContain("replay-opus-48000-mic-local-plain.summary.json"); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}, 20_000); diff --git a/test/audio-quality/clients/js/src/schema.test.ts b/test/audio-quality/clients/js/src/schema.test.ts index 3791de588f..0f887998b3 100644 --- a/test/audio-quality/clients/js/src/schema.test.ts +++ b/test/audio-quality/clients/js/src/schema.test.ts @@ -1,5 +1,15 @@ import { expect, test } from "bun:test"; -import { threadVoid } from "./schema.ts"; +import { frameFloor, threadVoid } from "./schema.ts"; + +test("the replay floor comes from distinct sorted media timestamps", () => { + expect(frameFloor([60.5, 0, 20.2, 20.2, 40.4])).toBe(21); + expect(frameFloor([12, 12])).toBeUndefined(); + expect(frameFloor([])).toBeUndefined(); +}); + +test("the replay floor accepts recordings too large for spread arguments", () => { + expect(frameFloor(Array.from({ length: 1_000_000 }, (_, i) => i * 20))).toBe(20); +}); test("audio fed by the page's worker over the lane's transport counts", () => { expect(threadVoid({ kind: "worker", transport: "webtransport" }, "webtransport")).toBeUndefined(); diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index c2dcf180b9..746042cc85 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -24,6 +24,17 @@ /** Milliseconds, as a float. The unit of every duration in this schema. */ export type Ms = number; +/** Sort media timestamps and return the smallest positive gap, rounded up to milliseconds. */ +export function frameFloor(media: number[]): Ms | undefined { + media.sort((a, b) => a - b); + let smallest = Number.POSITIVE_INFINITY; + for (let i = 1; i < media.length; i++) { + const gap = media[i] - media[i - 1]; + if (gap > 0 && gap < smallest) smallest = gap; + } + return Number.isFinite(smallest) ? Math.ceil(smallest) : undefined; +} + /** A fraction of 1. The unit of every share in this schema. */ export type Share = number; From e8e61e701f2865472bd96d3484cd95187e81048a Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:08:19 -0400 Subject: [PATCH 003/127] fix(shaper): verify traffic under each active profile Reject zero report intervals at argument parsing. Count datagrams by active profile without adding a lock or an extra atomic update to forwarding. Cover stepped impairments and benchmark clients against step counts in nightly CI. Fixes #47. Co-Authored-By: Codex --- .github/workflows/nightly.yml | 6 ++ Cargo.lock | 1 + rs/moq-shaper/Cargo.toml | 5 ++ rs/moq-shaper/README.md | 10 ++- rs/moq-shaper/benches/forward.rs | 84 +++++++++++++++++++ rs/moq-shaper/src/lib.rs | 134 ++++++++++++++++++++++++++++--- rs/moq-shaper/src/main.rs | 11 ++- rs/moq-shaper/tests/cli.rs | 12 +++ 8 files changed, 247 insertions(+), 16 deletions(-) create mode 100644 rs/moq-shaper/benches/forward.rs diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d1d34287eb..7b03884a4d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -90,6 +90,12 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just rs audit + - name: Shaper forwarding benchmark + if: ${{ !cancelled() }} + run: >- + nix develop --command cargo bench --locked -p moq-shaper --bench forward -- + --warm-up-time 1 --measurement-time 2 --sample-size 10 + # The OBS plugin and libmoq's C fixtures are the only things that link # libmoq.a from outside cargo, so a stale `rs/libmoq/native-libs/*.txt` # breaks them and nothing else. obs.yml lists the inputs that can do that, diff --git a/Cargo.lock b/Cargo.lock index 3e8b4b26fd..e4cf9dc097 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4722,6 +4722,7 @@ version = "0.0.1" dependencies = [ "anyhow", "clap", + "criterion", "humantime", "humantime-serde", "rand 0.10.3", diff --git a/rs/moq-shaper/Cargo.toml b/rs/moq-shaper/Cargo.toml index a18c3cc8fc..10a2e1a61f 100644 --- a/rs/moq-shaper/Cargo.toml +++ b/rs/moq-shaper/Cargo.toml @@ -28,4 +28,9 @@ tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-th toml = { workspace = true } [dev-dependencies] +criterion = { workspace = true } tokio = { workspace = true, features = ["test-util"] } + +[[bench]] +name = "forward" +harness = false diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md index 8eb817054a..0d942ef18c 100644 --- a/rs/moq-shaper/README.md +++ b/rs/moq-shaper/README.md @@ -40,12 +40,13 @@ profile never acted. | `--profile` | A built-in profile's name, or a profile TOML file, in place of the flags that shape the path. See [Profiles](#profiles). | | `--tcp-passthrough` | Also pipe TCP on the listening port to the target, untouched. See [TCP](#tcp). | | `--report` | Write the profile, seed and counters to this file as JSON at exit. See [Report](#report). | -| `--report-interval` | Print the same JSON as one line on stdout this often. | +| `--report-interval` | Print the same JSON as one line on stdout at this nonzero interval. | As a library, `Shaper::bind` takes a `Config`: where to listen and forward, the seed, and a `Profile` per direction. It also takes a `Setup`, which is a `Config` plus the opt-in options below, each off by default. `Shaper::verify` fails when the shaper stopped forwarding, or when an impairment the profile -configures never acted and the traffic makes that silence implausible. The relay's drills +configures never acted and the traffic makes that silence implausible. Each step is judged against +the traffic received while its profile was active. The relay's drills (`rs/moq-relay/tests/drills.rs`, described in `test/drill/README.md`) run every scenario through it. ## What a seed does and does not fix @@ -177,3 +178,8 @@ A profile that silently did nothing turns an impaired run into an unimpaired pas for the shaper's own exit; a harness grading a run should also check the report, where `near-zero` is the only profile for which a `delayed` of zero is the right answer. Keep the name, the seed and the counters with the run's artifacts: the seed is what turns a failing run into one that can be looked at again. + +## Benchmark + +`cargo bench -p moq-shaper --bench forward` measures UDP round trips across 1, 10, and 100 clients, +with 0, 4, and 64 scheduled profile steps. It checks packet contents and runs in nightly CI. diff --git a/rs/moq-shaper/benches/forward.rs b/rs/moq-shaper/benches/forward.rs new file mode 100644 index 0000000000..027a8ac40f --- /dev/null +++ b/rs/moq-shaper/benches/forward.rs @@ -0,0 +1,84 @@ +use std::{net::SocketAddr, sync::Arc, time::Duration}; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_shaper::{Config, Options, Profile, Setup, Shaper, Step}; +use tokio::{net::UdpSocket, runtime::Runtime, task::JoinSet}; + +fn forward(c: &mut Criterion) { + let runtime = Runtime::new().unwrap(); + let localhost: SocketAddr = "127.0.0.1:0".parse().unwrap(); + let mut group = c.benchmark_group("shaper_forward"); + for clients in [1, 10, 100] { + for steps in [0, 4, 64] { + let (shaper, sockets, echo_task) = runtime.block_on(async { + let echo = UdpSocket::bind(localhost).await.unwrap(); + let target = echo.local_addr().unwrap(); + let echo_task = tokio::spawn(async move { + let mut packet = [0; 64]; + loop { + let (size, from) = echo.recv_from(&mut packet).await.unwrap(); + echo.send_to(&packet[..size], from).await.unwrap(); + } + }); + let options = Options { + steps: (0..steps) + .map(|i| Step { + at: Duration::from_secs(3600 + i), + loss: Some(0.0), + ..Default::default() + }) + .collect(), + ..Default::default() + }; + let shaper = Shaper::bind(Setup { + up: options.clone(), + down: options, + ..Config { + bind: localhost, + target, + seed: 1, + up: Profile::default(), + down: Profile::default(), + } + .into() + }) + .await + .unwrap(); + let mut sockets = Vec::new(); + for _ in 0..clients { + let socket = UdpSocket::bind(localhost).await.unwrap(); + socket.connect(shaper.addr()).await.unwrap(); + sockets.push(Arc::new(socket)); + } + (shaper, sockets, echo_task) + }); + group.throughput(Throughput::Elements(clients * 32)); + group.bench_function(BenchmarkId::new(clients.to_string(), steps), |b| { + b.iter(|| { + runtime.block_on(async { + let mut tasks = JoinSet::new(); + for socket in &sockets { + let socket = socket.clone(); + tasks.spawn(async move { + let mut packet = [0; 64]; + for _ in 0..32 { + socket.send(&[7; 64]).await.unwrap(); + assert_eq!(socket.recv(&mut packet).await.unwrap(), 64); + assert_eq!(packet, [7; 64]); + } + }); + } + while let Some(result) = tasks.join_next().await { + result.unwrap(); + } + }) + }); + }); + shaper.verify().unwrap(); + echo_task.abort(); + } + } +} + +criterion_group!(benches, forward); +criterion_main!(benches); diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index fa84248661..493b67c74a 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -247,13 +247,24 @@ const IMPAIRMENTS: [Impairment; 3] = [ /// cannot, and that is when the silence means the profile is not in the path. const IMPLAUSIBLE: f64 = 1e-4; -/// The impairments `config` configures that `stats` shows implausibly never acted. -fn unapplied(config: &Config, stats: &Stats) -> Vec<&'static str> { +/// Impairments that implausibly never acted on traffic under their configured profile. +fn unapplied(setup: &Setup, stats: &Stats, segments: &[Vec; 2]) -> Vec<&'static str> { IMPAIRMENTS .iter() .filter(|(_, chance, acted)| { - let silence = (1.0 - chance(&config.up)).powf(stats.up.packets as f64) - * (1.0 - chance(&config.down)).powf(stats.down.packets as f64); + let silence: f64 = [(&setup.config.up, &setup.up), (&setup.config.down, &setup.down)] + .into_iter() + .zip(segments) + .map(|((profile, options), packets)| { + let mut profile = profile.clone(); + let mut silence = (1.0 - chance(&profile)).powf(packets[options.steps.len()] as f64); + for (step, &packets) in options.steps.iter().zip(packets.iter().rev().skip(1)) { + step.apply(&mut profile); + silence *= (1.0 - chance(&profile)).powf(packets as f64); + } + silence + }) + .product(); acted(&stats.up) + acted(&stats.down) == 0 && silence < IMPLAUSIBLE }) .map(|(name, ..)| *name) @@ -422,7 +433,7 @@ impl Shaper { false => None, }; - let tally = Arc::new([Tally::default(), Tally::default()]); + let tally = Arc::new([Tally::new(setup.up.steps.len()), Tally::new(setup.down.steps.len())]); let failed = Arc::new(OnceLock::new()); let task = tokio::spawn({ let setup = setup.clone(); @@ -473,7 +484,14 @@ impl Shaper { anyhow::bail!("the shaper stopped forwarding: {err} ({stats})"); } - let mut missing = unapplied(&self.setup.config, &stats); + let segments = self.tally.each_ref().map(|tally| { + tally + .packets + .iter() + .map(|packets| packets.load(Ordering::Relaxed)) + .collect() + }); + let mut missing = unapplied(&self.setup, &stats, &segments); if unbatched(&self.setup, &stats) { missing.push("batch"); } @@ -495,9 +513,9 @@ impl Drop for Shaper { const UP: usize = 0; const DOWN: usize = 1; -#[derive(Default)] struct Tally { - packets: AtomicU64, + /// Indexed by the number of profile steps still to come. + packets: Box<[AtomicU64]>, lost: AtomicU64, overflowed: AtomicU64, throttled: AtomicU64, @@ -505,10 +523,27 @@ struct Tally { reordered: AtomicU64, } +impl Default for Tally { + fn default() -> Self { + Self::new(0) + } +} + impl Tally { + fn new(steps: usize) -> Self { + Self { + packets: (0..=steps).map(|_| AtomicU64::new(0)).collect(), + lost: AtomicU64::new(0), + overflowed: AtomicU64::new(0), + throttled: AtomicU64::new(0), + delayed: AtomicU64::new(0), + reordered: AtomicU64::new(0), + } + } + fn snapshot(&self) -> Counters { Counters { - packets: self.packets.load(Ordering::Relaxed), + packets: self.packets.iter().map(|packets| packets.load(Ordering::Relaxed)).sum(), lost: self.lost.load(Ordering::Relaxed), overflowed: self.overflowed.load(Ordering::Relaxed), throttled: self.throttled.load(Ordering::Relaxed), @@ -870,7 +905,7 @@ impl Link { /// counted as delayed, or `None` when it is dropped. fn treat(&mut self, now: Instant, size: usize, tally: &Tally) -> Option<(Instant, bool)> { self.step(now); - bump(&tally.packets); + bump(&tally.packets[self.steps.len()]); // Every draw happens for every datagram, whatever the profile, so one // knob's outcome never shifts the stream another knob draws from. @@ -1111,6 +1146,13 @@ mod tests { #[test] fn silence_is_only_a_failure_once_it_is_implausible() { + let unapplied = |config: &Config, stats: &Stats| { + unapplied( + &config.clone().into(), + stats, + &[vec![stats.up.packets], vec![stats.down.packets]], + ) + }; let config = Config { bind: LOCALHOST, target: LOCALHOST, @@ -1153,6 +1195,74 @@ mod tests { assert_eq!(unapplied(&config, &quiet(1)), ["delay"]); } + #[tokio::test] + async fn verification_uses_the_profile_that_received_traffic() { + let echo = UdpSocket::bind(LOCALHOST).await.unwrap(); + let target = echo.local_addr().unwrap(); + let setup = Setup { + up: stepped(vec![Step { + at: Duration::ZERO, + loss: Some(0.0), + ..Default::default() + }]), + ..Config { + bind: LOCALHOST, + target, + seed: 1, + up: Profile { + loss: 1.0, + ..Default::default() + }, + down: Profile::default(), + } + .into() + }; + let shaper = Shaper::bind(setup).await.unwrap(); + let client = UdpSocket::bind(LOCALHOST).await.unwrap(); + client.send_to(b"ping", shaper.addr()).await.unwrap(); + let mut buf = [0; 4]; + let (size, from) = echo.recv_from(&mut buf).await.unwrap(); + assert_eq!(&buf[..size], b"ping"); + echo.send_to(&buf[..size], from).await.unwrap(); + let (size, _) = client.recv_from(&mut buf).await.unwrap(); + assert_eq!(&buf[..size], b"ping"); + let stats = shaper.verify().expect("loss ended before any traffic"); + assert_eq!(stats.up.packets, 1); + assert_eq!(stats.down.packets, 1); + assert_eq!(stats.up.lost, 0); + } + + #[test] + fn stepped_impairments_are_required_only_for_traffic_during_the_step() { + let setup = Setup { + up: stepped(vec![Step { + at: Duration::from_secs(1), + loss: Some(0.5), + ..Default::default() + }]), + ..Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 1, + up: Profile::default(), + down: Profile::default(), + } + .into() + }; + let mut stats = Stats { + up: Counters { + packets: 1000, + ..Default::default() + }, + ..Default::default() + }; + assert!(unapplied(&setup, &stats, &[vec![0, 1000], vec![0]]).is_empty()); + assert!(unapplied(&setup, &stats, &[vec![1, 999], vec![0]]).is_empty()); + assert_eq!(unapplied(&setup, &stats, &[vec![1000, 0], vec![0]]), ["loss"]); + stats.up.lost = 1; + assert!(unapplied(&setup, &stats, &[vec![1000, 0], vec![0]]).is_empty()); + } + #[tokio::test] async fn an_invalid_profile_is_refused() { let refused = |bad: Profile, why: &'static str| async move { @@ -1710,7 +1820,7 @@ mod tests { /// How long after `at` each of `sizes` leaves one link, fed at the same instant. fn owed(link: &mut Link, at: Instant, sizes: &[usize]) -> Vec { - let tally = Tally::default(); + let tally = Tally::new(link.steps.len()); sizes .iter() .map(|&size| link.treat(at, size, &tally).expect("dropped").0 - at) @@ -1771,7 +1881,7 @@ mod tests { let (queue, _) = mpsc::unbounded_channel(); let start = Instant::now(); let mut link = Link::new(&Profile::default(), &options, 7, 0, start, queue); - let tally = Tally::default(); + let tally = Tally::new(options.steps.len()); let mut lost = |at: Instant| (0..100).filter(|_| link.treat(at, 16, &tally).is_none()).count(); assert_eq!(lost(start), 0, "the profile opens clean"); diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index 0fac2dacd7..e9c2d3696b 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -71,11 +71,18 @@ struct Args { /// Write the profile, seed and counters to this file as JSON at exit. #[arg(long)] report: Option, - /// Print the same JSON as one line on stdout this often. - #[arg(long, value_parser = humantime::parse_duration)] + /// Print the same JSON as one line on stdout at this nonzero interval. + #[arg(long, value_parser = period)] report_interval: Option, } +fn period(value: &str) -> Result { + match humantime::parse_duration(value).map_err(|err| err.to_string())? { + Duration::ZERO => Err("must be greater than zero".to_string()), + period => Ok(period), + } +} + #[derive(Clone, Copy, clap::ValueEnum)] enum JitterModel { /// Uniform either way of the delay, per datagram, so datagrams can overtake each other. diff --git a/rs/moq-shaper/tests/cli.rs b/rs/moq-shaper/tests/cli.rs index fb37ba9d1f..169c5d9712 100644 --- a/rs/moq-shaper/tests/cli.rs +++ b/rs/moq-shaper/tests/cli.rs @@ -9,6 +9,18 @@ use std::{ time::Duration, }; +#[test] +fn a_zero_report_interval_is_refused() { + let output = Command::new(env!("CARGO_BIN_EXE_moq-shaper")) + .args(["--listen", "127.0.0.1:0", "--target", "127.0.0.1:9"]) + .args(["--report-interval", "0s"]) + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!(output.status.code(), Some(2), "{stderr}"); + assert!(stderr.contains("must be greater than zero"), "{stderr}"); +} + #[test] fn a_profile_run_reports_its_counters() { // The target answers TCP on the port number it takes datagrams on, as a relay does. From 227faa4896a5378a9f71c8b1f49d8bc122ef0842 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:01:53 -0400 Subject: [PATCH 004/127] fix(audio): preserve the noise estimate through digital silence Refs #31. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Codex --- rs/moq-audio/src/playout/noise.rs | 42 ++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/rs/moq-audio/src/playout/noise.rs b/rs/moq-audio/src/playout/noise.rs index 7f8beab818..5f71b48126 100644 --- a/rs/moq-audio/src/playout/noise.rs +++ b/rs/moq-audio/src/playout/noise.rs @@ -142,14 +142,14 @@ impl Channel { return false; } - // Quiet window: the level is worth having whatever the spectrum turns out to - // be, so record it before the flatness test can reject the filter. - self.threshold = energy.max(MIN_ENERGY); - let r0: f32 = window.iter().map(|x| x * x).sum(); + // Digital silence carries no room level and must not lower the search threshold. if r0 <= 0.0 { return false; } + + // Keep the quiet level even when the flatness test rejects its spectrum. + self.threshold = energy.max(MIN_ENERGY); let r1: f32 = window.windows(2).map(|w| w[0] * w[1]).sum(); let reflection = (r1 / r0).clamp(-0.99, 0.99); @@ -238,4 +238,38 @@ mod tests { let ratio = estimate.energy(0) / (level * level / 3.0); assert!(ratio > 0.25 && ratio < 4.0, "estimated {}", estimate.energy(0)); } + + /// One update per 20 ms block at 48 kHz, the way the engine feeds it. + fn feed(estimate: &mut Noise, pcm: &[f32]) { + for chunk in pcm.as_chunks::<960>().0 { + estimate.update(chunk); + } + } + + #[test] + fn zeros_do_not_lock_out() { + // Trained on a room, then half a second of exact zeros (a disabled mic, or a hole the + // ring filled), then the room comes back quieter. A quiet window is accepted on sight, + // so two seconds (a hundred updates) is plenty to be on the new level. + let mut estimate = Noise::new(1); + feed(&mut estimate, &noise(48_000, 2.0, 0.003, 1)); + assert!(estimate.initialised()); + feed(&mut estimate, &vec![0.0; 24_000]); + feed(&mut estimate, &noise(48_000, 2.0, 0.001, 1)); + let expected = 0.001f32 * 0.001 / 3.0; + let db = 10.0 * (estimate.energy(0) / expected).log10(); + assert!( + db.abs() < 3.0, + "estimate is {db:.2} dB off the room after digital silence" + ); + } + + #[test] + fn zeros_before_training_do_not_lock_out() { + // Zeros before any training: the first room tone should still train it. + let mut fresh = Noise::new(1); + feed(&mut fresh, &vec![0.0; 24_000]); + feed(&mut fresh, &noise(48_000, 1.0, 0.001, 1)); + assert!(fresh.initialised(), "a second of room tone after zeros should train it"); + } } From 0667ad6a9d9aea40e2dd63804e777cbb0a209db9 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:01:53 -0400 Subject: [PATCH 005/127] fix(audio): drain partial runs and park declared pauses Preserve decoded audio across bounded timeline holes. Commit short fragments before the splice instead of discarding them, and drain finite tracks without waiting for an impossible refill. Refs #32 and #33. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Codex --- doc/lib/rs/moq-audio.md | 2 +- rs/moq-audio/src/decode/consumer.rs | 228 ++++++++++++++++++++++++++-- rs/moq-audio/src/playout/buffer.rs | 12 ++ rs/moq-audio/src/playout/engine.rs | 139 ++++++++++++++++- 4 files changed, 364 insertions(+), 17 deletions(-) diff --git a/doc/lib/rs/moq-audio.md b/doc/lib/rs/moq-audio.md index 9d9b95e1c6..68636a23af 100644 --- a/doc/lib/rs/moq-audio.md +++ b/doc/lib/rs/moq-audio.md @@ -30,7 +30,7 @@ policy. Decoding likewise separates low-level `decode::Config`, PCM Highlights: - **`encode::Publication`** advertises the track and opens the microphone only while someone listens. Stop, swap devices, and restart without changing the track subscribers know; read a level meter for the UI. -- **A measured jitter buffer.** Set `decode::Options::delay` and `read()` hands back a fixed 10 ms block every call instead of whatever packet was decoded, so the speaker never waits on the network. How much it holds is measured from arrival timing rather than from a round trip ([Playout](/concept/playout)), and `delay` is only the floor under it. A gap is concealed rather than played as silence unless `conceal` is turned off. +- **A measured jitter buffer.** Set `decode::Options::delay` and `read()` hands back a fixed 10 ms block every call instead of whatever packet was decoded, so the speaker never waits on the network. How much it holds is measured from arrival timing rather than from a round trip ([Playout](/concept/playout)), and `delay` is only the floor under it. A gap is concealed rather than played as silence unless `conceal` is turned off. A declared endpoint drains the held audio and parks on silence without counting underruns. A finite track drains its final partial block before `read()` returns `None`. - **A/V sync signal.** `Consumer::playhead()` reports the media the speaker has actually been handed, which is what a video clock steers by; without a jitter buffer, `Sink::buffered()` says how far ahead the speaker is. - **Activity per packet**, read off the Opus stream, so a call UI shows who is talking without a second voice detector. - **One Linux build dependency**: ALSA headers, and only when `capture` or `playback` is enabled. diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 8d71da6189..4fdea2e022 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -360,7 +360,7 @@ impl Consumer { if let Some(frame) = mux_frame.as_ref() { self.observe(frame.timestamp); } - self.apply_discontinuity()?; + self.apply_discontinuity(mux_frame.as_ref().map(|frame| frame.timestamp))?; let Some(mux_frame) = mux_frame else { return std::task::Poll::Ready(self.flush()); }; @@ -513,10 +513,20 @@ impl Consumer { } } + // A playhead event the container raised with nothing after it yet is a declared endpoint: the + // marker group closed and the publisher paused. A hole or a skip only moves the cursor onto a + // group that already holds a frame, so it always arrives with one, and that frame is what + // applies it. The engine plays out what it holds and then the silence the pause is; the + // frame that ends the pause applies the event as usual. + let paused = self.track.discontinuity() != self.discontinuity; + let channels = self.resolved_layout.channels(); let rate = self.resolved_sample_rate; let format = self.options.output.format; let playout = self.playout.as_mut().expect("playout is on"); + if paused { + playout.engine.end(); + } if playout.ended && playout.engine.drained() { return Ok(None); } @@ -543,6 +553,7 @@ impl Consumer { fn end_playout(&mut self) { if let Some(playout) = self.playout.as_mut() { playout.ended = true; + playout.engine.end(); } } @@ -601,11 +612,16 @@ impl Consumer { /// A playhead event re-applies startup delay and skip. The decoder is not reset: /// the next group already starts on a keyframe, and pre-skip is a play-path concern. /// - /// A declared pause is heard differently here than in the browser. The marker never reaches - /// this consumer, so the engine keeps concealing until the resumed frame arrives and this - /// re-anchors on it, where the browser's ring is told the timeline ended and renders the pause - /// as the silence it is. Surfacing the marker and giving the engine an `end` is a follow-up. - fn apply_discontinuity(&mut self) -> Result<(), Error> { + /// Playout only starts over when `next`, the first frame after the event, does not + /// continue what it holds. A skipped group or a latency skip lands a hole no wider than + /// the budget on the wire, which is a hole the container could have waited out: the + /// audio held in front of it still plays and the splice carries the playhead across. + /// Wiping it would cut that audio and leave concealment nothing to study. + /// + /// A declared pause reaches playout before this does: [`read_playout`](Self::read_playout) sees + /// the event raised with nothing after it and parks the engine on the endpoint, the way the + /// browser's ring is told the timeline ended. The frame that resumes applies the event here. + fn apply_discontinuity(&mut self, next: Option) -> Result<(), Error> { let discontinuity = self.track.discontinuity(); if discontinuity == self.discontinuity { return Ok(()); @@ -613,7 +629,10 @@ impl Consumer { self.discontinuity = discontinuity; if let Some(playout) = self.playout.as_mut() { - playout.engine.reanchor(); + match next.is_some_and(|next| playout.engine.continues(next.into(), playout.budget)) { + true => playout.engine.jump(), + false => playout.engine.reanchor(), + } } self.next_start = None; self.spans.clear(); @@ -1458,12 +1477,8 @@ mod tests { /// media and nothing else. The endpoint bounds the run it ended, so what resumes has to play: /// the marker group closing is the playhead event, and it clears the endpoint with it. /// - /// The resumed run does play. What fails is the pause itself: see the `ignore` reason and the - /// note on [`Consumer::apply_discontinuity`]. + /// And the pause itself is silence, not concealment: nothing was lost, so nothing is an underrun. #[tokio::test] - #[ignore = "the engine conceals a declared pause rather than parking on it: the marker never \ - reaches this consumer, so 50 pulls across the pause are 49 underruns (measured \ - 2026-09-18). Surfacing the marker and an Engine::end() is the follow-up."] async fn a_declared_endpoint_then_resumed_media_decodes() { let input = Input { format: Format::F32, @@ -1549,10 +1564,24 @@ mod tests { .unwrap(); producer.cut(Some(end)).unwrap(); - // Half a second of the declared pause, pulled at the speaker's rate. + // Half a second of the declared pause, pulled at the speaker's rate. Past the tail of the run, + // it is silence, and the playhead parks on the endpoint rather than walking into the pause. + let mut silent = 0; for _ in 0..per_second / 2 { - consumer.read().await.unwrap().expect("a block of the declared pause"); + let block = consumer.read().await.unwrap().expect("a block of the declared pause"); + let pcm = Format::F32.as_interleaved_f32(&block.data, 1).unwrap(); + if pcm.iter().all(|sample| *sample == 0.0) { + silent += 1; + } } + assert!( + silent >= per_second / 2 - 3, + "only {silent} of the pause blocks were silence" + ); + assert!( + consumer.playhead().expect("audio has played") <= std::time::Duration::from(end), + "the playhead walked into the pause" + ); // Ten seconds later it unmutes. 48_000 samples per second, so 480_000 samples in. let resumed = 480_000 / frame_size as u64; @@ -1583,6 +1612,177 @@ mod tests { ); } + /// An Opus consumer with playout on, and the producer that feeds it. + async fn opus_playout( + delay: std::time::Duration, + ) -> ( + Encoder, + moq_mux::container::Producer, + Consumer, + moq_net::broadcast::Producer, + ) { + let encoder = Encoder::new(&Settings::new(48_000, Layout::Mono)).unwrap(); + let catalog = encoder.catalog(); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let consumer = Consumer::new( + &subscriber, + &catalog, + "audio", + Options { + max_age: std::time::Duration::from_secs(30), + delay: Some(delay), + ..Options::new() + }, + ) + .await + .unwrap(); + (encoder, producer, consumer, broadcast) + } + + /// One 20 ms Opus packet at `packet * 20 ms`, alone in its group, the way the browser + /// publisher writes audio. + fn write_opus( + producer: &mut moq_mux::container::Producer, + encoder: &mut Encoder, + packet: u64, + ) { + let frame_size = encoder.frame_size(); + let pcm = vec![0.25f32; frame_size]; + let payload = encoder.encode(&pcm).unwrap().payload; + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_scale(packet * frame_size as u64, 48_000).unwrap(), + duration: None, + payload, + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + } + + /// N1: a finite Opus track has to end. The pre-skip leaves every packet boundary 168 + /// frames off the 10 ms block, and live pacing (one packet per two pulls) keeps the + /// first fill under the trim that would realign it, so the last 168 frames are less + /// than a block. `read` returns `None` only once playout has drained. + #[tokio::test] + async fn a_finite_opus_track_ends() { + let (mut encoder, mut producer, mut consumer, _broadcast) = + opus_playout(std::time::Duration::from_millis(120)).await; + + write_opus(&mut producer, &mut encoder, 0); + consumer.read().await.unwrap().expect("first block"); + consumer.read().await.unwrap().expect("second block"); + for packet in 1..25 { + write_opus(&mut producer, &mut encoder, packet); + consumer.read().await.unwrap().expect("a block of the live run"); + consumer.read().await.unwrap().expect("a block of the live run"); + } + producer.finish().unwrap(); + + let mut reads = 0; + let mut ended = false; + while reads < 4_000 { + reads += 1; + if consumer.read().await.unwrap().is_none() { + ended = true; + break; + } + } + + let stats = consumer.playout.as_ref().unwrap().engine.stats(); + assert!( + ended, + "the track finished but read never returned None in {reads} reads: {stats:?}" + ); + assert_eq!( + stats.buffered, + std::time::Duration::ZERO, + "media was left unplayed: {stats:?}" + ); + } + + #[tokio::test(start_paused = true)] + async fn a_finite_track_below_the_refill_target_drains() { + let (mut encoder, mut producer, mut consumer, _broadcast) = + opus_playout(std::time::Duration::from_millis(120)).await; + write_opus(&mut producer, &mut encoder, 0); + consumer.read().await.unwrap().expect("initial stalled block"); + producer.finish().unwrap(); + for _ in 0..10 { + if consumer.read().await.unwrap().is_none() { + assert_eq!( + consumer.playout.as_ref().unwrap().engine.stats().buffered, + std::time::Duration::ZERO + ); + return; + } + } + panic!("a finished track waited for audio that cannot refill it"); + } + + /// N2: one skipped group is a hole in the timeline, not a new timeline. Eleven packets + /// (0 to 220 ms) arrive and playout starts on them; group 11 (220 ms) is never + /// published, and groups 12 onward arrive together, far enough past it that the + /// container gives up on 11 at once. The audio playout already held in front of the + /// hole still has to play before the audio behind it. + #[tokio::test] + async fn a_skipped_group_keeps_the_audio_already_held() { + let (mut encoder, mut producer, mut consumer, _broadcast) = + opus_playout(std::time::Duration::from_millis(60)).await; + + for packet in 0..=10 { + write_opus(&mut producer, &mut encoder, packet); + } + let first = consumer.read().await.unwrap().expect("first block"); + let held = consumer.playout.as_ref().unwrap().engine.stats(); + let budget = consumer.playout.as_ref().unwrap().budget; + + // Group 11 is lost. 12 to 22 land together (the burst after a stall), which puts the + // newest (440 ms) a budget past the hole, so the container gives up on 11 at once. It + // also sheds 12, a budget behind the newest, and resumes at 260 ms: 46.5 ms after the + // held audio ends, well inside the 185 ms budget. + producer.seek(12).unwrap(); + for packet in 12..=22 { + write_opus(&mut producer, &mut encoder, packet); + } + + let mut timestamps = vec![first.timestamp.as_micros()]; + let mut silent = 0; + for _ in 0..30 { + let frame = consumer.read().await.unwrap().expect("a block"); + let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap(); + if pcm.iter().all(|sample| *sample == 0.0) { + silent += 1; + } + timestamps.push(frame.timestamp.as_micros()); + } + let stats = consumer.playout.as_ref().unwrap().engine.stats(); + eprintln!("held before the hole: {held:?}, budget {budget:?}"); + eprintln!("block timestamps (us): {timestamps:?}"); + eprintln!("all-zero blocks: {silent}, stats after: {stats:?}"); + + // Opus pre-skip moves decoded audio 6.5 ms earlier, so the run before the hole + // ends at 213.5 ms. A block starting anywhere in its last 25 ms is that audio played. + let before_hole = timestamps + .iter() + .take_while(|at| **at < 240_000) + .copied() + .max() + .unwrap_or(0); + assert!( + before_hole >= 190_000, + "the audio held in front of the hole was thrown away: the last block before 240 ms \ + started at {before_hole} us (held {held:?})" + ); + } #[tokio::test] async fn a_playhead_event_reapplies_opus_pre_skip() { let input = Input { diff --git a/rs/moq-audio/src/playout/buffer.rs b/rs/moq-audio/src/playout/buffer.rs index cc99e6768a..fc0b26ea89 100644 --- a/rs/moq-audio/src/playout/buffer.rs +++ b/rs/moq-audio/src/playout/buffer.rs @@ -151,6 +151,18 @@ impl Buffer { ready } + /// Frames held in all, including any past a hole. + pub(crate) fn held(&self) -> usize { + self.packets.iter().map(|packet| packet.pcm.len() / self.channels).sum() + } + + /// Media time just past the newest sample held, or `None` when the buffer is empty. + pub(crate) fn end(&self) -> Option { + self.packets + .back() + .map(|packet| packet.timestamp + self.duration(packet.pcm.len() / self.channels)) + } + /// The contiguous run from the front, as media time. pub(crate) fn buffered(&self) -> Duration { self.duration(self.ready()) diff --git a/rs/moq-audio/src/playout/engine.rs b/rs/moq-audio/src/playout/engine.rs index 48172809bd..651898fac7 100644 --- a/rs/moq-audio/src/playout/engine.rs +++ b/rs/moq-audio/src/playout/engine.rs @@ -121,6 +121,10 @@ pub(crate) struct Engine { /// concealment off hears instead. silence: u32, + /// Whether the publisher declared its timeline finished here: once what is held has played, + /// the pause is silence on the wire rather than a gap, so there is nothing to conceal. + ended: bool, + /// Reused between blocks so a pull allocates nothing once it is warm. scratch: Vec, produced: Vec, @@ -165,6 +169,7 @@ impl Engine { segments: VecDeque::new(), playhead: None, silence: 0, + ended: false, scratch: Vec::new(), produced: Vec::new(), concealed: Vec::new(), @@ -204,6 +209,8 @@ impl Engine { } let count = pcm.len() / self.channels; + // Media is back, so the declared pause is over. + self.ended = false; self.decision.arrived(count); self.buffer.insert(timestamp, pcm); @@ -225,9 +232,42 @@ impl Engine { self.record_skip(dropped); } + /// Whether media starting at `at` continues the timeline held here across a hole + /// no wider than `within`, rather than starting a new one. + /// + /// Measured from the end of the newest audio held or already played, so a skipped + /// group reads as the hole it is and a publisher that restarted somewhere else, + /// earlier or far later, does not. + pub(crate) fn continues(&self, at: Duration, within: Duration) -> bool { + let reach = match (self.buffer.end(), self.played) { + (Some(end), Some(played)) => end.max(played), + (end, played) => match end.or(played) { + Some(reach) => reach, + None => return false, + }, + }; + at + self.buffer.duration(1) >= reach && at.saturating_sub(reach) <= within + } + + /// A hole in the timeline rather than a new one: what is held still plays, and the + /// splice carries the playhead over the hole. Only the arrival reference starts + /// over, since a measurement across the jump would read the hole as delay. + pub(crate) fn jump(&mut self) { + self.jitter.reanchor(); + self.observed = None; + } + + /// The publisher declared its timeline finished: play out what is held, then silence, and + /// count none of it as an underrun. Whatever comes next arrives on the far side of the pause, + /// so the next insert or [`reanchor`](Self::reanchor) takes it back. + pub(crate) fn end(&mut self) { + self.ended = true; + } + /// A timeline discontinuity: everything held describes a timeline that no longer /// exists. pub(crate) fn reanchor(&mut self) { + self.ended = false; self.jitter.reanchor(); self.buffer.clear(); self.sync.flush(); @@ -352,6 +392,25 @@ impl Engine { // One turn of the decision loop, committing at least one block to the output. fn produce(&mut self) { + let ready = self.buffer.ready(); + // A hole cannot grow the preceding fragment into a block. Commit that audio + // before splicing the next run, even when it fills only part of this pull. + if ready > 0 && ready < self.block && self.buffer.held() > ready { + self.play(ready); + return; + } + + // A declared endpoint cannot refill a stalled buffer. Drain every sample, + // pad the final block, then park without manufacturing an underrun. + if self.ended { + if ready > 0 { + self.play(self.block); + } else { + self.pause(); + } + return; + } + let front = self.buffer.front(); let contiguous = match (self.played, front) { (Some(played), Some(front)) => front <= played + self.buffer.duration(1), @@ -389,8 +448,10 @@ impl Engine { let at = start.expect("audio was taken"); self.played = Some(at + media); - let pcm = std::mem::take(&mut self.scratch); + let mut pcm = std::mem::take(&mut self.scratch); self.noise.update(&pcm); + // Pad the last partial block at a declared endpoint. + pcm.resize(count * self.channels, 0.0); self.commit(&pcm, media, 0, false); self.scratch = pcm; } @@ -454,6 +515,18 @@ impl Engine { self.scratch = input; } + // One block of the silence a declared pause is. It carries no media, and it leaves concealment + // and the level filter nothing from this side of the pause to build on. + fn pause(&mut self) { + self.produced.clear(); + let mut produced = std::mem::take(&mut self.produced); + produced.resize(self.block * self.channels, 0.0); + self.expand.reset(); + self.commit(&produced, Duration::ZERO, 0, false); + self.decision.reset(0); + self.produced = produced; + } + // Nothing to play: invent a block, or ramp into silence if the caller asked for // concealment to stay off. fn conceal(&mut self, underrun: bool) { @@ -578,7 +651,7 @@ impl Engine { // stays on the media timeline rather than on the output one. fn commit(&mut self, pcm: &[f32], media: Duration, shift: i64, concealed: bool) { let count = pcm.len() / self.channels; - debug_assert!(count >= self.block, "a produced block has to fill a pull"); + debug_assert!(count > 0, "a production step must advance the output"); self.sync.push(pcm); self.segments.push_back(Segment { frames: count, media }); @@ -1030,6 +1103,68 @@ mod tests { ); } + #[test] + fn a_fragment_before_a_hole_keeps_every_sample() { + let mut engine = Engine::new(config(1, false)).unwrap(); + engine.insert(Duration::ZERO, 0.0, &[0.25; 168]); + engine.insert(Duration::from_millis(20), 20.0, &[0.5; 960]); + let mut out = vec![0.0; engine.block()]; + engine.pull(&mut out); + assert_eq!(&out[..168], &[0.25; 168]); + assert_eq!(engine.stats().skipped, 0); + } + + /// What the Opus decoder hands playout: the first packet is short by the 312-frame + /// pre-skip, so every boundary after it sits 168 frames off the 480-frame block. + /// Returns packet `index`'s media time and samples. + fn opus_shaped(source: &[f32], index: usize) -> (Duration, Vec) { + const PRE_SKIP: usize = 312; + let packet = frames(RATE, PACKET); + let (start, count) = match index { + 0 => (0, packet - PRE_SKIP), + _ => (packet - PRE_SKIP + (index - 1) * packet, packet), + }; + ( + Duration::from_secs_f64(start as f64 / f64::from(RATE)), + source[start..start + count].to_vec(), + ) + } + + /// N1: a run shorter than one block at the front of the buffer, with a hole behind + /// it, never grows into a block. Live pacing (one 20 ms packet per two pulls) keeps + /// the first fill under the trim, so the play cursor stays on multiples of 480 and + /// the run before the hole ends 168 frames short of a block. The audio behind the + /// hole has to be played anyway. + #[test] + fn a_fragment_before_a_hole_does_not_stall_playout() { + let mut engine = Engine::new(config(1, true)).unwrap(); + let source = fixture::tone(RATE, 2.0, 997.0, 0.5, 1); + let mut out = vec![0.0; engine.block()]; + let mut now = 0.0; + + // Packet 12 never arrives. + const LOST: usize = 12; + const PACKETS: usize = 60; + for index in 0..PACKETS { + if index != LOST { + let (timestamp, pcm) = opus_shaped(&source, index); + engine.insert(timestamp, now, &pcm); + } + for _ in 0..2 { + engine.pull(&mut out); + now += BLOCK.as_secs_f64() * 1000.0; + } + } + + let (behind, _) = opus_shaped(&source, 30); + let playhead = engine.playhead().expect("audio has played"); + let stats = engine.stats(); + assert!( + playhead >= behind, + "playout never reached the audio behind the hole: playhead {playhead:?}, {stats:?}" + ); + } + /// A floor the age budget cannot hold is a contradiction the caller has to /// resolve, not something to clamp quietly. #[test] From 6e419705e005c1345cce58bb8211f0f3552f9629 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:01:53 -0400 Subject: [PATCH 006/127] fix(mux): avoid registering a reader for skip diagnostics Refs #39. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Codex --- rs/moq-mux/src/container/consumer.rs | 55 +++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index a158cecd5e..b51e8ec9a4 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -390,7 +390,7 @@ impl Consumer { Some(front) => ( front.min_timestamp, front.max_timestamp, - front.group.poll_finished(waiter).is_pending(), + front.group.poll_finished(&kio::Waiter::noop()).is_pending(), ), None => (None, None, false), }; @@ -2774,4 +2774,57 @@ mod tests { let event = kio::wait(|waiter| consumer.poll_event(waiter)).await.unwrap(); assert!(matches!(event, Some(Event::FrameEnd(end)) if end == ts(15_000))); } + + // review consumer-sync-video F16: the skip's log-only `open` field registers the read loop's + // waiter on the group it is about to drain. Counted as wake calls when that group's FIN lands + // afterwards. The same poll registers the waiter on group 0 twice elsewhere before it decides to + // skip, while group 0 is still what it waits on (two wakes with the log line's registration + // removed); a third is the log line's. + #[test] + fn a_skip_registers_no_extra_waker_on_the_group_it_drains() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct Count(AtomicUsize); + impl std::task::Wake for Count { + fn wake(self: std::sync::Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + fn wake_by_ref(self: &std::sync::Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let mut consumer = Consumer::new( + track.subscribe(moq_net::track::Subscription::default().with_max_age(Duration::from_millis(100))), + Container::Legacy(crate::container::Kind::Data), + ); + + // Group 0 stays open after its one frame, which the reader takes. + let mut open = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let frame = Frame { + timestamp: ts(0), + payload: Bytes::from_static(&[0xDE, 0xAD]), + keyframe: false, + duration: None, + }; + Container::Legacy(crate::container::Kind::Data) + .write(&mut open, &[frame]) + .unwrap(); + let first = kio::Waiter::noop(); + assert!(matches!(consumer.poll_read(&first), Poll::Ready(Ok(Some(f))) if f.timestamp == ts(0))); + + // Group 1 lands 300ms later, past the 100ms budget, so the next poll skips group 0. + write_group(&mut track, 1, &[ts(300_000)]); + let count = std::sync::Arc::new(Count(AtomicUsize::new(0))); + let waiter = kio::Waiter::new(std::task::Waker::from(count.clone())); + assert!(matches!(consumer.poll_read(&waiter), Poll::Ready(Ok(Some(f))) if f.timestamp == ts(300_000))); + + // The drained group's FIN arrives: nothing the reader is waiting for any more. + count.0.store(0, Ordering::SeqCst); + open.finish().unwrap(); + let wakes = count.0.load(Ordering::SeqCst); + assert!(wakes <= 2, "the drained group woke the reader {wakes} times"); + drop(waiter); + } } From f47fcee74443f9d890d59d340071010707486779 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:02:03 -0400 Subject: [PATCH 007/127] fix(relay): refuse new io_uring sessions during drain Refs #45. Verified raw QUIC and WebTransport admission against Linux io_uring workers. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: Codex --- doc/bin/relay/index.md | 2 +- rs/moq-relay/src/uring.rs | 6 +++ rs/moq-relay/tests/runtime_uring.rs | 84 +++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index 9bbd23666a..ffa967be9e 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -80,7 +80,7 @@ The accessors borrow and `run` consumes the relay, so clone `cluster`, tasks before calling it. `trigger.start()` drains every session with a GOAWAY and `run` returns once the drain window elapses, with the listeners released and the workers joined. A relay that has started draining refuses new sessions -with `503` on every transport, so a client redialing during a restart backs off +with `503` on every transport under both tokio and io_uring, so a client redialing during a restart backs off and keeps the session it is still being served on, instead of being handed one that is waved away on arrival. Build routes from `web().routes()` (or `internal().routes()`): `with_web` replaces the router, so `Router::new()` diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 4dae572f8d..3e0c662edf 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -660,6 +660,12 @@ async fn serve_connection( .await .context("moq handshake failed")?; + if serve.shutdown.draining() { + tracing::info!(id, "relay shutting down; refusing a new session"); + request.close(moq_net::Error::App(503)); + return Ok(()); + } + // The path + `?jwt=` ride the URL for WebTransport and the SETUP for raw // QUIC; either way the grant comes from the shared runtime, which owns the // auth client. diff --git a/rs/moq-relay/tests/runtime_uring.rs b/rs/moq-relay/tests/runtime_uring.rs index 6fbebee5c8..971faf1155 100644 --- a/rs/moq-relay/tests/runtime_uring.rs +++ b/rs/moq-relay/tests/runtime_uring.rs @@ -510,3 +510,87 @@ async fn spawn_auth_server(policy: moq_auth::serve::Policy) -> url::Url { tokio::spawn(async move { server.serve(listener).await }); url } + +/// F4 (review 2026-09-27): a draining relay refuses a new session on the io_uring workers too, +/// instead of admitting it and sending a GOAWAY on arrival, as `a_draining_relay_refuses_a_new_session` +/// (tests/shutdown_signal.rs) checks for the tokio listeners. +#[tokio::test] +async fn a_draining_uring_relay_refuses_a_new_session() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + if !supported() { + return; + } + + let dir = tempfile::tempdir().expect("tempdir"); + let (cert, key) = certificate(dir.path()); + let port = free_udp_port(); + let mut config = uring_config(&cert, &key, port); + config.drain_timeout = Duration::from_secs(5); + + let relay = Relay::load(config).await.expect("load relay"); + let origin = relay.cluster().origin.clone(); + let trigger = relay.shutdown_trigger().clone(); + let running = tokio::spawn(relay.run()); + + let broadcast = origin.create_broadcast("test").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + + // A session being served, so the drain finds one to GOAWAY. + let raw_url: url::Url = format!("moql://127.0.0.1:{port}/").parse().expect("parse url"); + let wt_url: url::Url = format!("https://127.0.0.1:{port}/").parse().expect("parse url"); + let subscriber = moq_tokio::origin::spawn(); + let consumer = subscriber.consume(); + let mut announced = consumer.announced(); + let first = connect(client().with_subscriber(subscriber), raw_url.clone()).await; + let draining = first.draining().expect("connected"); + let update = tokio::time::timeout(TIMEOUT, announced.next()) + .await + .expect("announcement timed out") + .expect("origin closed"); + assert!(update.kind.is_active(), "expected an announce, got a retraction"); + + trigger.start(); + let goaway = tokio::time::timeout(TIMEOUT, draining.recv()) + .await + .expect("no GOAWAY within the timeout of the trigger") + .expect("session closed without a GOAWAY"); + assert_eq!(goaway.uri(), "", "expected a reconnect-to-me GOAWAY"); + + // Both peer flavors the workers serve: raw QUIC and WebTransport. + let mut outcomes = Vec::new(); + for url in [&raw_url, &wt_url] { + let dialed = tokio::time::timeout( + TIMEOUT, + client().with_reconnect(false).connect(url.clone()).established(), + ) + .await + .expect("the second dial never settled"); + let outcome = match dialed { + Err(err) => format!("refused at dial: {err}"), + Ok(refused) => { + let drained = refused.draining(); + let closed = tokio::time::timeout(Duration::from_secs(2), refused.closed()) + .await + .is_ok(); + let goaway = drained.and_then(|goaway| goaway.peek()).is_some(); + match (closed, goaway) { + (true, false) => "closed on arrival without a GOAWAY".to_string(), + (_, true) => "admitted and sent a GOAWAY".to_string(), + (false, false) => "admitted and left open".to_string(), + } + } + }; + outcomes.push(format!("{}: {outcome}", url.scheme())); + } + + assert!( + outcomes.iter().all(|outcome| !outcome.contains("admitted")), + "a draining io_uring relay admitted a new session: {outcomes:?}" + ); + assert!(!running.is_finished(), "relay exited inside the drain window"); + + drop(first); + drop(broadcast); + running.abort(); + let _ = running.await; +} From 340f3cdf459b3580135a19dd6e3de6bf347cb778 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:56:17 -0400 Subject: [PATCH 008/127] fix(watch): preserve noise estimate across digital silence Fixes #31 for the JavaScript playout engine. Co-Authored-By: Codex --- js/watch/src/audio/playout/noise.ts | 2 +- js/watch/src/audio/playout/stretch.test.ts | 31 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/js/watch/src/audio/playout/noise.ts b/js/watch/src/audio/playout/noise.ts index 069346cd2b..9a3def4d72 100644 --- a/js/watch/src/audio/playout/noise.ts +++ b/js/watch/src/audio/playout/noise.ts @@ -97,6 +97,7 @@ class Channel { // Indexed rather than for-of: V8 boxes every element a for-of over a typed array yields. for (let i = 0; i < length; i++) energy += window[i] * window[i]; const r0 = energy; + if (r0 <= 0) return false; energy /= length; if (energy >= this.threshold) { @@ -112,7 +113,6 @@ class Channel { // Quiet window: the level is worth having whatever the spectrum turns out to be, so record // it before the flatness test can reject the filter. this.threshold = Math.max(energy, MIN_ENERGY); - if (r0 <= 0) return false; let r1 = 0; for (let i = 1; i < length; i++) r1 += window[i - 1] * window[i]; diff --git a/js/watch/src/audio/playout/stretch.test.ts b/js/watch/src/audio/playout/stretch.test.ts index c79a7bc7c4..1f8a890d09 100644 --- a/js/watch/src/audio/playout/stretch.test.ts +++ b/js/watch/src/audio/playout/stretch.test.ts @@ -213,6 +213,37 @@ describe("background noise", () => { for (let i = 256; i <= sine.length; i += 256) held.update([sine.subarray(i - 256, i)], 256); expect(held.initialised).toBe(false); }); + + // One update per 20ms block, the way the engine feeds it: 960 frames at 48kHz, of which the + // estimator reads the 256 frame tail. + const feed = (room: Noise, pcm: Float32Array) => { + for (let i = 960; i <= pcm.length; i += 960) room.update([pcm.subarray(i - 960, i)], 960); + }; + + it("digital silence does not stall the estimate", () => { + const rate = 48000; + const room = new Noise(1); + feed(room, noise(rate, 2, 0.003)[0]); + expect(room.initialised).toBe(true); + + // Half a second of exact zeros: a disabled mic, or a hole the ring filled. + feed(room, new Float32Array(rate / 2)); + + // The room comes back quieter. Two seconds is a hundred updates; a quiet window is accepted + // on sight, so the estimate should be on the new level well before that. + feed(room, noise(rate, 2, 0.001)[0]); + const expected = (0.001 * 0.001) / 3; + const db = 10 * Math.log10(room.energy(0) / expected); + expect(Math.abs(db)).toBeLessThan(3); + }); + + it("zeros before any training do not lock the estimate out", () => { + const rate = 48000; + const room = new Noise(1); + feed(room, new Float32Array(rate / 2)); + feed(room, noise(rate, 1, 0.001)[0]); + expect(room.initialised).toBe(true); + }); }); describe("buffer level filter", () => { From 09941b9bc57ac670586244edaa035be443355b82 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:57:22 -0400 Subject: [PATCH 009/127] fix(watch): terminate replaced audio worklet processors Fixes #43. Co-Authored-By: Codex --- js/watch/src/audio/decoder.offload.test.ts | 20 +++++++++++++++++++ js/watch/src/audio/decoder.ts | 8 +++++++- .../src/audio/render-worklet.port.test.ts | 17 ++++++++++++++++ js/watch/src/audio/render-worklet.ts | 14 +++++++++++++ js/watch/src/audio/render.ts | 11 +++++++++- 5 files changed, 68 insertions(+), 2 deletions(-) diff --git a/js/watch/src/audio/decoder.offload.test.ts b/js/watch/src/audio/decoder.offload.test.ts index b52c04f1b2..41f873ac7f 100644 --- a/js/watch/src/audio/decoder.offload.test.ts +++ b/js/watch/src/audio/decoder.offload.test.ts @@ -1153,6 +1153,26 @@ describe("one player's trouble", () => { }); }); +describe("a node the decoder replaces in the same context", () => { + it("lets its processor end", async () => { + tile({ offload: true }); + await until(() => handed()(), "the graph handed over"); + const [old] = Node.built; + const quantum = () => [new Float32Array(QUANTUM), new Float32Array(QUANTUM)]; + expect(old.render.process([], [quantum()], {})).toBe(true); + + // The page takes the audio back onto a fresh node of the same context. + fallbacks(); + const [player] = worker().told("player"); + worker().say({ type: "error", id: player.id, message: "TypeError: boom" }); + await until(() => Node.built.length === 2, "the page's own node"); + await sleep(20); + + // The context is still open, so the old node's processor would run until it closes. + expect(old.render.process([], [quantum()], {})).toBe(false); + }); +}); + describe("the thread a player's audio runs on", () => { it("is the page's, for no reason, when the player does not offload or has no relay to hand the worker", async () => { const off = tile({ offload: false }); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index f6a7f5d70e..7c0d118b7d 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -19,6 +19,7 @@ import type { Delay, Sync } from "../sync"; import { reportTransport, supportsSharedArrayBuffer } from "./buffer"; import { audioMaxAge, type DecoderConfig, decoderConfig } from "./config"; import type * as Playout from "./playout"; +import type { Close } from "./render"; // Compiled and inlined as a blob URL via vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; @@ -517,7 +518,12 @@ export class Decoder { channelCountMode: "explicit", outputChannelCount: [channelCount], }); - effect.cleanup(() => worklet.disconnect()); + effect.cleanup(() => { + // The context outlives this node, so the processor has to be told to end. See `Close`. + const close: Close = { type: "close" }; + worklet.port.postMessage(close); + worklet.disconnect(); + }); // Shared memory wherever the page can have it, for the page's own writes: the worker's ring is // messages on any page (see `Player` in `worker/host.ts`), so only the page's is worth naming. diff --git a/js/watch/src/audio/render-worklet.port.test.ts b/js/watch/src/audio/render-worklet.port.test.ts index 95c4de1d4b..3df3dc9273 100644 --- a/js/watch/src/audio/render-worklet.port.test.ts +++ b/js/watch/src/audio/render-worklet.port.test.ts @@ -169,4 +169,21 @@ describe("render worklet ports", () => { node.port2.close(); extra.port2.close(); }); + + it("lets its processor end once the node is closed", async () => { + // A processor whose `process` keeps returning true keeps running after its node is disconnected + // and dropped, for as long as the context is open. The decoder rebuilds nodes in a context that + // stays open, so it says when a node is done with. + if (!Render) throw new Error("render-worklet.ts registered no 'render' processor"); + const node = new MessageChannel(); + nextPort = node.port1; + const render = new Render(); + expect(render.process([], [[new Float32Array(QUANTUM)]], {})).toBe(true); + + node.port2.postMessage({ type: "close" }); + await settle(); + expect(render.process([], [[new Float32Array(QUANTUM)]], {})).toBe(false); + + node.port2.close(); + }); }); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index dcb7e131a9..daf55f769a 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -29,6 +29,8 @@ class Render extends AudioWorkletProcessor { #ports: MessagePort[] = []; // Whether a message has already failed to deserialize, which is said once. See #unreadable. #unread = false; + // Whether the node is done with, so `process` ends the processor. See `Close`. + #closed = false; constructor() { super(); @@ -65,6 +67,17 @@ class Render extends AudioWorkletProcessor { } else if (msg.type === "stall") { // Only meaningful in post mode; shared mode stalls via the control array. if (this.#backend instanceof AudioRingBuffer) this.#backend.stall(); + } else if (msg.type === "close") { + this.#closed = true; + this.#backend = undefined; + this.#engine = undefined; + this.#state = undefined; + for (const port of this.#ports) { + port.onmessage = null; + port.onmessageerror = null; + port.close(); + } + this.#ports.length = 0; } else if (msg.type === "end") { // Only meaningful in post mode; shared mode ends via the control array. if (this.#backend instanceof AudioRingBuffer) this.#backend.end(); @@ -103,6 +116,7 @@ class Render extends AudioWorkletProcessor { } process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record) { + if (this.#closed) return false; const output = outputs[0]; const backend = this.#backend; const engine = this.#engine; diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 9843df33a5..7746ed659e 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -4,7 +4,7 @@ import type { Snapshot } from "./playout"; import type { SharedRingBufferInit } from "./shared-ring-buffer"; /** Everything a writer sends the render worklet: over the node's own port, or over one handed to it as a {@link Port}. */ -export type Message = InitShared | InitPost | Data | End | Latency | Reset | Stall | Truncate | Port; +export type Message = InitShared | InitPost | Data | End | Latency | Reset | Stall | Truncate | Port | Close; /** Everything the render worklet sends back, to every port it holds. */ export type ToMain = State | Unreadable; @@ -74,6 +74,15 @@ export interface End { type: "end"; } +/** + * The node is done with: the processor stops rendering and lets the browser collect it. A processor + * whose `process` keeps returning true keeps running after its node is disconnected, for as long as the + * context is open, and the decoder rebuilds nodes in a context that stays open. + */ +export interface Close { + type: "close"; +} + /** * Drop buffered samples at or after `timestamp`, keeping what is already due (fallback path only; * the shared path truncates via Atomics). From 66c3930487f30a0f25051f9eaa88f6d3c61166dd Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:58:02 -0400 Subject: [PATCH 010/127] fix(watch): retain decoded sample rate during worker fallback Fixes #40. Co-Authored-By: Codex --- js/watch/src/audio/decoder.offload.test.ts | 41 ++++++++++++++++++++++ js/watch/src/audio/decoder.ts | 5 ++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/js/watch/src/audio/decoder.offload.test.ts b/js/watch/src/audio/decoder.offload.test.ts index 41f873ac7f..029aba9c83 100644 --- a/js/watch/src/audio/decoder.offload.test.ts +++ b/js/watch/src/audio/decoder.offload.test.ts @@ -1173,6 +1173,47 @@ describe("a node the decoder replaces in the same context", () => { }); }); +describe("the decoded rate", () => { + it("survives a fallback: the context the worker's rate built is kept", async () => { + // The catalog says 44.1 kHz; the decoder emits 48 kHz, as Chrome and Firefox do for Opus. + const rates: number[] = []; + class Counting extends MockContext { + constructor(options?: { sampleRate?: number }) { + super(options); + rates.push(this.sampleRate); + } + } + scope.AudioContext = Counting; + fallbacks(); + scope.crossOriginIsolated = true; + const t = tile({ offload: true }); + t.catalog.set({ + audio: { + renditions: { + audio: { codec: "opus", container: { kind: "legacy" }, sampleRate: 44_100, numberOfChannels: 2 }, + }, + }, + } as unknown as Catalog.Root); + await until(() => handed()(), "the graph handed over"); + for (let i = 0; i < 20; i++) t.write(i, i * 20_000); + await until(() => t.decoder.out.context.peek()?.sampleRate === RATE, "the context at the decoded rate"); + await sleep(50); + const before = rates.length; + + // The worker goes wrong for this player, and the page takes its audio back. + const [player] = worker().told("player"); + worker().say({ type: "error", id: player.id, message: "TypeError: boom" }); + await until(() => t.decoder.out.thread.peek()?.kind === "main", "the fallback"); + await until(() => t.page.live() === 1, "the page's own subscription"); + for (let i = 20; i < 40; i++) t.write(i, i * 20_000); + await sleep(150); + + // The page's decoder emits 48 kHz too, which the context already runs at. + expect(t.decoder.out.context.peek()?.sampleRate).toBe(RATE); + expect(rates.slice(before)).toEqual([]); + }); +}); + describe("the thread a player's audio runs on", () => { it("is the page's, for no reason, when the player does not offload or has no relay to hand the worker", async () => { const off = tile({ offload: false }); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 7c0d118b7d..12a771bf53 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -197,6 +197,8 @@ export class Decoder { // The rate the graph runs at: what the decoder turns out to emit, else what the catalog claims. // Deduped, so a decoded rate confirming the catalog's does not count as a change. readonly #rate: Computed; + // A replacement supply has not decoded yet, but the context already knows the stream's rate. + #decodedRate?: number; /** * The age budget for audio: `Sync.out.maxAge` plus what the ring can absorb past it. @@ -240,7 +242,8 @@ export class Decoder { if (!config) return undefined; const active = effect.get(this.#active); const decoded = active ? effect.get(active.supply.out.rate) : undefined; - return decoded ?? config.sampleRate; + if (decoded !== undefined) this.#decodedRate = decoded; + return this.#decodedRate ?? config.sampleRate; }); this.#mode = this.#signals.computed((effect) => { if (effect.get(this.#fallback) !== undefined) return "main"; From ee9d7cdc25b43a7be7adc24290933c5aa6f99a76 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:59:13 -0400 Subject: [PATCH 011/127] fix(watch): refuse workers that never finish starting Bound worker startup including lazy imports, terminate late workers, and preserve the refusal for later players. Fixes #42. Co-Authored-By: Codex --- js/watch/src/audio/worker/pool.test.ts | 59 ++++++++++++++++++- js/watch/src/audio/worker/protocol.ts | 78 ++++++++++++++++---------- 2 files changed, 106 insertions(+), 31 deletions(-) diff --git a/js/watch/src/audio/worker/pool.test.ts b/js/watch/src/audio/worker/pool.test.ts index 6576758f9e..1681d9430b 100644 --- a/js/watch/src/audio/worker/pool.test.ts +++ b/js/watch/src/audio/worker/pool.test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, jest, spyOn } from "bun:te import type * as Catalog from "@moq/hang/catalog"; import { Time } from "@moq/net"; import { LINGER, LIVENESS, Liveness, Pool } from "./pool"; -import { type FromWorker, type Report, type Support, support, TICK, type ToWorker } from "./protocol"; +import { AUDIO_DEADLINE, type FromWorker, type Report, type Support, support, TICK, type ToWorker } from "./protocol"; const FULL: Support = { audioDecoder: true, webTransport: true, webSocket: true }; @@ -239,6 +239,63 @@ describe("Pool", () => { expect(FakeWorker.created.length).toBe(1); }); + it("refuses and terminates a worker that is silent past the deadline", async () => { + // The worker, or the lazy chunk it comes in, never says ready: a stalled import, a script that hangs. + jest.useFakeTimers(); + FakeWorker.support = undefined; + const shared = pool(); + const a = shared.acquire(); + await ticks(); + + // Its player waits out its own deadline, falls back to the main thread and lets go. + jest.advanceTimersByTime(AUDIO_DEADLINE); + await ticks(); + a.release(); + jest.advanceTimersByTime(LINGER); + await ticks(); + + const [worker] = FakeWorker.created; + expect(worker.terminated).toBe(true); + }); + + it("does not make the next player wait on another worker after one was silent past the deadline", async () => { + jest.useFakeTimers(); + FakeWorker.support = undefined; + const shared = pool(); + const a = shared.acquire(); + await ticks(); + jest.advanceTimersByTime(AUDIO_DEADLINE); + await ticks(); + a.release(); + jest.advanceTimersByTime(LINGER); + await ticks(); + + // Whatever kept the first one from starting keeps the next one too, as a refusal does. + const b = shared.acquire(); + const settled = await Promise.race([b.ready, ticks().then(() => "still waiting")]); + expect(settled).not.toBe("still waiting"); + expect(settled).toBeDefined(); + expect(FakeWorker.created.length).toBe(1); + }); + + it("refuses a stalled import and terminates a worker created after its deadline", async () => { + jest.useFakeTimers(); + FakeWorker.support = undefined; + const loading = Promise.withResolvers(); + const shared = new Pool(() => loading.promise); + pools.push(shared); + const a = shared.acquire(); + jest.advanceTimersByTime(AUDIO_DEADLINE); + await ticks(); + const reason = await a.ready; + expect(reason).toContain("worker"); + expect(await shared.acquire().ready).toBe(reason); + loading.resolve(create()); + await ticks(); + expect(FakeWorker.created).toHaveLength(1); + expect(FakeWorker.created[0].terminated).toBe(true); + }); + it("refuses a message from a player before its worker is ready", async () => { FakeWorker.support = undefined; const shared = pool(); diff --git a/js/watch/src/audio/worker/protocol.ts b/js/watch/src/audio/worker/protocol.ts index e8a8b06f85..a2123b5d7a 100644 --- a/js/watch/src/audio/worker/protocol.ts +++ b/js/watch/src/audio/worker/protocol.ts @@ -226,44 +226,62 @@ export type Spawned = { handle: Handle } | { reason: string }; * `blob:` workers makes Chromium throw, and the lazy chunk it comes in can fail to load), reports a * load failure (the same CSP reaches Firefox as an `error` event), or lacks what {@link decide} needs. */ -export async function spawn(create: () => Worker | Promise, page: Transports = support()): Promise { - let worker: Worker; - try { - worker = await create(); - } catch (err) { - return { reason: `the worker could not be created: ${explain(err)}` }; - } - +export function spawn(create: () => Worker | Promise, page: Transports = support()): Promise { return new Promise((resolve) => { + let worker: Worker | undefined; + let settled = false; const fail = (reason: string) => { - detach(worker); - worker.terminate(); + if (settled) return; + settled = true; + clearTimeout(deadline); + if (worker) { + detach(worker); + worker.terminate(); + } resolve({ reason }); }; - - worker.onerror = (event: ErrorEvent) => { - // Before `ready` this can only be the script failing to load. - event.preventDefault(); - fail(`the worker failed to load: ${event.message || "error event"}`); - }; - worker.onmessageerror = () => fail("the worker's first message could not be deserialized"); - worker.onmessage = (event: MessageEvent) => { - const msg = event.data; - if (msg?.type !== "ready") { - fail(`the worker said ${JSON.stringify(msg?.type)} before it said ready`); + const deadline = setTimeout( + () => fail(`the audio worker played nothing in ${AUDIO_DEADLINE / 1000} s: ${STUCK.ready}`), + AUDIO_DEADLINE, + ); + + void (async () => { + let created: Worker; + try { + created = await create(); + } catch (err) { + fail(`the worker could not be created: ${explain(err)}`); return; } - - const reason = decide(msg.support, page); - if (reason !== undefined) { - fail(reason); + // A lazy import can complete after the startup deadline. + if (settled) { + created.terminate(); return; } - - const hello: ToWorker = { type: "hello", transports: page }; - worker.postMessage(hello); - resolve({ handle: handle(worker, msg.support) }); - }; + worker = created; + created.onerror = (event: ErrorEvent) => { + event.preventDefault(); + fail(`the worker failed to load: ${event.message || "error event"}`); + }; + created.onmessageerror = () => fail("the worker's first message could not be deserialized"); + created.onmessage = (event: MessageEvent) => { + const msg = event.data; + if (msg?.type !== "ready") { + fail(`the worker said ${JSON.stringify(msg?.type)} before it said ready`); + return; + } + const reason = decide(msg.support, page); + if (reason !== undefined) { + fail(reason); + return; + } + const hello: ToWorker = { type: "hello", transports: page }; + created.postMessage(hello); + settled = true; + clearTimeout(deadline); + resolve({ handle: handle(created, msg.support) }); + }; + })(); }); } From f8ee37a2c2a85a3254f44ea46cede0c55334bffe Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:59:28 -0400 Subject: [PATCH 012/127] fix(watch): wait for timing before building worker rings Fixes #39E. Co-Authored-By: Codex --- js/watch/src/audio/worker/host.test.ts | 23 +++++++++++++++++++++++ js/watch/src/audio/worker/host.ts | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/js/watch/src/audio/worker/host.test.ts b/js/watch/src/audio/worker/host.test.ts index 76697f0aef..8b844c50fa 100644 --- a/js/watch/src/audio/worker/host.test.ts +++ b/js/watch/src/audio/worker/host.test.ts @@ -667,6 +667,29 @@ describe("timing", () => { expect(read(1)).toEqual([Time.Milli(0), Time.Milli(20), Time.Milli(40)]); expect(read(2)).toEqual([Time.Milli(40)]); }); + + it("a graph before timing builds the ring in the mode timing asks for", async () => { + // The page sends timing before the graph today only because of the order its effects run in. + const { origin, dial } = relay(); + publish(origin); + const page = new Page(dial, 50); + page.post({ type: "hello", transports: TRANSPORTS }); + page.post(player(1)); + + // The worklet's end is this test: it records what the worker's ring tells it. + const { port1: worklet, port2: writer } = new MessageChannel(); + const told: Message[] = []; + worklet.onmessage = (event: MessageEvent) => told.push(event.data); + cleanup.push(() => worklet.close()); + page.post({ type: "graph", id: 1, ring: { port: writer, rate: RATE, channels: 2, conceal: false } }, [writer]); + await sleep(30); + page.post(timing(1, { buffer: Time.Milli(2_000), buffered: true })); + await sleep(100); + + const inits = told.filter((msg) => msg.type === "init-post"); + expect(inits.length).toBeGreaterThan(0); + expect(inits.at(-1)).toMatchObject({ type: "init-post", buffered: true }); + }); }); describe("the worker's own flushes", () => { diff --git a/js/watch/src/audio/worker/host.ts b/js/watch/src/audio/worker/host.ts index 9a62a83d46..21a43b64f5 100644 --- a/js/watch/src/audio/worker/host.ts +++ b/js/watch/src/audio/worker/host.ts @@ -271,7 +271,9 @@ class Player { source: this.#source, sync: this.#sync(), enabled: new Derived([this.#enabled, this.#timed] as const, (enabled, timed) => enabled && timed), - graph: this.#graph, + // The ring reads its depth and its buffered mode off timing once, when it is built, so it is + // not built before timing has arrived, whichever of the two the page sent first. + graph: new Derived([this.#graph, this.#timed] as const, (graph, timed) => (timed ? graph : undefined)), target: this.#target, maxAge: this.#maxAge, }); From 2e8bd11e41304d6fbb21c6260f11bc0efd32c686 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:00:25 -0400 Subject: [PATCH 013/127] fix(watch): keep worker ring reports off the page thread Keep the first playback report and errors on the page while the worker receives ongoing ring state. Fixes #39D. Co-Authored-By: Codex --- .../src/audio/render-worklet.port.test.ts | 56 +++++++++++++++++-- js/watch/src/audio/render-worklet.ts | 10 +++- js/watch/src/audio/render.ts | 4 +- js/watch/src/audio/worker/host.test.ts | 16 +++++- js/watch/src/audio/worker/remote.ts | 5 +- 5 files changed, 79 insertions(+), 12 deletions(-) diff --git a/js/watch/src/audio/render-worklet.port.test.ts b/js/watch/src/audio/render-worklet.port.test.ts index 3df3dc9273..3a09a1144c 100644 --- a/js/watch/src/audio/render-worklet.port.test.ts +++ b/js/watch/src/audio/render-worklet.port.test.ts @@ -112,13 +112,13 @@ describe("render worklet ports", () => { expect(loudest).toBeGreaterThan(0.2); await settle(); - // A state message every five quanta, to the node's port as always and to the writer's. - expect(page.length).toBeGreaterThanOrEqual(7); - expect(writer.length).toBe(page.length); + // A state message every five quanta to the writer's port; the node's port hears once that it played. + expect(writer.length).toBeGreaterThanOrEqual(7); const last = writer[writer.length - 1]; expect(last.type).toBe("state"); expect(last.debug.output).toBeGreaterThan(0); - expect(page[page.length - 1]).toEqual(last); + expect(page.length).toBe(1); + expect(page[0].debug.fresh).toBe(false); node.port2.close(); extra.port2.close(); @@ -186,4 +186,52 @@ describe("render worklet ports", () => { node.port2.close(); }); + + it("reports state only to a handed-over port, telling the node's own port once that the ring played", async () => { + // With a worker writing the ring, the page reads the node's port for two things only: that the + // ring played, and that a message was unreadable. Everything else is the worker's to hear, and a + // report every five quanta to a busy main thread is exactly what the offload is meant to spare it. + if (!Render) throw new Error("render-worklet.ts registered no 'render' processor"); + const node = new MessageChannel(); + nextPort = node.port1; + const render = new Render(); + const extra = new MessageChannel(); + const handoff: Port = { type: "port", port: extra.port1 }; + node.port2.postMessage(handoff, [extra.port1]); + await settle(); + + const page: ToMain[] = []; + const writer: ToMain[] = []; + node.port2.onmessage = (event: MessageEvent) => page.push(event.data); + extra.port2.onmessage = (event: MessageEvent) => writer.push(event.data); + + const init: InitPost = { + type: "init-post", + channels: 1, + rate: RATE, + latency: Time.Milli(20), + buffered: false, + conceal: false, + }; + extra.port2.postMessage(init); + const chunk = (RATE * 20) / 1000; + for (let i = 0; i < 20; i++) { + const samples = new Float32Array(chunk).fill(0.25); + const data: Data = { type: "data", data: [samples], timestamp: Time.Micro.fromMilli(Time.Milli(i * 20)) }; + extra.port2.postMessage(data, [samples.buffer]); + } + await settle(); + + // 50 quanta, about 133 ms of playing: ten reports. + pull(render, 50); + await settle(); + expect(writer.filter((msg) => msg.type === "state").length).toBe(10); + const told = page.filter((msg) => msg.type === "state"); + expect(told.length).toBeLessThanOrEqual(1); + // And what it was told is that the ring played. + expect(told.every((msg) => msg.type === "state" && !msg.debug.fresh)).toBe(true); + + node.port2.close(); + extra.port2.close(); + }); }); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index daf55f769a..e1e0c88bd8 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -31,6 +31,7 @@ class Render extends AudioWorkletProcessor { #unread = false; // Whether the node is done with, so `process` ends the processor. See `Close`. #closed = false; + #played = false; constructor() { super(); @@ -167,7 +168,14 @@ class Render extends AudioWorkletProcessor { this.#state.playhead = playhead; this.#state.debug = debug; } - for (const port of this.#ports) port.postMessage(this.#state); + for (const port of this.#ports) { + // The page only needs the first playback report when a worker owns the ring. + if (port === this.port && this.#ports.length > 1) { + if (this.#played || debug.fresh) continue; + this.#played = true; + } + port.postMessage(this.#state); + } } } diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 7746ed659e..8e48f51b4d 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -5,7 +5,7 @@ import type { SharedRingBufferInit } from "./shared-ring-buffer"; /** Everything a writer sends the render worklet: over the node's own port, or over one handed to it as a {@link Port}. */ export type Message = InitShared | InitPost | Data | End | Latency | Reset | Stall | Truncate | Port | Close; -/** Everything the render worklet sends back, to every port it holds. */ +/** Playback reports and errors sent by the render worklet. */ export type ToMain = State | Unreadable; /** @@ -23,7 +23,7 @@ export interface Unreadable { * * A writer that is not on the main thread (a dedicated worker) cannot reach the node's port, so the * page hands the worklet one end of a channel and the writer the other, and the ring's writes never - * wait on the page's event loop. {@link State} goes to every port the worklet holds. + * wait on the page's event loop. The writer gets every {@link State}; the page gets the first playback report. */ export interface Port { type: "port"; diff --git a/js/watch/src/audio/worker/host.test.ts b/js/watch/src/audio/worker/host.test.ts index 8b844c50fa..ec0facaa50 100644 --- a/js/watch/src/audio/worker/host.test.ts +++ b/js/watch/src/audio/worker/host.test.ts @@ -371,8 +371,22 @@ function graph(page: Page, id = 1, handed: Handed = direct()): { render: Process const node = new MessageChannel(); nextPort = node.port1; const render = new Render(); + // Everything the worklet reports: every state goes to the port it is handed, and the node's own port + // hears the rest (`unreadable`, and once that the ring played). const states: ToMain[] = []; - node.port2.onmessage = (event: MessageEvent) => states.push(event.data); + node.port2.onmessage = (event: MessageEvent) => { + if (event.data.type !== "state") states.push(event.data); + }; + node.port1.addEventListener("message", (event: MessageEvent) => { + if (event.data.type !== "port") return; + const port = event.data.port; + const post = port.postMessage.bind(port); + port.postMessage = ((msg: ToMain, transfer?: Transferable[]) => { + // Copied as postMessage copies it: the worklet refills one state object for every report. + if (msg.type === "state") states.push(structuredClone(msg)); + post(msg, transfer ?? []); + }) as MessagePort["postMessage"]; + }); cleanup.push(() => node.port2.close()); // The worklet's own copy of the port it is handed, as its listener receives it. diff --git a/js/watch/src/audio/worker/remote.ts b/js/watch/src/audio/worker/remote.ts index 4c5eed7905..da29adf5c9 100644 --- a/js/watch/src/audio/worker/remote.ts +++ b/js/watch/src/audio/worker/remote.ts @@ -255,10 +255,7 @@ export class Remote { const graph = effect.get(this.in.graph); if (!graph) return; - // The worklet reports to every port it holds, the node's own included: read here, where the page - // sees the ring play, or hears that the worklet could not read what it was sent, without waiting - // on the worker, and read at all, since a port never started would queue every report for the - // node's life. + // The node reports its first playback and any unreadable message directly to the page. const node = graph.target.port; effect.event(node, "message", (event) => { const msg = (event as MessageEvent).data; From 2c607c97c3bdeb52348fb05ddf8893d0ce16a672 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:00:25 -0400 Subject: [PATCH 014/127] fix(watch): avoid waking video for submillisecond clock noise Fixes #39A. Co-Authored-By: Codex --- js/watch/src/sync.test.ts | 50 +++++++++++++++++++++++++++++++++++++++ js/watch/src/sync.ts | 31 +++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 5ef380eb60..89b410aa7c 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -838,3 +838,53 @@ describe("wait", () => { sync.close(); }); }); + +// --- review consumer-sync-video F10: clock samples waking waiting frames --- + +describe("Sync wakes a waiting frame only when its deadline moves", () => { + // Counts the timers `wait()` arms while one frame waits and the audio clock republishes its + // playhead `count` times, 5ms apart, each sample off the ideal line by `jitter(i)` ms. Every timer + // after the first is a wake: the sleep was cut short, its listeners dropped and re-armed. + async function timersWhileSampling(jitter: (i: number) => number, count = 20): Promise { + clock = fakeClock(1000); + const sync = new Sync({ delay: Time.Milli(100) }); + const audio = sync.track("audio"); + audio.clock.set(sample(500, clock.at)); + await flush(); + + const real = globalThis.setTimeout; + let armed = 0; + globalThis.setTimeout = ((fn: () => void, ms?: number, ...rest: unknown[]) => { + // Only the sleep's own timer (a positive deadline), not the test's zero-delay flushes. + if (ms !== undefined && ms > 0) armed++; + return real(fn, ms, ...rest); + }) as typeof setTimeout; + try { + // A frame 200ms ahead of the playhead waits. + const waiting = sync.wait(Time.Milli(700)); + await flush(); + for (let i = 1; i <= count; i++) { + clock.advance(5); + audio.clock.set(sample(500 + 5 * i + jitter(i), clock.at)); + await flush(); + } + sync.close(); + await waiting; + } finally { + globalThis.setTimeout = real; + } + return armed; + } + + it("samples exactly on the extrapolated line do not wake it", async () => { + // The review's literal case: the playhead advances exactly as extrapolated. + expect(await timersWhileSampling(() => 0)).toBeLessThanOrEqual(2); + }); + + it("samples within a millisecond of the extrapolated line do not wake it", async () => { + // What a real clock delivers: the worklet's playhead and the reference it is stamped with + // never line up to the microsecond, so each sample lands a fraction of a millisecond off. + const jitter = (i: number) => [0.25, -0.25, 0.1, -0.1][i % 4]; + expect(await timersWhileSampling(jitter)).toBeLessThanOrEqual(2); + }); +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 90b772e221..85d11651ae 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -68,6 +68,16 @@ const OFFSET_WINDOW = Time.Milli(2_000); // that a track that has really gone stops holding the buffer open. const SPREAD_WINDOW = Time.Milli(2_000); +// How far a new playhead sample has to move the derived reference before waiting frames are woken. +// +// Every sample re-derives the reference, and a real clock never lands on the extrapolated line to the +// microsecond: the worklet's position and the time it is stamped with are read at different +// instants. Republishing each sub-millisecond wobble woke every frame parked in `wait()` on every +// sample (tens per second, times every frame in the trail) to recompute a deadline that had not +// moved by anything a display can show. A frame waits on the latest sample either way, since +// `#playhead` extrapolates from `#clock`, not from the reference. +const REFERENCE_SLACK = Time.Milli(1); + /** * One track's arrival floor, as two rotating windows. * @@ -239,6 +249,9 @@ export class Sync { // When the hold last moved, so it moves by one bucket at a time rather than in a burst. #stepped: Time.Milli | undefined; + // The trail the published reference was derived with, so a new trail always republishes it. + #referenceTrail: Time.Milli | undefined; + // The last reading each track published, and when it stops counting once the track has stopped // publishing it. See SPREAD_WINDOW. #spreads = new Map<"audio" | "video" | "text", { spread: Time.Milli; until?: Time.Milli }>(); @@ -391,6 +404,7 @@ export class Sync { const clock = audio ?? video; const previous = this.#clock; + const was = this.#out.clock.peek(); this.#clock = clock; this.#out.clock.set(source); @@ -399,7 +413,22 @@ export class Sync { if (!sample) return; const now = Time.Milli.now(); - this.#out.reference.set(Time.Milli.sub(Time.Milli.sub(now, delay), extrapolate(sample, now))); + const reference = Time.Milli.sub(Time.Milli.sub(now, delay), extrapolate(sample, now)); + // A new sample from the same clock, at the same rate and trail, that lands where the last one + // already put playback is not news. Anything else (a handover, a park or resume, a new trail) + // is always published. + const current = this.#out.reference.peek(); + const same = + current !== undefined && + clock !== undefined && + source === was && + previous !== undefined && + previous.rate === clock.rate && + delay === this.#referenceTrail && + Math.abs(reference - current) < REFERENCE_SLACK; + this.#referenceTrail = delay; + if (same) return; + this.#out.reference.set(reference); } /** From 4c5a1bf5d609b5be575b61f37d731b25ff894e5b Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:03:57 -0400 Subject: [PATCH 015/127] fix(watch): map playback clocks only through valid device time Reject incomplete device timestamps, map shared rings to output time, and sample again on worker progress after resume. Fixes #37, #41, #56, and #39B. Co-Authored-By: Codex --- js/watch/src/audio/buffer.test.ts | 114 ++++++++++++++++++++- js/watch/src/audio/buffer.ts | 35 +++++-- js/watch/src/audio/decoder.offload.test.ts | 25 +++++ js/watch/src/audio/worker/remote.ts | 21 ++-- 4 files changed, 178 insertions(+), 17 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 59b10fb10b..8ef15dcbde 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it } from "bun:test"; +import { afterEach, describe, expect, it, jest } from "bun:test"; import { Time } from "@moq/net"; import { Effect } from "@moq/signals"; import { type AudioBuffer, ClockSource, createAudioBuffer } from "./buffer"; @@ -25,6 +25,7 @@ let clock: ReturnType | undefined; afterEach(() => { clock?.restore(); clock = undefined; + jest.useRealTimers(); }); function playhead(media: number, rate: number): Playhead { @@ -292,6 +293,37 @@ describe("AudioBuffer output clock", () => { }); }); +describe("AudioBuffer output clock, device starting", () => { + // Chromium 153 answered the first getOutputTimestamp() of a fresh AudioContext (a + // reattached after 5 s) with { contextTime: 0.00535, performanceTime: 0 }: a context time with no + // device time yet. Mapping through it anchors the playhead to performance time ~0, so `Sync` put + // the playhead as far ahead as the page was old (about 50 s) until the next sample. + it("publishes no clock while the output timestamp has no performance time", () => { + clock = fakeClock(50_000); + const worklet = new FakeWorklet(); + const output = { contextTime: 0.00535, performanceTime: 0 }; + const buffer = createAudioBuffer(worklet as unknown as AudioWorkletNode, { + context: { getOutputTimestamp: () => output }, + channels: 1, + rate: 48000, + latency: 4800, + buffered: false, + conceal: true, + shared: false, + }); + try { + worklet.deliver({ ...state(worklet, playhead(500, 1), false), contextTime: Time.Second(0.02) }); + expect(buffer.clock.peek()).toBeUndefined(); + output.contextTime = 0; + output.performanceTime = 50_000; + worklet.deliver({ ...state(worklet, playhead(500, 1), false), contextTime: Time.Second(0.02) }); + expect(buffer.clock.peek()).toEqual({ timestamp: Time.Micro(500_000), reference: Time.Milli(50_020), rate: 1 }); + } finally { + buffer.close(); + } + }); +}); + describe("AudioBuffer, flushed", () => { it("never reports the old playhead once the postMessage ring is flushed", async () => { const worklet = new FakeWorklet(); @@ -393,3 +425,83 @@ describe("AudioBuffer, flushed", () => { buffer.close(); }); }); + +// --- review consumer-sync-video F9 --- + +describe("AudioBuffer output clock, shared ring", () => { + it("the shared ring's clock is anchored to when its samples leave the output device", async () => { + // The page polls at a fixed instant, 1000ms. The device is 40ms behind the render graph: + // what was rendered up to context time 1.04s is only now (1000ms) at 1.00s on the output. + jest.useFakeTimers(); + clock = fakeClock(1000); + const worklet = new FakeWorklet(); + const buffer = createAudioBuffer(worklet as unknown as AudioWorkletNode, { + context: { + getOutputTimestamp: () => ({ contextTime: 1, performanceTime: 1000 }), + currentTime: 1.04, + } as unknown as AudioContext, + channels: 1, + rate: 48000, + latency: 4800, + buffered: false, + conceal: true, + shared: true, + }); + try { + buffer.insert(3_000_000 as Time.Micro, [new Float32Array(4800)]); + // The shared ring is polled rather than pushed, so wait out a poll interval (real time). + jest.advanceTimersByTime(50); + + const sampled = buffer.clock.peek(); + expect(sampled).toBeDefined(); + // The post ring maps its playhead through getOutputTimestamp; the shared ring must too. + // Stamped at the poll instead, video leads the sound by the device's output latency. + expect(sampled?.reference).toBe(Time.Milli(1040)); + } finally { + buffer.close(); + } + }); +}); + + +// --- review consumer-sync-video F15 --- + +describe("AudioBuffer, partial output timestamp", () => { + it("a partial output timestamp neither throws nor strands backpressure", async () => { + const worklet = new FakeWorklet(); + let output: Partial = { contextTime: 1, performanceTime: 1000 }; + const buffer = createAudioBuffer(worklet as unknown as AudioWorkletNode, { + context: { getOutputTimestamp: () => output as AudioTimestamp }, + channels: 1, + rate: 48000, + latency: 4800, + buffered: true, + conceal: true, + shared: false, + }); + try { + // Playing, 500ms in. A frame at 1s is more than the 100ms floor ahead, so it is held. + worklet.deliver(state(worklet, playhead(500, 1), false)); + let released = false; + const waiting = buffer.wait(1_000_000 as Time.Micro).then(() => { + released = true; + }); + await Promise.resolve(); + expect(released).toBe(false); + + // A runtime hands back only half of the output timestamp, and the playhead reaches the frame. + output = { performanceTime: 1000 }; + let thrown: unknown; + try { + worklet.deliver(state(worklet, playhead(950, 1), false)); + } catch (err) { + thrown = err; + } + await Promise.resolve(); + expect({ thrown: (thrown as Error | undefined)?.message, released }).toEqual({ thrown: undefined, released: true }); + await waiting; + } finally { + buffer.close(); + } + }); +}); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index aabf2c64cb..e2cc6ed0b2 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -232,10 +232,18 @@ export function supportsSharedArrayBuffer(): boolean { return true; } +/** Read the device clock once it has a complete timestamp. */ +export function outputTimestamp(context: Pick): Required | undefined { + const { contextTime, performanceTime } = context.getOutputTimestamp?.() ?? {}; + // Chromium can expose context time before the device has produced a performance timestamp. + if (contextTime === undefined || performanceTime === undefined || performanceTime === 0) return undefined; + return { contextTime, performanceTime }; +} + /** How the ring behind the worklet is built. */ export interface AudioBufferProps { /** Maps render time to the audio device's output clock. */ - context: Pick; + context: Pick & Partial>; /** Channels of planar PCM the graph runs at. */ channels: number; /** Samples per second per channel. */ @@ -358,7 +366,16 @@ class SharedAudioBuffer implements AudioBuffer { this.#stalled.set(stalled); this.#underruns.set(this.#ring.underruns); this.#debug.set(this.#ring.debug()); - this.#clock.set(this.#clockSource.sample(playhead)); + const output = outputTimestamp(props.context); + const rendered = props.context.currentTime; + this.#clock.set( + output && rendered !== undefined + ? this.#clockSource.sample( + playhead, + Time.Milli(output.performanceTime + (rendered - output.contextTime) * 1000), + ) + : undefined, + ); // While stalled the playhead is parked, so release the decode loop to refill the floor; // once playing, hold it to ~the floor ahead. if (stalled || !playhead) this.#backpressure.flush(); @@ -494,17 +511,15 @@ class PostAudioBuffer implements AudioBuffer { this.#stalled.set(data.debug.stalled); this.#underruns.set(data.debug.underruns); this.#debug.set(data.debug); - const { contextTime, performanceTime } = props.context.getOutputTimestamp(); - if (contextTime === undefined || performanceTime === undefined) { - throw new Error("Audio output timestamp is missing"); - } + const output = outputTimestamp(props.context); // Message delivery varies with main-thread load. Anchor the playhead to when its // samples reach the output device so that delivery jitter does not pace video. - const reference = Time.Milli(performanceTime + (data.contextTime - contextTime) * 1000); this.#clock.set( - contextTime === 0 && performanceTime === 0 - ? undefined - : this.#clockSource.sample(data.playhead, reference), + output && + this.#clockSource.sample( + data.playhead, + Time.Milli(output.performanceTime + (data.contextTime - output.contextTime) * 1000), + ), ); // While stalled the playhead is parked, so release the decode loop to refill the floor; // once playing, hold it to ~the floor ahead. diff --git a/js/watch/src/audio/decoder.offload.test.ts b/js/watch/src/audio/decoder.offload.test.ts index 029aba9c83..d9ab16bc75 100644 --- a/js/watch/src/audio/decoder.offload.test.ts +++ b/js/watch/src/audio/decoder.offload.test.ts @@ -647,6 +647,7 @@ describe("what the page tells the worker", () => { it("samples its output clock when it hands over the graph, when the context changes state, and every second", async () => { jest.useFakeTimers(); + jest.advanceTimersByTime(1); scope.crossOriginIsolated = false; const t = tile({ offload: true }); for (let i = 0; i < 200 && !InProcessWorker.created[0]?.told("graph").length; i++) await immediate(); @@ -1243,3 +1244,27 @@ describe("the thread a player's audio runs on", () => { expect(t.page.total()).toBe(0); }); }); + + +describe("the worker output clock after resume", () => { + it("rejects a missing device timestamp and refreshes it on the next worker report", async () => { + jest.useFakeTimers(); + jest.advanceTimersByTime(50_000); + const t = tile({ offload: true }); + for (let i = 0; i < 200 && !InProcessWorker.created[0]?.told("graph").length; i++) await immediate(); + const context = t.decoder.out.context.peek(); + if (!context) throw new Error("no audio context"); + let output: AudioTimestamp = { contextTime: 0.971, performanceTime: 0 }; + context.getOutputTimestamp = () => output; + context.dispatchEvent(new Event("statechange")); + expect(worker().told("output").at(-1)?.output).toBeUndefined(); + + output = { contextTime: 1.02, performanceTime: performance.now() }; + jest.advanceTimersByTime(50); + for (let i = 0; i < 20; i++) await immediate(); + expect(worker().told("output").at(-1)?.output).toEqual({ + contextTime: 1.02, + at: performance.timeOrigin + (output.performanceTime ?? 0), + }); + }); +}); diff --git a/js/watch/src/audio/worker/remote.ts b/js/watch/src/audio/worker/remote.ts index da29adf5c9..ebdb5fdd23 100644 --- a/js/watch/src/audio/worker/remote.ts +++ b/js/watch/src/audio/worker/remote.ts @@ -18,6 +18,7 @@ import type * as Moq from "@moq/net"; import { Time } from "@moq/net"; import { type Computed, Effect, type Getter, type Readonlys, readonlys, Signal } from "@moq/signals"; import type { Clock, Sync } from "../../sync"; +import { outputTimestamp } from "../buffer"; import type { Stats } from "../decoder"; import type { Snapshot } from "../playout"; import type { Port, ToMain } from "../render"; @@ -159,6 +160,7 @@ export class Remote { // report, whether audio reached it since the last check, and whether the ring has ever played. readonly #deadline = new Deadline(); #last?: Report; + #sampleOutput?: () => void; #heard = false; readonly #played = new Signal(false); @@ -278,7 +280,15 @@ export class Remote { effect.set(this.#out.ring, this.#ring, undefined); // Only the page can read its output clock, which the postMessage ring maps its playhead through. - const output = () => this.#lease.post({ type: "output", id: this.#lease.id, output: sample(graph.context) }); + const output = () => { + const sampled = sample(graph.context); + this.#lease.post({ type: "output", id: this.#lease.id, output: sampled }); + // The device clock can appear after statechange. Sample as the worker reports progress. + this.#sampleOutput = !sampled && graph.context.state === "running" ? output : undefined; + }; + effect.cleanup(() => { + this.#sampleOutput = undefined; + }); output(); effect.event(graph.context, "statechange", output); effect.interval(output, OUTPUT_INTERVAL); @@ -376,6 +386,7 @@ export class Remote { } #report(report: Report): void { + this.#sampleOutput?.(); // What the deadline goes on, whichever flush it was composed under. this.#last = report; if (report.arrivals.length > 0) this.#heard = true; @@ -435,11 +446,9 @@ export class Remote { /** * When the context's current sample leaves the output device, or undefined while the device has not - * started, which `getOutputTimestamp` reads as both zero. + * started, which `getOutputTimestamp` reads with a zero performance time. */ function sample(context: Pick): Output | undefined { - const { contextTime, performanceTime } = context.getOutputTimestamp(); - if (contextTime === undefined || performanceTime === undefined) return undefined; - if (contextTime === 0 && performanceTime === 0) return undefined; - return { contextTime, at: performance.timeOrigin + performanceTime }; + const output = outputTimestamp(context); + return output && { contextTime: output.contextTime, at: performance.timeOrigin + output.performanceTime }; } From 6a6ee9dd3c914afc503ff0da30eab6b9068d1691 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:03:57 -0400 Subject: [PATCH 016/127] test(watch): run decoder recovery against virtual time Distinguish codec recovery from the stall watchdog and remove load-sensitive deadlines. Fixes the watch portion of #53. Co-Authored-By: Codex --- js/watch/src/video/decoder.test.ts | 53 ++++++++++++++++-------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 4973e8285b..3b00bd6384 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, expect, test } from "bun:test"; +import { afterEach, beforeEach, expect, jest, test } from "bun:test"; import { Container } from "@moq/hang"; import * as Catalog from "@moq/hang/catalog"; import * as Moq from "@moq/net"; @@ -23,28 +23,20 @@ type Built = { let built: Built[] = []; -// Wall time runs this many times faster inside a test, so BUFFERING lands at 20ms, RETRY at 40ms and -// RECOVER at 200ms. -const SPEEDUP = 25; - const real = { VideoDecoder: globalThis.VideoDecoder, EncodedVideoChunk: globalThis.EncodedVideoChunk, - setTimeout: globalThis.setTimeout, }; beforeEach(() => { built = []; - // The retry and recovery windows are seconds by design, so the timers run on a compressed clock - // rather than being waited out. `Effect.timer` calls the global, so scaling it here is enough; - // microtasks and the signal graph stay real. Same shape as the `performance.now` stub in - // `sync.replay.test.ts`. Scaling rather than clamping keeps the order between the windows, which - // is what tells a rebuild driven by the codec error from one driven by the stall watchdog. - globalThis.setTimeout = ((fn: () => void, ms?: number, ...rest: unknown[]) => { - const scaled = ms === undefined ? ms : Math.max(1, Math.round(ms / SPEEDUP)); - return real.setTimeout(fn, scaled, ...rest); - }) as typeof setTimeout; + // The retry and recovery windows are seconds by design, so they run on bun's fake clock and only + // move when a test advances it. The old version scaled real timers 25x, which put BUFFERING at + // 20ms of wall time: a loaded machine let it fire between two statements and flaked the stall + // assertions. `flush` below still yields a real turn, so the signal graph and the track + // plumbing run as before; only deadlines are virtual. + jest.useFakeTimers(); class FakeVideoFrame { readonly displayWidth = 16; @@ -111,12 +103,22 @@ beforeEach(() => { }); afterEach(() => { + jest.useRealTimers(); globalThis.VideoDecoder = real.VideoDecoder; globalThis.EncodedVideoChunk = real.EncodedVideoChunk; - globalThis.setTimeout = real.setTimeout; }); -const flush = () => new Promise((resolve) => real.setTimeout(resolve, 0)); +// setImmediate, not setTimeout: bun's fake timers take over every setTimeout, including one captured +// before they were installed, but leave setImmediate alone, so this still yields a real turn. +const flush = () => new Promise((resolve) => setImmediate(resolve)); + +/** Move the fake clock forward in small steps, letting the signal graph and the plumbing react. */ +async function advance(ms: number, step = 25): Promise { + for (let elapsed = 0; elapsed < ms; elapsed += step) { + jest.advanceTimersByTime(Math.min(step, ms - elapsed)); + await flush(); + } +} async function settle(rounds = 10): Promise { for (let i = 0; i < rounds; i++) await flush(); @@ -205,9 +207,9 @@ function fixture() { for (let i = 0; i < 400 && opened.length < count; i++) await flush(); return opened.length; }, - /** Wait until `count` codecs have been built, or give up. One per rebuilt track. */ - async decoders(count: number): Promise { - for (let i = 0; i < 400 && built.length < count; i++) await flush(); + /** Advance the fake clock until `count` codecs have been built, or give up after `within` ms of it. */ + async decoders(count: number, within = 10_000): Promise { + for (let elapsed = 0; elapsed < within && built.length < count; elapsed += 25) await advance(25); return built.length; }, close(): void { @@ -290,8 +292,9 @@ test("a codec error rebuilds the track instead of stranding the subscription", a built[0].fail(new Error("DataError")); // A replacement track, with its own subscription and its own codec, rather than a dead - // one left in place. - expect(await fx.decoders(2)).toBeGreaterThanOrEqual(2); + // one left in place. Within 2s: past RETRY, well short of the stall watchdog's + // BUFFERING + RECOVER, so only the codec error can have caused it. + expect(await fx.decoders(2, 2_000)).toBeGreaterThanOrEqual(2); fx.served[0].encode(payload(16), Time.Micro(1_000_000), true); await settle(); @@ -381,7 +384,7 @@ test("the arrival estimator survives the rendition leaving the catalog and comin try { fx.served[0].encode(payload(16), Time.Micro(0), true); fx.served[0].encode(payload(16), Time.Micro(20_000), false); - await new Promise((resolve) => real.setTimeout(resolve, 600)); + await advance(600); fx.served[0].encode(payload(16), Time.Micro(40_000), false); await settle(); @@ -541,13 +544,13 @@ test("a rendition that left the catalog is not a stall", async () => { // The camera goes, and nothing arrives for several watchdog windows. fx.config.set(undefined); - await new Promise((resolve) => real.setTimeout(resolve, 100)); + await advance(2_500); await settle(); expect(fx.decoder.out.stalled.peek()).toBe(false); // It comes back, so a picture is due again and the watchdog arms with it. fx.config.set(rendition); - await new Promise((resolve) => real.setTimeout(resolve, 60)); + await advance(1_500); await settle(); expect(fx.decoder.out.stalled.peek()).toBe(true); } finally { From 10d8751829c251f6d2e0f21c1c58ca49fd8889d5 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:06:54 -0400 Subject: [PATCH 017/127] fix(watch): pace rendition promotion and learn sparse video cadence Keep preview timestamps out of promotion and preserve healthy sparse subscriptions. Reset learned cadence when the source changes. Fixes #36 and #38A. Co-Authored-By: Codex --- js/watch/src/video/decoder.test.ts | 86 ++++++++++++++++++++++++++++++ js/watch/src/video/decoder.ts | 36 +++++++++++-- 2 files changed, 119 insertions(+), 3 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 3b00bd6384..ab011d9fee 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -197,6 +197,8 @@ function fixture() { return { served, + broadcast, + rendition: source.out.track as Signal, decoder, sync, track, @@ -557,3 +559,87 @@ test("a rendition that left the catalog is not a stall", async () => { fx.close(); } }); + + +test("a pending rendition waits until its preview picture is due", async () => { + const fx = fixture(); + const second = new Moq.Track.Producer("second").accept({}); + fx.broadcast.insertTrack(second); + const encoded = new Container.Legacy.Producer(second, new Container.Legacy.Format("video")); + try { + expect(await fx.subscriptions(1)).toBe(1); + const audio = fx.sync.track("audio"); + audio.clock.set({ timestamp: Time.Micro(1_000_000), reference: Time.Milli.now(), rate: 0 }); + fx.served[0].encode(payload(16), Time.Micro(1_000_000), true); + await settle(); + built[0].emit(1_000_000); + await settle(); + expect(fx.decoder.out.timestamp.peek()).toBe(Time.Milli(1_000)); + + fx.rendition.set("second"); + await settle(); + expect(built).toHaveLength(2); + encoded.encode(payload(16), Time.Micro(1_150_000), true); + await settle(); + built[1].emit(1_150_000); + await settle(); + expect(fx.decoder.out.timestamp.peek()).toBe(Time.Milli(1_000)); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(1_000_000); + + audio.clock.set({ timestamp: Time.Micro(1_150_000), reference: Time.Milli.now(), rate: 0 }); + await settle(); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(1_150_000); + } finally { + fx.close(); + } +}); + +test("a sparse rendition is not rebuilt at every healthy silence", async () => { + const fx = fixture(); + try { + expect(await fx.subscriptions(1)).toBe(1); + for (let n = 0; n < 6; n++) { + const timestamp = Time.Micro(n * 6_000_000); + fx.served[0].encode(payload(16), timestamp, true); + await settle(); + built.at(-1)?.emit(timestamp); + await settle(); + await advance(6_000); + } + expect(await fx.subscriptions(2)).toBe(2); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(30_000_000); + } finally { + fx.close(); + } +}); + + +test("a new rendition does not inherit a sparse rendition's recovery window", async () => { + const fx = fixture(); + const second = new Moq.Track.Producer("second").accept({}); + fx.broadcast.insertTrack(second); + const encoded = new Container.Legacy.Producer(second, new Container.Legacy.Format("video")); + try { + expect(await fx.subscriptions(1)).toBe(1); + for (let n = 0; n < 2; n++) { + const timestamp = Time.Micro(n * 6_000_000); + fx.served[0].encode(payload(16), timestamp, true); + await settle(); + built.at(-1)?.emit(timestamp); + await settle(); + await advance(6_000); + } + fx.rendition.set("second"); + await settle(); + encoded.encode(payload(16), Time.Micro(12_000_000), true); + await settle(); + built.at(-1)?.emit(12_000_000); + await advance(100); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(12_000_000); + const before = built.length; + await advance(6_000); + expect(built).toHaveLength(before + 1); + } finally { + fx.close(); + } +}); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index ff667c3dbe..34f58eea69 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -150,6 +150,14 @@ export class Decoder { // set. #recover = RECOVER; + // The longest wall-clock wait seen between two successive new pictures, and the last new one. + // A rendition that only sends a frame when its content changes (a static screen share) is + // healthy through silences longer than RECOVER; once one such gap has been seen, the recovery + // window stretches to cover it, so the watchdog stops rebuilding a subscription that is fine. + #cadence = Time.Milli.zero; + #painted?: { at: Time.Milli; timestamp: number }; + #cadenceSource?: { broadcast: Moq.Broadcast.Consumer; track: string }; + #signals = new Effect(); #clearCurrentFrame(): void { @@ -322,6 +330,11 @@ export class Decoder { } effect.cleanup(() => active.close()); + if (this.#cadenceSource?.broadcast !== active.broadcast || this.#cadenceSource.track !== active.track) { + this.#cadenceSource = { broadcast: active.broadcast, track: active.track }; + this.#cadence = Time.Milli.zero; + this.#painted = undefined; + } // Clone the frame so we own it independently of the DecoderTrack. // proxy() would share the same reference, allowing the source to close our frame. @@ -381,6 +394,15 @@ export class Decoder { // the full window again. this.#recover = RECOVER; + // Only a newer picture says how long the source goes between frames: a rebuilt subscription + // repainting the picture already held says nothing about the cadence. + const now = Time.Milli.now(); + const painted = this.#painted; + if (!painted || frame.timestamp > painted.timestamp) { + if (painted) this.#cadence = Time.Milli.max(this.#cadence, Time.Milli.sub(now, painted.at)); + this.#painted = { at: now, timestamp: frame.timestamp }; + } + effect.timer(() => { this.#out.stalled.set(true); }, BUFFERING); @@ -395,7 +417,9 @@ export class Decoder { if (!effect.get(this.#active)) return; if (!effect.get(this.#out.stalled)) return; - const after = this.#recover; + // Twice the longest gap a healthy source has already shown, so a sparse rendition is not + // rebuilt at every silence; still bounded by the ceiling for a subscription that really died. + const after = Time.Milli(Math.min(RECOVER_MAX, Math.max(this.#recover, 2 * this.#cadence))); effect.timer(() => { this.#recover = Time.Milli(Math.min(RECOVER_MAX, after * 2)); this.#rebuild(`no frame for ${after}ms`); @@ -457,6 +481,11 @@ class DecoderTrack { // so in-flight decodes from before a rewind can be dropped on output. #discontinuity = 0; + // The timestamp of the preview picture: the first decoded frame, shown before the shared clock + // reaches it. Older backlog is dropped against it, but it is not `timestamp`: that one says a + // picture is due, which is what the parent's promotion reads. + #preview?: Time.Milli; + #signals = new Effect(); constructor(props: DecoderTrackProps) { @@ -488,7 +517,7 @@ class DecoderTrack { const generation = this.#discontinuity; const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); - if (timestamp < (this.timestamp.peek() ?? 0)) { + if (timestamp < Math.max(this.timestamp.peek() ?? 0, this.#preview ?? 0)) { // Late frame, don't render it. return; } @@ -502,7 +531,7 @@ class DecoderTrack { if (this.frame.peek() === undefined) { // This preview is already visible. Older backlog must not replace it // while its timestamp is still waiting for the shared clock. - this.timestamp.set(timestamp); + this.#preview = timestamp; this.frame.set(frame.clone()); } @@ -756,6 +785,7 @@ class DecoderTrack { if (count === this.#discontinuity) return false; this.#discontinuity = count; this.timestamp.set(undefined); + this.#preview = undefined; this.#buffered.set([]); // A video delivery gap must not reset the audio track's running clock. if (this.sync.out.clock.peek() !== "audio") this.sync.reset(); From ad40bf861820946e30203c580c0f2d031e404c83 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:06:54 -0400 Subject: [PATCH 018/127] fix(watch): present frame pairs without accumulating refresh lag Fixes #38B. Co-Authored-By: Codex --- js/watch/src/video/renderer.test.ts | 120 ++++++++++++++++++++++++++++ js/watch/src/video/renderer.ts | 25 +++++- 2 files changed, 142 insertions(+), 3 deletions(-) diff --git a/js/watch/src/video/renderer.test.ts b/js/watch/src/video/renderer.test.ts index df56b915e2..b3b0513647 100644 --- a/js/watch/src/video/renderer.test.ts +++ b/js/watch/src/video/renderer.test.ts @@ -227,3 +227,123 @@ describe("Renderer", () => { } }); }); + +// --- review consumer-sync-video F11 --- + +describe("Renderer frame pairs", () => { + let callbacks: Map; + let nextCallback: number; + let originalRequest: PropertyDescriptor | undefined; + let originalCancel: PropertyDescriptor | undefined; + + beforeEach(() => { + callbacks = new Map(); + nextCallback = 0; + originalRequest = Object.getOwnPropertyDescriptor(globalThis, "requestAnimationFrame"); + originalCancel = Object.getOwnPropertyDescriptor(globalThis, "cancelAnimationFrame"); + Object.defineProperty(globalThis, "requestAnimationFrame", { + configurable: true, + value: (callback: FrameRequestCallback) => { + const id = ++nextCallback; + callbacks.set(id, callback); + return id; + }, + }); + Object.defineProperty(globalThis, "cancelAnimationFrame", { + configurable: true, + value: (id: number) => callbacks.delete(id), + }); + }); + + afterEach(() => { + if (originalRequest) Object.defineProperty(globalThis, "requestAnimationFrame", originalRequest); + else Reflect.deleteProperty(globalThis, "requestAnimationFrame"); + if (originalCancel) Object.defineProperty(globalThis, "cancelAnimationFrame", originalCancel); + else Reflect.deleteProperty(globalThis, "cancelAnimationFrame"); + }); + + // One display refresh at 60Hz: rAF hands the callbacks the refresh's timestamp. + let refreshAt = 0; + function paint(): void { + refreshAt += 1000 / 60; + const pending = [...callbacks.values()]; + callbacks.clear(); + for (const callback of pending) callback(refreshAt); + } + + function setup(first: number) { + const drawn: number[] = []; + const frame = (timestamp: number) => + ({ + timestamp, + clone() { + return this; + }, + close() {}, + }) as unknown as VideoFrame; + const context = { + canvas: { width: 640, height: 360 }, + save() {}, + restore() {}, + fillRect() {}, + drawImage(value: VideoFrame) { + drawn.push(value.timestamp); + }, + }; + const frames = new Signal(frame(first)); + const decoder = { + in: { enabled: new Signal(true) }, + out: { display: new Signal(undefined), frame: frames }, + source: { out: { catalog: new Signal(undefined) } }, + } as unknown as Decoder; + const renderer = new Renderer({ + decoder, + canvas: { getContext: () => context } as unknown as HTMLCanvasElement, + visible: "never", + }); + return { drawn, renderer, show: (timestamp: number) => frames.set(frame(timestamp)) }; + } + + it("presents both frames of a 30 fps pair released within one refresh", async () => { + const { drawn, renderer, show } = setup(-33_333); + try { + await settle(); + paint(); + // A late timer releases two adjacent 30fps frames inside one refresh. + show(0); + await settle(); + show(33_333); + await settle(); + paint(); + paint(); + expect(drawn.slice(1)).toEqual([0, 33_333]); + } finally { + renderer.close(); + } + }); + + it("does not keep a standing one-refresh lag", async () => { + const { drawn, renderer, show } = setup(0); + try { + await settle(); + paint(); + // A 60fps pair lands inside one refresh, then one frame per refresh as usual. + show(16_667); + await settle(); + show(33_333); + await settle(); + paint(); + let latest = 33_333; + for (let i = 0; i < 4; i++) { + latest += 16_667; + show(latest); + await settle(); + paint(); + } + // Within two refreshes of the pair the picture is back to the newest released frame. + expect(drawn.at(-1)).toBe(latest); + } finally { + renderer.close(); + } + }); +}); diff --git a/js/watch/src/video/renderer.ts b/js/watch/src/video/renderer.ts index 8bb158216e..34ba6cc82a 100644 --- a/js/watch/src/video/renderer.ts +++ b/js/watch/src/video/renderer.ts @@ -150,15 +150,31 @@ export class Renderer { let dirty = false; let source: VideoFrame | undefined; const frames: VideoFrame[] = []; + // The display's refresh interval, measured from rAF timestamps (60Hz until measured), and + // whether the last refresh left a frame queued. A pair released inside one refresh is shown + // over two; a queue that is still non-empty a refresh later is a standing lag, so it drains. + let refresh = 1000 / 60; + let last: number | undefined; + let carried = false; const clear = () => { for (const frame of frames) frame.close(); frames.length = 0; + carried = false; }; - const render = () => { + const render = (now?: number) => { animate = undefined; + if (now !== undefined && last !== undefined) { + const delta = now - last; + if (delta > 0 && delta < 100) refresh = delta; + } + if (now !== undefined) last = now; + if (carried && frames.length > 1) { + while (frames.length > 1) frames.shift()?.close(); + } if (dirty || frames.length) { dirty = false; const pending = frames.shift(); + carried = frames.length > 0; const frame = pending ?? (source ? this.#out.frame.peek() : undefined); const video = this.decoder.source.out.catalog.peek(); try { @@ -186,8 +202,11 @@ export class Renderer { if (frame && (frame !== source || reset)) { frames.push(frame.clone()); // Timers can release adjacent frames on opposite sides of a display refresh. - // Preserve that pair, but never turn it into a stale presentation backlog. - while (frames.length > 2 || (frames[0] && frame.timestamp - frames[0].timestamp > 20_000)) { + // Preserve that pair, but never turn it into a stale presentation backlog. Adjacent + // is sized from the display, not a fixed 20ms of media: two refreshes' worth keeps a + // 30fps pair on a 60Hz display. + const adjacent = Math.max(20_000, 2.1 * refresh * 1000); + while (frames.length > 2 || (frames[0] && frame.timestamp - frames[0].timestamp > adjacent)) { frames.shift()?.close(); } } From 88bba04fe00423304dcafe5ae4fcb3795889ade1 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:10:21 -0400 Subject: [PATCH 019/127] test(shaper): bound forwarding benchmark completion Fail an interrupted UDP benchmark batch instead of waiting until the nightly job timeout. Co-Authored-By: Codex --- rs/moq-shaper/benches/forward.rs | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/rs/moq-shaper/benches/forward.rs b/rs/moq-shaper/benches/forward.rs index 027a8ac40f..d8dd0f2b98 100644 --- a/rs/moq-shaper/benches/forward.rs +++ b/rs/moq-shaper/benches/forward.rs @@ -56,21 +56,25 @@ fn forward(c: &mut Criterion) { group.bench_function(BenchmarkId::new(clients.to_string(), steps), |b| { b.iter(|| { runtime.block_on(async { - let mut tasks = JoinSet::new(); - for socket in &sockets { - let socket = socket.clone(); - tasks.spawn(async move { - let mut packet = [0; 64]; - for _ in 0..32 { - socket.send(&[7; 64]).await.unwrap(); - assert_eq!(socket.recv(&mut packet).await.unwrap(), 64); - assert_eq!(packet, [7; 64]); - } - }); - } - while let Some(result) = tasks.join_next().await { - result.unwrap(); - } + tokio::time::timeout(Duration::from_secs(10), async { + let mut tasks = JoinSet::new(); + for socket in &sockets { + let socket = socket.clone(); + tasks.spawn(async move { + let mut packet = [0; 64]; + for _ in 0..32 { + socket.send(&[7; 64]).await.unwrap(); + assert_eq!(socket.recv(&mut packet).await.unwrap(), 64); + assert_eq!(packet, [7; 64]); + } + }); + } + while let Some(result) = tasks.join_next().await { + result.unwrap(); + } + }) + .await + .expect("shaper round trips did not finish"); }) }); }); From 26f71e4e7aa92f43c4054145dca8fd6aba21c323 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:09:48 -0400 Subject: [PATCH 020/127] fix(net): release expiry guards at FIN and use Old resets Start guarded writes only while the group is live. Release partial subscriptions when setup fails, and keep rejected signal listeners out of the subscriber set. Preserve lite priority updates through FIN acknowledgment without expiry listeners. Map local expiry and remote OLD resets to Expired, matching Rust and allowing JSON window consumers to resume at checkpoints. DELIVERY_TIMEOUT remains a distinct stream error. Co-Authored-By: Codex --- doc/lib/js/net.md | 2 +- drafts/draft-lcurley-moq-lite.md | 4 +- js/json/src/window/window.test.ts | 36 ++++++++++++ js/net/src/error.test.ts | 16 +++-- js/net/src/error.ts | 9 ++- js/net/src/group.test.ts | 34 +++++++++++ js/net/src/group.ts | 24 ++++---- js/net/src/ietf/publisher.test.ts | 10 +--- js/net/src/ietf/publisher.ts | 7 +-- js/net/src/internal.ts | 4 +- js/net/src/lite/publisher.test.ts | 97 ++++++++++++++++++++++++++++--- js/net/src/lite/publisher.ts | 22 +++---- js/net/src/track.test.ts | 23 +++++--- js/signals/src/index.test.ts | 18 ++++++ js/signals/src/index.ts | 9 ++- 15 files changed, 246 insertions(+), 69 deletions(-) diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index f376076d2f..9ca26bd2e6 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -51,7 +51,7 @@ for (;;) { - **Connections** race WebTransport against WebSocket. `new Connection({ url })` pools one connection per relay URL and reconnects with backoff, which the elements use. Supplying WebTransport/WebSocket options, discovery, delay, or a caller-owned origin selects a private loop; explicit `share: true` refuses those options. `closed` settles when the handle is released (`null` on a clean close); the failure that stopped retrying the current URL is `error`, and a new URL recovers the same handle. A connection owns one send-rate sampler and one `Bandwidth.Allocator`; publishers reserve against it so their encoder targets sum to the estimate instead of each matching it. - **Bandwidth** (`Bandwidth.Allocator`) divides the connection's send-rate estimate by track priority, max-min fair within a tier. An idle track claims nothing. The receive side is untouched. - **Discovery** by any pattern scope (`origin.announced(scope)`, such as `room/*/chat`; default everything). Each event's `prefix` is the covered prefix relative to the origin, `captures` reports what the scope's wildcards matched when the prefix pins them, and `kind` says whether it was announced, updated, or retracted. The consumer is an async iterable. `origin.broadcasts(scope)` is a live `Getter>` of the same covered prefixes for UIs that need the current set. A borrowed `Connection.origin` also exposes `dynamic(prefix, route)` for serving paths on demand. -- **Subscriptions** carry a priority, a `Time.Milli` max age, and optional `groups` bounds. Groups arrive out of order and are read frame by frame, with `Error.TooFarBehind` when a reader asks for a frame the group never held, `Error.Expired` when the max age gives up on a group that still held content, and `Error.GroupTooLarge` when a write exceeds the cache budget and aborts the group. +- **Subscriptions** carry a priority, a `Time.Milli` max age, and optional `groups` bounds. Groups arrive out of order and are read frame by frame, with `Error.TooFarBehind` when a reader asks for a frame the group never held, `Error.Expired` with `StreamCode.Old` when the max age gives up on a group that still held content, and `Error.GroupTooLarge` when a write exceeds the cache budget and aborts the group. - **Datagrams** on moq-lite 05+ and fetch-by-sequence for history. - **Errors** live under one namespace: a stream reset throws `Error.Stream` with a `StreamCode`, while a session close gives `Error.Session` with a `SessionCode`. The registries are disjoint, so the same number means different things in each, and 64+ is yours. Named conditions such as `Error.TooFarBehind`, `Error.Expired`, `Error.FrameTooLarge`, and `Error.GroupTooLarge` subclass `Error.Stream`, so one `code` check handles a condition raised here or reported by the peer. IETF streams use their own mapping: cancellation sends CANCELLED, other local failures send INTERNAL\_ERROR, and received codes remain opaque. - **Paths** with `Path.relative` for the cross-broadcast catalog references hang uses. Path patterns (`Path.Pattern`, `Path.Patterns`) are re-exported from [`@moq/pattern`](https://www.npmjs.com/package/@moq/pattern). Literal `Path` stays a coordinate. diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 8c5c464c08..63a484960d 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -313,7 +313,7 @@ Sent when resetting a stream (RESET_STREAM), or when refusing to receive one (ST | ------- | ------------- | ----------- | | 0x33 | NOT_FOUND | The requested group, track, or broadcast is not here. | | ------- | ------------- | ----------- | -| 0x34 | OLD | The group was superseded by a newer group and dropped. | +| 0x34 | OLD | The group was superseded by a newer group and dropped, including when it exceeds the subscription max age. | | ------- | ------------- | ----------- | | 0x35 | EVICTED | The group was dropped under memory pressure. Unlike OLD it was still current, so it can be re-fetched. | | ------- | ------------- | ----------- | @@ -1330,6 +1330,8 @@ The `Message Length` describes the payload size on the wire. ## moq-lite-07 +- Clarified that a group reset for exceeding the subscription max age uses OLD. + - Assigned `moq-lite-07` as this draft's protocol identifier. - Hid routes with a `.`-prefixed segment below the requested prefix from announce discovery, and added the ANNOUNCE_REQUEST `Hidden` field to opt in. diff --git a/js/json/src/window/window.test.ts b/js/json/src/window/window.test.ts index 95d2f0bedd..9e24b40cc4 100644 --- a/js/json/src/window/window.test.ts +++ b/js/json/src/window/window.test.ts @@ -418,3 +418,39 @@ test("an uncommitted edit leaves the window unchanged", () => { popped.commit(); expect(encoder.window).toEqual([2, 3]); }); + +// F7 (review 2026-09-27): the same max-age verdict, reset by a JS publisher and by a Rust one, +// as the JS subscriber decodes each RESET_STREAM (js/net fromTransport). The Rust publisher sends +// Old (0x34), which reads back as Stream(Old). The JS publisher sends the code of its `Expired` +// verdict, which reads back as `Expired`. Both are a group the publisher gave up on, so both must +// be a gap the window resyncs from, not a fatal error. +test("F7: an expired group is a gap whichever publisher reset it", async () => { + for (const [publisher, error] of [ + ["rust (Error::Old, 0x34)", new NetError.Stream(StreamCode.Old)], + ["js (Expired)", new NetError.Expired()], + ] as const) { + const track = new Track.Producer("test"); + const consumer = new Consumer({ track: track.subscribe() }); + const encoder = new Encoder({ opRatio: 0 }); + + let frame = encoder.push({ n: 0 }); + let group = track.appendGroup(); + group.writeFrame({ payload: frame.payload, timestamp: Time.Timestamp.now() }); + frame.commit(); + expect(await consumer.next()).toEqual({ push: { index: 0, value: { n: 0 } } }); + group.close(error); + + frame = encoder.push({ n: 1 }); + group = track.appendGroup(); + group.writeFrame({ payload: frame.payload, timestamp: Time.Timestamp.now() }); + frame.commit(); + group.close(); + track.close(); + + const next = await consumer.next().then( + (event) => event, + (err: unknown) => `fatal: ${String(err)} (code ${(err as { code?: number }).code})`, + ); + expect({ publisher, next }).toEqual({ publisher, next: { push: { index: 1, value: { n: 1 } } } }); + } +}); diff --git a/js/net/src/error.test.ts b/js/net/src/error.test.ts index e22b1d0081..32af7d8766 100644 --- a/js/net/src/error.test.ts +++ b/js/net/src/error.test.ts @@ -73,11 +73,11 @@ afterEach(() => { }); test("fromTransport: a stream reset keeps the peer's code verbatim", () => { - const src = fake("stream", 0x34); + const src = fake("stream", 0x35); const err = fromTransport(src); expect(err).toBeInstanceOf(StreamError); - expect((err as StreamError).code).toBe(StreamCode.Old); - expect(err.message).toBe("remote error: 52"); + expect((err as StreamError).code).toBe(StreamCode.Evicted); + expect(err.message).toBe("remote error: 53"); // The original error stays reachable for logging. expect(err.cause).toBe(src); }); @@ -303,7 +303,7 @@ test("toTransport: works with no WebTransportError global", () => { // lagging reader or an unknown broadcast reads as a crash on the sender's side. test("toStreamCode: a local condition maps to the code the peer can act on", () => { expect(toStreamCode(new Lagged())).toBe(StreamCode.TooFarBehind); - expect(toStreamCode(new Expired())).toBe(StreamCode.DeliveryTimeout); + expect(toStreamCode(new Expired())).toBe(StreamCode.Old); expect(toStreamCode(new FrameTooLarge())).toBe(StreamCode.FrameTooLarge); expect(toStreamCode(new GroupTooLarge())).toBe(StreamCode.GroupTooLarge); expect(toStreamCode(new NotFound("broadcast x"))).toBe(StreamCode.NotFound); @@ -381,9 +381,9 @@ test("toStreamCode and fromTransport agree on what a code means", () => { expect(new FrameTooLarge()).toBeInstanceOf(StreamError); expect(fromTransport(toTransport(StreamCode.GroupTooLarge, "overflow"))).toBeInstanceOf(GroupTooLarge); expect(new GroupTooLarge()).toBeInstanceOf(StreamError); - expect(fromTransport(toTransport(StreamCode.DeliveryTimeout, "too late"))).toBeInstanceOf(Expired); + expect(fromTransport(toTransport(StreamCode.Old, "superseded"))).toBeInstanceOf(Expired); expect(new Expired()).toBeInstanceOf(StreamError); - expect(new Expired().code).toBe(StreamCode.DeliveryTimeout); + expect(new Expired().code).toBe(StreamCode.Old); // The values the four codes were sent from before they were assigned stay reserved: // a peer still emitting one is not read as anything. @@ -407,3 +407,7 @@ test("toStreamCode: lite-only codes do not reach an IETF peer", () => { expect(toStreamCode(new StreamError(code), { version: Version.DRAFT_20 })).toBe(StreamCode.Internal); } }); + +test("delivery timeout does not claim the subscription age verdict", () => { + expect(fromTransport(toTransport(StreamCode.DeliveryTimeout, "too late"))).not.toBeInstanceOf(Expired); +}); diff --git a/js/net/src/error.ts b/js/net/src/error.ts index 63063ff688..8d560613d1 100644 --- a/js/net/src/error.ts +++ b/js/net/src/error.ts @@ -181,17 +181,16 @@ export class Stream extends Error { } /** - * The content missed its delivery deadline, so what was still unread is gone. + * Newer content exceeded this group's age budget, so its unread content is gone. * * Raised locally when a subscription's max age budget gives up on a group that still held - * content, and decoded from a moq-lite peer's `DELIVERY_TIMEOUT` reset, since a deadline the - * sender gave up on truncates the reader the same way. + * content, and decoded from a moq-lite peer's `OLD` reset. * * @public */ export class Expired extends Stream { constructor(options?: { cause?: unknown }) { - super(StreamCode.DeliveryTimeout, { + super(StreamCode.Old, { ...options, message: "expired: group exceeded the subscription max age budget", }); @@ -377,7 +376,7 @@ export function fromTransport(err: unknown, options?: TransportErrorOptions): Er if (options?.version !== undefined && !sharedStreamCode(code, options.version) && claimedLocally(code)) { return new Stream(StreamCode.Internal, { cause: err, message: `remote error: ${code}` }); } - if (code === StreamCode.DeliveryTimeout) return new Expired({ cause: err }); + if (code === StreamCode.Old) return new Expired({ cause: err }); if (code === StreamCode.TooFarBehind) return new TooFarBehind({ cause: err }); if (code === StreamCode.FrameTooLarge) return new FrameTooLarge({ cause: err }); if (code === StreamCode.GroupTooLarge) return new GroupTooLarge({ cause: err }); diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index 5bb5187199..9a6b6fdd6b 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -1,6 +1,8 @@ import { expect, test } from "bun:test"; +import { Signal } from "@moq/signals"; import { FrameTooLarge, GroupTooLarge } from "./error.ts"; import { MAX_GROUP_CACHE_BYTES, MAX_GROUP_FRAMES, Producer } from "./group.ts"; +import { hooks } from "./internal.ts"; import { Timestamp } from "./time.ts"; const dec = new TextDecoder(); @@ -226,3 +228,35 @@ test("a frame larger than the cache is rejected rather than silently dropped", ( producer.close(); expect(consumer.tryReadFrame()).toBeUndefined(); }); + +test("an expiry subscription failure starts no write and releases earlier subscriptions", async () => { + const group = new Producer(0).consume(); + const changed = new Signal(0); + const crowded = new Signal(0); + const listeners = Array.from({ length: 99 }, () => crowded.subscribe(() => {})); + let checks = 0; + let writes = 0; + hooks.expireGroup(group, { + expired: () => { + checks++; + return false; + }, + changed: [changed, crowded], + }); + try { + await expect( + hooks.guardGroup(group, async () => { + writes++; + }), + ).rejects.toThrow("too many subscribers"); + const before = checks; + changed.set(1); + crowded.set(1); + await Promise.resolve(); + expect(checks).toBe(before); + expect(writes).toBe(0); + } finally { + for (const dispose of listeners) dispose(); + group.close(); + } +}); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 9d98202730..e3014ee572 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -418,9 +418,11 @@ export class Consumer { return true; } - #guard(operation: Promise): Promise { + #guard(operation: () => Promise): Promise { + this.#expire(true); + if (this.#terminal) return Promise.reject(this.#terminal); const expiry = this.#expiry; - if (!expiry && !this.#terminal) return operation; + if (!expiry) return operation(); return new Promise((resolve, reject) => { let settled = false; @@ -439,14 +441,16 @@ export class Consumer { } }; - for (const changed of expiry?.changed ?? []) disposes.push(changed.subscribe(check)); - // The write has already started. Observe it before an expiry verdict can reset - // the stream and reject it, including when the group expired before this call. - operation.then( - (value) => finish(() => resolve(value)), - (error: unknown) => finish(() => reject(error)), - ); - check(); + try { + for (const changed of expiry.changed) disposes.push(changed.subscribe(check)); + operation().then( + (value) => finish(() => resolve(value)), + (error: unknown) => finish(() => reject(error)), + ); + check(); + } catch (error) { + finish(() => reject(error)); + } }); } diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index 44a05f5d7e..bf46972f4e 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -176,7 +176,7 @@ test("TRACK_STATUS gets exact NOT_SUPPORTED refusal bytes on every draft", async // The header is part of the group's lifetime too. If it blocks on flow control, advancing // the live edge must reset the stream without waiting for that write to finish. -test.each(["header", "FIN"] as const)("a blocked group %s is reset when the group expires", async (phase) => { +test.each(["header"] as const)("a blocked group %s is reset when the group expires", async (phase) => { const pair = createMockTransportPair(ALPN.DRAFT_19); let started!: () => void; @@ -191,7 +191,7 @@ test.each(["header", "FIN"] as const)("a blocked group %s is reset when the grou const streamReset = new Promise((resolve) => { reset = resolve; }); - const closed = phase === "FIN" ? blocked : new Promise(() => {}); + const closed = new Promise(() => {}); const writable = { getWriter: () => ({ closed, @@ -200,11 +200,7 @@ test.each(["header", "FIN"] as const)("a blocked group %s is reset when the grou started(); await blocked; }, - close: async () => { - if (phase !== "FIN") return; - started(); - await blocked; - }, + close: async () => {}, abort: async () => { reset(); }, diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index 6771bda4af..e9183e0c1e 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -476,7 +476,7 @@ export class Publisher { }); try { - await hooks.guardGroup(group, header.encode(stream, this.#session.version)); + await hooks.guardGroup(group, () => header.encode(stream, this.#session.version)); // The first written object goes on the wire as its absolute id, so a trimmed // head shows the true numbering rather than a silently renumbered group. let first = true; @@ -505,8 +505,7 @@ export class Publisher { const obj = new Frame({ payload: read.frame.payload, timestamp: read.frame.timestamp }); const delta = first ? read.sequence : 0; first = false; - await hooks.guardGroup( - group, + await hooks.guardGroup(group, () => obj.encode(stream, header.flags, timescale, this.#session.version, delta), ); } finally { @@ -515,8 +514,6 @@ export class Publisher { } stream.close(); - // The transport may still hold data after FIN is queued. - await hooks.guardGroup(group, stream.closed); } catch (err: unknown) { stream.reset(error(err)); } diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 3f235938cd..d924d348fa 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -137,8 +137,8 @@ export const hooks: { group: GroupConsumer, expiry: { expired: () => boolean; changed: readonly Getter[] }, ) => void; - /** Stop an in-flight group operation if the handed-out group expires. */ - guardGroup: (group: GroupConsumer, operation: Promise) => Promise; + /** Start a group operation within its age budget and stop it if the group expires. */ + guardGroup: (group: GroupConsumer, operation: () => Promise) => Promise; /** Read a frame the wire publisher completes (or skips) once written. */ readGroupFrame: (group: GroupConsumer, from?: number) => Promise; /** Make an evicted mirror terminal while its track timeline still contains it. */ diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 86c318a598..d5b13085f7 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -1240,7 +1240,7 @@ test("lite draft-05: group streams do not ask the transport to wait for a slot", // The header is part of the group's lifetime too. If it blocks on flow control, advancing // the live edge must reset the stream without waiting for that write to finish. -test.each(["header", "FIN"] as const)("a blocked group %s is reset when the group expires", async (phase) => { +test.each(["header"] as const)("a blocked group %s is reset when the group expires", async (phase) => { const pair = createMockTransportPair(ALPN_05); let started!: () => void; @@ -1255,7 +1255,7 @@ test.each(["header", "FIN"] as const)("a blocked group %s is reset when the grou const streamReset = new Promise((resolve) => { reset = resolve; }); - const closed = phase === "FIN" ? blocked : new Promise(() => {}); + const closed = new Promise(() => {}); const writable = { getWriter: () => ({ closed, @@ -1264,11 +1264,7 @@ test.each(["header", "FIN"] as const)("a blocked group %s is reset when the grou started(); await blocked; }, - close: async () => { - if (phase !== "FIN") return; - started(); - await blocked; - }, + close: async () => {}, abort: async () => { reset(); }, @@ -1508,3 +1504,90 @@ test("a repeat subscription fans out from the live track instead of raising a re first.close(); } }); + +async function settleMicrotasks() { + for (let i = 0; i < 200; i++) await Promise.resolve(); +} + +// F6 (review 2026-09-27): every group waiting for its FIN acknowledgement holds one guard, and +// each guard subscribes to the track's expiry signals. In a dev build @moq/signals throws at the +// 100th subscriber of one signal ("may be leaking"). A publisher with 100+ groups in flight +// (congestion, one group per audio frame) must not fail a group, or leave a promise unobserved, +// because of that cap. +test("120 groups waiting for their FIN do not trip the dev subscriber cap", async () => { + const N = 120; + const pair = createMockTransportPair(ALPN_05); + + // Every group stream sends its data at once and never has its FIN acknowledged. + const acks: PromiseWithResolvers[] = []; + let closing = 0; + pair.server.createUnidirectionalStream = async () => { + const ack = Promise.withResolvers(); + // A stream the publisher never closes never hands its ack to the sink; rejecting it + // below must not count as the publisher's unhandled rejection. + ack.promise.catch(() => {}); + acks.push(ack); + return new WritableStream({ + close: () => { + closing++; + return ack.promise; + }, + }); + }; + + const resets: unknown[] = []; + const reset = Writer.prototype.reset; + const resetSpy = spyOn(Writer.prototype, "reset"); + resetSpy.mockImplementation(function (this: Writer, reason: unknown) { + resets.push(reason); + reset.call(this, reason); + }); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => { + unhandled.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + + const origin = new OriginProducer(); + const publisher = new Publisher(pair.server, Version.DRAFT_05, randomHop(), origin.consume()); + const broadcast = publish(origin, Path.from("test")); + const track = broadcast.createTrack("audio"); + const client = await Stream.open(pair.client); + const server = await Stream.accept(pair.server); + if (!server) throw new Error("publisher never accepted the subscribe stream"); + + try { + void publisher.runSubscribe( + new Subscribe({ id: 0n, broadcast: Path.from("test"), track: "audio", priority: 0, maxAge: 60_000 }), + server, + ); + + // One group per 1ms audio frame: none is anywhere near the 60s max-age budget. + for (let i = 0; i < N; i++) { + const group = new GroupProducer(i); + group.writeFrame({ payload: new Uint8Array([i]), timestamp: Timestamp.fromMillis(i) }); + group.close(); + track.writeGroup(group); + await settleMicrotasks(); + } + await settleMicrotasks(); + + // The peer then stops every stream: each FIN wait rejects. + for (const ack of acks) ack.reject(new Error("STOP_SENDING")); + for (let i = 0; i < 20; i++) await settleMicrotasks(); + + const capped = resets.filter((reason) => String(reason).includes("too many subscribers")); + expect({ + streams: acks.length, + closing, + cappedGroups: capped.map(String), + unhandled: unhandled.map(String), + }).toEqual({ streams: N, closing: N, cappedGroups: [], unhandled: [] }); + } finally { + process.off("unhandledRejection", onUnhandled); + resetSpy.mockRestore(); + publisher.close(); + client.close(); + broadcast.close(); + } +}); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index b35f624f64..f35ba42608 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1025,13 +1025,10 @@ export class Publisher { // follows it too rather than keeping a stale rank until it finishes. priority.add(stream, group.sequence); - await hooks.guardGroup( - group, - (async () => { - await stream.u53(0); // stream type - await msg.encode(stream, this.version); - })(), - ); + await hooks.guardGroup(group, async () => { + await stream.u53(0); // stream type + await msg.encode(stream, this.version); + }); // Lite05+ prefixes every frame with a zigzag-delta timestamp at the track's // advertised timescale; older drafts omit it. @@ -1063,22 +1060,21 @@ export class Publisher { if (timestamps) { // Convert each frame to the track's advertised timescale. const ts = BigInt(Math.round(read.frame.timestamp.as(timescale))); - await hooks.guardGroup(group, stream.u62(zigzag(ts - prevTs))); + await hooks.guardGroup(group, () => stream.u62(zigzag(ts - prevTs))); prevTs = ts; } - await hooks.guardGroup(group, stream.u53(read.frame.payload.byteLength)); - await hooks.guardGroup(group, stream.write(read.frame.payload)); + await hooks.guardGroup(group, () => stream.u53(read.frame.payload.byteLength)); + await hooks.guardGroup(group, () => stream.write(read.frame.payload)); } finally { read.complete(); } } stream.close(); - // FIN can still be queued behind congestion. Keep expiry and priority active - // until the transport finishes sending, so stale buffered groups can be reset. - await hooks.guardGroup(group, stream.closed); group.close(); + // Queued bytes still follow subscription priority changes after FIN. + await stream.closed; } catch (err: unknown) { const e = error(err); stream.reset(e); diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 06e4fe8510..ac812cdb69 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -808,7 +808,7 @@ test("a handed-out frame cancels its in-flight operation when it expires", async const operation = new Promise((resolve) => { release = resolve; }); - const guarded = hooks.guardGroup(group, operation); + const guarded = hooks.guardGroup(group, () => operation); producer.writeString("new"); await expect(guarded).rejects.toThrow("max age budget"); @@ -816,7 +816,7 @@ test("a handed-out frame cancels its in-flight operation when it expires", async }); for (const alreadyExpired of [false, true]) { - test(`a guarded write handles its rejection when the group ${alreadyExpired ? "already expired" : "expires before guarding"}`, async () => { + test(`a guarded write never starts when the group ${alreadyExpired ? "already expired" : "expires before guarding"}`, async () => { const producer = new TrackProducer("test").accept({ maxAge: Milli(5000) }); const track = producer.subscribe(); producer.writeString("old"); @@ -826,7 +826,7 @@ for (const alreadyExpired of [false, true]) { producer.writeString("new"); if (alreadyExpired) { - await expect(hooks.guardGroup(group, Promise.resolve())).rejects.toBeInstanceOf(Expired); + await expect(hooks.guardGroup(group, () => Promise.resolve())).rejects.toBeInstanceOf(Expired); } const stream = new TransformStream(); @@ -834,11 +834,16 @@ for (const alreadyExpired of [false, true]) { const reader = stream.readable.getReader(); try { // No reader drains the stream, so resetting it rejects the pending write. - const guarded = hooks.guardGroup(group, writer.write(enc.encode("old"))); + let writes = 0; + const guarded = hooks.guardGroup(group, () => { + writes++; + return writer.write(enc.encode("old")); + }); const verdict = await guarded.catch((err: unknown) => err); expect(verdict).toBeInstanceOf(Expired); + expect(writes).toBe(0); writer.reset(verdict); - await expect(reader.read()).rejects.toMatchObject({ streamErrorCode: StreamCode.DeliveryTimeout }); + await expect(reader.read()).rejects.toMatchObject({ streamErrorCode: StreamCode.Old }); await settle(); const next = await track.recvGroup(); @@ -867,7 +872,7 @@ test("a budget verdict on unread content is Expired, an eviction is TooFarBehind const operation = new Promise((resolve) => { release = resolve; }); - const guarded = hooks.guardGroup(group, operation); + const guarded = hooks.guardGroup(group, () => operation); producer.writeString("new"); const expired = await guarded.then( @@ -875,7 +880,7 @@ test("a budget verdict on unread content is Expired, an eviction is TooFarBehind (err: unknown) => err, ); expect(expired).toBeInstanceOf(Expired); - expect((expired as Expired).code).toBe(StreamCode.DeliveryTimeout); + expect((expired as Expired).code).toBe(StreamCode.Old); release(); // Same shape of loss, different reason: retention drops the unread tail while the @@ -924,7 +929,7 @@ test("a guarded write keeps the position of the frame removed from the buffer", const operation = new Promise((resolve) => { release = resolve; }); - const guarded = hooks.guardGroup(group, operation); + const guarded = hooks.guardGroup(group, () => operation); producer.writeFrame({ payload: enc.encode("edge"), timestamp: Timestamp.fromMillis(1_000) }); // A group beyond the edge, so group 0's reach (1s) is provably behind it: a group is @@ -953,7 +958,7 @@ test("clean source closure stays provisional while a frame write can expire", as const operation = new Promise((resolve) => { release = resolve; }); - const guarded = hooks.guardGroup(group, operation); + const guarded = hooks.guardGroup(group, () => operation); const edge = producer.appendGroup(); edge.writeFrame({ payload: enc.encode("edge"), timestamp: Timestamp.fromMillis(1_000) }); diff --git a/js/signals/src/index.test.ts b/js/signals/src/index.test.ts index 8491741c57..b53d088767 100644 --- a/js/signals/src/index.test.ts +++ b/js/signals/src/index.test.ts @@ -1239,3 +1239,21 @@ describe("spawn retention", () => { effect.close(); }); }); + +test("a rejected subscription does not retain its callback", async () => { + const signal = new Signal(0); + const listeners = Array.from({ length: 99 }, () => signal.subscribe(() => {})); + let notified = false; + try { + expect(() => + signal.subscribe(() => { + notified = true; + }), + ).toThrow("too many subscribers"); + signal.set(1); + await settle(); + expect(notified).toBe(false); + } finally { + for (const dispose of listeners) dispose(); + } +}); diff --git a/js/signals/src/index.ts b/js/signals/src/index.ts index 5b4681faf9..6d5e8b8d1f 100644 --- a/js/signals/src/index.ts +++ b/js/signals/src/index.ts @@ -176,10 +176,13 @@ export class Signal implements Getter, Setter { /** Calls `fn` every time the value changes. Returns a function to unsubscribe. */ subscribe(fn: Subscriber): Dispose { - this.#subscribers.add(fn); - if (DEV && this.#subscribers.size >= 100 && Number.isInteger(Math.log10(this.#subscribers.size))) { - throw new Error("signal has too many subscribers; may be leaking"); + if (DEV && !this.#subscribers.has(fn)) { + const size = this.#subscribers.size + 1; + if (size >= 100 && Number.isInteger(Math.log10(size))) { + throw new Error("signal has too many subscribers; may be leaking"); + } } + this.#subscribers.add(fn); return () => this.#subscribers.delete(fn); } From f3e2f8a7810d081c339dd28bd217ae3707503423 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:09:48 -0400 Subject: [PATCH 021/127] fix(hang): bound paused readers and preserve marker events Apply positive age budgets to completed groups when the reader stops pulling. Preserve zero-budget burst behavior. Report covered marker groups as discontinuities and avoid mistaking sparse arrivals for event-loop stalls under timer throttling. Pin arrival measurement before parsing and preserve a long GOP after a shorter completed group. Co-Authored-By: Codex --- js/hang/src/container/consumer.test.ts | 105 ++++++++++++++++++++++++- js/hang/src/container/consumer.ts | 28 ++++++- js/hang/src/container/stall.test.ts | 45 +++++++++++ js/hang/src/container/stall.ts | 30 +++++-- 4 files changed, 196 insertions(+), 12 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 26e4f7f43f..7d4c00e408 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -665,7 +665,7 @@ test("Consumer counts a group the budget abandoned instead of failing its task", await settle(); // The verdict a subscription that gave up on this group reaches, and the same one a peer's - // DELIVERY_TIMEOUT reset decodes back into. + // OLD reset decodes back into. abandoned.close(new NetError.Expired()); writeGroupWithLegacyFrames(track, 1, [40_000 as Time.Micro]); @@ -2366,3 +2366,106 @@ for (const end of [ } }); } + +test.each([false, true])("a paused reader bounds finished backlog with a partial head: %s", async (partial) => { + let clock = performance.now(); + const now = spyOn(performance, "now").mockImplementation(() => clock); + const track = new Track.Producer("audio"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxAge: Time.Milli(100) }); + try { + writeGroupWithLegacyFrames(track, 0, partial ? [Time.Micro.zero, Time.Micro(10_000)] : [Time.Micro.zero]); + expect((await consumer.next())?.frame?.timestamp).toBe(Time.Micro.zero); + clock += 150; + for (let sequence = 1; sequence <= 20; sequence++) { + writeGroupWithLegacyFrames(track, sequence, [Time.Micro(sequence * 20_000)]); + } + for (let i = 0; i < 200; i++) await Promise.resolve(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(Time.Micro(300_000)); + expect(consumer.skipped.peek()).toBeGreaterThan(0); + } finally { + consumer.close(); + track.close(); + now.mockRestore(); + } +}); + +test("a marker group skipped as covered still raises its playhead event", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxAge: Time.Milli(2_000) }); + const marker = new Group.Producer(1); + try { + writeGroupWithLegacyFrames(track, 0, [Time.Micro.zero]); + expect((await consumer.next())?.frame?.timestamp).toBe(Time.Micro.zero); + expect((await consumer.next())?.frame).toBeUndefined(); + + track.writeGroup(marker); + marker.writeFrame({ + payload: encodeLegacyFrame(Time.Micro(20_000), new Uint8Array()), + timestamp: Time.Timestamp.fromMicros(Time.Micro(20_000)), + }); + const endpoint = await consumer.next(); + expect(endpoint?.end).toBe(Time.Micro(20_000)); + expect(endpoint?.discontinuity).toBe(0); + + writeGroupWithLegacyFrames(track, 2, [Time.Micro(20_000)]); + const resumed = await nextFrame(consumer); + expect(resumed?.frame?.timestamp).toBe(Time.Micro(20_000)); + expect(resumed?.discontinuity).toBe(1); + } finally { + marker.close(); + consumer.close(); + track.close(); + } +}); + +test("Consumer stamps an arrival before the container parses it", async () => { + let clock = 1000; + const now = spyOn(performance, "now").mockImplementation(() => clock); + const observe = spyOn(Jitter.prototype, "observe"); + const track = new Track.Producer("audio"); + const legacy = new LegacyFormat("audio"); + const format: ContainerFormat = { + decode(payload) { + clock += 30; + return legacy.decode(payload); + }, + end: (frame) => legacy.end(frame), + }; + const consumer = new Consumer(replay(track), { format }); + try { + writeGroupWithLegacyFrames(track, 0, [Time.Micro.zero]); + expect((await consumer.next())?.frame?.timestamp).toBe(Time.Micro(0)); + expect(observe.mock.calls[0]?.[1]).toBe(Time.Milli(1000)); + } finally { + consumer.close(); + track.close(); + observe.mockRestore(); + now.mockRestore(); + } +}); + +test("a completed short group does not bound a later open GOP", async () => { + const track = new Track.Producer("video"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxAge: Time.Milli(200) }); + const long = new Group.Producer(5); + try { + writeGroupWithLegacyFrames(track, 0, [Time.Micro.zero]); + expect((await consumer.next())?.frame?.timestamp).toBe(Time.Micro(0)); + await consumer.next(); + track.writeGroup(long); + long.writeFrame({ payload: encodeLegacy(Time.Micro(5_000_000)), timestamp: Time.Timestamp.now() }); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(Time.Micro(5_000_000)); + + writeGroupWithLegacyFrames(track, 10, [Time.Micro(10_000_000)]); + writeGroupWithLegacyFrames(track, 11, [Time.Micro(10_020_000)]); + for (let i = 0; i < 100; i++) await Promise.resolve(); + long.writeFrame({ payload: encodeLegacy(Time.Micro(9_990_000)), timestamp: Time.Timestamp.now() }); + long.close(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(Time.Micro(9_990_000)); + expect(consumer.skipped.peek()).toBe(0); + } finally { + long.close(); + consumer.close(); + track.close(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 2e94ae1602..0ae50ec413 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -111,6 +111,7 @@ export class Consumer { #discontinuity = 0; // A group below the live edge aborts the track. #error?: Error; + #pulled?: Time.Milli; // Wake up the consumer when a new frame is available. #notify?: () => void; @@ -385,6 +386,11 @@ export class Consumer { let skipped = false; let walked = false; let hole = false; + const paused = + this.#maxAge.peek() > 0 && + this.#pulled !== undefined && + this.#notify === undefined && + Moq.Time.Milli.now() - this.#pulled >= this.#maxAge.peek(); // Walk the delivery cursor forward while what the oldest group could still present has aged // past the budget. This is also what ends the wait on a gap in group sequence numbers: if @@ -395,7 +401,7 @@ export class Consumer { const threshold = Moq.Time.Micro.fromMilli(this.#maxAge.peek()); const first = this.#groups[0]; // Where delivery stands, which decides what a verdict against the head means. - const cursor = this.#active; + const cursor: number | undefined = this.#active; // A group is measured by how far it could still reach, not by how far behind it // started: it cannot present past where the next group holding a frame begins, so @@ -445,14 +451,26 @@ export class Consumer { // contiguous successor one frame later is judged a hole, which re-anchors the reader. // `rs/moq-mux`'s consumer cannot reach either verdict: its read arm returns a buffered // frame, and closes out a spent group as `GroupEnd`, before the budget is consulted. - if (first.done && cursor !== undefined && first.consumer.sequence <= cursor) break; + if (first.done && first.frames.length === 0 && paused) { + this.#groups.shift(); + if (first.consumer.sequence === cursor) { + this.#recordPresented(first); + this.#active = continues(first, this.#groups[0]) ? this.#groups[0].consumer.sequence : cursor + 1; + } + if (first.truncated) this.#gap = true; + if (!first.empty && !first.media) hole = true; + first.consumer.close(); + walked = true; + continue; + } + if (first.done && !paused && cursor !== undefined && first.consumer.sequence <= cursor) break; // Above the cursor the group it sits on never arrived, and now it never will. Give up // on those sequences rather than on the media that did arrive: walk the cursor onto the // head, the way the same consumer walks onto the first arrived group instead of // dropping it. A head that finished holding nothing cannot be walked onto, so it is // convicted below along with a head that is still downloading. - if (first.done && first.frames.length > 0) { + if (first.done && first.frames.length > 0 && !paused) { // Whether that cost anything is the one thing the reader has to be told: a head // that continues the timeline we left off at means the sequence numbers merely // jumped, and a head that does not means a span of media is missing. @@ -528,6 +546,7 @@ export class Consumer { console.debug(`skipping covered group: ${active.consumer.sequence} -> ${next.consumer.sequence}`); this.#recordPresented(active); this.#active = next.consumer.sequence; + if (!active.empty && !active.media) this.#markPlayhead(); active.consumer.close(); active.frames.length = 0; @@ -660,6 +679,7 @@ export class Consumer { this.#liveEdge = { group: seq, timestamp: end }; } this.#updateBuffered(); + this.#pulled = Moq.Time.Milli.now(); return { frame: undefined, group: seq, @@ -677,6 +697,7 @@ export class Consumer { this.#liveEdge = { group: seq, timestamp: frame.timestamp }; } this.#updateBuffered(); + this.#pulled = Moq.Time.Milli.now(); return { frame, group: seq, discontinuity: this.#discontinuity, continuous }; } @@ -707,6 +728,7 @@ export class Consumer { // once every terminal packet behind the marker has been delivered. if (!group.empty && !group.media) this.#markPlayhead(); this.#updateBuffered(); + this.#pulled = Moq.Time.Milli.now(); return { frame: undefined, group: seq, diff --git a/js/hang/src/container/stall.test.ts b/js/hang/src/container/stall.test.ts index 7c08868bc4..459044f216 100644 --- a/js/hang/src/container/stall.test.ts +++ b/js/hang/src/container/stall.test.ts @@ -315,3 +315,48 @@ describe("the event loop monitor", () => { restarted.close(); }); }); + +// review consumer-sync-video F12. A hidden tab rations the tick to one a second while an +// audio-only track delivers a PES of several frames every 200ms: the loop is fine, the arrivals +// are simply further apart than the gap threshold, and the tick is no longer there to vouch for +// the loop in between. Flagging each of them drops the estimator's reference on every arrival, +// so its target decays to one bucket; its own idle rule would wait 500ms for that. +it("a rationed tick does not flag arrivals spaced under the idle threshold", () => { + const timer = fake(); + const stall = new Stall(timer); + + timer.throttle(1000); + // Long enough for the rationed tick to have run a few times. + timer.advance(3000); + + const flags: boolean[] = []; + for (let i = 0; i < 25; i++) { + timer.advance(200); + flags.push(stall.blocked(timer.at())); + } + + // The track's own cadence has to be seen before a gap can be told apart from a block, so the + // first two arrivals are allowed either answer. None after that is a block. + expect({ flagged: flags.slice(2).filter((f) => f).length, of: flags.length - 2 }).toEqual({ flagged: 0, of: 23 }); + + stall.close(); +}); + +it("two blocks during a rationed tick do not become the arrival cadence", () => { + const timer = fake(); + const stall = new Stall(timer); + try { + timer.throttle(1000); + timer.advance(3000); + for (let i = 0; i < 5; i++) { + timer.advance(20); + stall.blocked(timer.at()); + } + for (let i = 0; i < 2; i++) { + timer.block(400); + expect(stall.blocked(timer.at())).toBe(true); + } + } finally { + stall.close(); + } +}); diff --git a/js/hang/src/container/stall.ts b/js/hang/src/container/stall.ts index 8ad6bc4a96..23cd3d73ca 100644 --- a/js/hang/src/container/stall.ts +++ b/js/hang/src/container/stall.ts @@ -117,6 +117,10 @@ export class Stall { // When the loop last came back from a block, if it ever has. #ended?: number; + #ticked?: number; + #slowTicks = 0; + #queried?: number; + #spacing?: number; #handle: unknown; #holders = 1; @@ -155,6 +159,12 @@ export class Stall { // the probe wins the race out of a block, which nothing orders. this.#see(now); this.#arm(now); + if (this.#queried !== undefined && now > this.#queried) { + const spacing = now - this.#queried; + // A detected block must not become the cadence used to excuse the next block. + if (this.#spacing === undefined || spacing <= this.#spacing + THRESHOLD) this.#spacing = spacing; + } + if (this.#queried === undefined || now > this.#queried) this.#queried = now; return this.#ended !== undefined && now - this.#ended <= TICK; } @@ -178,14 +188,14 @@ export class Stall { const gap = now - this.#alive; if (now > this.#alive) this.#alive = now; - // Two shapes of a receiver that is not keeping up, and a rationed timer is neither of them. No - // turn of the loop at all for longer than the threshold means it stopped; the tick is the - // witness between arrivals, until a hidden tab rations that too and a track whose arrivals are - // further apart than the threshold is left with none, which is the answer the estimator's own - // idle rule already gives such a track. Or the loop is running and the task it was handed - // before this turn is still waiting, which is how long it kept this arrival waiting too. + // A rationed timer cannot witness the healthy gaps between sparse arrivals. + const cadence = + this.#slowTicks >= 2 && + this.#spacing !== undefined && + this.#queried !== undefined && + now - this.#queried <= this.#spacing + THRESHOLD; const sent = this.#sent; - if (gap > THRESHOLD || (sent !== undefined && now - sent > THRESHOLD)) this.#ended = now; + if ((gap > THRESHOLD && !cadence) || (sent !== undefined && now - sent > THRESHOLD)) this.#ended = now; } // Hands the loop a task to be timed by, at most one a tick. @@ -214,7 +224,11 @@ export class Stall { #tick = (): void => { // Being a turn of the loop is all this is for. How late it runs says nothing: a hidden tab // rations it to one a second with the loop running fine underneath. - this.#see(this.#timer.now()); + const now = this.#timer.now(); + this.#slowTicks = + this.#ticked !== undefined && now - this.#ticked > 2 * TICK ? Math.min(2, this.#slowTicks + 1) : 0; + this.#ticked = now; + this.#see(now); this.#handle = this.#timer.schedule(this.#tick, TICK); }; } From ad1380ba6d21aefbe423a3bd748d1525d2aa91ff Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:09:48 -0400 Subject: [PATCH 022/127] fix(publish): accept reordered capture callbacks after worker ticks Keep new frames readable when a returning frame callback predates the last worker tick by no more than one frame period. Preserve rejection of larger clock reversals and make the suspended-callback grace test use a controlled clock. Co-Authored-By: Codex --- js/publish/src/video/processor.test.ts | 77 +++++++++++++++++++++++++- js/publish/src/video/processor.ts | 8 ++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/js/publish/src/video/processor.test.ts b/js/publish/src/video/processor.test.ts index 7c53b02877..22a717b31e 100644 --- a/js/publish/src/video/processor.test.ts +++ b/js/publish/src/video/processor.test.ts @@ -135,11 +135,15 @@ class FakeVideo { this.#callbacks.delete(handle); } - /** A new picture arrives. Returns the instant it was reported at, which is what stamps it. */ - advance(metadata: { captureTime?: number } = {}): number { + /** + * A new picture arrives. Returns the instant it was reported at, which is what stamps it. `at` + * overrides that instant, for a callback whose timestamp is the frame's start (vsync) rather + * than when it ran. + */ + advance(metadata: { captureTime?: number } = {}, at?: number): number { this.currentTime += 1 / RATE; - const now = performance.now(); + const now = at ?? performance.now(); if (this.suspended) return now; // The real callback re-registers itself, so hand out the pending ones and start empty. @@ -296,6 +300,9 @@ test("keeps capturing when the frame callback is suspended", async () => { arrivalStep = 0; spawned.length = 0; + const realNow = performance.now.bind(performance); + let clock = realNow(); + performance.now = () => clock; const dom = install(); try { const track = new FakeTrack(); @@ -310,6 +317,7 @@ test("keeps capturing when the frame callback is suspended", async () => { for (let i = 0; i < 3; i++) { const read = reader.read(); await drain(); + clock += 1000 / RATE; const now = video.advance(); expect((await read).value?.timestamp).toBe(now * 1000); } @@ -326,18 +334,21 @@ test("keeps capturing when the frame callback is suspended", async () => { // A tick inside the grace is normal jitter, not a suspended callback. const early = reader.read(); await drain(); + clock += 1000 / RATE; video.advance(); ticker.tick(); expect(await pending(early)).toBe("pending"); // Past the grace the tick drives capture: one frame per picture, none of them repeats. const stamps: number[] = []; + clock += 2000 / RATE; ticker.tick(); stamps.push((await early).value?.timestamp ?? 0); for (let i = 0; i < 9; i++) { const read = reader.read(); await drain(); + clock += 1000 / RATE; video.advance(); ticker.tick(); const frame = await read; @@ -353,6 +364,7 @@ test("keeps capturing when the frame callback is suspended", async () => { await reader.cancel(); expect(ticker.terminated).toBe(true); } finally { + performance.now = realNow; dom.restore(); } }); @@ -434,3 +446,62 @@ test("stops rather than hand the encoder a timeline that goes backwards", async dom.restore(); } }); + +// N10: when a hidden window comes back, the worker tick has been capturing, stamped with +// performance.now() when its message was handled. The first frame callback after that carries the +// refresh tick's timestamp, taken when the frame began, which can sit just before the tick's stamp +// if the tick ran between vsync and the callbacks. That is one new picture half a millisecond +// "early", not a clock fault, and it must not tear the capture down. +test("a callback stamped just before the last tick keeps the stream readable", async () => { + supported = false; + advance = 1000; + arrivalStep = 0; + spawned.length = 0; + + const realNow = performance.now.bind(performance); + let clock = 1000; + performance.now = () => clock; + const dom = install(); + try { + const track = new FakeTrack(); + const stream = TrackProcessor(track as unknown as Parameters[0]); + const reader = stream.getReader(); + + await drain(); + const video = dom.video(); + + // On screen, one picture through the callback. + const shown = reader.read(); + await drain(); + video.advance(); + expect((await shown).value).toBeDefined(); + + // Hidden: past the grace, the tick takes the next picture and stamps it with its own clock. + video.suspended = true; + const ticker = spawned.at(-1) as FakeWorker; + const hidden = reader.read(); + await drain(); + expect(await pending(hidden)).toBe("pending"); + clock += 2000 / RATE; + video.advance(); + ticker.tick(); + const stamped = ((await hidden).value?.timestamp ?? 0) / 1000; + + // Visible again: the next new picture's callback reports the frame start, 0.5 ms before that. + video.suspended = false; + const next = reader.read(); + await drain(); + video.advance({}, stamped - 0.5); + + const result = await next.then( + (read) => ({ readable: true, frame: read.value !== undefined }), + (err: Error) => ({ readable: false, error: err.message }), + ); + expect(result).toEqual({ readable: true, frame: true }); + + await reader.cancel(); + } finally { + performance.now = realNow; + dom.restore(); + } +}); diff --git a/js/publish/src/video/processor.ts b/js/publish/src/video/processor.ts index ca00a6acec..7c99a422ae 100644 --- a/js/publish/src/video/processor.ts +++ b/js/publish/src/video/processor.ts @@ -244,8 +244,12 @@ function videoProcessor(track: StreamTrack): ReadableStream { waiting = undefined; if (at <= stamped) { - pull.reject(new Error(`video capture went backwards: ${at}ms after ${stamped}ms`)); - return; + // A returning callback can carry a frame-start time just before the last worker tick. + if (stamped - at > 1000 / rate) { + pull.reject(new Error(`video capture went backwards: ${at}ms after ${stamped}ms`)); + return; + } + at = Time.Milli(stamped + 0.001); } taken = video.currentTime; From 3f1b7543faa7391ded8ee032b8fc3f3a5f93fc0c Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:08:21 -0400 Subject: [PATCH 023/127] style(watch): format regression fixes Co-Authored-By: Codex --- js/watch/src/audio/buffer.test.ts | 12 +++++++++--- js/watch/src/audio/buffer.ts | 4 +++- js/watch/src/audio/decoder.offload.test.ts | 1 - js/watch/src/video/decoder.test.ts | 2 -- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 8ef15dcbde..43ede89ccd 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -317,7 +317,11 @@ describe("AudioBuffer output clock, device starting", () => { output.contextTime = 0; output.performanceTime = 50_000; worklet.deliver({ ...state(worklet, playhead(500, 1), false), contextTime: Time.Second(0.02) }); - expect(buffer.clock.peek()).toEqual({ timestamp: Time.Micro(500_000), reference: Time.Milli(50_020), rate: 1 }); + expect(buffer.clock.peek()).toEqual({ + timestamp: Time.Micro(500_000), + reference: Time.Milli(50_020), + rate: 1, + }); } finally { buffer.close(); } @@ -463,7 +467,6 @@ describe("AudioBuffer output clock, shared ring", () => { }); }); - // --- review consumer-sync-video F15 --- describe("AudioBuffer, partial output timestamp", () => { @@ -498,7 +501,10 @@ describe("AudioBuffer, partial output timestamp", () => { thrown = err; } await Promise.resolve(); - expect({ thrown: (thrown as Error | undefined)?.message, released }).toEqual({ thrown: undefined, released: true }); + expect({ thrown: (thrown as Error | undefined)?.message, released }).toEqual({ + thrown: undefined, + released: true, + }); await waiting; } finally { buffer.close(); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index e2cc6ed0b2..1119de7e7b 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -233,7 +233,9 @@ export function supportsSharedArrayBuffer(): boolean { } /** Read the device clock once it has a complete timestamp. */ -export function outputTimestamp(context: Pick): Required | undefined { +export function outputTimestamp( + context: Pick, +): Required | undefined { const { contextTime, performanceTime } = context.getOutputTimestamp?.() ?? {}; // Chromium can expose context time before the device has produced a performance timestamp. if (contextTime === undefined || performanceTime === undefined || performanceTime === 0) return undefined; diff --git a/js/watch/src/audio/decoder.offload.test.ts b/js/watch/src/audio/decoder.offload.test.ts index d9ab16bc75..43ae6440b4 100644 --- a/js/watch/src/audio/decoder.offload.test.ts +++ b/js/watch/src/audio/decoder.offload.test.ts @@ -1245,7 +1245,6 @@ describe("the thread a player's audio runs on", () => { }); }); - describe("the worker output clock after resume", () => { it("rejects a missing device timestamp and refreshes it on the next worker report", async () => { jest.useFakeTimers(); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index ab011d9fee..bd45272460 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -560,7 +560,6 @@ test("a rendition that left the catalog is not a stall", async () => { } }); - test("a pending rendition waits until its preview picture is due", async () => { const fx = fixture(); const second = new Moq.Track.Producer("second").accept({}); @@ -613,7 +612,6 @@ test("a sparse rendition is not rebuilt at every healthy silence", async () => { } }); - test("a new rendition does not inherit a sparse rendition's recovery window", async () => { const fx = fixture(); const second = new Moq.Track.Producer("second").accept({}); From d209e30ea94d6f464e243eb66e02c78dfe2faf74 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:11:49 -0400 Subject: [PATCH 024/127] fix(watch): retain decoded rates only for matching configurations Extend #40 coverage to detach and reattach with a different codec configuration. Co-Authored-By: Codex --- js/watch/src/audio/decoder.offload.test.ts | 35 +++++++++++++++++++++- js/watch/src/audio/decoder.ts | 7 +++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/js/watch/src/audio/decoder.offload.test.ts b/js/watch/src/audio/decoder.offload.test.ts index 43ae6440b4..44e226f52f 100644 --- a/js/watch/src/audio/decoder.offload.test.ts +++ b/js/watch/src/audio/decoder.offload.test.ts @@ -1,5 +1,5 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, jest, mock, spyOn } from "bun:test"; -import type * as Catalog from "@moq/hang/catalog"; +import * as Catalog from "@moq/hang/catalog"; import type * as Moq from "@moq/net"; import { Group, Origin, Path, Time, Varint } from "@moq/net"; import { Effect, Signal } from "@moq/signals"; @@ -1267,3 +1267,36 @@ describe("the worker output clock after resume", () => { }); }); }); + +describe("the decoded rate's rendition", () => { + it("does not carry a learned rate into a different decoder configuration", async () => { + const rates: number[] = []; + scope.AudioContext = class extends MockContext { + constructor(options?: { sampleRate?: number }) { + super(options); + rates.push(this.sampleRate); + } + }; + const t = tile({ offload: true }); + const config = (sampleRate: number): Catalog.Root => + Catalog.RootSchema.parse({ + audio: { + renditions: { + audio: { codec: "opus", container: { kind: "legacy" }, sampleRate, numberOfChannels: 2 }, + }, + }, + }); + t.catalog.set(config(44_100)); + await until(() => handed()(), "the graph handed over"); + for (let i = 0; i < 20; i++) t.write(i, i * 20_000); + await until(() => t.decoder.out.context.peek()?.sampleRate === RATE, "the decoded rate"); + t.attached.set(false); + await until(() => t.decoder.out.context.peek() === undefined, "the context closed"); + t.catalog.set(config(32_000)); + await sleep(25); + const before = rates.length; + t.attached.set(true); + await until(() => rates.length > before, "the replacement context"); + expect(rates[before]).toBe(32_000); + }); +}); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 12a771bf53..d9cacd1046 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -198,7 +198,7 @@ export class Decoder { // Deduped, so a decoded rate confirming the catalog's does not count as a change. readonly #rate: Computed; // A replacement supply has not decoded yet, but the context already knows the stream's rate. - #decodedRate?: number; + #decodedRate?: { config: string; rate: number }; /** * The age budget for audio: `Sync.out.maxAge` plus what the ring can absorb past it. @@ -242,8 +242,9 @@ export class Decoder { if (!config) return undefined; const active = effect.get(this.#active); const decoded = active ? effect.get(active.supply.out.rate) : undefined; - if (decoded !== undefined) this.#decodedRate = decoded; - return this.#decodedRate ?? config.sampleRate; + const key = JSON.stringify(config); + if (decoded !== undefined) this.#decodedRate = { config: key, rate: decoded }; + return this.#decodedRate?.config === key ? this.#decodedRate.rate : config.sampleRate; }); this.#mode = this.#signals.computed((effect) => { if (effect.get(this.#fallback) !== undefined) return "main"; From c6c842c443f4234b17d1f92644032b5a1b261497 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 07:09:30 -0700 Subject: [PATCH 025/127] fix(net): hold a parked track's warm cache until the upstream confirms it (#4104) Co-authored-by: Claude Opus 5.5 --- doc/lib/c/index.md | 2 +- rs/moq-net/src/lite/subscriber.rs | 15 +- rs/moq-net/src/model/front.rs | 10 +- rs/moq-net/src/model/origin.rs | 370 ++++++++++++++++++++++++++++-- rs/moq-net/src/model/resume.rs | 114 ++++++++- rs/moq-net/src/model/track.rs | 57 ++++- rs/moq-tokio/tests/broadcast.rs | 240 +++++++++++++++++++ 7 files changed, 776 insertions(+), 32 deletions(-) diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index b77ae8a4bd..7a3c5b08dd 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -40,7 +40,7 @@ and `target/include/moq.h`. - **Encoded video metadata.** `moq_video_init.hint` is a zero-initialized `moq_video_hint` with `has_*` flags for coded dimensions, bitrate (bits per second), frame rate, and latency preference. Hints seed a video codec track's catalog; detected dimensions take precedence. - **Client config.** A zeroed `moq_client_config` means the defaults for every knob, which is what lets a new one be appended without disturbing callers. Fields cover protocol (`versions`), TLS (`tls_fingerprints`, `tls_roots`, `tls_cert`/`_key`, `tls_host_name`), transport (`bind`, `connect_timeout_us`, the Happy Eyeballs delays, `websocket_enabled`/`_delay_us`), and tuning (reconnect backoff, `quic_*`). Every duration is in microseconds. A knob whose default isn't zero carries a `has_*` flag, so setting `backoff_timeout_us = 0` needs `has_backoff_timeout = true` to mean "retry forever" rather than "use the default". `moq_client_defaults()` reports what a NULL config dials with. - **Server.** `moq_server_listen` binds before it returns (a bad address or certificate fails there) and hands each incoming session to `on_request` as a request handle. Read `moq_session_request_path` and `_query` to route and authenticate, then `moq_session_request_accept` (a session handle, with origins like `moq_session_connect`) or `moq_session_request_reject` with an HTTP-style code (401 and 403 become the protocol's unauthorized close). An accepted session reports `1` once SETUP completes and never reconnects. `moq_server_addr` reports an ephemeral port and `moq_server_fingerprints` the hashes a client pins for a `tls_generate` certificate. `moq_server_close` stops listening; its terminal callback fires once the sockets are released. -- **Demand.** A watcher on a published track (`moq_publish_track_demand`, `moq_publish_media_demand`, `moq_encode_video_demand`, `moq_encode_audio_demand`) calls `on_demand` with `MOQ_DEMAND_USED` or `MOQ_DEMAND_UNUSED` right away and again on every change, so an encoder on a battery-powered device runs only while someone is watching. The first call is the current state, so a track that went unused before the watcher existed still reports it. `moq_publish_demand_cancel` stops it; the terminal callback still fires. A container has no single demand and is refused. Demand follows the last real subscriber: an origin that served the track drops its source copy on the unused edge and keeps only the finished groups it already cached warm for 30 seconds, so the cache linger does not delay the unused edge. +- **Demand.** A watcher on a published track (`moq_publish_track_demand`, `moq_publish_media_demand`, `moq_encode_video_demand`, `moq_encode_audio_demand`) calls `on_demand` with `MOQ_DEMAND_USED` or `MOQ_DEMAND_UNUSED` right away and again on every change, so an encoder on a battery-powered device runs only while someone is watching. The first call is the current state, so a track that went unused before the watcher existed still reports it. `moq_publish_demand_cancel` stops it; the terminal callback still fires. A container has no single demand and is refused. Demand follows the last real subscriber: an origin that served the track drops its source copy on the unused edge, so a cache linger never delays it. Only a relay keeps what it already delivered warm for 30 seconds, and a returning subscriber is served that cache only once the publisher confirms it is still current. - **Requests.** `moq_publish_dynamic` serves subscriptions to tracks the broadcast never declared: each arrives as a request handle, read its name with `moq_track_request_name`, then `moq_track_request_accept` (a raw track handle), `moq_track_request_video` / `_audio` (the media handle `moq_publish_video` / `_audio` return), or `moq_track_request_abort` with an application code the subscriber sees. Without a live handler an unknown name is refused. `moq_publish_track_dynamic` does the same for fetches of groups a track no longer has cached, delivered as `moq_group_request_*` (`sequence`, `priority`, `frame_start`); `moq_group_request_accept` starts the producer at `frame_start` so written frames keep their group indices. Register it with `moq_track_request_dynamic` before accepting a track that was itself requested by a fetch, so that pending group survives the transition. Both handlers stop with `moq_publish_dynamic_cancel`. - **Everything the bindings can do** ([list](/lib/#what-every-binding-can-do)): media publish and consume with the catalog managed for you, raw pixels and PCM with the codec inside (`moq_encode_video`, `moq_encode_audio`, and the `moq_decode_*` mirrors), raw tracks with timestamps and datagrams, JSON and binary data tracks (snapshot or stream, each advertised in the catalog for as long as it lives), group fetch, catalog sections, shared video properties, and stalled hints. The three advertising operations are `moq_origin_create_broadcast` (unannounced producer, invisible to everyone), `moq_publish_announce` / `moq_publish_unannounce` (exact-path advertisement), and `moq_origin_dynamic` (a claim over a path prefix and everything beneath it; `""` for everything). A route is a capability, not an inventory. `moq_origin_announced` takes a literal prefix and an optional relative pattern filter; `moq_announce_update.prefix` stays relative to the origin, while `captures` reports what each wildcard matched when `has_captures` is true. Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts); name the dot segment in `prefix` to list them. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 387ba189b8..9013f381cc 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -2777,8 +2777,13 @@ impl TrackServe { // The floor tracks the requested start until this subscription's own // SUBSCRIBE_START refines it: the peer never serves below the request, a // previous subscription's declaration must not outlive its demand, and - // live-edge demand (None) starts with no floor at all. - let _ = producer.start_at(subscription.start.map(|start| start.group)); + // live-edge demand (None) starts with no floor at all. Lite-06+ only resolves + // the start with its SUBSCRIBE_START, so until then the floor is just a request. + let floor = subscription.start.map(|start| start.group); + let _ = match self.subscriber.version.resolves_start() { + true => producer.request_start(floor), + false => producer.start_at(floor), + }; tracing::info!(id, broadcast = %self.subscriber.log_path(&self.path), track = %self.name, "subscribe started"); @@ -3200,6 +3205,12 @@ impl ServeLoop { // request's own fetch gate, and a cache-miss fetch queued while TRACK_INFO // was in flight would be drained as NotFound in the gap. let dynamic = request.dynamic(); + // Lite-06+ resolves each subscription's start from its budget, so the floor + // the SUBSCRIBE asks for is not where delivery begins until SUBSCRIBE_START says. + let request = match serve.subscriber.version.resolves_start() { + true => request.resolving_start(), + false => request, + }; let serving = request.accept(info); Self { serving, diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index c651082fb7..df6a582a65 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -105,7 +105,8 @@ pub(super) enum Action { /// Drop the source copy of `track` but keep the delivered groups spliced, /// so resume stays seamless while nobody reads. Park { track: Arc }, - /// Drop the delivered groups of `track` too: the linger expired unread. + /// Drop the source copy of `track` and every delivered group: the linger expired + /// unread, or the source is local and keeps its own cache. Release { track: Arc }, /// The logical track completed. Finish { track: Arc }, @@ -587,6 +588,13 @@ impl Front { }; track.used = false; match track.state { + // A local source keeps its own cache, so a warm copy would only be a staler + // duplicate of it: drop the copy outright, and a returning reader re-splices + // the source and reads its cache against the real live edge. + TrackState::Spliced { .. } if self.identity == Identity::Local => { + track.state = TrackState::Idle; + actions.push(Action::Release { track: name }); + } // Drop the copy so the source goes idle at once; the delivered // groups stay spliced for the linger. TrackState::Spliced { .. } => { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 8beaa32a75..2f4c0af840 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -1,4 +1,4 @@ -use crate::{broadcast, cache, stats, track}; +use crate::{broadcast, cache, group, stats, track}; use kio::Pollable; use std::{ cmp::Reverse, @@ -1815,6 +1815,9 @@ const TRACK_IDLE_LINGER: Duration = Duration::from_secs(30); struct WarmCopy { track: track::Producer, _dynamic: track::Dynamic, + /// The newest group, where the copy spliced after this one picks up (see + /// [`TrackIo::head`]). + edge: Option, } impl Drop for WarmCopy { @@ -1823,25 +1826,142 @@ impl Drop for WarmCopy { } } -/// Cache `source`'s finished groups on a new local track the origin owns. -fn warm_copy(source: &track::Consumer) -> Option { +/// A warm copy's newest group, which the copy spliced after it continues. +/// +/// Kept past that splice: an open one must stay open for the continuation (dropping +/// an unfinished producer clears its frames), and either kind supplies the head the +/// continuation lacks when the next park rebuilds the group. Nothing will ever finish +/// an open one, so it aborts on drop. +struct WarmGroup(group::Producer); + +impl Drop for WarmGroup { + fn drop(&mut self) { + if !self.0.is_finished() { + let _ = self.0.clone().abort(Error::Cancel); + } + } +} + +/// Cache what `source` delivered on a new local track the origin owns: its complete +/// groups, and its open live edge rebuilt from the frames already delivered. +/// +/// `head` is the previous park's edge. A copy spliced after a warm cache continues its +/// edge group from the next frame, so its copy of that group lacks the head, which +/// `head` supplies. +fn warm_copy(source: &track::Consumer, head: Option<&WarmGroup>) -> Option { let info = source.cached_info()?; let mut track = track::Producer::new(Arc::new(source.broadcast().clone()), source.name(), info); - for (group, visible) in source.cached_groups() { - // An open group is left for the re-splice to deliver whole. Dropping the source - // copy resets it mid-transfer, and its dead head would anchor the next takeover - // mid-group, asking upstream for a tail no returning reader can use. - if group.is_finished() { - let _ = track.adopt_group(group, visible); + let head = head.map(|head| &head.0); + let groups = source.cached_groups(); + // Not `source.latest()`: datagrams share the sequence counter and can run past it. + let latest = groups + .iter() + .filter(|(_, visible)| *visible) + .map(|(group, _)| group.sequence) + .max(); + let mut edge = None; + + // A spliced copy hides the group it continued (its halves sit in two segments), so + // carry the previous edge over when the copy has no version of it at all. First, + // since it arrived before anything the copy holds. + if let Some(head) = head + && !groups.iter().any(|(group, _)| group.sequence == head.sequence) + { + let is_latest = latest.is_none_or(|latest| head.sequence >= latest); + if head.is_finished() { + let _ = track.adopt_group(head.clone(), true); + if is_latest { + edge = Some(WarmGroup(head.clone())); + } + } else if is_latest { + edge = warm_rebuild(&track, head, None); + } + } + + for (group, visible) in groups { + let finished = group.is_finished(); + let whole = group.live_first_frame() == Some(0); + let is_latest = visible && Some(group.sequence) == latest; + let warm = if finished && whole { + let _ = track.adopt_group(group.clone(), visible); + Some(WarmGroup(group)) + } else if (finished || is_latest) && (whole || head.is_some_and(|head| head.sequence == group.sequence)) { + // Dropping the source copy resets an open live edge mid-transfer, so rebuild + // it from the frames already delivered: the re-splice asks for the next + // frame, and a group that stays open for good (a JSON log in group 0) + // continues instead of being re-sent whole on every resume. A continuation + // is rebuilt whole from the previous edge's head the same way. + warm_rebuild(&track, &group, head) + } else { + // Mid-transfer backlog, or a continuation with no head to complete it. + None + }; + if is_latest { + edge = warm; } } let dynamic = track.dynamic(); Some(WarmCopy { track, _dynamic: dynamic, + edge, }) } +/// Rebuild `live` on `track` from its delivered frames, prefixed by `head`'s when `live` +/// only holds a continuation of it. Finished like `live`, or left open. +fn warm_rebuild(track: &track::Producer, live: &group::Producer, head: Option<&group::Producer>) -> Option { + // A continuation holds nothing below its offset. + let mut start = live.live_first_frame()? as u64; + let mut tail = live.consume(); + tail.start_at(start); + let mut frames = Vec::new(); + if start > 0 + && let Some(head) = head.filter(|head| head.sequence == live.sequence) + && let Some(head_start) = head.live_first_frame() + { + let mut head = head.consume(); + head.start_at(head_start as u64); + while head.index() < start { + match head.poll_read_frame(&kio::Waiter::noop()) { + Poll::Ready(Ok(Some(frame))) => frames.push(frame), + _ => break, + } + } + match head.index() == start { + true => start = head_start as u64, + // The head doesn't reach the continuation: keep only the continuation. + false => frames.clear(), + } + } + while let Poll::Ready(Ok(Some(frame))) = tail.poll_read_frame(&kio::Waiter::noop()) { + frames.push(frame); + } + if frames.is_empty() { + return None; + } + + // Wrapped first, so a failed write aborts it rather than dropping it unfinished. + let rebuilt = WarmGroup( + track + .create_group(group::Info { + sequence: live.sequence, + }) + .ok()?, + ); + let mut writer = rebuilt.0.clone(); + if start > 0 { + writer.start_at(start).ok()?; + } + for frame in frames { + writer.write_frame(frame.timestamp, frame.payload).ok()?; + } + if live.is_finished() { + writer.finish().ok()?; + } + Some(rebuilt) +} + /// Everything [`run_front`] owns, queued by [`Consumer::request_broadcast`]. struct FrontTask { /// The route table the front selects from. @@ -1879,6 +1999,9 @@ struct TrackIo { /// Delivered groups kept after the copy was dropped, so resume stays spliced /// through the linger without pinning the source as a reader. warm: Option, + /// The last warm copy's newest group, outliving it so the copy spliced after it + /// can continue the group (see [`WarmGroup`]). Released at the next park. + head: Option, /// Whether the track had a reader as of the last demand edge. used: bool, } @@ -2106,7 +2229,7 @@ async fn run_front(task: FrontTask) { tracks.remove(&name); continue; } - io.warm = None; + io.head = io.warm.take().and_then(|mut warm| warm.edge.take()); // The new segment has produced nothing yet: this is the // edge the copy is asked to advance. io.edge = io.resume.resume_position(); @@ -2118,24 +2241,25 @@ async fn run_front(task: FrontTask) { // Drop the source copy so its producer goes idle at once; keep // the groups it delivered on a local track so resume stays // spliced until the linger expires. - let warm = warm_copy(©); + let warm = warm_copy(©, io.head.as_ref()); drop(copy); - if io.resume.release().is_err() { + io.head = None; + let parked = match &warm { + Some(warm) => io.resume.park(&warm.track), + None => io.resume.release(), + }; + if parked.is_err() { tracks.remove(&name); continue; } - if let Some(warm) = warm { - if let Err(err) = io.resume.takeover(&warm.track) { - let _ = io.resume.abort(err); - tracks.remove(&name); - continue; - } - io.warm = Some(warm); - } + io.warm = warm; } Action::Release { track: name } => { let Some(io) = tracks.get_mut(&name) else { continue }; + // A local source releases straight from the spliced copy. + io.copy = None; io.warm = None; + io.head = None; if io.resume.release().is_err() { tracks.remove(&name); } @@ -2252,6 +2376,7 @@ async fn run_front(task: FrontTask) { copy: None, edge: None, warm: None, + head: None, used: false, }, ); @@ -4615,6 +4740,211 @@ mod tests { assert!(matches!(subscription.recv_group().await, Ok(None)), "ends cleanly"); } + /// A front resolved through a served route, as a relay's upstream session serves + /// one: the upstream broadcast's track requests arrive on the returned handle. + async fn served_front() -> (Dynamic, broadcast::Producer, broadcast::Dynamic, broadcast::Consumer) { + let producer = origin(1).produce(); + let consumer = producer.consume(); + let server = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10]))) + .unwrap(); + let pending = consumer.request_broadcast("room/alice"); + let upstream = broadcast::Info::new().produce(); + let dynamic = upstream.dynamic(); + queued(&server).await.accept(&upstream); + let resolved = pending.await.expect("resolves"); + (server, upstream, dynamic, resolved) + } + + /// A reader returning to a parked track waits for the fresh copy to resolve its + /// start, and skips the warm cache when the copy resolves past it: the source + /// judged the groups in between stale, so the older cache is stale too. Without the + /// hold the reader was handed the whole warm cache first, seconds behind live. + #[tokio::test] + async fn returning_reader_skips_a_warm_cache_the_copy_resolved_past() { + let ms = |v: u64| crate::Timestamp::from_millis(v).unwrap(); + let (_server, _upstream, mut dynamic, resolved) = served_front().await; + let budget = track::Subscription::default().with_max_age(Duration::from_millis(100)); + + let track = resolved.track("audio").unwrap(); + let b = budget.clone(); + let subscribing = tokio::spawn(async move { track.subscribe(b).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source") + .expect("request"); + let source = request.resolving_start().accept(None); + for seq in 0..4u64 { + let mut group = source.create_group(seq.into()).unwrap(); + group.write_frame(ms(seq * 20), b"old".as_ref()).unwrap(); + group.finish().unwrap(); + } + let mut subscription = subscribing.await.unwrap().expect("subscribe"); + subscription.recv_group().await.unwrap().expect("the live group"); + drop(subscription); + tokio::time::timeout(Duration::from_secs(1), source.unused()) + .await + .expect("parked") + .expect("source open"); + drop(source); + + let track = resolved.track("audio").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(budget).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source again") + .expect("request"); + let mut source = request.resolving_start().accept(None); + let mut subscription = subscribing.await.unwrap().expect("resubscribe"); + + // The copy has not resolved its start: nothing is handed out yet. + assert!( + tokio::time::timeout(Duration::from_millis(50), subscription.recv_group()) + .await + .is_err(), + "the warm cache was served before the copy resolved its start" + ); + + // The source resolves past the floor (lite-06 skipped 4..20 as stale). + source.start_at(20).unwrap(); + let mut group = source.create_group(20u64.into()).unwrap(); + group.write_frame(ms(2000), b"new".as_ref()).unwrap(); + group.finish().unwrap(); + let group = subscription.recv_group().await.unwrap().expect("the live group"); + assert_eq!(group.sequence, 20, "a stale warm group was served"); + } + + /// A warm cache whose newest group finished still resumes when the source has + /// nothing newer: the re-splice asks for that group's tail, which a source that + /// resolves starts lazily (with its first served group) can answer at once. Asking + /// past it left a returning catalog reader waiting for the next catalog change. + #[tokio::test] + async fn returning_reader_replays_a_current_warm_cache() { + let (_server, _upstream, mut dynamic, resolved) = served_front().await; + + let track = resolved.track("catalog").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(None).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source") + .expect("request"); + let source = request.resolving_start().accept(None); + let mut group = source.create_group(0u64.into()).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"snapshot".as_ref()).unwrap(); + group.finish().unwrap(); + let mut subscription = subscribing.await.unwrap().expect("subscribe"); + subscription.recv_group().await.unwrap().expect("the catalog"); + drop(subscription); + tokio::time::timeout(Duration::from_secs(1), source.unused()) + .await + .expect("parked") + .expect("source open"); + drop(source); + + let track = resolved.track("catalog").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(None).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source again") + .expect("request"); + let mut source = request.resolving_start().accept(None); + let mut subscription = subscribing.await.unwrap().expect("resubscribe"); + + // The source still has group 0 as its newest: it serves the empty tail, and + // that is when its start resolves. + let reading = tokio::spawn(async move { + let mut group = subscription.recv_group().await.unwrap().expect("the catalog"); + assert_eq!(group.sequence, 0); + group.read_frame().await.unwrap().expect("the snapshot").payload + }); + tokio::task::yield_now().await; + assert_eq!( + source.subscription().and_then(|sub| sub.start), + Some(track::Position { group: 0, frame: 1 }), + "the re-splice asked past the cached catalog" + ); + source.start_at(0).unwrap(); + let mut tail = source.create_group(0u64.into()).unwrap(); + tail.start_at(1).unwrap(); + tail.finish().unwrap(); + let payload = tokio::time::timeout(Duration::from_secs(1), reading) + .await + .expect("the returning reader never got the catalog") + .unwrap(); + assert_eq!(&payload[..], b"snapshot"); + } + + /// A group that stays open for good (a JSON log in group 0) survives a park: the + /// returning reader gets the frames delivered before it from the warm cache, and the + /// re-splice asks the source only for the frames after them, across repeated parks. + /// A datagram sequenced past the group does not hide it as the live edge. + #[tokio::test] + async fn returning_reader_continues_an_open_warm_group() { + let (_server, _upstream, mut dynamic, resolved) = served_front().await; + + async fn read(group: &mut group::Consumer) -> Vec { + let frame = tokio::time::timeout(Duration::from_secs(1), group.read_frame()) + .await + .expect("frame") + .unwrap() + .expect("group ended"); + frame.payload.to_vec() + } + + let mut expect: Vec<&[u8]> = Vec::new(); + let mut floor: Option = None; + for (round, payload) in [b"a".as_ref(), b"b", b"c"].into_iter().enumerate() { + let track = resolved.track("log").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(None).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source") + .expect("request"); + let mut source = request.resolving_start().accept(None); + let mut subscription = subscribing.await.unwrap().expect("subscribe"); + + // The source resolves at the floor's group, continuing group 0. + source.start_at(0).unwrap(); + let mut group = source.create_group(0u64.into()).unwrap(); + if let Some(floor) = floor { + group.start_at(floor.frame).unwrap(); + } + group.write_frame(crate::Timestamp::ZERO, payload).unwrap(); + expect.push(payload); + source + .insert_datagram(10, crate::Timestamp::ZERO, b"datagram".as_ref()) + .unwrap(); + + let mut reading = tokio::time::timeout(Duration::from_secs(1), subscription.recv_group()) + .await + .expect("group 0") + .unwrap() + .expect("track ended"); + assert_eq!(reading.sequence, 0); + for frame in &expect { + assert_eq!(read(&mut reading).await, *frame, "round {round}"); + } + assert_eq!( + source.subscription().and_then(|sub| sub.start), + floor, + "round {round} asked for the wrong continuation" + ); + + drop(reading); + drop(subscription); + tokio::time::timeout(Duration::from_secs(1), source.unused()) + .await + .expect("parked") + .expect("source open"); + drop(group); + drop(source); + floor = Some(track::Position { + group: 0, + frame: expect.len() as u64, + }); + } + } + /// The same holds for a reader returning to a parked track: its warm cache /// does not stand in for the copy it is waiting on. #[tokio::test] diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index da250e34be..2fa3d0c048 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -43,6 +43,12 @@ struct Segment { end: Option, /// The underlying per-session track. track: track::Consumer, + /// Where the source is asked to start: `start`, except after a warm cache (see + /// [`Segment::warm_edge`]). + ask: Option, + /// A parked track's warm cache (see [`Producer::park`]), which live readers hold + /// until the next segment's copy resolves its start. + warm: bool, } impl Segment { @@ -72,6 +78,23 @@ impl Segment { }) } + /// Where a copy spliced after this warm cache is asked to start: the end of the + /// cache's newest group, even a finished one, rather than the head of the next. + /// + /// A source only resolves a start once it has a group to serve, so asking past its + /// newest group would leave a returning reader waiting on the next one. Asking for the + /// newest group's tail lets a source that is still there answer at once, and one that + /// moved on answer past it. Only the ask moves: the boundary stays where the cache + /// stops, so whatever the source sends for a finished group's empty tail (a FIN, or a + /// reset from a publisher that refuses empty ranges) sits outside the new segment. + fn warm_edge(&self) -> Option { + let group = self.track.peek_latest()?; + Some(Position { + group: group.sequence, + frame: group.frame_count() as u64, + }) + } + /// The newest cached group this segment serves below `before` (exclusive), or its /// newest cached group at all when `before` is `None`. /// @@ -234,6 +257,8 @@ impl ResumeState { start, end: None, track, + ask: start, + warm: false, }); self.epoch += 1; self.prune(); @@ -339,11 +364,23 @@ impl Producer { // segments at all) there is nothing to splice around, so the replacement // replaces them outright and starts unbounded, exactly like a first splice. // `switch` rejects a `None` start once a segment exists, hence the clear. + let ask = state + .segments + .last() + .filter(|last| last.warm) + .and_then(Segment::warm_edge); let start = state.resume_position(); if start.is_none() { state.segments.clear(); } - state.switch(track, start) + state.switch(track, start)?; + if let Some(ask) = ask + && start.is_some() + && let Some(last) = state.segments.last_mut() + { + last.ask = Some(ask); + } + Ok(()) } /// Drop every segment, releasing the underlying tracks while keeping the @@ -370,6 +407,30 @@ impl Producer { Ok(()) } + /// Replace every segment with `warm`, a cache of what they delivered, for a track + /// nobody reads anymore: [`Self::release`] followed by an unbounded first splice. + /// + /// Live readers hold the cache until the next [`Self::takeover`]'s copy resolves + /// where its feed starts. A copy that picks up at the cache's edge proves the cache + /// still leads into the live feed, so it is read as usual (and a group the cache + /// holds open continues from its next frame). A copy that starts past the edge + /// skipped groups its source already judged stale, so the older cache is stale too + /// and live readers skip it. + pub(crate) fn park(&mut self, warm: impl super::origin_impl::Consume) -> Result<()> { + let track = warm.consume(); + let mut state = self.state.write().map_err(|_| Error::Dropped)?; + if state.finished || state.abort.is_some() { + return Err(Error::Closed); + } + state.segments.clear(); + state.pruned = None; + state.switch(track, None)?; + if let Some(segment) = state.segments.last_mut() { + segment.warm = true; + } + Ok(()) + } + /// Whether any segment is spliced in, and so whether there is anything for /// [`Self::release`] to drop. /// @@ -1250,6 +1311,8 @@ struct SegmentSub { id: u64, start: Option, end: Option, + /// Where the source is asked to start; see [`Segment::ask`]. + ask: Option, sub: SubState, /// A completed segment's cursor, retained while parked groups may need their /// max age budget re-evaluated after the outer cap rises. @@ -1264,6 +1327,16 @@ struct SegmentSub { /// the lowest is re-offered first; holding them here (rather than blocking on /// the first) keeps in-range groups that arrive behind a capped one flowing. parked: BTreeMap, + /// Set while this is a warm segment (see [`Producer::park`]) that has not been + /// cleared for live reads: the copy spliced after it, once there is one. + warm: Option, +} + +/// A warm segment waiting on the copy spliced after it; see [`Subscriber::poll_activate`]. +struct Warm { + /// The cache's newest group. + edge: Option, + next: Option, } impl SegmentSub { @@ -1432,7 +1505,7 @@ impl Subscriber { }; self.last_prefs = prefs; for seg in &mut self.segments { - let prefs = slice(&self.last_prefs, seg.start, seg.end); + let prefs = slice(&self.last_prefs, seg.ask, seg.end); if let Some(sub) = seg.stale_sub_mut() { let _ = sub.update(prefs); } @@ -1499,9 +1572,14 @@ impl Subscriber { self.segments.retain(|s| !s.retired()); let anchor = self.anchor_end(); - for segment in segments { + let nexts: Vec<_> = segments.iter().skip(1).map(|next| Some(next.track.clone())).collect(); + let nexts = nexts.into_iter().chain(std::iter::once(None)); + for (segment, next) in segments.into_iter().zip(nexts) { match self.segments.iter_mut().find(|s| s.id == segment.id) { Some(existing) => { + if let Some(warm) = &mut existing.warm { + warm.next = next; + } if existing.end != segment.end { existing.end = segment.end; let cap = Self::stale_cap(existing, anchor); @@ -1512,7 +1590,7 @@ impl Subscriber { // read bounds stay on this subscriber (see `poll_recv_group`): // an inner `end_at` would park boundary-crossing groups in the // inner cursor, hiding the segment's completion. - let _ = sub.update(slice(&self.last_prefs, segment.start, segment.end)); + let _ = sub.update(slice(&self.last_prefs, segment.ask, segment.end)); } // A still-pending subscription picks the moved boundary up // when it activates (see `poll_activate`). Groups already handed @@ -1523,15 +1601,20 @@ impl Subscriber { None => { let sub = segment .track - .subscribe(slice(&self.last_prefs, segment.start, segment.end)); + .subscribe(slice(&self.last_prefs, segment.ask, segment.end)); self.segments.push(SegmentSub { id: segment.id, start: segment.start, end: segment.end, + ask: segment.ask, sub: SubState::Pending(sub), terminal: None, pruned: false, parked: BTreeMap::new(), + warm: segment.warm.then(|| Warm { + edge: segment.track.latest(), + next, + }), }); } } @@ -1657,6 +1740,25 @@ impl Subscriber { anchor_end: Option, waiter: &kio::Waiter, ) -> Poll<()> { + if matches!(seg.sub, SubState::Pending(_)) + && let Some(warm) = &seg.warm + { + // Nothing spliced after the cache yet, so nothing says it still leads into + // the live feed. A splice bumps the epoch, which wakes this waiter. + let Some(next) = &warm.next else { + return Poll::Pending; + }; + let start = ready!(next.poll_start(waiter)); + let edge = warm.edge; + seg.warm = None; + // The copy was asked for the cache's newest group and started past it: its + // source judged that group stale, so the older cache is no use to live reads. + if start.is_some_and(|start| edge.is_some_and(|edge| start > edge)) { + seg.complete(None); + return Poll::Ready(()); + } + } + if let SubState::Pending(pending) = &mut seg.sub { match ready!(pending.poll_ok(waiter)) { Ok(mut sub) => { @@ -1669,7 +1771,7 @@ impl Subscriber { // budget and floor, and this must not rewind past it. sub.raise_start_to(seg.first_group().max(min_sequence)); sub.set_stale_cap(Self::stale_cap(seg, anchor_end)); - let _ = sub.update(slice(prefs, seg.start, seg.end)); + let _ = sub.update(slice(prefs, seg.ask, seg.end)); seg.sub = SubState::Active(Box::new(sub)); } // The underlying track was rejected or closed: stall, not error. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 531a71db12..17bec9149b 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -223,6 +223,12 @@ pub(crate) struct TrackState { // fetch can still create them. start_sequence: Option, + // Whether `start_sequence` is only the floor a subscription asked for, still + // waiting on the serving session to resolve where the live feed begins (a + // lite-06+ SUBSCRIBE_START). Readers that must know the resolved start (see + // [`Consumer::poll_start`]) wait on it; everything else treats the floor as usual. + start_pending: bool, + // Where production stopped, snapshotted when the cached groups are released (an // abort, or the last producer dropping). Computed live from the cache otherwise; // see [`Self::resume_position`]. @@ -986,8 +992,9 @@ impl TrackState { /// declaration: the signal is scoped to the current subscription's demand, /// which may legitimately move in either direction. `None` clears it (the /// demand dropped to the live edge, whose floor is unknown until declared). - fn set_start(&mut self, start_sequence: Option) { + fn set_start(&mut self, start_sequence: Option, pending: bool) { self.start_sequence = start_sequence; + self.start_pending = pending; } /// Record the exclusive final sequence, rejecting a re-finish or a boundary that @@ -1381,7 +1388,16 @@ impl Producer { /// still promised. Pass `None` to clear it, for demand at the live edge: /// its floor is unknown until the feed declares one. pub fn start_at(&mut self, sequence: impl Into>) -> Result<()> { - self.modify()?.set_start(sequence.into()); + self.modify()?.set_start(sequence.into(), false); + Ok(()) + } + + /// Declare the floor a subscription asked for while the serving session has yet to + /// resolve its start: nothing below `sequence` arrives, exactly as [`Self::start_at`], + /// but [`Consumer::poll_start`] keeps waiting until a later [`Self::start_at`] + /// resolves it. + pub(crate) fn request_start(&mut self, sequence: Option) -> Result<()> { + self.modify()?.set_start(sequence, true); Ok(()) } @@ -2318,6 +2334,28 @@ impl Consumer { }) } + /// Poll for the first group the live feed serves, once the serving session has + /// resolved it: `Some` for a declared start, `None` for none (the live edge, or a + /// source that never declares one). Parks while a lite-06+ session still owes its + /// SUBSCRIBE_START (see [`Producer::request_start`]); a closed track is ready with + /// whatever it last declared, since nothing will resolve it anymore. + pub(crate) fn poll_start(&self, waiter: &kio::Waiter) -> Poll> { + match &self.inner { + ConsumerKind::Plain(state) => { + let res = state.poll(waiter, |state| match state.start_pending && state.abort.is_none() { + true => Poll::Pending, + false => Poll::Ready(state.start_sequence), + }); + match res { + Poll::Ready(Ok(start)) => Poll::Ready(start), + Poll::Ready(Err(state)) => Poll::Ready(state.start_sequence), + Poll::Pending => Poll::Pending, + } + } + ConsumerKind::Spliced(_) => Poll::Ready(None), + } + } + /// The newest group, when it is already cached: resolved synchronously, without /// counting as a fetch or a delivery. The IETF publisher snapshots its frame count to /// resolve Largest Object; a group that is not immediately available reads as no edge. @@ -3908,6 +3946,10 @@ pub struct Request { // Ingress stats scope, threaded into the accepted [`Producer`]. Empty (no-op) // unless this request was reserved on a tagged broadcast. stats: stats::Scope, + + // The serving session resolves the start of each subscription itself, so the + // accepted track's start is unknown until it says (see [`Self::resolving_start`]). + resolving_start: bool, } impl Request { @@ -3924,9 +3966,19 @@ impl Request { alive, _dynamic: dynamic, stats: stats::Scope::default(), + resolving_start: false, } } + /// Mark the track as served by a session that resolves each subscription's start + /// (lite-06+), so [`Consumer::poll_start`] waits for its declaration instead of + /// reading the requested floor as the start. Applied atomically with + /// [`Self::accept`], before any reader can see the track. + pub(crate) fn resolving_start(mut self) -> Self { + self.resolving_start = true; + self + } + /// Attach an ingress stats scope, applied to the [`Producer`] on accept. Set by /// a tagged [`broadcast::Producer::reserve_track`]. pub(crate) fn with_stats(mut self, scope: stats::Scope) -> Self { @@ -3991,6 +4043,7 @@ impl Request { // tolerate it: the Producer we hand back simply can't write. if let Ok(mut state) = self.state.write() { state.accept(info.clone()); + state.start_pending = self.resolving_start; } // Accepting the request creates the track producer: count it as one ingress // subscription (closed when the last handle drops). No-op when untagged. diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 5b7f244665..4c2953a72d 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -994,6 +994,246 @@ async fn broadcast_route_migration() { handle_b.await.expect("server b panicked").expect("server b failed"); } +/// A subscriber returning to a parked track is not handed the parked cache when the +/// upstream resolves its start past it (lite-06+ resolves the start from the budget). +/// +/// The front keeps what an unread track delivered as a warm cache. While parked, the +/// publisher moved on, so the resumed upstream subscription starts well past that cache. +/// The cache was only fresh against its own frozen edge; serving it first put a +/// rejoining player seconds behind live. +#[tracing_test::traced_test] +#[tokio::test] +async fn broadcast_rejoin_skips_a_stale_warm_cache() { + use moq_net::Timestamp; + + let ms = |ms: u64| Timestamp::from_millis(ms).unwrap(); + let write = |track: &moq_net::track::Producer, sequence: u64, at: u64| { + let mut group = track + .create_group(moq_net::group::Info { sequence }) + .expect("create group"); + group.write_frame(ms(at), b"frame".as_ref()).expect("write frame"); + group.finish().expect("finish group"); + }; + + let pub_origin = moq_tokio::origin::spawn(); + let broadcast = pub_origin.create_broadcast("test").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + let track = broadcast.create_track("audio", None).expect("create track"); + let live = track.clone(); + for sequence in 0..4u64 { + write(&track, sequence, sequence * 20); + } + + let mut config = moq_tokio::listen::Config::default(); + config.bind = Some("[::]:0".parse().unwrap()); + config.tls.generate = vec!["localhost".into()]; + let mut server = config + .init(Default::default()) + .expect("init server") + .listen() + .await + .expect("listen"); + let addr = server.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + let request = server.accept().await.expect("accept"); + let session = request.with_publisher(&pub_origin).ok().await?; + let _broadcast = broadcast; + let _track = track; + let _ = session.closed().await; + Ok::<_, anyhow::Error>(()) + }); + + let sub_origin = moq_tokio::origin::spawn(); + let sub_consumer = sub_origin.consume(); + let mut announcements = sub_consumer.announced(); + let mut config = moq_tokio::connect::Config::default(); + config.tls.insecure = Some(true); + let client = config.init(Default::default()).expect("init client"); + let url: url::Url = format!("moqt://localhost:{}", addr.port()).parse().unwrap(); + let (_client, session) = tokio::time::timeout(TIMEOUT, connect_once(client.with_subscriber(sub_origin), url)) + .await + .expect("connect timeout") + .expect("connect failed"); + + assert!(next_announce(&mut announcements).await.kind.is_active()); + let remote = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + .await + .expect("request timeout") + .expect("broadcast resolves"); + let budget = moq_net::track::Subscription::default().with_max_age(Duration::from_millis(100)); + async fn recv(sub: &mut moq_net::track::Subscriber) -> u64 { + tokio::time::timeout(TIMEOUT, sub.recv_group()) + .await + .expect("recv timeout") + .expect("recv failed") + .expect("track ended") + .sequence + } + + let mut sub = remote + .track("audio") + .unwrap() + .subscribe(budget.clone()) + .await + .expect("subscribe"); + recv(&mut sub).await; + drop(sub); + + // The front parks the track and cancels upstream, while the publisher moves on. + tokio::time::timeout(TIMEOUT, live.unused()) + .await + .expect("upstream never canceled") + .expect("track open"); + for sequence in 4..=20u64 { + write(&live, sequence, 10_000 + (sequence - 4) * 20); + } + + let mut sub = remote + .track("audio") + .unwrap() + .subscribe(budget) + .await + .expect("resubscribe"); + let first = recv(&mut sub).await; + assert!( + first > 4, + "a rejoining reader was served the stale cache first: group {first}" + ); + let mut sequence = first; + while sequence < 20 { + sequence = recv(&mut sub).await; + assert!(sequence >= 4, "a rejoining reader was served stale group {sequence}"); + } + + drop(sub); + drop(session); + server.await.expect("server panicked").expect("server failed"); +} + +/// A subscriber returning to a parked track whose newest group is still current gets it +/// back, then the live feed resumes: the re-splice asks for that group's tail, which the +/// publisher answers at once. Asking past it would wait for a group that a quiet track (a +/// catalog) may never send. Covers a finished newest group (a catalog) and one that stays +/// open (a JSON log appending frames to group 0), on every version. +#[tracing_test::traced_test] +#[tokio::test] +async fn broadcast_rejoin_replays_a_current_warm_cache() { + for version in moq_net::Version::names() { + for open in [false, true] { + rejoin_replays_a_current_warm_cache(version, open).await; + } + } +} + +async fn rejoin_replays_a_current_warm_cache(version: &str, open: bool) { + let pub_origin = moq_tokio::origin::spawn(); + let broadcast = pub_origin.create_broadcast("test").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + let track = broadcast.create_track("catalog.json", None).expect("create track"); + let live = track.clone(); + let mut group = live.append_group().expect("append group"); + group + .write_frame(moq_net::Timestamp::ZERO, b"v0".as_ref()) + .expect("write frame"); + if !open { + group.finish().expect("finish group"); + } + + let mut config = moq_tokio::listen::Config::default(); + config.bind = Some("[::]:0".parse().unwrap()); + config.tls.generate = vec!["localhost".into()]; + config.version = vec![version.parse().unwrap()]; + let mut server = config + .init(Default::default()) + .expect("init server") + .listen() + .await + .expect("listen"); + let addr = server.local_addr().expect("local addr"); + let server = tokio::spawn(async move { + let request = server.accept().await.expect("accept"); + let session = request.with_publisher(&pub_origin).ok().await?; + let _broadcast = broadcast; + let _track = track; + let _ = session.closed().await; + Ok::<_, anyhow::Error>(()) + }); + + let sub_origin = moq_tokio::origin::spawn(); + let sub_consumer = sub_origin.consume(); + let mut announcements = sub_consumer.announced(); + let mut config = moq_tokio::connect::Config::default(); + config.tls.insecure = Some(true); + config.version = vec![version.parse().unwrap()]; + let client = config.init(Default::default()).expect("init client"); + let url: url::Url = format!("moqt://localhost:{}", addr.port()).parse().unwrap(); + let (_client, session) = tokio::time::timeout(TIMEOUT, connect_once(client.with_subscriber(sub_origin), url)) + .await + .expect("connect timeout") + .expect("connect failed"); + + assert!(next_announce(&mut announcements).await.kind.is_active()); + let remote = tokio::time::timeout(TIMEOUT, sub_consumer.request_broadcast("test")) + .await + .expect("request timeout") + .expect("broadcast resolves"); + + async fn recv(sub: &mut moq_net::track::Subscriber, ctx: &str) -> moq_net::group::Consumer { + tokio::time::timeout(TIMEOUT, sub.recv_group()) + .await + .unwrap_or_else(|_| panic!("{ctx}: recv_group timeout")) + .expect("recv_group failed") + .expect("track closed") + } + async fn read(group: &mut moq_net::group::Consumer, ctx: &str) -> String { + let frame = tokio::time::timeout(TIMEOUT, group.read_frame()) + .await + .unwrap_or_else(|_| panic!("{ctx}: read_frame timeout")) + .expect("read_frame failed") + .expect("group ended"); + String::from_utf8(frame.payload.to_vec()).unwrap() + } + + for round in 1..=3 { + let ctx = format!("{version} open={open} round {round}"); + let update = format!("v{round}"); + let mut sub = remote + .track("catalog.json") + .unwrap() + .subscribe(None) + .await + .expect("subscribe"); + let mut reading = recv(&mut sub, &ctx).await; + if open { + for frame in 0..round { + assert_eq!(read(&mut reading, &ctx).await, format!("v{frame}"), "{ctx}"); + } + group + .write_frame(moq_net::Timestamp::ZERO, update.as_bytes()) + .expect("write frame"); + } else { + assert_eq!(read(&mut reading, &ctx).await, format!("v{}", round - 1), "{ctx}"); + let mut next = live.append_group().expect("append group"); + next.write_frame(moq_net::Timestamp::ZERO, update.as_bytes()) + .expect("write frame"); + next.finish().expect("finish group"); + reading = recv(&mut sub, &ctx).await; + } + // The live feed resumed behind the replayed cache. + assert_eq!(read(&mut reading, &ctx).await, update, "{ctx}"); + drop(reading); + drop(sub); + // The front parks the track and cancels upstream before the next round rejoins. + tokio::time::timeout(TIMEOUT, live.unused()) + .await + .unwrap_or_else(|_| panic!("{ctx}: upstream never canceled")) + .expect("track open"); + } + + drop(session); + server.await.expect("server panicked").expect("server failed"); +} + /// A publisher-side route update re-advertises downstream as a restart. /// /// The publisher re-prices its announced route with a longer chain; the From 2fe47e4581aa3e1ef1244d267be8fe4890d5b857 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 08:48:50 -0700 Subject: [PATCH 026/127] fix(net): judge spliced staleness against the logical live edge (#4103) Co-authored-by: Claude Opus 5.5 Co-authored-by: Grok 4.7 --- rs/moq-net/src/model/origin.rs | 74 +++++- rs/moq-net/src/model/resume.rs | 374 +++++++++++++++++--------- rs/moq-net/src/model/track.rs | 466 ++++++++++++++++++++++++++++----- 3 files changed, 719 insertions(+), 195 deletions(-) diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 2f4c0af840..52c06cc5f6 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -6291,6 +6291,77 @@ mod tests { assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"live"); } + /// A returning reader judges the warm cache against the logical track's live + /// edge, not the parked segment's own frozen one: groups the fresh source + /// has left behind by more than the budget are skipped, exactly as they would + /// be on one unspliced track. + /// + /// A local source is released rather than parked (it keeps its own cache), so + /// this goes through a served front, which is what actually holds the warm copy. + /// The copy resolves at the cached edge, not past it: resolving past it drops + /// the cache outright, which is a different case. + #[tokio::test] + async fn resumed_reader_skips_warm_groups_behind_the_new_edge() { + let ms = |v: u64| crate::Timestamp::from_millis(v).unwrap(); + let (_server, _upstream, mut dynamic, resolved) = served_front().await; + let budget = track::Subscription::default().with_max_age(Duration::from_millis(100)); + let write = |source: &track::Producer, sequence: u64, millis: u64| { + let mut group = source.create_group(sequence.into()).unwrap(); + group.write_frame(ms(millis), b"x".as_ref()).unwrap(); + group.finish().unwrap(); + }; + let drain = |subscription: &mut track::Subscriber| { + let mut sequences = Vec::new(); + while let Poll::Ready(group) = subscription.poll_recv_group(&kio::Waiter::noop()) { + sequences.push(group.unwrap().expect("track ended").sequence); + } + sequences + }; + + let track = resolved.track("video").unwrap(); + let first = budget.clone(); + let subscribing = tokio::spawn(async move { track.subscribe(first).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source") + .expect("request"); + let source = request.resolving_start().accept(None); + for (sequence, millis) in [(0, 0), (1, 20), (2, 40), (3, 60)] { + write(&source, sequence, millis); + } + let mut subscription = subscribing.await.unwrap().expect("subscribe"); + next_group(&mut subscription).await.unwrap().expect("a cached group"); + drain(&mut subscription); + drop(subscription); + + tokio::time::timeout(Duration::from_secs(1), source.unused()) + .await + .expect("parked") + .expect("source open"); + drop(source); + + let track = resolved.track("video").unwrap(); + let second = budget.clone(); + let subscribing = tokio::spawn(async move { track.subscribe(second).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source again") + .expect("request"); + let mut source = request.resolving_start().accept(None); + for (sequence, millis) in [(20, 400), (21, 420), (22, 440)] { + write(&source, sequence, millis); + } + // At the cached edge, not past it, so the warm copy stays and the budget + // decides. Past it, the copy has already judged the cache stale. + source.start_at(3).unwrap(); + let mut subscription = subscribing.await.unwrap().expect("resubscribe"); + settle(|| subscription.latest() == Some(22)).await; + + // Groups 0..=2 reach at most 60ms against an edge at 440ms. Group 3 reaches + // where group 20 starts, 40ms behind that edge, inside the 100ms budget. + assert_eq!(drain(&mut subscription), [3, 20, 21, 22]); + } + /// A front serving from another front's spliced copy has no snapshot to keep: /// it still drops upstream on the unused edge, so the publisher's `unused()` /// resolves far below `TRACK_IDLE_LINGER` through the whole chain. The next @@ -6345,10 +6416,11 @@ mod tests { "every front keeps the delivered groups after releasing its source" ); + // A budget spanning the cache, so the returning reader replays it. let mut subscription = edge_resolved .track("video") .unwrap() - .subscribe(None) + .subscribe(track::Subscription::default().with_max_age(Duration::from_secs(3600))) .await .expect("resubscribe"); tokio::time::timeout(Duration::from_secs(5), track.used()) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 2fa3d0c048..e2227a17f1 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -26,7 +26,10 @@ use std::collections::{BTreeMap, HashSet}; use std::ops::Bound; use std::task::{Poll, ready}; +#[cfg(test)] +use crate::Timestamp; use crate::{Datagram, Error, Result, frame, group, track}; +use track::{Anchor, LiveEdge, Successor}; use super::subscription::{Cap, Position, Subscription, max_some, min_some}; @@ -141,6 +144,18 @@ fn slice(prefs: &Subscription, start: Option, end: Option) - } } +/// The first servable group in `from..cap` across `segments`, with the slot identity a +/// later judgment needs. An unstamped group stops the search: skipping it for a later +/// start would shrink a reach that is not yet proven. +fn served_start(segments: &[Segment], from: u64, cap: Option) -> Option { + segments.iter().find_map(|segment| { + let start = segment.start.map_or(0, |start| start.group).max(from); + segment + .track + .served_start(start, min_some(cap, last_group(segment.end))) + }) +} + /// How many segments a logical track keeps before pruning terminal ones from the /// front: the live segment plus a couple of predecessors still draining to slow /// readers. Without a bound, every failover leaves one dead segment (pinning a @@ -219,6 +234,28 @@ impl ResumeState { } } + /// The newest live edge across the segments below the exclusive `cap`, each clamped + /// to its own range: the edge the logical track measures drift against. + fn live_edge(&self, cap: Option) -> Option { + self.segments + .iter() + .filter_map(|segment| { + let edge = segment.track.live_edge(min_some(cap, last_group(segment.end)))?; + // Out-of-range content below the segment is not its to serve. + let start = segment.start.map_or(0, |start| start.group); + (edge.sequence >= start).then_some(edge) + }) + .max_by_key(|edge| edge.sequence) + } + + /// Where the logical track continues past the exclusive group `boundary` of segment + /// `id`, below the reader's `cap`: the start of the first group the later segments + /// serve there. `None` while none is cached, or it has no frame yet. + fn successor(&self, id: u64, boundary: u64, cap: Option) -> Option { + let index = self.segments.iter().position(|segment| segment.id == id)?; + served_start(&self.segments[index + 1..], boundary, cap) + } + /// Append a segment serving the track from `start` onward, capping (or replacing) /// the previous segments so the ranges stay disjoint and ascending. fn switch(&mut self, track: track::Consumer, start: Option) -> Result<()> { @@ -591,8 +628,8 @@ impl Consumer { next_sequence: 0, min_sequence: 0, end_sequence: None, - stale_cap: None, - drift_cap: kio::Producer::new(None), + outer: Anchor::default(), + drift_anchor: kio::Producer::new(Anchor::default()), } } @@ -659,6 +696,17 @@ impl Consumer { self.state.read().resume_position() } + /// The newest live edge across the segments; see [`track::Consumer::live_edge`]. + pub(crate) fn live_edge(&self, cap: Option) -> Option { + self.state.read().live_edge(cap) + } + + /// Where the first servable group in `from..cap` starts, with the identity to + /// revalidate it; see [`track::Consumer::served_start`]. + pub(crate) fn served_start(&self, from: u64, cap: Option) -> Option { + served_start(&self.state.read().segments, from, cap) + } + /// The newest cached group across every spliced segment; see /// [`track::Consumer::peek_latest`]. pub(crate) fn peek_latest(&self) -> Option { @@ -865,8 +913,8 @@ pub(crate) struct Group { state: kio::Consumer, /// The logical subscription's live max age budget. subscription: kio::Consumer, - /// The logical reader's group cap, used only to bound each route's drift anchor. - cap: kio::Consumer>, + /// The logical reader's drift anchor, used only to judge each route's copy. + anchor: kio::Consumer, /// The logical group being assembled. sequence: u64, @@ -921,7 +969,7 @@ impl Clone for Group { Self { state: self.state.clone(), subscription: self.subscription.clone(), - cap: self.cap.clone(), + anchor: self.anchor.clone(), sequence: self.sequence, index: self.index, end: self.end, @@ -936,14 +984,14 @@ impl Group { fn new( state: kio::Consumer, subscription: kio::Consumer, - cap: kio::Consumer>, + anchor: kio::Consumer, sequence: u64, index: u64, ) -> Self { Self { state, subscription, - cap, + anchor, sequence, index, end: None, @@ -1136,7 +1184,7 @@ impl Group { // `start_at` clamps up to the first frame the copy still holds, so landing // higher than asked means this route can't cover the seam after all. Treat it // like a dead copy and wait for one that can. - let mut group = track.guard_group(group, self.subscription.clone(), self.cap.clone(), bound); + let mut group = track.guard_group(group, self.subscription.clone(), self.anchor.clone(), bound); group.set_stale_meter(self.stale_stats.clone()); group.start_at(self.index); if group.index() != self.index { @@ -1295,7 +1343,7 @@ impl Group { // already includes the frames it skipped. Some(continuation) => { let mut continuation = - track.guard_group(continuation, self.subscription.clone(), self.cap.clone(), bound); + track.guard_group(continuation, self.subscription.clone(), self.anchor.clone(), bound); continuation.set_stale_meter(self.stale_stats.clone()); return continuation.poll_finished(waiter); } @@ -1317,6 +1365,9 @@ struct SegmentSub { /// A completed segment's cursor, retained while parked groups may need their /// max age budget re-evaluated after the outer cap rises. terminal: Option, + /// The drift anchor for this segment's cursor as of the last + /// [`Subscriber::refresh_anchor`], applied when a pending cursor activates. + anchor: Anchor, /// The producer dropped this segment (pruned, or replaced before producing). /// The cursor drains what it already holds, then retires; see /// [`Self::retired`]. @@ -1437,14 +1488,15 @@ pub struct Subscriber { min_sequence: u64, /// Exclusive cap for [`Self::next_group`], set by [`Self::end_at`]. end_sequence: Option, - /// A cap imposed by a reader wrapping this subscriber (a nested splice segment), - /// folded into every drift anchor pushed onto the segments. Never bounds delivery: - /// the outer reader enforces its own window, and an inner cap would hide a - /// segment's completion from it. - stale_cap: Option, - /// Shared copy of the effective cap ([`Self::end_sequence`] and [`Self::stale_cap`] - /// combined) for groups that outlive this cursor poll. - drift_cap: kio::Producer>, + /// The anchor imposed by a reader wrapping this subscriber (a nested splice + /// segment), folded into every drift anchor pushed onto the segments. Its cap never + /// bounds delivery: the outer reader enforces its own window, and an inner cap would + /// hide a segment's completion from it. + outer: Anchor, + /// The logical drift anchor ([`Self::end_sequence`] and [`Self::outer`] combined, + /// with the newest edge across the segments), shared with groups that outlive this + /// cursor poll. Refreshed on every [`Self::poll_sync`]. + drift_anchor: kio::Producer, } impl Subscriber { @@ -1453,6 +1505,7 @@ impl Subscriber { fn poll_sync(&mut self, waiter: &kio::Waiter) { self.sync(waiter); self.reap(); + self.refresh_anchor(); } /// Reap retired cursors, then bound the live stragglers: a pruned segment's @@ -1571,7 +1624,6 @@ impl Subscriber { } self.segments.retain(|s| !s.retired()); - let anchor = self.anchor_end(); let nexts: Vec<_> = segments.iter().skip(1).map(|next| Some(next.track.clone())).collect(); let nexts = nexts.into_iter().chain(std::iter::once(None)); for (segment, next) in segments.into_iter().zip(nexts) { @@ -1581,11 +1633,10 @@ impl Subscriber { warm.next = next; } if existing.end != segment.end { + // The boundary bounds the drift anchor as well as the demand; the + // anchor follows in `refresh_anchor`. existing.end = segment.end; - let cap = Self::stale_cap(existing, anchor); if let Some(sub) = existing.stale_sub_mut() { - // The boundary bounds the drift anchor as well as the demand. - sub.set_stale_cap(cap); // Shrink the demand so the session can cap upstream. The // read bounds stay on this subscriber (see `poll_recv_group`): // an inner `end_at` would park boundary-crossing groups in the @@ -1609,6 +1660,7 @@ impl Subscriber { ask: segment.ask, sub: SubState::Pending(sub), terminal: None, + anchor: Anchor::default(), pruned: false, parked: BTreeMap::new(), warm: segment.warm.then(|| Warm { @@ -1642,7 +1694,7 @@ impl Subscriber { let spliced = Group::new( self.state.clone(), self.prefs.consume(), - self.drift_cap.consume(), + self.drift_anchor.consume(), sequence, 0, ) @@ -1650,41 +1702,75 @@ impl Subscriber { Some(group.into_spliced(spliced)) } - /// The highest sequence a segment could hand its reader: the reader's own cap and - /// the segment's boundary, whichever is lower. + /// The anchor a segment's cursor measures drift against: the logical `anchor`, with + /// its cap lowered to the segment's boundary. /// /// A segment's inner cursor is deliberately left uncapped (an inner `end_at` would /// park boundary-crossing groups where its completion can't be seen), so this is how /// both bounds reach the drift anchor. Without them a segment measures staleness /// against groups it will never surface: the route running past the boundary, or the - /// reader's own cap holding content back. `anchor_end` is [`Self::anchor_end`], so a - /// cap imposed on this subscriber from outside is included. - fn stale_cap(seg: &SegmentSub, anchor_end: Option) -> Option { - min_some(anchor_end, seg.last_group()) + /// reader's own cap holding content back. The edge is left as is: it is the newest + /// content the logical track holds, which a segment's own track never sees. + /// + /// When the boundary lowers the cap, the reader's next group past it lives in a + /// later segment, so `state` supplies where it starts ([`Anchor::successor`]). + /// Otherwise the logical anchor's own successor (a wrapping splice's) still holds. + fn segment_anchor(seg: &SegmentSub, anchor: Anchor, state: &ResumeState) -> Anchor { + let Some(boundary) = seg.last_group() else { + return anchor; + }; + let cap = anchor.cap; + let mut capped = anchor.capped(Some(boundary)); + if capped.cap != cap { + capped.successor = state.successor(seg.id, boundary, cap); + } + capped } - /// The tightest cap any reader of this subscriber imposes: its own [`Self::end_at`] - /// and whatever a wrapping splice pushed down. What every segment's drift anchor is - /// bounded by. - fn anchor_end(&self) -> Option { - min_some(self.stale_cap, self.end_sequence) + /// The logical drift anchor, as of the last [`Self::refresh_anchor`]. + fn anchor(&self) -> Anchor { + self.drift_anchor.read().clone() } - /// Bound every segment's drift anchor from outside; the spliced arm of - /// [`track::Subscriber::set_stale_cap`], for this subscriber nested as a segment of - /// another splice. - pub(crate) fn set_stale_cap(&mut self, cap: Option) { - self.stale_cap = cap; - self.update_drift_cap(); - let anchor = self.anchor_end(); + /// Re-derive the logical drift anchor and push it onto every segment cursor. + /// + /// The cap is the tightest any reader of this subscriber imposes: its own + /// [`Self::end_at`] and whatever a wrapping splice pushed down. The edge is the + /// newest across the producer's segments within that cap, or a wrapping splice's if + /// newer. Resolved on every sync, since each segment is a separate track and the + /// logical edge moves whenever any of them grows. + fn refresh_anchor(&mut self) { + let outer = self.outer.clone().capped(self.end_sequence); + let state = self.state.read(); + let edge = state + .live_edge(outer.cap) + .into_iter() + .chain(outer.edge.clone()) + .max_by_key(|edge| edge.sequence); + let anchor = Anchor { edge, ..outer }; + // Skip a no-op write: every handed-out group's expiry watches this channel. + if self.anchor() != anchor + && let Ok(mut current) = self.drift_anchor.write() + { + *current = anchor.clone(); + } for seg in &mut self.segments { - let cap = Self::stale_cap(seg, anchor); + let anchor = Self::segment_anchor(seg, anchor.clone(), &state); + seg.anchor = anchor.clone(); if let Some(sub) = seg.stale_sub_mut() { - sub.set_stale_cap(cap); + sub.set_anchor(anchor); } } } + /// Bound every segment's drift anchor from outside; the spliced arm of + /// [`track::Subscriber::set_anchor`], for this subscriber nested as a segment of + /// another splice. + pub(crate) fn set_anchor(&mut self, anchor: Anchor) { + self.outer = anchor; + self.refresh_anchor(); + } + /// Count each segment's seek convictions behind the deliverer's `committed` /// watermark; the spliced arm of [`track::Subscriber::commit_seek_stale`]. pub(crate) fn commit_seek_stale(&mut self, committed: u64) { @@ -1723,23 +1809,10 @@ impl Subscriber { Poll::Ready(Ok(false)) } - /// Publish the effective cap for groups that outlive this cursor poll. - fn update_drift_cap(&mut self) { - if let Ok(mut cap) = self.drift_cap.write() { - *cap = self.anchor_end(); - } - } - /// Resolve a segment's pending subscription, if any. Ready once the segment is /// `Active` or `Done`; a rejected or closed track becomes `Done` (stall, not /// error). Never consumes groups, so terminal-state pollers can share it. - fn poll_activate( - seg: &mut SegmentSub, - prefs: &Subscription, - min_sequence: u64, - anchor_end: Option, - waiter: &kio::Waiter, - ) -> Poll<()> { + fn poll_activate(seg: &mut SegmentSub, prefs: &Subscription, min_sequence: u64, waiter: &kio::Waiter) -> Poll<()> { if matches!(seg.sub, SubState::Pending(_)) && let Some(warm) = &seg.warm { @@ -1770,7 +1843,7 @@ impl Subscriber { // assigned: the inner subscription resolved its own start from its // budget and floor, and this must not rewind past it. sub.raise_start_to(seg.first_group().max(min_sequence)); - sub.set_stale_cap(Self::stale_cap(seg, anchor_end)); + sub.set_anchor(seg.anchor.clone()); let _ = sub.update(slice(prefs, seg.ask, seg.end)); seg.sub = SubState::Active(Box::new(sub)); } @@ -1787,13 +1860,12 @@ impl Subscriber { seg: &mut SegmentSub, prefs: &Subscription, min_sequence: u64, - anchor_end: Option, waiter: &kio::Waiter, ) -> Poll> { loop { match &mut seg.sub { SubState::Pending(_) => { - ready!(Self::poll_activate(seg, prefs, min_sequence, anchor_end, waiter)); + ready!(Self::poll_activate(seg, prefs, min_sequence, waiter)); } SubState::Active(sub) => match ready!(sub.poll_recv_group(waiter)) { Ok(Some(group)) => { @@ -1859,7 +1931,6 @@ impl Subscriber { self.commit_seek_stale(committed); } - let anchor = self.anchor_end(); let mut floor = floor; 'retry: loop { let mut all_done = true; @@ -1867,14 +1938,8 @@ impl Subscriber { for index in 0..self.segments.len() { if matches!(self.segments[index].sub, SubState::Pending(_)) - && Self::poll_activate( - &mut self.segments[index], - &self.last_prefs, - self.min_sequence, - anchor, - waiter, - ) - .is_pending() + && Self::poll_activate(&mut self.segments[index], &self.last_prefs, self.min_sequence, waiter) + .is_pending() { all_done = false; continue; @@ -1966,7 +2031,6 @@ impl Subscriber { self.poll_sync(waiter); let end_sequence = self.end_sequence; - let anchor = self.anchor_end(); let min_sequence = self.min_sequence; let beyond_cap = |sequence: u64| !super::subscription::before_end(sequence, end_sequence); @@ -2021,13 +2085,7 @@ impl Subscriber { } loop { - let polled = Self::poll_segment( - &mut self.segments[index], - &self.last_prefs, - min_sequence, - anchor, - waiter, - ); + let polled = Self::poll_segment(&mut self.segments[index], &self.last_prefs, min_sequence, waiter); match polled { Poll::Ready(Some(group)) => { if beyond_cap(group.sequence) { @@ -2142,9 +2200,8 @@ impl Subscriber { // datagrams must still resolve the subscription (registering demand) and // be woken when it activates. let mut pending_activation = false; - let anchor = self.anchor_end(); if let Some(seg) = self.segments.last_mut() { - if Self::poll_activate(seg, &self.last_prefs, self.min_sequence, anchor, waiter).is_pending() { + if Self::poll_activate(seg, &self.last_prefs, self.min_sequence, waiter).is_pending() { pending_activation = true; } else if let SubState::Active(sub) = &mut seg.sub && let Ok(Some(datagram)) = ready!(sub.poll_recv_datagram(waiter)) @@ -2203,17 +2260,10 @@ impl Subscriber { /// completing the segment, would steal them from a `recv_group` caller on the /// same subscriber. fn poll_final(&mut self, waiter: &kio::Waiter) -> Poll> { - let anchor = min_some(self.stale_cap, self.end_sequence); let Some(seg) = self.segments.last_mut() else { return Poll::Ready(None); }; - ready!(Self::poll_activate( - seg, - &self.last_prefs, - self.min_sequence, - anchor, - waiter - )); + ready!(Self::poll_activate(seg, &self.last_prefs, self.min_sequence, waiter)); match &mut seg.sub { SubState::Done(count) => Poll::Ready(*count), // Observe only: the cursor may still hold groups, so the read path @@ -2269,16 +2319,9 @@ impl Subscriber { /// re-offers it. pub fn end_at(&mut self, end: impl Into) { self.end_sequence = end.into().exclusive(); - self.update_drift_cap(); // The cap bounds each segment's drift anchor as well as this reader's own // delivery: a segment must not measure against groups this cap hides. - let anchor = self.anchor_end(); - for seg in &mut self.segments { - let cap = Self::stale_cap(seg, anchor); - if let Some(sub) = seg.stale_sub_mut() { - sub.set_stale_cap(cap); - } - } + self.refresh_anchor(); } /// The shared preferences channel, so `track::Control` can wrap it. @@ -2323,7 +2366,7 @@ impl Subscriber { #[cfg(test)] mod test { use super::*; - use crate::{Timestamp, broadcast}; + use crate::broadcast; use futures::FutureExt; use std::sync::Arc; use std::time::Duration; @@ -2693,7 +2736,7 @@ mod test { let mut producer = Producer::new(); producer.switch(&consumer_a, None).unwrap(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); write_group(&mut track_a, 0, "a0"); producer.switch(&consumer_b, Position::group(1)).unwrap(); @@ -3126,9 +3169,11 @@ mod test { "the arrival path skips straight to the live edge" ); - // Spliced: the same backlog split across a takeover boundary. - let (mut track_a, consumer_a) = track_pair("a"); - let (mut track_b, consumer_b) = track_pair("b"); + // Spliced: the same backlog split across a takeover boundary. Retained long + // enough that the replay budget below is not clamped to the default window. + let retain = track::Info::default().with_max_age(Duration::from_secs(60)); + let (mut track_a, consumer_a) = track_pair_with("a", retain.clone()); + let (mut track_b, consumer_b) = track_pair_with("b", retain); let mut producer = Producer::new(); producer.switch(&consumer_a, None).unwrap(); producer.switch(&consumer_b, Position::group(2)).unwrap(); @@ -3147,11 +3192,11 @@ mod test { }) .collect(); - // Group 1 survives where the plain cursor drops it: each segment measures drift - // against its own boundary, since a segment cannot serve content past it, and - // group 1 is the newest thing segment A has. The sequence path inherits that - // from the arrival path rather than inventing its own anchor, so the two agree. - assert_eq!(spliced, vec![1, 3], "each segment is judged within its own boundary"); + // Segment A's groups are judged as the plain cursor judges them: against the + // logical edge (group 3), with group 1's reach bounded by its successor in + // segment B. The sequence path inherits that from the arrival path rather than + // inventing its own anchor, so the two agree. + assert_eq!(spliced, baseline, "a splice sheds the backlog like one track"); let mut arrival = producer.consume().subscribe(None); let arrival: Vec = std::iter::from_fn(|| { @@ -3177,38 +3222,101 @@ mod test { assert_eq!(replayed, vec![0, 1, 2, 3], "a backlog inside the budget crosses whole"); } + /// A segment's track never sees the groups of the segments after it: its own edge + /// freezes once a takeover caps it, and its last group has no successor there. + /// Both spliced cursors judge its backlog as one plain track holding the same groups + /// would: against the logical edge, with the last group's reach bounded by where the + /// next segment picks up. + #[tokio::test] + async fn a_capped_segment_is_judged_like_one_track() { + let budget = Subscription::default().with_max_age(Duration::from_millis(100)); + let old = [(0, 0), (1, 20), (2, 40), (3, 60)]; + let new = [(20, 400), (21, 420), (22, 440), (23, 460), (24, 480), (25, 500)]; + + let (mut plain, _plain_consumer) = track_pair("plain"); + for (sequence, millis) in old.into_iter().chain(new) { + write_group_at(&mut plain, sequence, "p", Duration::from_millis(millis)); + } + let mut baseline = plain.subscribe(budget.clone()); + let baseline: Vec = std::iter::from_fn(|| { + baseline + .recv_group() + .now_or_never()? + .expect("should not error") + .map(|group| group.sequence) + }) + .collect(); + // Group 3 reaches group 20's start at 400ms, 100ms behind the edge at 500ms. + assert_eq!(baseline, vec![20, 21, 22, 23, 24, 25]); + + let (mut track_a, consumer_a) = track_pair("a"); + let (mut track_b, consumer_b) = track_pair("b"); + let mut producer = Producer::new(); + producer.switch(&consumer_a, None).unwrap(); + producer.switch(&consumer_b, Position::group(4)).unwrap(); + for (sequence, millis) in old { + write_group_at(&mut track_a, sequence, "a", Duration::from_millis(millis)); + } + for (sequence, millis) in new { + write_group_at(&mut track_b, sequence, "b", Duration::from_millis(millis)); + } + + let mut arrival = producer.consume().subscribe(budget.clone()); + let arrival: Vec = std::iter::from_fn(|| { + kio::wait(|waiter| arrival.poll_recv_group(waiter)) + .now_or_never()? + .expect("should not error") + .map(|group| group.sequence) + }) + .collect(); + assert_eq!(arrival, baseline); + + let mut ordered = producer.consume().subscribe(budget); + let ordered: Vec = std::iter::from_fn(|| { + kio::wait(|waiter| ordered.poll_next_group(waiter)) + .now_or_never()? + .expect("should not error") + .map(|group| group.sequence) + }) + .collect(); + assert_eq!(ordered, baseline); + } + /// A nested splice's leaves are judged within the *outer* boundary. The outer /// window reaches them two ways, through the seek's own `end` and through - /// [`track::Subscriber::set_stale_cap`] recursing into the spliced segment; without + /// [`track::Subscriber::set_anchor`] recursing into the spliced segment; without /// either, a leaf anchors drift on a group past the outer boundary and convicts /// everything the outer segment still owes its reader. #[tokio::test] async fn nested_splice_judges_within_the_outer_boundary() { let stamp = |sequence: u64| Duration::from_secs(10 * sequence); + let retain = track::Info::default().with_max_age(Duration::from_secs(60)); + let budget = Subscription::default().with_max_age(Duration::from_secs(15)); // Inner splice: one segment carrying groups 0..=2. Group 2 sits past the - // outer boundary below, so it is exactly the anchor the outer window must - // hide from the inner leaves. - let (mut track_a, consumer_a) = track_pair("a"); + // outer boundary below and rewinds to 0s, so it is exactly the successor the + // outer window must hide from the inner leaves: taken as group 1's, it would + // collapse group 1's reach to 0s, 30s behind the edge. + let (mut track_a, consumer_a) = track_pair_with("a", retain.clone()); let mut inner = Producer::new(); inner.switch(&consumer_a, None).unwrap(); write_group_at(&mut track_a, 0, "a0", stamp(0)); write_group_at(&mut track_a, 1, "a1", stamp(1)); - write_group_at(&mut track_a, 2, "a2", stamp(2)); + write_group_at(&mut track_a, 2, "a2", stamp(0)); // Outer splice: the inner spliced track up to group 2, then a plain track. let inner_track = track::Consumer::spliced("inner".into(), Arc::new(broadcast::Info::default()), inner.consume()); - let (mut track_b, consumer_b) = track_pair("b"); + let (mut track_b, consumer_b) = track_pair_with("b", retain); let mut outer = Producer::new(); outer.switch(&inner_track, None).unwrap(); outer.switch(&consumer_b, Position::group(2)).unwrap(); write_group_at(&mut track_b, 2, "b2", stamp(2)); write_group_at(&mut track_b, 3, "b3", stamp(3)); - // Group 1 is the newest group the outer window can serve from the nested - // segment, so it is its own live edge there and survives a zero budget. - let mut sub = outer.consume().subscribe(None); + // Group 0 is 20s behind the edge, past the 15s budget. Group 1 reaches B's + // group 2 at 20s, only 10s behind, so it survives. + let mut sub = outer.consume().subscribe(budget.clone()); let sequences: Vec = std::iter::from_fn(|| { kio::wait(|waiter| sub.poll_next_group(waiter)) .now_or_never()? @@ -3218,11 +3326,11 @@ mod test { .collect(); assert_eq!( sequences, - vec![1, 3], + vec![1, 2, 3], "the nested segment is judged within the outer boundary" ); - let mut arrival = outer.consume().subscribe(None); + let mut arrival = outer.consume().subscribe(budget); let arrival: Vec = std::iter::from_fn(|| { kio::wait(|waiter| arrival.poll_recv_group(waiter)) .now_or_never()? @@ -3239,7 +3347,9 @@ mod test { /// the group after all, and delivered content never counts. #[tokio::test] async fn seek_conviction_counts_only_once_committed() { - let stamp = |sequence: u64| Duration::from_secs(10 * sequence); + // Group 2 starts after group 3 (a rewind), so group 1's reach runs past the live + // edge and it survives a zero budget, while groups 0 and 2 are convicted. + let stamp = |sequence: u64| Duration::from_secs([0, 10, 40, 30][sequence as usize]); let (mut track_a, consumer_a) = track_pair("a"); let (mut track_b, consumer_b) = track_pair("b"); @@ -3284,7 +3394,9 @@ mod test { /// jumped past it, and still be deliverable (uncounted) once the budget widens. #[tokio::test] async fn a_reversible_floor_does_not_commit_a_conviction() { - let stamp = |sequence: u64| Duration::from_secs(10 * sequence); + // Group 2 starts after group 3 (a rewind), so group 1's reach runs past the live + // edge and it survives a zero budget, while groups 0 and 2 are convicted. + let stamp = |sequence: u64| Duration::from_secs([0, 10, 40, 30][sequence as usize]); let (mut track_a, consumer_a) = track_pair("a"); let (mut track_b, consumer_b) = track_pair("b"); @@ -3333,7 +3445,9 @@ mod test { /// that content, so the conviction must be discarded rather than counted. #[tokio::test] async fn a_delivered_continuation_is_not_counted_stale() { - let stamp = |sequence: u64| Duration::from_secs(10 * sequence); + // Group 2 starts after group 3 (a rewind), so group 1's reach runs past the live + // edge and it survives a zero budget, while groups 0 and 2 are convicted. + let stamp = |sequence: u64| Duration::from_secs([0, 10, 40, 30][sequence as usize]); let (mut track_a, consumer_a) = track_pair("a"); let (mut track_b, consumer_b) = track_pair("b"); @@ -3357,9 +3471,9 @@ mod test { .sequence }; - // Group 1 wins from segment A (it holds the head copy) while segment B has - // convicted both its continuation copy of 1 and group 2. Delivering 1 must - // discard B's copy, not count it. + // Group 1 wins from segment A (it holds the head copy) while segment B convicts + // group 2. B's continuation copy of 1 is delivered through the head, so it + // must not be counted. assert_eq!(next(&mut sub), 1); assert_eq!(next(&mut sub), 3); assert_eq!( @@ -3376,12 +3490,16 @@ mod test { /// commit may count; a conviction never committed is dropped with the cursor. #[tokio::test] async fn a_finalized_segment_flushes_nothing_without_a_delivery() { - let stamp = |sequence: u64| Duration::from_secs(10 * sequence); + // Group 1 starts after group 3 (a rewind), so group 0's reach runs past the live + // edge and it survives a zero budget, while groups 1 and 2 are convicted. + let stamp = |sequence: u64| Duration::from_secs([0, 40, 20, 30][sequence as usize]); // C owns group 0, A owns group 1's head, finalized B owns its tail onward. - let (mut track_c, consumer_c) = track_pair("c"); - let (mut track_a, consumer_a) = track_pair("a"); - let (mut track_b, consumer_b) = track_pair("b"); + // Retained long enough that the widened budget below is not clamped. + let retain = track::Info::default().with_max_age(Duration::from_secs(60)); + let (mut track_c, consumer_c) = track_pair_with("c", retain.clone()); + let (mut track_a, consumer_a) = track_pair_with("a", retain.clone()); + let (mut track_b, consumer_b) = track_pair_with("b", retain); let mut producer = Producer::new(); producer.switch(&consumer_c, None).unwrap(); producer.switch(&consumer_a, Position::group(1)).unwrap(); @@ -4273,7 +4391,7 @@ mod test { #[tokio::test] async fn capped_subscriber_bounds_parked_segments() { let mut producer = Producer::new(); - let mut sub = producer.consume().subscribe(None); + let mut sub = producer.consume().subscribe(replay()); sub.end_at(..1); // Every round parks one group beyond the cap, then fails over to a live @@ -5082,8 +5200,8 @@ mod test { let mut producer = Producer::new(); producer.switch(&consumer_a, None).unwrap(); let consumer = producer.consume(); - let mut sub1 = consumer.subscribe(None); - let mut sub2 = consumer.subscribe(None); + let mut sub1 = consumer.subscribe(replay()); + let mut sub2 = consumer.subscribe(replay()); recv_pending(&mut sub1); recv_pending(&mut sub2); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 17bec9149b..ba83c78728 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -477,9 +477,8 @@ impl TrackState { /// it, so the groups it still wants aren't stale just because the route ran on. /// Fetched backfill is absent from `arrival`, so it cannot age subscription content /// as though it were a live replacement. - fn live_edge(&self, cap: Option) -> Option { - let presentation = self - .lookup + fn live_edge(&self, cap: Option) -> Option { + self.lookup .range(..) .rev() .filter(|(seq, _)| super::subscription::before_end(**seq, cap)) @@ -495,9 +494,26 @@ impl TrackState { stamp: slot.stamp, timestamp: slot.group.latest().unwrap_or(timestamp), }) - }); + }) + } + + /// This track's own edge under the exclusive `cap`, for measuring drift. An outer + /// edge and a successor live on other tracks; the caller revalidates those before + /// taking this lock and passes them in, so the locks never nest. + fn drift_edge(&self, cap: Option, outer: Option<(u64, Timestamp)>, successor: Option) -> Edge { + Edge { + presentation: self.live_edge(cap), + outer, + cap, + successor, + } + } - presentation.map(|presentation| Edge { presentation, cap }) + /// Whether `sequence` still holds the servable incarnation `stamp`. + fn holds(&self, sequence: u64, stamp: u32) -> bool { + self.lookup + .get(&sequence) + .is_some_and(|slot| slot.stamp == stamp && !slot.group.is_aborted()) } /// The furthest presentation time the group at `sequence` could still reach: where @@ -510,14 +526,45 @@ impl TrackState { /// so a later stamped group proves nothing about where an unstamped successor will /// begin, and shrinking the bound is the unsafe direction. An unstamped successor /// therefore leaves the reach unbounded until it presents its first frame. - fn reach(&self, sequence: u64, cap: Option) -> Option { - let successor = self + /// + /// With no servable successor below `cap`, the reader's next group is past the cap, + /// and `beyond` is where it starts when another track serves it (a splice's next + /// segment; see [`Anchor::successor`]). + fn reach(&self, sequence: u64, cap: Option, beyond: Option) -> Option { + match self.first_start(sequence.saturating_add(1), cap) { + Some(start) => start, + None => beyond, + } + } + + /// Where the first servable group in `from..cap` starts presenting: `None` when no + /// such group is cached, `Some(None)` while it has no frame yet. + fn first_start(&self, from: u64, cap: Option) -> Option> { + let slot = self + .lookup + .range(from..) + .map(|(_, slot)| slot) + .take_while(|slot| super::subscription::before_end(slot.group.sequence, cap)) + .find(|slot| slot.visible && !slot.group.is_aborted())?; + Some(slot.group.timestamp()) + } + + /// The first servable group's start in `from..cap`, with the slot identity a later + /// judgment needs to tell that group from whatever replaces it. `None` when no such + /// group is cached or it has no frame yet: an unstamped successor leaves reach + /// unbounded, and this does not skip past it to a later group. + fn served_start(&self, from: u64, cap: Option) -> Option { + let slot = self .lookup - .range(sequence.saturating_add(1)..) + .range(from..) .map(|(_, slot)| slot) .take_while(|slot| super::subscription::before_end(slot.group.sequence, cap)) .find(|slot| slot.visible && !slot.group.is_aborted())?; - successor.group.timestamp() + Some(ServedStart { + sequence: slot.group.sequence, + stamp: slot.stamp, + timestamp: slot.group.timestamp()?, + }) } /// Whether the group at `sequence` has drifted further behind `edge` than `budget` @@ -543,11 +590,10 @@ impl TrackState { /// /// The edge must sit strictly above the candidate. The live edge is never late /// against itself, and backfill or the tail of a rewound timeline can carry a high - /// timestamp on a low sequence without being an edge at all. - fn is_stale(&self, sequence: u64, edge: Option<&Edge>, budget: Duration) -> bool { - let Some(edge) = edge else { - return false; - }; + /// timestamp on a low sequence without being an edge at all. Of the edges that + /// qualify, the highest sequence is the newest content, whichever track holds it: + /// this one's, or the `outer` edge a splice pushed from another segment. + fn is_stale(&self, sequence: u64, edge: &Edge, budget: Duration) -> bool { if !self.lookup.contains_key(&sequence) { return false; } @@ -555,16 +601,23 @@ impl TrackState { // The anchor was resolved under an earlier lock, so confirm it still names // the same servable incarnation before it convicts a candidate. Failing safe // (delivering) is right, since the next poll resolves fresh anchors. - let live_edge = &edge.presentation; - let reach = self.reach(sequence, edge.cap); - live_edge.sequence > sequence - && self - .lookup - .get(&live_edge.sequence) - .is_some_and(|live| live.stamp == live_edge.stamp && !live.group.is_aborted()) - && reach.is_some_and( - |reach| matches!(live_edge.timestamp.checked_sub(reach), Ok(age) if Duration::from(age) >= budget), - ) + let local = edge + .presentation + .filter(|live| self.holds(live.sequence, live.stamp)) + .map(|live| (live.sequence, live.timestamp)); + // `outer` and `successor` were revalidated on their own tracks before this lock + // was taken ([`LiveEdge::is_live`], [`Successor::start`]). There is no slot for + // them here, and taking their locks here would nest. + let Some((_, timestamp)) = local + .into_iter() + .chain(edge.outer) + .filter(|(live, _)| *live > sequence) + .max_by_key(|(live, _)| *live) + else { + return false; + }; + self.reach(sequence, edge.cap, edge.successor) + .is_some_and(|reach| matches!(timestamp.checked_sub(reach), Ok(age) if Duration::from(age) >= budget)) } /// Resolve a one-shot fetch from the track side: the cached group, or an [`Error`] @@ -1554,7 +1607,7 @@ impl Producer { let min_sequence = floor_of(&preferences); let subscription = kio::Producer::new(preferences); register_subscription(self.state.read(), &subscription); - let drift_cap = kio::Producer::new(None); + let drift_anchor = kio::Producer::new(Anchor::default()); // Hoisted: an inline `read()` guard would live to the end of the struct literal, // deadlocking against the `consume()` below. @@ -1573,7 +1626,7 @@ impl Producer { end_sequence: None, parked: BTreeMap::new(), stale_cap: None, - drift_cap, + drift_anchor, stale: stats::Content::default(), seek_pending: BTreeMap::new(), }), @@ -2369,6 +2422,40 @@ impl Consumer { } } + /// The live edge below the exclusive `cap` that drift is measured against; see + /// [`TrackState::live_edge`]. A splice reports the newest across its segments. + pub(crate) fn live_edge(&self, cap: Option) -> Option { + match &self.inner { + ConsumerKind::Plain(state) => { + let edge = state.read().live_edge(cap)?; + Some(LiveEdge { + sequence: edge.sequence, + timestamp: edge.timestamp, + stamp: edge.stamp, + track: state.weak(), + }) + } + ConsumerKind::Spliced(resume) => resume.live_edge(cap), + } + } + + /// Where the first servable group in `from..cap` starts, with enough identity to + /// revalidate it later. A splice answers from the first segment holding one. + pub(crate) fn served_start(&self, from: u64, cap: Option) -> Option { + match &self.inner { + ConsumerKind::Plain(state) => { + let served = state.read().served_start(from, cap)?; + Some(Successor { + sequence: served.sequence, + timestamp: served.timestamp, + stamp: served.stamp, + track: state.weak(), + }) + } + ConsumerKind::Spliced(resume) => resume.served_start(from, cap), + } + } + /// The nearest cached group below `sequence`, under the same terms as /// [`Self::peek_group`]. Walks the cache's own order, so gaps in the group numbering /// are crossed and aborted (evicted) entries are skipped. @@ -2411,7 +2498,7 @@ impl Consumer { &self, group: group::Consumer, subscription: kio::Consumer, - cap: kio::Consumer>, + anchor: kio::Consumer, bound: Option, ) -> group::Consumer { let ConsumerKind::Plain(state) = &self.inner else { @@ -2421,7 +2508,7 @@ impl Consumer { group.with_expiry(Arc::new(GroupExpiry { state: state.weak(), subscription, - cap, + anchor, bound, sequence, })) @@ -2672,7 +2759,7 @@ impl Subscribing { let info = ready!(state.poll(waiter, |state| state.poll_info())) .map_err(|e| e.abort.clone().unwrap_or(Error::Dropped))??; - let drift_cap = kio::Producer::new(None); + let drift_anchor = kio::Producer::new(Anchor::default()); let min_sequence = floor_of(&self.subscription.read()); Poll::Ready(Ok(Subscriber { name: self.name.clone(), @@ -2688,7 +2775,7 @@ impl Subscribing { end_sequence: None, parked: BTreeMap::new(), stale_cap: None, - drift_cap, + drift_anchor, stale: stats::Content::default(), seek_pending: BTreeMap::new(), }), @@ -2978,11 +3065,14 @@ enum SubscriberKind { /// One poll's view of how far this subscription may drift: the clamped budget and the /// live edge to measure a candidate group against. Resolved once, then applied to every -/// group that poll considers. +/// group that poll considers. `outer` and `successor` are revalidated per candidate, +/// outside this track's lock. #[derive(Clone)] struct Drift { budget: Duration, - edge: Option, + edge: Edge, + outer: Option, + successor: Option, } /// Keeps one handed-out group tied to the subscription whose cursor selected it. @@ -2994,7 +3084,7 @@ struct GroupExpiry { /// the last real subscriber. state: kio::ConsumerWeak, subscription: kio::Consumer, - cap: kio::Consumer>, + anchor: kio::Consumer, bound: Option, sequence: u64, } @@ -3007,19 +3097,26 @@ impl group::Expiry for GroupExpiry { Poll::<()>::Pending }); - let mut cap = None; - let _ = self.cap.poll(waiter, |current| { - cap = **current; + let mut anchor = Anchor::default(); + let _ = self.anchor.poll(waiter, |current| { + anchor = (**current).clone(); Poll::<()>::Pending }); - let cap = super::subscription::min_some(cap, self.bound); + let anchor = anchor.capped(self.bound); + let cap = anchor.cap; + // Before this track's lock: both may name another track, and nesting deadlocks. + let outer = anchor + .edge + .filter(LiveEdge::is_live) + .map(|live| (live.sequence, live.timestamp)); + let successor = anchor.successor.as_ref().and_then(Successor::start); let mut expired = false; let _ = self.state.poll(waiter, |state| { let budget = clamp_max_age(max_age, state.max_age_bound()); loop { - let edge = state.live_edge(cap); - expired = state.is_stale(self.sequence, edge.as_ref(), budget); + let edge = state.drift_edge(cap, outer, successor); + expired = state.is_stale(self.sequence, &edge, budget); if expired { break; } @@ -3073,12 +3170,121 @@ impl group::Expiry for GroupExpiry { /// The group a poll's drift is measured against, identified well enough to tell it apart /// from whatever may occupy its sequence by the time a candidate is judged. -#[derive(Clone)] +#[derive(Clone, Copy)] struct Edge { - presentation: PresentationEdge, + /// This track's own edge, revalidated before it convicts anything. + presentation: Option, + /// A newer edge on another track, already revalidated: its sequence and the newest + /// frame it had presented. See [`Anchor::edge`]. + outer: Option<(u64, Timestamp)>, /// The cap the edge was resolved under, so per-candidate reach lookups measure /// against the same servable window. cap: Option, + /// Where the next group past `cap` starts, already revalidated. See [`Anchor::successor`]. + successor: Option, +} + +/// How a reader wrapping a cursor bounds its drift anchor from outside: pushed by a +/// splice onto each segment's cursor, and shared with the groups a cursor hands out. +#[derive(Clone, Default, PartialEq)] +pub(crate) struct Anchor { + /// The exclusive sequence cap on what the reader could be handed; see [`servable_cap`]. + pub cap: Option, + /// The newest edge across a splice's segments. Each segment is a separate track + /// that only sees its own groups, so without this a parked segment measures against + /// its own frozen edge while the logical track has moved on. Revalidated on its own + /// track before it convicts anything, since it may be judged long after it was pushed. + pub edge: Option, + /// Where the reader's next group past `cap` starts presenting, when another track + /// serves it (a splice's next segment). The last group below the cap has no + /// successor in its own track, so without this nothing bounds its reach and it is + /// never judged stale. `None` while unknown or unstamped. Revalidated like `edge`: + /// a cached start must not convict once that group is gone. + pub successor: Option, +} + +impl Anchor { + /// This anchor under a further `cap`. A lower cap drops the successor: it named + /// where the reader continues past the old cap, which is no longer served. + pub fn capped(mut self, cap: Option) -> Self { + let capped = servable_cap(cap, self.cap); + if capped != self.cap { + self.cap = capped; + self.successor = None; + } + self + } +} + +/// The newest stamped group of a track: its sequence and the newest frame it presented, +/// plus enough identity for a reader on another track to revalidate it. +#[derive(Clone)] +pub(crate) struct LiveEdge { + pub sequence: u64, + pub timestamp: Timestamp, + stamp: u32, + track: kio::ConsumerWeak, +} + +impl LiveEdge { + /// Whether the edge still names the same servable group on its own track, the + /// check [`TrackState::is_stale`] runs on a local edge. An eviction or abort since + /// the splice resolved it must not convict anything. Takes that track's lock, so + /// never call it under another's. + fn is_live(&self) -> bool { + self.track.read().holds(self.sequence, self.stamp) + } +} + +impl PartialEq for LiveEdge { + fn eq(&self, other: &Self) -> bool { + self.sequence == other.sequence + && self.timestamp == other.timestamp + && self.stamp == other.stamp + && self.track.same_channel(&other.track) + } +} + +/// The first servable group past a segment boundary: where it starts, and which slot +/// that start was read from. +struct ServedStart { + sequence: u64, + stamp: u32, + timestamp: Timestamp, +} + +/// A successor pushed onto another track's cursor. The timestamp alone is not enough: +/// once the group is evicted, a later group can keep the outer edge valid while this +/// start is no longer where the track continues. +#[derive(Clone)] +pub(crate) struct Successor { + sequence: u64, + timestamp: Timestamp, + stamp: u32, + track: kio::ConsumerWeak, +} + +impl Successor { + /// The start this still names, re-read from its own track, or `None` once that + /// group is gone or no longer stamped. Takes that track's lock, so never call it + /// under another's. + fn start(&self) -> Option { + let state = self.track.read(); + let slot = state.lookup.get(&self.sequence)?; + if slot.stamp != self.stamp || slot.group.is_aborted() { + return None; + } + slot.group.timestamp() + } +} + +impl PartialEq for Successor { + fn eq(&self, other: &Self) -> bool { + self.sequence == other.sequence + && self.timestamp == other.timestamp + && self.stamp == other.stamp + && self.track.same_channel(&other.track) + } } /// The newest servable group that has presented at least one frame. @@ -3124,8 +3330,9 @@ struct PlainSubscriber { /// segment), folded into the drift anchor only. Delivery is still bounded by /// `end_sequence`, which stays unset on a segment so its completion is visible. stale_cap: Option, - /// Shared effective cap used by groups after this cursor hands them out. - drift_cap: kio::Producer>, + /// Shared effective anchor used by groups after this cursor hands them out. The + /// only copy of the outer edge a wrapping reader pushed (see [`Anchor::edge`]). + drift_anchor: kio::Producer, /// Groups the drift budget skipped since the count was last drained. Accumulated /// here rather than metered in place because the handle that owns the stats scope /// is the outer [`Subscriber`], which may be reading this cursor through a @@ -3138,9 +3345,20 @@ struct PlainSubscriber { } impl PlainSubscriber { - fn update_drift_cap(&mut self) { - if let Ok(mut cap) = self.drift_cap.write() { - *cap = servable_cap(self.end_sequence, self.stale_cap); + /// The drift anchor for a read bounded by `end`. Every read folds `end_sequence` + /// into `end`, so capping the shared anchor (which already holds it) is exact. + fn anchor(&self, end: Option) -> Anchor { + self.drift_anchor.read().clone().capped(end) + } + + /// Publish the `outer` anchor under this cursor's own cap. + fn update_drift_anchor(&mut self, outer: Anchor) { + let anchor = outer.capped(self.end_sequence); + // Skip a no-op write: every handed-out group's expiry watches this channel. + if *self.drift_anchor.read() != anchor + && let Ok(mut current) = self.drift_anchor.write() + { + *current = anchor; } } @@ -3177,16 +3395,23 @@ impl PlainSubscriber { /// discarding a backlog of N groups costs one scan rather than N. Only ever /// [`Poll::Ready`]; the track ending surfaces as the error the caller was going to /// get anyway. - fn poll_drift(&self, cap: Option, waiter: &kio::Waiter) -> Poll> { + fn poll_drift(&self, anchor: Anchor, waiter: &kio::Waiter) -> Poll> { let mut max_age = Duration::default(); let _ = self.subscription.poll(waiter, |subscription| { max_age = subscription.max_age; Poll::<()>::Pending }); - self.poll(waiter, move |state| { + let cap = anchor.cap; + let outer = anchor.edge; + let successor = anchor.successor; + self.poll(waiter, |state| { + // Local edge only. The pushed edge and successor are revalidated in + // [`Self::poll_stale`], outside this lock. Poll::Ready(Ok(Drift { budget: clamp_max_age(max_age, state.max_age_bound()), - edge: state.live_edge(cap), + edge: state.drift_edge(cap, None, None), + outer: outer.clone(), + successor: successor.clone(), })) }) } @@ -3194,8 +3419,25 @@ impl PlainSubscriber { /// Whether the drift budget says to skip `group`, against a [`Drift`] already resolved /// for this poll. fn poll_stale(&self, group: &group::Consumer, drift: &Drift, waiter: &kio::Waiter) -> Poll> { + // Revalidate before this track's lock. Both can name another track, including + // one whose own judgment is waiting on this one. + let outer = drift + .outer + .as_ref() + .filter(|live| live.is_live()) + .map(|live| (live.sequence, live.timestamp)); + let successor = drift.successor.as_ref().and_then(Successor::start); + let presentation = drift.edge.presentation; + let cap = drift.edge.cap; + let budget = drift.budget; self.poll(waiter, move |state| { - Poll::Ready(Ok(state.is_stale(group.sequence, drift.edge.as_ref(), drift.budget))) + let edge = Edge { + presentation, + outer, + cap, + successor, + }; + Poll::Ready(Ok(state.is_stale(group.sequence, &edge, budget))) }) } @@ -3204,7 +3446,7 @@ impl PlainSubscriber { group.with_expiry(Arc::new(GroupExpiry { state: self.state.weak(), subscription: self.subscription.consume(), - cap: self.drift_cap.consume(), + anchor: self.drift_anchor.consume(), bound: None, sequence, })) @@ -3232,7 +3474,7 @@ impl PlainSubscriber { .retain(|sequence, group| *sequence >= min_sequence && watch(group)); // One scan for the whole poll, so walking a backlog off stays linear in its size. - let drift = ready!(self.poll_drift(servable_cap(self.end_sequence, self.stale_cap), waiter))?; + let drift = ready!(self.poll_drift(self.anchor(self.end_sequence), waiter))?; loop { // Re-offer the lowest parked group back inside the cap once it rises, @@ -3331,7 +3573,7 @@ impl PlainSubscriber { let mut floor = floor.max(self.min_sequence); let end = super::subscription::min_some(end, self.end_sequence); // One scan for the whole poll, so walking a backlog off stays linear in its size. - let drift = ready!(self.poll_drift(servable_cap(end, self.stale_cap), waiter))?; + let drift = ready!(self.poll_drift(self.anchor(end), waiter))?; loop { let Some(producer) = ready!(self.poll(waiter, |state| state.poll_next_in_range(floor, end))?) else { @@ -3462,21 +3704,22 @@ impl Subscriber { } /// Bound the drift anchor from outside, for a reader that caps this subscriber - /// without capping its cursor. + /// without capping its cursor, and splices it with other tracks. /// /// A [`super::resume::Subscriber`] segment is deliberately left uncapped /// ([`Self::set_groups`] would park boundary-crossing groups where its completion can't /// be seen), so its own cap has to reach the anchor this way or the segment measures - /// drift against groups its reader will never be served. A spliced segment folds the - /// cap into what it pushes onto its own segments, so the bound reaches the plain - /// cursors at the leaves however deep the splices nest. - pub(crate) fn set_stale_cap(&mut self, cap: Option) { + /// drift against groups its reader will never be served. The same goes for the edge: + /// a segment's track never sees the groups of the segments after it. A spliced + /// segment folds the anchor into what it pushes onto its own segments, so it reaches + /// the plain cursors at the leaves however deep the splices nest. + pub(crate) fn set_anchor(&mut self, anchor: Anchor) { match &mut self.inner { SubscriberKind::Plain(plain) => { - plain.stale_cap = cap; - plain.update_drift_cap(); + plain.stale_cap = anchor.cap; + plain.update_drift_anchor(anchor); } - SubscriberKind::Spliced(spliced) => spliced.set_stale_cap(cap), + SubscriberKind::Spliced(spliced) => spliced.set_anchor(anchor), } } @@ -3514,7 +3757,7 @@ impl Subscriber { SubscriberKind::Plain(plain) => plain, SubscriberKind::Spliced(spliced) => return spliced.poll_stale(group, waiter), }; - let drift = ready!(plain.poll_drift(servable_cap(plain.end_sequence, plain.stale_cap), waiter))?; + let drift = ready!(plain.poll_drift(plain.anchor(plain.end_sequence), waiter))?; let stale = ready!(plain.poll_stale(group, &drift, waiter))?; if stale { plain.note_stale(group); @@ -3741,7 +3984,13 @@ impl Subscriber { match &mut self.inner { SubscriberKind::Plain(plain) => { plain.end_sequence = end.exclusive(); - plain.update_drift_cap(); + // A successor dropped by a lower cap stays dropped until the wrapping + // reader pushes its anchor again, which it does on every poll. + let outer = Anchor { + cap: plain.stale_cap, + ..plain.drift_anchor.read().clone() + }; + plain.update_drift_anchor(outer); } SubscriberKind::Spliced(spliced) => spliced.end_at(end), } @@ -6018,10 +6267,12 @@ mod test { let state = producer.state.read(); let drift = Drift { budget: Duration::ZERO, - edge: state.live_edge(None), + edge: state.drift_edge(None, None, None), + outer: None, + successor: None, }; assert!( - state.is_stale(0, drift.edge.as_ref(), drift.budget), + state.is_stale(0, &drift.edge, drift.budget), "stale against a live edge" ); drop(state); @@ -6032,11 +6283,94 @@ mod test { let state = producer.state.read(); assert!( - !state.is_stale(0, drift.edge.as_ref(), drift.budget), + !state.is_stale(0, &drift.edge, drift.budget), "a vanished edge is no reason to drop what is left" ); } + /// The same holds for an edge a splice pushed from another segment: a handed-out + /// group judges it long after the splice resolved it, so it is revalidated on its own + /// track before it convicts anything. + #[tokio::test] + async fn an_evicted_outer_edge_convicts_nothing() { + let mut producer = track_producer("a", None); + let mut subscriber = producer.subscribe(Subscription::default().with_max_age(Duration::from_secs(1))); + let mut open = producer.append_group().unwrap(); + open.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"a")) + .unwrap(); + let mut group = subscriber.recv_group().await.unwrap().expect("group"); + assert!(group.read_frame().await.unwrap().is_some()); + // A successor bounds the open group's reach, without being late against it. + append_at(&mut producer, 10); + + let mut next = track_producer("b", None); + append_at(&mut next, 30_000); + let edge = append_at(&mut next, 30_010); + subscriber.set_anchor(Anchor { + cap: None, + edge: next.consume().live_edge(None), + successor: None, + }); + + let mut control = group.clone(); + assert!( + matches!(control.read_frame().now_or_never(), Some(Ok(None))), + "the open group ends against the outer edge" + ); + + // The outer edge dies before the group is judged again. + let slot = next.modify().unwrap().lookup.remove(&edge).unwrap(); + let _ = slot.group.abort(Error::Evicted); + + assert!( + group.read_frame().now_or_never().is_none(), + "a vanished outer edge is no reason to drop what is left" + ); + assert!(!group.latency_expired()); + open.finish().unwrap(); + } + + /// A pushed successor is the same kind of cached fact. Evicting it must not keep + /// bounding the previous segment's last group while a later group still anchors the + /// outer edge. + #[tokio::test] + async fn an_evicted_successor_convicts_nothing() { + let producer = track_producer("a", None); + let mut subscriber = producer.subscribe(Subscription::default().with_max_age(Duration::from_secs(1))); + let mut open = producer.append_group().unwrap(); + open.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"a")) + .unwrap(); + let mut group = subscriber.recv_group().await.unwrap().expect("group"); + assert!(group.read_frame().await.unwrap().is_some()); + + let mut next = track_producer("b", None); + // Early enough that group 0 is already past the budget, and not itself the edge. + let successor = append_at(&mut next, 10); + append_at(&mut next, 30_000); + let consumer = next.consume(); + subscriber.set_anchor(Anchor { + cap: None, + edge: consumer.live_edge(None), + successor: consumer.served_start(successor, None), + }); + + let mut control = group.clone(); + assert!( + matches!(control.read_frame().now_or_never(), Some(Ok(None))), + "the open group ends against the successor" + ); + + let slot = next.modify().unwrap().lookup.remove(&successor).unwrap(); + let _ = slot.group.abort(Error::Evicted); + + assert!( + group.read_frame().now_or_never().is_none(), + "a vanished successor is no reason to drop what is left" + ); + assert!(!group.latency_expired()); + open.finish().unwrap(); + } + #[tokio::test] async fn a_lower_sequence_is_never_the_live_edge() { let producer = track_producer("test", None); From 294ee03064b3bd4aa68d761d02fb9d7d6cc72e0b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 13:58:36 -0700 Subject: [PATCH 027/127] fix(net): skip a stale warm cache on an IETF rejoin (#4150) An IETF live join delivers nothing below the group its SUBSCRIBE_OK names as Largest, so declare that as the copy's start. A parked track's warm cache waiting on the copy then sees the upstream moved past it and live readers skip it, as they already do on lite-06+. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/ietf/subscriber.rs | 11 +++++++++++ rs/moq-tokio/tests/broadcast.rs | 27 ++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 93d0c2d4d6..6185af5443 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -1564,6 +1564,8 @@ where } let subscription = request.subscription(); + // A live join delivers nothing below the group SUBSCRIBE_OK names as Largest. + let live = subscription.as_ref().and_then(|s| s.start).is_none(); let join = match subscribe_join( subscription.as_ref().and_then(|s| s.start), subscription.as_ref().and_then(|s| s.end), @@ -1700,7 +1702,16 @@ where .with_timescale(Timescale::MICRO) .with_max_age(self.origin.default_max_age()) .with_priority(super::priority::from_wire(priority.unwrap_or(128))); + // Declared before the track is released to readers, so a warm cache waiting on + // this copy judges itself against where the live feed actually starts. + let request = match live { + true => request.resolving_start(), + false => request, + }; let mut track = request.accept(info); + if live { + let _ = track.start_at(largest.map(|largest| largest.group)); + } let mut fetching: Option> = None; { let mut state = self.state.lock(); diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 4c2953a72d..69140e2d87 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -995,15 +995,31 @@ async fn broadcast_route_migration() { } /// A subscriber returning to a parked track is not handed the parked cache when the -/// upstream resolves its start past it (lite-06+ resolves the start from the budget). +/// upstream resolves its start past it: lite-06+ resolves the start from the budget, and +/// an IETF live join starts at the group SUBSCRIBE_OK names as Largest. /// /// The front keeps what an unread track delivered as a warm cache. While parked, the /// publisher moved on, so the resumed upstream subscription starts well past that cache. /// The cache was only fresh against its own frozen edge; serving it first put a /// rejoining player seconds behind live. +/// +/// Pre-06 lite is skipped: nothing on those wires says where a subscription starts. #[tracing_test::traced_test] #[tokio::test] async fn broadcast_rejoin_skips_a_stale_warm_cache() { + let pre06 = [ + "moq-lite-01", + "moq-lite-02", + "moq-lite-03", + "moq-lite-04", + "moq-lite-05", + ]; + for version in moq_net::Version::names().filter(|version| !pre06.contains(version)) { + rejoin_skips_a_stale_warm_cache(version).await; + } +} + +async fn rejoin_skips_a_stale_warm_cache(version: &str) { use moq_net::Timestamp; let ms = |ms: u64| Timestamp::from_millis(ms).unwrap(); @@ -1027,6 +1043,7 @@ async fn broadcast_rejoin_skips_a_stale_warm_cache() { let mut config = moq_tokio::listen::Config::default(); config.bind = Some("[::]:0".parse().unwrap()); config.tls.generate = vec!["localhost".into()]; + config.version = vec![version.parse().unwrap()]; let mut server = config .init(Default::default()) .expect("init server") @@ -1048,6 +1065,7 @@ async fn broadcast_rejoin_skips_a_stale_warm_cache() { let mut announcements = sub_consumer.announced(); let mut config = moq_tokio::connect::Config::default(); config.tls.insecure = Some(true); + config.version = vec![version.parse().unwrap()]; let client = config.init(Default::default()).expect("init client"); let url: url::Url = format!("moqt://localhost:{}", addr.port()).parse().unwrap(); let (_client, session) = tokio::time::timeout(TIMEOUT, connect_once(client.with_subscriber(sub_origin), url)) @@ -1097,12 +1115,15 @@ async fn broadcast_rejoin_skips_a_stale_warm_cache() { let first = recv(&mut sub).await; assert!( first > 4, - "a rejoining reader was served the stale cache first: group {first}" + "{version}: a rejoining reader was served the stale cache first: group {first}" ); let mut sequence = first; while sequence < 20 { sequence = recv(&mut sub).await; - assert!(sequence >= 4, "a rejoining reader was served stale group {sequence}"); + assert!( + sequence >= 4, + "{version}: a rejoining reader was served stale group {sequence}" + ); } drop(sub); From b3da59228be3a2f324275d0a5b86b0736af2f5f1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 26 Sep 2026 18:57:25 -0700 Subject: [PATCH 028/127] fix(cli): close the relay connection on SIGINT and SIGTERM (#4287) Co-authored-by: Claude Opus 5.5 --- rs/moq-cli/src/main.rs | 98 ++++++++++++++++++++++------------- rs/moq-tokio/src/client.rs | 17 ++++++ rs/moq-tokio/src/noq.rs | 8 +++ rs/moq-tokio/tests/backend.rs | 58 +++++++++++++++++++++ 4 files changed, 145 insertions(+), 36 deletions(-) diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 27ee260451..a3730144ac 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -402,12 +402,12 @@ impl Directions { async fn spawn_moq( moq: &MoqSide, net: &Net, + client: moq_tokio::Client, cluster: moq_relay::cluster::Cluster, directions: Directions, tasks: &mut JoinSet>, ) -> anyhow::Result<(moq_net::bandwidth::Allocator, moq_net::origin::Producer)> { let mut bandwidth = moq_net::bandwidth::Allocator::unlimited(); - let client = net.client(moq.client.clone())?; let cluster = cluster .with_client(client.clone()) .with_client_tls(moq.client.tls.build()?) @@ -471,7 +471,8 @@ async fn run_play(moq: MoqSide, args: play::Args, net: Net) -> anyhow::Result<() consume: true, ..Default::default() }; - let (_, origin) = spawn_moq(&moq, &net, cluster, directions, &mut tasks).await?; + let client = net.client(moq.client.clone())?; + let (_, origin) = spawn_moq(&moq, &net, client, cluster, directions, &mut tasks).await?; play::run(origin.consume(), name, args, tasks) } @@ -479,7 +480,7 @@ async fn run_play(moq: MoqSide, args: play::Args, net: Net) -> anyhow::Result<() /// Run every stage over one Origin and one MoQ attachment. /// /// Stages are independent: each names its own broadcast and owns its own endpoint, -/// and the first to finish (stdin EOF, Ctrl-C, or an error) ends the process. +/// and the first to finish (stdin EOF, SIGINT, SIGTERM, or an error) ends the process. async fn run_stages(moq: MoqSide, stages: Vec, net: Net) -> anyhow::Result<()> { let cluster = moq.cluster()?; let mut tasks: JoinSet> = JoinSet::new(); @@ -489,40 +490,50 @@ async fn run_stages(moq: MoqSide, stages: Vec, net: Net) -> anyhow::Res // The stage combinations were refused up front by `Invocation::validate`, before // anything bound a port or dialed out. - let (bandwidth, origin) = spawn_moq(&moq, &net, cluster, Directions::of(&stages), &mut tasks).await?; - - // stdin and stdout are one resource each, so two stages can't share them. - let mut stdin = None; - let mut stdout = None; - - for stage in stages { - let name = stage.broadcast(&moq); - match stage { - Command::Import(import) => { - if import.source.stdin_format().is_some() { - claim("stdin", &mut stdin, &name)?; - } - if let Some(publish) = spawn_import(&origin, import, name, bandwidth.clone(), &mut tasks)? { - locals.push(publish); + let client = net.client(moq.client.clone())?; + let result = async { + let (bandwidth, origin) = + spawn_moq(&moq, &net, client.clone(), cluster, Directions::of(&stages), &mut tasks).await?; + + // stdin and stdout are one resource each, so two stages can't share them. + let mut stdin = None; + let mut stdout = None; + + for stage in stages { + let name = stage.broadcast(&moq); + match stage { + Command::Import(import) => { + if import.source.stdin_format().is_some() { + claim("stdin", &mut stdin, &name)?; + } + if let Some(publish) = spawn_import(&origin, import, name, bandwidth.clone(), &mut tasks)? { + locals.push(publish); + } } - } - Command::Export(export) => { - if export.sink.is_stdout() { - claim("stdout", &mut stdout, &name)?; + Command::Export(export) => { + if export.sink.is_stdout() { + claim("stdout", &mut stdout, &name)?; + } + spawn_export(&origin, export, name, &mut tasks)?; } - spawn_export(&origin, export, name, &mut tasks)?; + other => unreachable!("`{}` is not a stage", other.name()), } - other => unreachable!("`{}` is not a stage", other.name()), } - } - if locals.is_empty() { - return drive(tasks).await; + if locals.is_empty() { + drive(tasks).await + } else { + let local = tokio::task::LocalSet::new(); + supervise(&local, locals.into_iter().map(Publish::run), &mut tasks); + local.run_until(drive(tasks)).await + } } + .await; - let local = tokio::task::LocalSet::new(); - supervise(&local, locals.into_iter().map(Publish::run), &mut tasks); - local.run_until(drive(tasks)).await + // The process exits next, even on a setup error, so the relay only hears we left + // if the close goes out now. + client.close().await; + result } /// Run the non-Send pipelines on `local`, reporting each into `tasks`. @@ -740,13 +751,10 @@ async fn run_stdout(consumer: moq_net::origin::Consumer, name: String, args: Sub Subscribe::new(source, catalog, args).run().await } -/// Run every endpoint until the first finishes (stdin EOF, Ctrl-C, or an error), -/// then drop the rest. +/// Run every endpoint until the first finishes (stdin EOF, SIGINT, SIGTERM, or an +/// error), then drop the rest. async fn drive(mut tasks: JoinSet>) -> anyhow::Result<()> { - tasks.spawn(async { - let _ = tokio::signal::ctrl_c().await; - Ok(()) - }); + tasks.spawn(shutdown_signal()); while let Some(res) = tasks.join_next().await { match res { @@ -760,6 +768,24 @@ async fn drive(mut tasks: JoinSet>) -> anyhow::Result<()> { Ok(()) } +/// Resolve on SIGINT or, on unix, SIGTERM (what process supervisors send on stop). +async fn shutdown_signal() -> anyhow::Result<()> { + #[cfg(unix)] + { + let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .context("failed to listen for SIGTERM")?; + tokio::select! { + res = tokio::signal::ctrl_c() => res.context("failed to listen for SIGINT")?, + _ = term.recv() => {} + } + Ok(()) + } + #[cfg(not(unix))] + { + tokio::signal::ctrl_c().await.context("failed to listen for SIGINT") + } +} + /// The listener / HTTP-serving endpoints bridge one named broadcast, so an /// empty `--broadcast` is rejected rather than silently defaulting to the root. fn require_broadcast(name: String, endpoint: &str) -> anyhow::Result { diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index e9133a8e89..562d471e48 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -237,6 +237,23 @@ impl Client { Connection::new(self.clone(), addrs.into()) } + /// Close every QUIC connection this client dialed, once each peer has been sent the + /// close. + /// + /// Clones share one endpoint, so this closes theirs too. Dropping the connections + /// only queues the close, which nothing sends once the runtime stops: a process + /// that exits without this leaves each peer waiting out its idle timeout. + /// + /// Only the noq endpoint is closed. WebSocket, TCP, and UDS sessions end when their + /// [`Connection`] is dropped (the kernel closes the socket on exit), and an iroh + /// endpoint passed to `with_iroh` is closed by its owner. + pub async fn close(self) { + #[cfg(feature = "noq")] + if let Some(noq) = self.noq { + noq.close().await; + } + } + /// Connect to the configured [`connect.url`](crate::connect::Config::url) URL, publishing /// `origin` to it. /// diff --git a/rs/moq-tokio/src/noq.rs b/rs/moq-tokio/src/noq.rs index e75d8bd708..99b4bc48aa 100644 --- a/rs/moq-tokio/src/noq.rs +++ b/rs/moq-tokio/src/noq.rs @@ -325,6 +325,14 @@ impl NoqClient { }) } + /// Close every connection, then wait until each has sent its close to the peer. + pub async fn close(self) { + self.quic.close(noq::VarInt::from_u32(0), b"client shutdown"); + // Not `wait_idle`, which also sits out each connection's 3 PTO closing + // period: that only repeats the close to a peer that already has it. + self.quic.wait_all_draining().await; + } + pub async fn connect( &self, tls: &rustls::ClientConfig, diff --git a/rs/moq-tokio/tests/backend.rs b/rs/moq-tokio/tests/backend.rs index 15d850c24b..b68936208b 100644 --- a/rs/moq-tokio/tests/backend.rs +++ b/rs/moq-tokio/tests/backend.rs @@ -670,6 +670,64 @@ async fn iroh_connect() { // ── Noq backend ───────────────────────────────────────────────────── +/// A client that closes before its runtime stops tells the server at once, instead of +/// leaving it to the idle timeout, which is what a process exiting on a signal does. +#[cfg(feature = "noq")] +#[tracing_test::traced_test] +#[tokio::test] +async fn noq_client_close_reaches_server() { + let quic = moq_tokio::quic::Config::default(); + assert!( + quic.idle_timeout > TIMEOUT, + "an idle timeout inside TIMEOUT would hide a lost close" + ); + + let mut server_config = moq_tokio::listen::Config::default(); + server_config.bind = Some("127.0.0.1:0".parse().unwrap()); + server_config.tls.generate = vec!["localhost".into()]; + let server = server_config.init(quic.clone()).expect("failed to init server"); + let mut server = server.listen().await.expect("failed to listen"); + let url: url::Url = format!("moqt://localhost:{}", server.local_addr().unwrap().port()) + .parse() + .unwrap(); + + // The client gets a runtime of its own, gone as soon as the client returns: nothing + // drives its endpoint afterwards, exactly as when a process exits. + let client = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("client runtime"); + runtime.block_on(async move { + let mut config = moq_tokio::connect::Config::default(); + config.tls.insecure = Some(true); + config.bind = Some("127.0.0.1:0".parse().unwrap()); + let client = config + .init(quic) + .expect("failed to init client") + .with_subscriber(moq_tokio::origin::spawn()); + let (client, connection) = connect_once(client, url).await.expect("client connect failed"); + drop(connection); + client.close().await; + }); + }); + + let request = tokio::time::timeout(TIMEOUT, server.accept()) + .await + .expect("accept timed out") + .expect("no incoming connection"); + let session = request.ok().await.expect("server handshake failed"); + tokio::task::spawn_blocking(move || client.join()) + .await + .unwrap() + .expect("client thread panicked"); + + let err = tokio::time::timeout(TIMEOUT, session.closed()) + .await + .expect("the server never heard the close"); + assert!(!err.to_string().contains("timed out"), "{err}"); +} + #[cfg(feature = "noq")] #[tracing_test::traced_test] #[tokio::test] From 722f22a0d9006a1475a7fd2df3e30fd190a77ea2 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:15:56 -0400 Subject: [PATCH 029/127] fix(cli): handle SIGTERM during native playback Use the shared shutdown signal in the player and send the QUIC close after the window exits. Exercise both process signals with real signals in isolated nextest processes. Refs #51. Co-Authored-By: Codex --- doc/bin/cli.md | 4 ++++ rs/moq-cli/src/main.rs | 39 +++++++++++++++++++++++++++++++++-- rs/moq-cli/src/play/window.rs | 8 ++++--- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 4af62528dc..f02d906144 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -10,6 +10,10 @@ itself) and moves media into MoQ from a source, out of MoQ to a sink, or plays it locally. Install it with `cargo install moq-cli`, brew, apt, dnf, winget, or Docker; see [Install](/setup/install). +`SIGINT` and, on Unix, `SIGTERM` stop the running stages or native player. +The CLI sends the QUIC connection close before exiting, so the relay does not +wait for an idle timeout to notice the disconnect. + ## What it does | Verb | Endpoint | | diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index a3730144ac..dbb1562e08 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -472,9 +472,11 @@ async fn run_play(moq: MoqSide, args: play::Args, net: Net) -> anyhow::Result<() ..Default::default() }; let client = net.client(moq.client.clone())?; - let (_, origin) = spawn_moq(&moq, &net, client, cluster, directions, &mut tasks).await?; + let (_, origin) = spawn_moq(&moq, &net, client.clone(), cluster, directions, &mut tasks).await?; - play::run(origin.consume(), name, args, tasks) + let result = play::run(origin.consume(), name, args, tasks); + client.close().await; + result } /// Run every stage over one Origin and one MoQ attachment. @@ -823,6 +825,39 @@ mod tests { type Pipeline = Pin>>>; + #[cfg(unix)] + async fn signal_exits(signal: &str) { + let mut waiting = std::pin::pin!(shutdown_signal()); + std::future::poll_fn(|cx| { + assert!(waiting.as_mut().poll(cx).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + assert!( + std::process::Command::new("/bin/kill") + .args([signal, &std::process::id().to_string()]) + .status() + .unwrap() + .success() + ); + tokio::time::timeout(std::time::Duration::from_secs(5), waiting) + .await + .expect("shutdown signal was not handled") + .unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn sigterm_stops_the_cli() { + signal_exits("-TERM").await; + } + + #[cfg(unix)] + #[tokio::test] + async fn sigint_stops_the_cli() { + signal_exits("-INT").await; + } + /// A local pipeline that dies takes the process with it, even while another one is /// still running. Reporting completion from inside the task instead would miss /// this: a panic skips the report, leaving the survivor to keep the process alive diff --git a/rs/moq-cli/src/play/window.rs b/rs/moq-cli/src/play/window.rs index 89e8ed8ba9..81317a2ece 100644 --- a/rs/moq-cli/src/play/window.rs +++ b/rs/moq-cli/src/play/window.rs @@ -78,9 +78,11 @@ pub fn run( let signal = tokio::spawn({ let proxy = proxy.clone(); async move { - if tokio::signal::ctrl_c().await.is_ok() { - let _ = proxy.send_event(Event::Finished); - } + let event = match crate::shutdown_signal().await { + Ok(()) => Event::Finished, + Err(error) => Event::Failed(format!("shutdown listener failed: {error:#}")), + }; + let _ = proxy.send_event(event); } }); From 9ed7494e9543e900238944e4194aa837c0e82968 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:17:28 -0400 Subject: [PATCH 030/127] fix(audio): preserve media time while draining across holes Continue splicing across holes when an endpoint or finite track drains the buffer, so the playhead still reaches the final media timestamp. Refs #32 and #33. Co-Authored-By: Codex --- rs/moq-audio/src/playout/engine.rs | 51 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/rs/moq-audio/src/playout/engine.rs b/rs/moq-audio/src/playout/engine.rs index 651898fac7..2e462e8bb7 100644 --- a/rs/moq-audio/src/playout/engine.rs +++ b/rs/moq-audio/src/playout/engine.rs @@ -390,27 +390,9 @@ impl Engine { self.decision.target(self.target); } - // One turn of the decision loop, committing at least one block to the output. + // Commit enough output for the next pull, possibly over multiple runs. fn produce(&mut self) { let ready = self.buffer.ready(); - // A hole cannot grow the preceding fragment into a block. Commit that audio - // before splicing the next run, even when it fills only part of this pull. - if ready > 0 && ready < self.block && self.buffer.held() > ready { - self.play(ready); - return; - } - - // A declared endpoint cannot refill a stalled buffer. Drain every sample, - // pad the final block, then park without manufacturing an underrun. - if self.ended { - if ready > 0 { - self.play(self.block); - } else { - self.pause(); - } - return; - } - let front = self.buffer.front(); let contiguous = match (self.played, front) { (Some(played), Some(front)) => front <= played + self.buffer.duration(1), @@ -418,6 +400,19 @@ impl Engine { _ => false, }; + // A hole cannot extend its preceding fragment, and an endpoint cannot refill + // a stalled buffer. Drain those runs while preserving each splice's media time. + if self.ended || (ready > 0 && ready < self.block && self.buffer.held() > ready) { + if ready == 0 { + self.pause(); + } else if !contiguous { + self.splice(); + } else { + self.play(if self.ended { self.block } else { ready }); + } + return; + } + // Audio that has run too far ahead of the playhead is dropped back to the level // playout holds: it is going to be late either way, and playing it is the delay // the target exists to bound. The same rule as on the way in, so a target that @@ -1103,6 +1098,24 @@ mod tests { ); } + #[test] + fn a_finished_run_keeps_the_playhead_across_a_hole() { + let mut engine = Engine::new(config(1, false)).unwrap(); + engine.insert(Duration::ZERO, 0.0, &[0.25; 960]); + engine.insert(Duration::from_millis(40), 40.0, &[0.5; 960]); + engine.end(); + let mut out = vec![0.0; engine.block()]; + for _ in 0..10 { + if engine.drained() { + break; + } + engine.pull(&mut out); + } + assert!(engine.drained()); + assert_eq!(engine.playhead(), Some(Duration::from_millis(60))); + assert_eq!(engine.stats().underruns, 0); + } + #[test] fn a_fragment_before_a_hole_keeps_every_sample() { let mut engine = Engine::new(config(1, false)).unwrap(); From faea8b97500755a8f6962b3630adf116b4a38af6 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:23:34 -0400 Subject: [PATCH 031/127] refactor(watch): trim playback code and cover 100 viewers Remove redundant code, comments, and the duplicate worklet report test. Close benchmark audio channels and extend report scaling coverage while retaining its existing budgets. Co-Authored-By: Codex --- js/watch/README.md | 8 +++ js/watch/src/audio/buffer.test.ts | 5 -- js/watch/src/audio/decoder.ts | 20 +------ .../src/audio/render-worklet.port.test.ts | 54 ++----------------- .../src/audio/worker/remote.bench.test.ts | 19 ++++--- js/watch/src/audio/worker/remote.ts | 17 +++--- js/watch/src/sync.test.ts | 2 - js/watch/src/sync.ts | 9 +--- js/watch/src/video/decoder.test.ts | 6 +-- 9 files changed, 34 insertions(+), 106 deletions(-) diff --git a/js/watch/README.md b/js/watch/README.md index 1bb610d5a6..3dce955838 100644 --- a/js/watch/README.md +++ b/js/watch/README.md @@ -150,6 +150,14 @@ The `` element automatically discovers the nested `` el - **Quality selection**: Switch between available renditions - **Custom tracks**: Unknown catalog sections pass through, and `broadcast.out.active` subscribes your own tracks +## Report benchmark + +Run `bun test js/watch/src/audio/worker/remote.bench.test.ts` from the repository root. +The benchmark measures main-thread report processing for 1 to 100 viewers at 10, 20, +and 50 ms report intervals. It runs the real `Decoder`, `Remote`, and `Sync` with a +mock worker and audio device, so it does not measure codec or relay throughput. +The existing JavaScript test suite runs it in CI. + ## License Licensed under either: diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 43ede89ccd..c7aa4b2f59 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -430,8 +430,6 @@ describe("AudioBuffer, flushed", () => { }); }); -// --- review consumer-sync-video F9 --- - describe("AudioBuffer output clock, shared ring", () => { it("the shared ring's clock is anchored to when its samples leave the output device", async () => { // The page polls at a fixed instant, 1000ms. The device is 40ms behind the render graph: @@ -453,7 +451,6 @@ describe("AudioBuffer output clock, shared ring", () => { }); try { buffer.insert(3_000_000 as Time.Micro, [new Float32Array(4800)]); - // The shared ring is polled rather than pushed, so wait out a poll interval (real time). jest.advanceTimersByTime(50); const sampled = buffer.clock.peek(); @@ -467,8 +464,6 @@ describe("AudioBuffer output clock, shared ring", () => { }); }); -// --- review consumer-sync-video F15 --- - describe("AudioBuffer, partial output timestamp", () => { it("a partial output timestamp neither throws nor strands backpressure", async () => { const worklet = new FakeWorklet(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index d9cacd1046..5aefb5cc0c 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -200,23 +200,10 @@ export class Decoder { // A replacement supply has not decoded yet, but the context already knows the stream's rate. #decodedRate?: { config: string; rate: number }; - /** - * The age budget for audio: `Sync.out.maxAge` plus what the ring can absorb past it. - * - * See {@link audioMaxAge} for what it adds and why. - * - * Audio only, and deliberately not `Sync.out.maxAge` itself. That value is also the lookahead - * cap in `Sync.received`, which bounds how far ahead of the playhead an *early* frame may be - * held before playback skips forward. Widening it would let buffered playback drift a headroom - * further from the live edge to solve a problem that only exists for late arrivals. Video is - * untouched for the same reason: it drops a late frame at render rather than losing the group. - * - * It reads synchronously from the first peek, which is what `subscribeMedia` and - * `Container.Consumer` need at construction. - */ - // The ring's depth: `sync.out.delay` plus `sync.out.offset`. See the constructor. + // The ring's depth: `sync.out.delay` plus `sync.out.offset`. readonly #target: Derived, Getter], Time.Milli>; + // The subscription age budget includes what the audio ring can absorb; see audioMaxAge. readonly #maxAge: Derived< readonly [Getter, Getter, Getter], Time.Milli @@ -491,9 +478,6 @@ export class Decoder { // less efficient for video-only playback but makes muting/unmuting instant, since the first // gesture on the page builds a context for every tile whether or not it is the one clicked. - //const enabled = effect.get(this.enabled); - //if (!enabled) return; - const config = effect.get(this.#config); if (!config) return; diff --git a/js/watch/src/audio/render-worklet.port.test.ts b/js/watch/src/audio/render-worklet.port.test.ts index 3a09a1144c..70715149da 100644 --- a/js/watch/src/audio/render-worklet.port.test.ts +++ b/js/watch/src/audio/render-worklet.port.test.ts @@ -106,14 +106,14 @@ describe("render worklet ports", () => { } await settle(); - // 40 quanta is about 107 ms: past the fill, well inside the 200 ms written. - const played = pull(render, 40); + // 50 quanta is about 133 ms: past the fill, well inside the 200 ms written. + const played = pull(render, 50); const loudest = played.reduce((max, v) => Math.max(max, Math.abs(v)), 0); expect(loudest).toBeGreaterThan(0.2); await settle(); // A state message every five quanta to the writer's port; the node's port hears once that it played. - expect(writer.length).toBeGreaterThanOrEqual(7); + expect(writer.length).toBe(10); const last = writer[writer.length - 1]; expect(last.type).toBe("state"); expect(last.debug.output).toBeGreaterThan(0); @@ -186,52 +186,4 @@ describe("render worklet ports", () => { node.port2.close(); }); - - it("reports state only to a handed-over port, telling the node's own port once that the ring played", async () => { - // With a worker writing the ring, the page reads the node's port for two things only: that the - // ring played, and that a message was unreadable. Everything else is the worker's to hear, and a - // report every five quanta to a busy main thread is exactly what the offload is meant to spare it. - if (!Render) throw new Error("render-worklet.ts registered no 'render' processor"); - const node = new MessageChannel(); - nextPort = node.port1; - const render = new Render(); - const extra = new MessageChannel(); - const handoff: Port = { type: "port", port: extra.port1 }; - node.port2.postMessage(handoff, [extra.port1]); - await settle(); - - const page: ToMain[] = []; - const writer: ToMain[] = []; - node.port2.onmessage = (event: MessageEvent) => page.push(event.data); - extra.port2.onmessage = (event: MessageEvent) => writer.push(event.data); - - const init: InitPost = { - type: "init-post", - channels: 1, - rate: RATE, - latency: Time.Milli(20), - buffered: false, - conceal: false, - }; - extra.port2.postMessage(init); - const chunk = (RATE * 20) / 1000; - for (let i = 0; i < 20; i++) { - const samples = new Float32Array(chunk).fill(0.25); - const data: Data = { type: "data", data: [samples], timestamp: Time.Micro.fromMilli(Time.Milli(i * 20)) }; - extra.port2.postMessage(data, [samples.buffer]); - } - await settle(); - - // 50 quanta, about 133 ms of playing: ten reports. - pull(render, 50); - await settle(); - expect(writer.filter((msg) => msg.type === "state").length).toBe(10); - const told = page.filter((msg) => msg.type === "state"); - expect(told.length).toBeLessThanOrEqual(1); - // And what it was told is that the ring played. - expect(told.every((msg) => msg.type === "state" && !msg.debug.fresh)).toBe(true); - - node.port2.close(); - extra.port2.close(); - }); }); diff --git a/js/watch/src/audio/worker/remote.bench.test.ts b/js/watch/src/audio/worker/remote.bench.test.ts index f76650e4ff..7159f050aa 100644 --- a/js/watch/src/audio/worker/remote.bench.test.ts +++ b/js/watch/src/audio/worker/remote.bench.test.ts @@ -69,9 +69,13 @@ class Context extends EventTarget { } class Node { - readonly port = new MessageChannel().port1; + readonly #channel = new MessageChannel(); + readonly port = this.#channel.port1; connect(): void {} - disconnect(): void {} + disconnect(): void { + this.port.close(); + this.#channel.port2.close(); + } } const scope = globalThis as unknown as Record; @@ -223,7 +227,7 @@ it("costs the page the same per report however many players share the worker", a const rows: Row[] = []; for (const interval of [50, 20, 10]) { - for (const players of [1, 2, 4, 8, 16]) rows.push(await measure(players, interval)); + for (const players of [1, 2, 4, 8, 16, 32, 64, 100]) rows.push(await measure(players, interval)); } console.log("players interval per report per second of reports"); @@ -236,11 +240,14 @@ it("costs the page the same per report however many players share the worker", a for (const interval of [50, 20, 10]) { const at = (players: number) => rows.find((row) => row.players === players && row.interval === interval); const one = at(1); - const sixteen = at(16); - if (!one || !sixteen) throw new Error("missing a row"); + if (!one) throw new Error("missing a row"); // No slope with the page's players: a report costs what it costs whoever else is on the worker. The // ratio is loose because one player's timings are a handful of samples against a bun process's pauses. - expect(sixteen.perReport).toBeLessThan(3 * one.perReport + 20); + for (const players of [16, 32, 64, 100]) { + const many = at(players); + if (!many) throw new Error("missing a row"); + expect(many.perReport).toBeLessThan(3 * one.perReport + 20); + } } // Sixteen players at the host's own cadence cost the page's main thread well under a twentieth of it. diff --git a/js/watch/src/audio/worker/remote.ts b/js/watch/src/audio/worker/remote.ts index ebdb5fdd23..37ccd4478c 100644 --- a/js/watch/src/audio/worker/remote.ts +++ b/js/watch/src/audio/worker/remote.ts @@ -25,7 +25,7 @@ import type { Port, ToMain } from "../render"; import type { Source } from "../source"; import type { Graph, RingState, SupplyOutput } from "../supply"; import { type Lease, Pool } from "./pool"; -import { Deadline, type FromWorker, type Output, type Report, type Stage, TICK, type ToWorker } from "./protocol"; +import { Deadline, type FromWorker, type Report, type Stage, TICK, type ToWorker } from "./protocol"; // How often the page samples its output clock for the worker. Device and system clocks drift apart by // tens of parts per million, so a sample a second old maps a playhead to well under a millisecond. @@ -281,7 +281,11 @@ export class Remote { // Only the page can read its output clock, which the postMessage ring maps its playhead through. const output = () => { - const sampled = sample(graph.context); + const timestamp = outputTimestamp(graph.context); + const sampled = timestamp && { + contextTime: timestamp.contextTime, + at: performance.timeOrigin + timestamp.performanceTime, + }; this.#lease.post({ type: "output", id: this.#lease.id, output: sampled }); // The device clock can appear after statechange. Sample as the worker reports progress. this.#sampleOutput = !sampled && graph.context.state === "running" ? output : undefined; @@ -443,12 +447,3 @@ export class Remote { this.#signals.close(); } } - -/** - * When the context's current sample leaves the output device, or undefined while the device has not - * started, which `getOutputTimestamp` reads with a zero performance time. - */ -function sample(context: Pick): Output | undefined { - const output = outputTimestamp(context); - return output && { contextTime: output.contextTime, at: performance.timeOrigin + output.performanceTime }; -} diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 89b410aa7c..eed240765e 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -839,8 +839,6 @@ describe("wait", () => { }); }); -// --- review consumer-sync-video F10: clock samples waking waiting frames --- - describe("Sync wakes a waiting frame only when its deadline moves", () => { // Counts the timers `wait()` arms while one frame waits and the audio clock republishes its // playhead `count` times, 5ms apart, each sample off the ideal line by `jitter(i)` ms. Every timer diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 85d11651ae..2a26024780 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -68,14 +68,7 @@ const OFFSET_WINDOW = Time.Milli(2_000); // that a track that has really gone stops holding the buffer open. const SPREAD_WINDOW = Time.Milli(2_000); -// How far a new playhead sample has to move the derived reference before waiting frames are woken. -// -// Every sample re-derives the reference, and a real clock never lands on the extrapolated line to the -// microsecond: the worklet's position and the time it is stamped with are read at different -// instants. Republishing each sub-millisecond wobble woke every frame parked in `wait()` on every -// sample (tens per second, times every frame in the trail) to recompute a deadline that had not -// moved by anything a display can show. A frame waits on the latest sample either way, since -// `#playhead` extrapolates from `#clock`, not from the reference. +// Submillisecond clock noise does not move waiting frames' display deadlines. const REFERENCE_SLACK = Time.Milli(1); /** diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index bd45272460..8e260432cb 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -31,11 +31,7 @@ const real = { beforeEach(() => { built = []; - // The retry and recovery windows are seconds by design, so they run on bun's fake clock and only - // move when a test advances it. The old version scaled real timers 25x, which put BUFFERING at - // 20ms of wall time: a loaded machine let it fire between two statements and flaked the stall - // assertions. `flush` below still yields a real turn, so the signal graph and the track - // plumbing run as before; only deadlines are virtual. + // Only advance() moves decoder recovery deadlines. jest.useFakeTimers(); class FakeVideoFrame { From 61564015a5877703cc26f5c1f69c8b9591452b8d Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:24:25 -0400 Subject: [PATCH 032/127] docs(watch): remove stale playback comments Co-Authored-By: Codex --- js/watch/src/audio/supply.ts | 2 -- js/watch/src/video/renderer.test.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/js/watch/src/audio/supply.ts b/js/watch/src/audio/supply.ts index 875718f5a5..6cd326b7b1 100644 --- a/js/watch/src/audio/supply.ts +++ b/js/watch/src/audio/supply.ts @@ -416,8 +416,6 @@ export class Supply { this.#terminal.clear(preSkip); const format = config.container.kind === "loc" ? new Container.Loc.Format("audio") : new Container.Legacy.Format(config); - // Create consumer with slightly less latency than the render worklet to avoid underflowing. - // TODO include JITTER_UNDERHEAD const consumer = new Container.Consumer(sub, { format, maxAge: this.in.maxAge, diff --git a/js/watch/src/video/renderer.test.ts b/js/watch/src/video/renderer.test.ts index b3b0513647..b4eaeca62c 100644 --- a/js/watch/src/video/renderer.test.ts +++ b/js/watch/src/video/renderer.test.ts @@ -228,8 +228,6 @@ describe("Renderer", () => { }); }); -// --- review consumer-sync-video F11 --- - describe("Renderer frame pairs", () => { let callbacks: Map; let nextCallback: number; From aa816c79ed6fe3a5700290128814b2de8cf54d55 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:27:02 -0400 Subject: [PATCH 033/127] chore: remove stale fork investigation notes and comments Delete the unreferenced session handoff and shorten worktree comments without changing recipe code. Remove review labels and stale test commentary. Co-Authored-By: Codex --- HANDOFF.md | 5221 --------------------------- js/hang/src/container/stall.test.ts | 6 +- js/json/src/window/window.test.ts | 7 +- js/net/src/lite/publisher.test.ts | 7 +- justfile | 69 +- 5 files changed, 17 insertions(+), 5293 deletions(-) delete mode 100644 HANDOFF.md diff --git a/HANDOFF.md b/HANDOFF.md deleted file mode 100644 index b817cfa65a..0000000000 --- a/HANDOFF.md +++ /dev/null @@ -1,5221 +0,0 @@ -# Real-time audio playout: what was wrong, what this branch does about it - -Branch: `fperex/moq` `debug-findings-solution`, based on `upstream/dev` `712ffd810`. It was rebased -there on 2026-09-18 from `61da0d247`, twenty-five upstream commits below, and every conflict and -adaptation that took is recorded under "The rebase onto upstream dev, 2026-09-18"; the rebase before -it, onto `61da0d247` on 2026-09-17, keeps its own section under that one. The code tip is -`f4b1e0081`, 125 commits above `upstream/dev` `712ffd810`, and this document sits in the docs commit -directly above it. The last change -to the player is `1c2605450`. The five commits above the previous docs pass are the first ten seconds -after an event: a declared pause that reopens, a spent head that is no longer convicted, a rebuilt -tile that keeps what the viewer chose, a hidden camera that is not a stall, a shaper that can drop, -and the unit this repository packages the relay with. They touch `js/hang`, `js/watch`, -`js/publish`, `demo/web`, `rs/moq-shaper`, `packaging/`, the hang draft and `doc/concept/`, and they -are findings 36 to 44 below. The six commits above `17d32eae4`, the five code ones and the docs -commit of findings 36 to 44, carry no `Claude-Session` trailer because the session URL was not at -hand when they were written, and the 2026-09-18 rebase did not add one; a tree-preserving rebase -adds it and rewrites all six hashes again, and the commit table below is refreshed then. - -This is a **draft pull request on a fork, opened so the work can be read, and not a submission**. No -pull request on `moq-dev/moq` is intended and nothing here asks to be merged as a unit. It exists so -that the investigation behind moq-dev/moq#2812 and #3477 is readable and measurable, and so the -slices worth taking can be taken one commit at a time, in any order. It is a proof of concept, and -if a slice should be reshaped before it is adopted, say which. - -It is one document on purpose. It was three files under `debug-findings/`: the full report, a short -form written as an issue comment, and an operational handoff. They said the same things at three -lengths, and they are folded into this one file at the root of the branch so there is one thing to -read. Where they overlapped, the report's fuller wording is the one kept. - -**How to read it.** "How to read the branch" below is the map: one commit per quest slice, each one -compiling and testable alone, with a suggested reading order of the eight commits that are the fix, -and the same commits grouped by the quest they come from. The tables are the evidence, and every -number in them is read out of a run's row JSON rather than off a printed line. "Findings for the -maintainer" is the part that is worth reading even if none of the code is adopted. - -**What everything is graded against**, at the auto preset: a jitter target of 20 to 100 ms, a held -total under 150 ms on the LAN and under 200 ms through the public relay, audio and video within one -frame, and zero underruns after convergence. Firefox and Safari are graded like Chromium. - -**State at this tip.** The four topics an earlier revision listed as in progress are done, and they -are findings 18 to 25 below, each with the commit that fixes it: what a stopped video track does to a -watcher and what a capture stop does to a publisher's catalog, the fill and the spinner after an -unmute, hiding and showing video, and the console noise a watcher produces. The cross-browser -resilience matrix has since run: 49 rows over five engines, publishing and watching, hide and show, -mute and unmute, a device change, an impaired path, a relay restart and a 30 minute run. Its -failures were sorted into defects and bench artefacts, the defects were fixed and the affected rows -re-run, and findings 26 to 31 are what came out. Ten of those rows have since been run again on a -machine with nothing else on it, and that run is the last table in the Evidence section. What is -still open is listed under "Open items and follow-ups": real Safari and Playwright WebKit as -publishers, the 30 minute long run on a quiet machine, and the maintainer questions each fix left -behind. - -Since then the branch has been **rebased onto `upstream/dev` `61da0d247`** and re-gated end to end -on the rebased tree: `fix`, `check`, `test default`, the replay lane, `nextest`, `smoke-full` and -the drafts, all green, plus three bench rows. **The user listened on 2026-09-17 at the tip that -`e6be6a775` records, and reported it clean**: no stutter, no spinner on an unmute, video and audio -together, and hide and show and self-publish working, on both served pages with a real microphone -and camera. They then reported two transients that round had not covered, the first seconds of their -own self-publish and the stutter after an unmute, and both are fixed and measured here as findings -32 and 33. Two more came out of the multi-watcher round the user asked for, findings 34 and 35. -**The user has since listened to this tip remotely, in Brave over a wide-area path**: the mute and -reload family is fixed to the ear, and the first seconds of a fresh publish are not, which is -finding 45 and the open item this document ends on. The four transients the user reported after the last round they did hear -are findings 36 and 38 to 40, and each is fixed and measured below. That is one listener, one machine and one session either way, so the measured rows below are -the evidence and the ear is the confirmation. - -**Contents** - -- [Summary](#summary) -- [How to read the branch](#how-to-read-the-branch) -- [Root causes](#root-causes) -- [What changed, per stage](#what-changed-per-stage) -- [Evidence](#evidence) -- [Findings for the maintainer](#findings-for-the-maintainer) -- [Public API and wire impact](#public-api-and-wire-impact) -- [Departures from the quests](#departures-from-the-quests) -- [The rebase onto upstream dev, 2026-09-18](#the-rebase-onto-upstream-dev-2026-09-18) -- [The rebase onto upstream dev, 2026-09-17](#the-rebase-onto-upstream-dev-2026-09-17) -- [Verification and limitations](#verification-and-limitations) -- [How to run](#how-to-run) -- [Open items and follow-ups](#open-items-and-follow-ups) -- [Attribution and licensing](#attribution-and-licensing) - -## Summary - -1. `` at the "Real-time" / auto preset stutters (moq-dev/moq#2812, #3477). - -2. Five separate defects produce it, and each one alone is enough to be heard. - -3. The auto delay came from the round trip, which cannot see how unevenly a publisher flushes. - -4. The audio ring skipped whenever it ran long and never re-stalled after running dry. - -5. The age budget was also the container's skip threshold, so the estimator under it could only confirm the budget it was cut to. - -6. This branch replaces the estimator with the algorithm NetEq uses, written down in `doc/concept/playout.md` and held by a checked-in corpus that both languages replay. - -7. The ring then converges by time stretch rather than by skipping, and conceals an outage rather than playing silence. - -8. `rs/moq-audio` gets the same engine, so `moq play` has a jitter buffer for the first time. - -9. `test/audio-quality` measures all of it over a seeded impaired path, in Chromium, in real Safari, and against the recorded traces on a simulated clock, and grades it nightly. - -10. On the public relay at auto, over 120 s: 36 underruns before the engine, 1 after. - -11. A user then muted a real microphone and kept hearing the room. Concealment had no end: it faded - into the measured background level and stayed there. It now ends in silence, as current WebRTC - does, and a muted publisher says on the wire that its timeline paused instead of leaving the - watcher to guess. See finding 10. - -12. Firefox then underran where Chromium did not, on the same broadcast over the same relay at the - same moment. That is not the path either: the Firefox content process stops executing for two - to five hundred milliseconds at a time, and the estimator was reading the block as delay. The - estimator now takes an explicit "the receiver was blocked" input. See finding 12. - -13. A viewer then left, and every later viewer of the same browser publisher got a tile, an accepted - subscription and no catalog. The publisher's catalog track stayed cached with nobody writing to - it. See finding 13. - -14. A browser publish stamped its microphone and its camera on two different epochs, so sound - led picture by a third of a second for every viewer, and by the whole of it for a second after - every unmute. See finding 14. - -15. And real Safari played one run in eight in silence. The click started the audio context; the - next catalog frame, the one that carries the codec description, closed it and built a - replacement that was born suspended with the gesture already spent. See finding 15. - -## How to read the branch - -One commit per quest slice, in the order they were written. Each one compiles, tests, and is -readable alone. The two oldest are the maintainer's own, cherry-picked from the closed PR #3517 -with their author intact. - -| # | Commit | Quest / stage | What it is | -| --- | --- | --- | --- | -| 1 | `e4b7e9eff` | #3517 (kixelated) | Auto sized from measured arrivals, ring slack, re-stall, `stall()`, underrun counter | -| 2 | `bec43e540` | #3517 (kixelated) | Short quantum counts as an underrun, stale arrival minimum expires | -| 3 | `4c488b08a` | `audio-jitter-target/spec.md` | `doc/concept/playout.md`, the NetEq estimator, the 14-case corpus and its generator | -| 4 | `2d86e147a` | `audio-jitter-target/watch.md` | Auto sized from the target, render quantum moved into the ring, advertised jitter the floor and 2 s the ceiling | -| 5 | `bd27266cc` | review follow-up | Arrival clock read before the container is parsed; the arrival test made real | -| 6 | `953bb3c29` | `transport-impairment-profile.md` | `rs/moq-shaper`, a seeded userspace UDP path impairment with counters | -| 7 | `8eb68753c` | `watch-audio-time-stretch.md` (native half) | Accelerate, preemptive expand, expand, merge, background noise, in Rust | -| 8 | `d3a53cc97` | review follow-up | `Container.Jitter` is the class, not a namespace | -| 9 | `c0f623941` | budget finding, part 1 | `Expired` on `StreamCode.DeliveryTimeout` | -| 10 | `9918f15fb` | budget finding, part 2 | A censored group is counted, not a `spawn error` | -| 11 | `48bdbea5e` | budget finding, part 3 (fork addition) | Audio subscribes and consumes at `maxAge + headroom` | -| 12 | `47fbfb5c2` | finding | `demo/web` tiles start in auto instead of a hard-coded `100ms` | -| 13 | `1a6c1fe45` | CodeRabbit | Buzz fixture period sized from the signal it built | -| 14 | `cf0436ae0` | `m1/plan-av-clock.md` | The audio playhead drives `Sync.reference`; `sync.track()` replaces the flat inputs | -| 15 | `46d8081e0` | tune-in defect (browser) | A receiver's own reading stall stays out of the target; proportional fall | -| 16 | `f44ce10a4` | `audio-jitter-target/native.md` | The native estimator, same corpus, `f64` | -| 17 | `90ac1fa96` | `audio-jitter-target/native.md` | The native engine: frame buffer, decision loop, playhead | -| 18 | `954f64090` | tune-in defect (native) | The same two rules in Rust | -| 19 | `47f6b3cba` | `audio-jitter-target/native.md` | `decode::Config::delay` turns it on; `moq play --delay` becomes a floor | -| 20 | `fe316b664` | native test | The native guards hold to the fourteenth corpus case | -| 21 | `23b9ffa86` | `watch-audio-time-stretch.md` | The ring reader splits into `view`, `peek`, `commit` | -| 22 | `ca06c55c8` | `watch-audio-time-stretch.md` | The browser engine: the ring converges by stretch rather than by skipping | -| 23 | `558a679cc` | `qa-failure-artifacts.md` | A failing run keeps its directory | -| 24 | `db3ce6e5d` | `audio-quality-harness/browser.md` | `test/audio-quality`: page, probe, driver, sink, analyzer, grader, schema | -| 25 | `69724d938` | harness | The page denies the WebSocket fallback so the shaper is in the path | -| 26 | `7a1850c90` | harness | Budgets recorded from the full matrix, not enforced | -| 27 | `caf5d93de` | harness | The probe can read a build it was not written against | -| 28 | `8dddb900d` | harness | The probe's generic helper is a function declaration, not an arrow | -| 29 | `0e2ffde66` | harness | `skipped_groups` and the clock source become real numbers | -| 30 | `d5a0b834a` | CodeRabbit | A void row fails the run; the probe's remaining reads are guarded | -| 31 | `b7b3806b7` | concealment (user decision) | The browser twin of expand and merge, behind `conceal`, default on | -| 32 | `76c0cdab5` | re-land | MPEG-TS importer keeps fractional sample ticks | -| 33 | `395daac3d` | re-land | Authored audio gaps render as silence before encoding | -| 34 | `c38170a1f` | re-land | A file's audio format is learned from its first decoded sample | -| 35 | `e6f072009` | re-land | The video encoder's admission queue is bounded | -| 36 | `1ecf18151` | re-land | Native chunks bridge into the Libav polyfill when one codec is missing | -| 37 | `02a117006` | `audio-quality-harness/browser.md` | A real Safari lane over safaridriver | -| 38 | `c784a1cdc` | `audio-quality-harness/browser.md` | A deterministic replay lane, graded on its own rows | -| 39 | `78a4010c9` | `audio-quality-harness/browser.md` | The nightly `audio-quality` job | -| 40 | `514db7c72` | harness | The device's own rate is sampled; the unearned Safari budgets go | -| 41 | `3a02a1afa` | harness | An underrun episode ends at the last sample that underran | -| 42 | `b23179839` | concealment (user decision) | `conceal` as an element attribute, settable before connect | -| 43 | `d862f4ab5` | flake | The `console.error` spy is isolated in the consumer test | -| 44 | `b34f8c952` | harness | The budgets enforced, measured on the playout engine | -| 45 | `cd746e434` | review | The review pass over the branch | -| 46 | `3f5828da7` | delivery | The report and the issue comment | -| 47 | `f86598325` | shaper defect | Jitter varies the delay without reordering datagrams | -| 48 | `69fe766f9` | harness | The budgets re-recorded and enforced on the fixed shaper | -| 49 | `edff16dd1`, `400e8e3fa` | delivery | The report, filled in with the enforced budgets | -| 50 | `fdc9a2ae1` | `m1/plan-av-clock.md`, user report | A hole in the source reaches the decoder, and a ring nothing is draining is flushed | -| 51 | `a6142a6b3` | delivery | The report and the issue comment, with the A/V desync fix | -| 52 | `a21ea530e` | mic hold (user report) | Both rings hold one chunk above the playout target | -| 53 | `fffdeec14` | mic hold (user report) | The same one-frame hold in the native engine | -| 54 | `0d4625677` | harness | The real-microphone traces replayed and graded | -| 55 | `fccd59eaa` | delivery | The report and the issue comment, with the one-frame hold finding | -| 56 | `6d7abd6ba` | mute (user report) | A long outage ends in silence, not comfort noise | -| 57 | `3f6bc1cdd` | mute (user report) | A muted audio track declares its endpoint, both ends | -| 58 | `d3dc1bcbb` | mute (user report) | The same silence in the native engine | -| 59 | `0bb50e38f` | delivery | The report and the issue comment, with the mute finding | -| 60 | `e044c97a4` | cold start (user decision) | The catalog jitter seeds the playout target instead of flooring it | -| 61 | `8245d8f07` | harness | The budgets re-recorded on the rebased tree | -| 62 | `55e8f8685` | delivery | The report and the issue comment, with the cold-start rule | -| 63 | `31b38aed6` | cold start (user decision) | The first measurement replaces the seeded playout target, as NetEq does | -| 64 | `4df92b41e` | harness (intermittent) | The re-anchored decoder is never handed a chunk it refuses | -| 65 | `063cd3031` | harness | The budgets re-recorded after the seed and the decoder fixes | -| 66 | `d2edb4b10` | delivery | Cross-browser results: four engines, three sources | -| 67 | `004721b0c` | delivery | The Safari catalog defect retracted | -| 68 | `4450407cb` | delivery | The handoff for whoever continues the branch | -| 69 | `499d7806c` | harness (rebase) | The harness relay admits anonymous sessions under upstream's lease model | -| 70 | `19edd01ce` | unmute spike (user report) | A flushed ring reports no playhead until it is re-anchored | -| 71 | `29af6fb9d` | delivery | remark's own formatting over the report | -| 72 | `a1499011c` | Safari unlock (user report) | The audio unlock is armed on the first gesture, so one click plays | -| 73 | `d8d78ead0` | CodeRabbit | Each shaper direction draws its own seed, and a reorder without a delay is refused | -| 74 | `220b09934` | CodeRabbit | A capture that ends while still enabled declares an audio endpoint | -| 75 | `1b90331be` | CodeRabbit | The grader, beacon, probe and Safari lane fail loudly where they were silent | -| 76 | `6efdb3c73` | CodeRabbit | The stereo merge case uses a distinct signal per channel | -| 77 | `c3571e072` | CodeRabbit | The native replay module's doc comment says four traces, not two | -| 78 | `0b80a0257` | finding 12 | The estimator is told when the receiver itself was blocked | -| 79 | `b2099c477` | finding 12 | The recorded Firefox window replayed and graded | -| 80 | `116d4ad88` | CI | The nightly job runs the replay lane ahead of the Chromium matrix | -| 81 | `7d06c66c6` | delivery | The report, the issue comment and the handoff, refreshed for the rebased tip | -| 82 | `3f83de0fc` | finding 13 | A served catalog track is released when its last subscriber leaves | -| 83 | `c53ecfe49` | finding 14 | Captured audio is stamped on the context clock, so both tracks share one epoch | -| 84 | `f846462e5` | finding 15 | The audio context is keyed on its rate alone, so a later catalog frame cannot spend the gesture that started it | -| 85 | `c096a1ace` | delivery | The quiet re-measure, the audio-context fix, and the gates at the tip | -| 86 | `8ea416d7d` | delivery | The tip filled into the short form, and the handoff pointer | -| 87 | `462637ef7` | delivery | This document: the report, the short form and the handoff folded into one file at the repository root | -| 88 | `c90eec220` | finding 20 | The arrival estimate lives as long as the rendition, so an unmute continues it instead of starting over at the declaration | -| 89 | `86baf84e7` | finding 20 | The spinner shows an interruption, not the fill a cold start or an unmute costs | -| 90 | `6e04673c9` | finding 24 | The audio transport is named once per page, at info level, rather than once per player at warning level | -| 91 | `b366db8bb` | finding 18 | A video track that stopped producing is rebuilt at the live edge, and a capture that stopped says why | -| 92 | `97f32ea66` | finding 21 | A media subscription reads from the live edge rather than replaying the window its predecessor filled | -| 93 | `3539e4ef2` | finding 22 | A camera that is only busy is asked again instead of spending the retry budget, and the reason reaches the buttons | -| 94 | `002ae00e1` | finding 19 | A video rendition's track stays open while it is not encoding, so a re-subscription is not answered from a finished track | -| 95 | `16e6242aa` | finding 23 | The audio context is built on the first gesture, or when the app turns audio on, and never at load for a muted tile | -| 96 | `c2e248ec5` | delivery | Findings 18 to 25, the measurements behind them, the API impact and the gates | -| 97 | `d2443eba9` | finding 26 | A group is judged by how far it could still reach, so a long GOP whose tail is late is not convicted, and the video estimator lives as long as the rendition | -| 98 | `208755120` | finding 27 | A test that a replaced session does raise a second video request, which refuted the reading of the relay-restart row | -| 99 | `804c13a20` | finding 27 | The video download gate is re-armed when the tile reconnects, so a rebuilt or moved tile asks for video again | -| 100 | `dc6f8d38c` | finding 30 | The broadcast stays announced while a device is replaced, so a switch does not drop every subscription | -| 101 | `69a89f17c` | finding 28 | `Sync.out.offset`: the sound is held for a picture that arrives later | -| 102 | `8b64a6aa6` | finding 28 | That hold is capped at what lip sync is worth | -| 103 | `5be36750f` | finding 31 | A draining relay refuses new sessions with 503, a drain is named as a drain, and the give-up window outlasts a restart | -| 104 | `eec9016d9` | finding 31 | The FFI and the Go wrapper follow the native give-up default | -| 105 | `b6c5a56f2` | delivery | The resilience matrix, findings 26 to 31, the API impact and the gates | -| 106 | `b80a2084a` | finding 31 | The reconnect test measures the outage rather than the paused clock's jumps | -| 107 | `b8596d0c2` | delivery | The quiet confirmation rows and the gates at the tip | -| 108 | `e6be6a775` | delivery | The listening round at this tip | -| 109 | `090965c10` | cleanup | Exports nothing consumes, and one ring test the surviving case already brackets | -| 110 | `c55af8566` | cleanup | The comments a later commit on this branch had made false | -| 111 | `988609dbc` | delivery | The cleanup sweep and the final counts | -| 112 | `50b0a72b9` | finding 32 | The two terms a tune-in and an unmute restart walk in rather than land in one step | -| 113 | `7168d1a8b` | finding 34 | Three watchers joining and leaving audio and video, and a fourth arriving after them | -| 114 | `e437fef5a` | finding 33 | Playout starts on the level the ring holds, so a first fill past it is trimmed rather than stretched away | -| 115 | `91c5878bd` | finding 33 | The same trim in the native engine | -| 116 | `088752023` | finding 35 | A rendition's arrival estimate outlives a gap in the rendition, so a camera hide does not reseed it | -| 117 | `cd43ff0a8` | finding 35 | A departed track's last reading stays in the shared delay for a window | -| 118 | `17d32eae4` | delivery | The rebase onto upstream dev, and findings 32 to 35 | -| 119 | `63ee6639c` | findings 36, 37 | The endpoint alone in its group is the discontinuity, raised when the group closes, so a declared pause plays out and the run after it reopens | -| 120 | `1c2605450` | findings 38, 39, 40, 44 | A finished head the cursor has reached is never convicted and one above it is walked onto; a broadcast that changes under a pinned tile is a tune-in; a rebuilt tile keeps the viewer's preset and the viewer's choice; a rendition that left the catalog is not a stall | -| 121 | `ca8d90857` | congestion campaign | The shaper takes a list of steps and its rate bucket can drop, so a capped link can be a tail-drop link | -| 122 | `1b0d8d831` | findings 41, 43 | The packaged relay unit passes its config file positionally, which is the only way the relay has ever read it | -| 123 | `54a6844ce` | delivery | The replay budgets for the two rows the declared pause added | -| 124 | `f4b1e0081` | delivery | Findings 36 to 44, the before, after, after2, after3, after4 and regression rows behind them, and the gates at that tip | -| 125 | this one | delivery | The rebase onto upstream dev of 2026-09-18, the watcher-side correction, and findings 45 to 47 | - -Row 49 carries two commits, so the numbered rows cover one hash more than there are rows. Every hash -`git log --oneline upstream/dev..HEAD` prints is in the table, in that order, and the last row is -this commit. Every hash above is the rebased one: the rebase of 2026-09-17 rewrote all of them, and -the section below records what it resolved. - -Suggested reading order for review: 3, 4, 9 to 11, 21, 22, 31, 78, and 119 and 120. The first eight -are the fix; the last two change how a marker group and a spent head are read, which is the part -worth reviewing even if nothing else is adopted. The -native half (7, 16 to 20) is the same algorithm again and can be read second or skipped entirely. -The harness (6, 23 to 30, 37 to 39, 44, 47, 48, 65, 69, 75, 79, 80) stands alone and can be adopted -without any of the player changes. - -The same commits, grouped by the quest they come from: - -| Quest | Commits | -| --- | --- | -| [`audio-jitter-target/spec.md`](https://github.com/moq-dev/moq/blob/dev/quest/m2/audio-jitter-target/spec.md) | `4c488b08a` (doc, estimator, 14-case corpus) | -| [`audio-jitter-target/watch.md`](https://github.com/moq-dev/moq/blob/dev/quest/m2/audio-jitter-target/watch.md) | `e4b7e9eff`, `bec43e540` (kixelated), `2d86e147a`, `bd27266cc`, `d3a53cc97` | -| the budget finding (root cause 3 above) | `c0f623941`, `9918f15fb`, `48bdbea5e` | -| [`m1/plan-av-clock.md`](https://github.com/moq-dev/moq/blob/dev/quest/m1/plan-av-clock.md) | `cf0436ae0`, `fdc9a2ae1` (a hole in the source reaches the decoder) | -| tune-in defect, found by the harness | `46d8081e0` (browser), `954f64090` (native) | -| [`audio-jitter-target/native.md`](https://github.com/moq-dev/moq/blob/dev/quest/m2/audio-jitter-target/native.md) | `f44ce10a4`, `90ac1fa96`, `47f6b3cba`, `fe316b664` | -| [`watch-audio-time-stretch.md`](https://github.com/moq-dev/moq/blob/dev/quest/m2/watch-audio-time-stretch.md) | `8eb68753c`, `23b9ffa86`, `ca06c55c8` | -| concealment (a departure, see "Departures from the quests") | `b7b3806b7`, `b23179839` (the element attribute) | -| [`transport-impairment-profile.md`](https://github.com/moq-dev/moq/blob/dev/quest/m2/transport-impairment-profile.md) | `953bb3c29` (`rs/moq-shaper`) | -| [`audio-quality-harness/browser.md`](https://github.com/moq-dev/moq/blob/dev/quest/m2/audio-quality-harness/browser.md) | `db3ce6e5d`, `69724d938`, `7a1850c90`, `caf5d93de`, `8dddb900d`, `0e2ffde66`, `d5a0b834a`, `02a117006`, `c784a1cdc` (replay lane), `78a4010c9` (nightly), `514db7c72`, `3a02a1afa`, `b34f8c952`, `69fe766f9` | -| [`qa-failure-artifacts.md`](https://github.com/moq-dev/moq/blob/dev/quest/m2/qa-failure-artifacts.md) | `558a679cc` | -| re-landed narrow fixes, each with a failing-then-passing test | `76c0cdab5`, `395daac3d`, `c38170a1f`, `e6f072009`, `1ecf18151` | -| the shaper defect found while re-measuring | `f86598325` | -| the one-frame hold, and the mute (root causes 4 and 5 in the branch's own numbering) | `a21ea530e`, `fffdeec14`, `0d4625677`, `6d7abd6ba`, `3f6bc1cdd`, `d3dc1bcbb` | -| the cold start, so a declaration is a prior the first measurement replaces | `e044c97a4`, `8245d8f07` (the budgets it moved), `31b38aed6` (the first measurement replaces the seed) | -| the unmute transient and the Safari unlock | `19edd01ce` (a flushed ring reports no playhead), `a1499011c` (the unlock is armed on the first gesture), `f846462e5` (the context outlives a catalog update) | -| the receiver stall (Firefox), found by watching two engines at once | `0b80a0257` (the estimator takes a stalled input), `b2099c477` (the recorded window replayed and graded) | -| the browser publisher's catalog and its clock | `3f83de0fc` (a served catalog track is released when its last subscriber leaves), `c53ecfe49` (captured audio is stamped on the context clock) | -| CodeRabbit fixes over the whole branch | `d8d78ead0`, `220b09934`, `1b90331be`, `6efdb3c73`, `c3571e072` | -| CI, and the rebase onto the lease model | `78a4010c9`, `116d4ad88` (the nightly job), `499d7806c` (the harness relay under `moq-auth`) | -| the listening round of 2026-09-17, findings 18 to 25 | `c90eec220`, `86baf84e7` (the unmute fill and the spinner), `6e04673c9` (the ring fallback line), `b366db8bb` (the video track that stopped), `97f32ea66` (the live edge), `3539e4ef2` (a busy camera), `002ae00e1` (the rendition's track stays open), `16e6242aa` (the context on the gesture) | -| the two transients the same listener reported next, findings 32 and 33 | `50b0a72b9` (the terms a tune-in restarts walk in), `e437fef5a` and `91c5878bd` (playout starts on the level the ring holds, both engines) | -| the multi-watcher round, findings 34 and 35 | `7168d1a8b` (three watchers and a fourth), `088752023` (the estimate outlives a gap in the rendition), `cd43ff0a8` (a departed track's reading is held for a window) | -| review, flake and delivery | `d862f4ab5`, `cd746e434`, `3f5828da7`, `edff16dd1`, `d2edb4b10`, `004721b0c`, `4450407cb`, `29af6fb9d`, `7d06c66c6`, `c096a1ace`, `8ea416d7d`, `462637ef7`, `b6c5a56f2`, `090965c10` and `c55af8566` (the cleanup sweep), `988609dbc`, plus this one | - -## Root causes - -Five defects. The first three were confirmed against `upstream/dev` when it was at `8f41d4d82`; the -last two are this branch's own, found by a listener on builds of it. - -### 1. The ring cap has no hysteresis and no refill - -`js/watch/src/audio/shared-ring-buffer.ts` skipped ahead whenever `buffered > latency`, and never -re-stalled after running dry. `ring-buffer.ts` mirrored it. Early arrivals were discarded, and the -moment the ring emptied it resumed on whatever landed next, on an empty cushion, so the next late -arrival was another stall. That is the stutter that never settles: at auto on the public relay, -most of a minute could be silence. - -### 2. The auto target came from the round trip - -`js/watch/src/sync.ts` computed `max(20ms, 1.25 x minRtt)`, plus a codec floor, with a flat 100 ms -fallback. The round trip says nothing about how evenly a publisher emits frames. The public -`bbb.hang` arrives in 7-frame bursts of about 162 ms, because ffmpeg packs roughly seven AAC frames -per PES and the TS importer forwards every frame of a PES in one pass. No round-trip formula can see -that, so the ring was sized for a retransmit and never held enough to play through a flush. - -### 3. The budget censored the estimator - -`Sync.out.maxAge` was both the wire `Subscription.maxAge` and the `Container.Consumer` skip -threshold. Any estimator measuring below it can only ever confirm the budget it was cut to. Worse, -`js/net/src/group.ts` raised a bare `Error` for a group that missed its deadline, while the sibling -verdicts `Lagged` and `GroupTooLarge` carried stream codes. `Container.Consumer` rethrew it, -`Effect.spawn` logged `spawn error`, the `finally` advanced the cursor anyway, and content -disappeared above the decoder with no counter anywhere. - -### 4. The ring held the target minus one frame - -Reported after the three above, from a browser publisher's own microphone: 20 ms Opus, one frame per -group, watched at auto on a local relay. The Latency panel read `jitter buffer 20ms (auto)`, `total -buffer 20ms`, and the audio stuttered through continuous speech, while the `bbb` sources with their -300 ms floors played clean on the same build. - -The estimator was right. 20 ms is its lowest value, the upper edge of the first histogram bucket, and -that microphone path really does deliver inside 20 ms. The ring was what was wrong: it un-stalled at -`WRITE - READ >= LATENCY`, refilled to the same after an underrun, and the level filter's lower bound -was the target itself. But the ring holds *unplayed* audio, and the engine takes a whole chunk out of -it to produce each block. A ring holding exactly the target therefore holds nothing unplayed at the -moment the engine asks, so it runs dry on the first arrival a millisecond late, re-stalls, refills to -the same empty level, and stutters there for as long as the call lasts. - -NetEq does not have this: its target delay counts *the packet being played* as well as the ones -waiting, because its buffer level is the `packet_buffer` span plus what the sync buffer still holds. -The maintainer's #3517 carried a "+ one frame" for exactly this reason; the objection recorded in -`quest/m2/audio-jitter-target/spec.md` was to *learning* that frame from timestamp gaps, not to the -term. The branch's own `doc/concept/playout.md` had already decided that the term belongs to the -ring's slack, but the slack had only ever been applied to the skip band, never to the level the ring -holds. - -### 5. The advertised jitter was carried twice - -The rendition's catalog `jitter` is the publisher's declared flush span. `Sync` used it twice: as a -floor under `auto` and as a term added to a fixed delay. The TS importer advertises 302 to 372 ms on -the `bbb` sources; the estimator on the same path reads 40 to 180. So `auto` never came below the -declaration for the length of a session however well the path delivered, and the demo's `100ms` -preset actually waited 402 ms. - -It is one quantity, not two. A flush span is exactly what the estimator goes on to measure, published -by the party that already knows it before the first frame lands. That makes it the best prior a -receiver has and a poor floor: a floor asserts something about the path that the receiver has since -measured for itself. NetEq guesses 80 ms (`kStartDelayMs`) at the same point only because RTP carries -nothing like a declared flush span. - -### What upstream had already done - -\#3478 is fixed by #3508. #3479 is fixed by #3513 and #3516. #3477 was closed as a re-plan (#3576). -\#2812 is still open. PR #3517 was closed unmerged: its estimator learned a frame duration from the -gap between consecutive media timestamps, so a tune-in that saw one frame of a stale group and then -the live edge read 14560 ms on its own head, and was still at 14360 ms ten seconds later. Everything -else in #3517 is kept here. - -## What changed, per stage - -### The written algorithm and the corpus - -`doc/concept/playout.md` is the algorithm, 450 lines, written once so two languages can be held to -it: the observation point, relative arrival delay against a min-deque over 2 s of media time, the -reordered-arrival rule without sequence numbers, 500 ms max-over-interval resampling, the 100 x -20 ms histogram with its seeded prior and its cold-start forget ramp, the 95th percentile walk, the -rise and fall rules, re-anchoring, the "plus one frame" decision, a parity-trap list, and the corpus -contract. - -The corpus is `rs/moq-audio/tests/playout-01.json`, 17 cases, generated by -`js/hang/src/container/jitter.vectors.ts` and freshness-checked in both languages. Every expected -target is an integer multiple of 20 ms, so an `f64` difference between V8 and rustc cannot break -parity. Both implementations replay it exactly, 17 of 17. - -The corpus lives in `rs/moq-audio` for the same reason `moq-json`'s vectors live where they do: the -crate that must `include_str!` it owns it, and `cargo package` cannot reach outside a package root. -`js/hang` generates it, and the JS scope in `js/justfile` covers that one path. - -### The estimator and its wiring - -`js/hang/src/container/jitter.ts` is the estimator. It observes at container frame arrival, before -the age budget can skip a group, one clock read per wire frame, taken before `Format.decode` runs so -this receiver's own parsing cost is not folded into a measurement of the network. - -`Sync` now has a floor and a ceiling rather than a formula. The rendition's advertised catalog jitter -is a publisher-declared flush span and the measured target is a measurement of the network, so auto -is the larger of the two, a fixed delay is their sum, and the ceiling is the estimator's own -histogram range. The render quantum left `playbackJitter()` and went into both rings' skip band, -where it was always describing something: a ring drains in whole blocks, so it sits a block above the -target between reads, and cutting audio for that is cutting audio the reader was about to play. -Native has no worklet, which is the other half of why it cannot be in a number two languages have to -agree on. - -The shared ring is now sized for the 2 s ceiling at construction. It used to be sized from the -initial delay, so an auto target climbing past it would have been capped by the overflow path with -nothing saying why. A target above capacity throws instead. - -### The budget - -Three commits, the third droppable. - -`Expired` is a `StreamError` on `StreamCode.DeliveryTimeout`, the code already registered for this, -so a caller that checks `err.code` reads the same answer whether the local budget gave up or a peer -reset the stream with it. `fromTransport` decodes it back. - -`Container.Consumer` then marks the group truncated, counts it, and rethrows only a real failure. -`#checkMaxAge` feeds the same counter once per shifted group. The counter reaches -`Audio.Decoder.out.skipped` and `Video.Decoder.out.skipped` and shows as a Skipped row beside -Underruns. An underrun count alone cannot show this: the ring never ran dry, the frames simply never -got there. Every `instanceof StreamError` catch in `js/` was audited for the new subclass; -`watch/text/renderer.ts` and `watch/broadcast.ts` got the same correction, and nothing else changed. - -The third commit is this fork's addition. Audio subscribes and consumes at `maxAge + headroom`, where -the headroom is the rounding between the two numbers and nothing more: one histogram bucket, one -frame duration, one stretch bound. `Sync.out.maxAge` itself is untouched, so video and the -`Sync.received` lookahead cap keep their semantics. - -### The A/V clock - -`Sync.reference` follows the audio playhead while audio plays. Both transports already knew where the -reader was, the shared ring through its read cursor and the postMessage ring through its state -message; each now reports it as a media position with the rate it is advancing at. Sync extrapolates -locally between samples, so the per-frame `wait()` never crosses a thread. A re-stall reports a rate -of zero and playback parks with it. A mute, an ended track, or a park that stops looking like a -refill hands the clock back to the wall clock at the last value the playhead gave. - -`SyncInput` is down to `{delay, buffer}`. The advertised delay and the measured spread are per-track -handles now, from `sync.track("audio" | "video" | "text")`, which is also where a track nominates its -playhead. Captions join without another pair of inputs. - -#### The mute that put video 2.8 s ahead - -Reported against the frozen site build after the branch was finished: changing the volume and the -latency presets a few times left the Latency tab showing video about 2.8 s ahead of audio, the two -visibly out of sync, and the audio bar still sitting on its target. - -Neither bar was lying. The audio playhead was three seconds behind the live edge, and since this -stage the playhead is the clock, video was held back there with it. What put it there is a WebCodecs -behaviour one layer up. An `AudioDecoder` timestamps its output by accumulating decoded frame -durations from the chunk that opened its run rather than by copying each chunk's own timestamp, so a -hole in the source is swallowed. Muting stops the download (`emitter.ts` clears `enabled`), so the -subscription that resumes starts at the live edge and the media covering the mute was never sent. -Wrapping `AudioDecoder` on the page shows the swallow exactly: the chunks in step from 54357386 to -54360428 us across the mute, and the samples out carry on at 54357409, 54357433, 54357456, 23.2 ms -apart. Every sample after that lands in the ring three seconds in the past, the ring plays a stream -that is contiguous and so never skips, and the playhead stays behind the live edge for good. It is -not specific to a mute: a group the age budget skips is the same hole one frame wide, which is why -the reported sequence ends 843 ms behind live rather than back where it started. - -The decode loop on `upstream/dev` has no hole handling either, and `terminal.ts` there is byte -identical, so the collapse itself predates this branch. What this stage changed is who follows it: -before, video paced against a wall clock and only the audio was late. - -Two changes, both in `fdc9a2ae1`. `Terminal.continues()` measures each frame against the last one at the -decoder's own frame duration and reports a hole; `Audio.Decoder.#reanchor` drains the decoder so the -frames in flight keep the run that is ending, then restarts it so the next chunk's own timestamp -opens the new one. `Audio.Decoder.#runFlush` drops what the ring still holds when the download -stops, because nothing drains it while the graph is disconnected and replaying it on the way back in -steps the playhead back to where the mute started. A flushed ring stops reporting a clock, so `Sync` -runs on the wall clock at the last audio-derived value until the ring anchors again. - -Driving the reported sequence on this tree's `demo/web` against the local relay in headless -Chromium, sampling the painted video timestamp against `sync.now()` every 250 ms over 58 s: - -| | before | after | -| --- | ---: | ---: | -| worst video-minus-audio skew | 3418 ms | 50 ms | -| samples worse than 200 ms, of 232 | 19 | 0 | -| video buffer at the end, at auto | 843 ms | 236 ms | -| underruns over the run | 6 | 0 | - -The shared-memory ring and `conceal=false` give the same answer (64 ms and 46 ms worst skew, zero -underruns). `sync.replay.test.ts` carries the sequence as a regression test: a real ring, a real -estimator, a real `Sync`, a model decoder that collapses a hole the way Chromium does, and one -`sync.wait()` per video frame. It asserts the painted frame stays within a video frame plus a poll -of the playhead and that audio ends at the live edge, then reruns with each of the two changes -switched off as the control, where the playhead ends 5.9 s behind live and the picture 5.8 s ahead -of it. `terminal.test.ts` covers the hole test and its re-anchor; `sync.test.ts` covers a clock that -is lost and re-nominated at a different playhead. - -Re-anchoring mid-group also has to hand the restarted decoder a chunk it will accept, which this -stage did not, and which killed the decode loop outright about one harness run in three. See finding -11\. - -### The tune-in defect - -This one was found by the harness and is worth reading on its own, because it is the bug that made -the first estimator look like it did not work. - -A viewer tuning in on a busy machine sat at a 1.2 to 1.9 s auto target twenty seconds later, falling -at exactly 20 ms a second. Neither number came from the network. The receiver's main thread blocks -for about 1.5 s at tune-in, on the decoder polyfill, the worklet start, and the first keyframe. -Nothing about the path changes: frames land on time and queue, and the read loop stamps every one of -them with the clock it reads when it runs again. The first frame out of the queue measured 1590 ms -against a reference taken before the block, the 500 ms resample interval committed that as its -maximum, and within 258 ms the receiver had drained 1834 ms of media and was back to a 13 ms steady -delay. Blocking the main thread for 1500 ms against a local relay reproduces it at 1620 ms. - -Two rules, one at each end. - -An arrival that follows a gap in the receiver's own reading longer than the resample interval, and -that covers less media than the gap was long, drops the arrival reference. The backlog behind it is -then measured against the path as the receiver can now see it. Dropping the reference and not the one -arrival is the point: every frame in the backlog is equally late against a reference from before the -gap, so skipping the first would leave the second to set the same interval maximum. The media term is -what separates the receiver from the publisher: a 1 fps track is idle for longer than the interval on -every arrival while its timeline advances just as far. - -The fall bound closes a sixth of the remaining distance per second, one bucket at the floor. What a -ring refills in a second scales with what it holds, so a fixed step brakes hardest exactly where the -target is furthest from what the histogram asks for. A sixth closes the histogram's whole range -inside the 29 s the histogram remembers, so the limiter can never outlast the observation that raised -the target. - -Measured on one set of live arrivals from a local relay with a 1500 ms block, the target every two -seconds for the first thirty: - -``` -before 80 140 160 1480 1480 1480 1460 1440 1400 1340 1300 1260 1240 1180 1140 1120 -fall 80 140 160 1480 1480 1480 1280 1120 860 600 480 400 380 320 280 260 -both 80 140 160 160 160 240 240 240 220 180 160 160 180 180 180 160 -``` - -All thirteen existing corpus cases are byte identical under both rules. The fourteenth, -`tune-in-stall`, records the shape and reads 1540 ms without the fix and 140 ms with it, in both -languages, to the millisecond. - -### Time stretch - -`js/watch/src/audio/playout/` holds one file per NetEq role and mirrors `rs/moq-audio/src/playout/` -file for file and constant for constant: - -| File | Mirrors | -| --- | --- | -| `level.ts` | `buffer_level_filter.cc` | -| `decision.ts` | `decision_logic.cc` | -| `stretch.ts` | `time_stretch.cc`, `accelerate.cc`, `preemptive_expand.cc` | -| `expand.ts` | `expand.cc` | -| `merge.ts` | `merge.cc` | -| `noise.ts` | `background_noise.cc` | -| `sync.ts` | `sync_buffer.cc` | -| `index.ts` | the loop that runs them | - -The ring reader split into `view`, `peek`, and `commit` first, with `read` as the three in a row, so -the worklet can look at the ring before deciding what to do with the media. No behaviour change: -every #3517 ring test passes unchanged. - -Per 20 ms output block the engine filters the buffer level, and if the level sits above -`target + chunk + 20ms` it drops one pitch period, or below the target and above half of it repeats -one, by the WSOLA splice NetEq uses: a 4 kHz coarse search over 2.5 to 15 ms lags, one full-rate -normalised correlation at the peak, and a passive gate at eight times the background so room tone -splices without a correlation at all. At most 15 ms per 100 ms of output, so convergence is a few -percent and inaudible. - -The skip-ahead is now the last resort. Its band is `target + chunk + 75ms`, and it rules on the -trough between two flushes rather than on the peak, so a flush that drains again is played rather -than discarded. The same 75 ms is the stretch term of the audio age budget's headroom, exported once -as `STRETCH_BOUND`. - -Both rings publish what the engine did, so the media playhead is `READ - queued` and its rate the -measured one rather than an output frame count a stretch has moved. - -### Concealment - -When the ring runs dry mid-playback the reader carries the audio on instead of ramping to silence: -the pitch period of the last real audio, repeated with a voiced and unvoiced mix set by how periodic -the signal was, muted block by block under NetEq's muting slope until it is digital silence, and -giving up entirely after two seconds. When the media comes back it is aligned against the -concealment and crossfaded in. That is NetEq's expand and merge. (An earlier commit on this branch -faded the tail into the measured room tone instead of to silence; finding 10 is why it no longer -does.) A hole that reaches the playhead with nothing behind it is no longer queued -as silence by the ring either: playing it would make a listener wait for the same missing audio -twice, once while the reader covers for it and again when the zeros arrive. - -`conceal` on the audio decoder, default on, turns it off and the ramp is back byte for byte. - -The invariant grows a term, since concealed frames are output but not media: - -``` -READ == output - concealed + stretched + queued + skipped -``` - -and the playhead rate follows at `1 + (dSTRETCHED - dCONCEALED)/dOUTPUT`, so video waits with the -audio rather than running through a concealed gap. - -### The native engine - -`rs/moq-audio/src/playout/` is the same engine in Rust, 4480 lines across 14 files, with the module -header carrying the role-to-`.cc` table. Three things differ from NetEq by design: PCM in and PCM -out, because decoding stays in `decode`; the sink pulls, because the speaker asks for one block at a -time on the device clock; and `f32` samples with `f64` arithmetic, because NetEq is fixed point only -because it targets DSPs. - -`decode::Config::delay` turns it on, and `Consumer::read` then hands back one 10 ms block per call -rather than whatever packet came out of the decoder. The block is always there: what arrived is -folded in, what did not is concealed, and nothing waits on the network. That is what lets the -speaker's own clock pace the pulls. - -It runs on the decode task, ahead of a 30 ms device cushion, not inside the device callback. The -callback mixes every sink on the device and may not allocate, lock, or log, and the engine does all -three, so a stream that had to conceal would cost every other stream on the device a dropout. - -Arrivals are measured off the container, before the age budget. The budget then follows the target -rather than the config, and `Config::max_age` becomes the ceiling on how deep the buffer may grow -rather than the budget itself. A floor that does not fit under the budget is refused at construction: -playout claims three quarters of it, so a budget four thirds of the floor would pin the target to the -floor, and a measured buffer that can never measure anything is a fixed one with extra steps. - -Replaying the same trimmed traces, the native engine settles on the same 80 ms and 240 ms targets the -browser does, and the corpus passes 17 of 17. - -### The chunk above the target - -One rule, in the same four places in both engines: the level a ring holds is `target + chunk`, where -`chunk` is the size of the most recent insert, measured on the way in and republished on every -arrival rather than learned from timestamp arithmetic. - -- Both rings un-stall at `WRITE - READ >= target + chunk` and a refill after an underrun reaches the - same level, bounded by what the ring can physically hold so an oversized decode cannot name a level - no refill could reach. -- The buffer level filter's band becomes `low = target + chunk`, `high = low + 20 ms`. `high` does - not move: it was already `target + chunk + 20 ms`. What moves is the floor, so a ring sitting - exactly where it is meant to sit is no longer read as one that needs expanding. -- The skip band stays where it was, so the band above the level the ring holds is exactly the stretch - bound: everything inside it is something the time stretch closes without dropping a sample. -- `rs/moq-audio` gets the same rule in `decision.rs` (`Decision::hold`), and the engine's two flush - paths are collapsed onto one so a target that has just fallen does not convict from the pull side - what it tolerates on the insert side. - -The audio age budget's headroom is unchanged and still exactly covers it: `Jitter.BUCKET + frame + -STRETCH_BOUND` is the bucket the estimator rounded up by, the chunk held on top of the target, and -the stretch band. - -`Sync.out.delay` stays the estimator's answer, because it is also the wire subscription's age budget -and what video paces against. The player's Latency panel now separates the two: "Jitter buffer" is -that value and "Total buffer" is `delay + chunk`, which is what a listener actually waits. The chunk -comes from the ring's own `audio.out.debug` snapshot, so no new public signal was added for it. - -### The cold start - -The declaration moves from `Sync` into the estimator, in both languages. `Container.Jitter` takes a -`start`, rounded up to a whole bucket, never below NetEq's 80 ms guess and never above the -histogram's 2 s range; `Container.Consumer` takes it as a `jitter` prop from the rendition config, -and `decode::Consumer` reads `catalog.jitter` for `Jitter::seeded`. - -**The first measurement replaces that start outright**, however far below it the measurement lands. -A seed is a prior, not an observation: the fall bound exists to keep one measurement from yanking -the buffer out from under the last one, and a declaration is not a measurement for it to protect. -NetEq is the same shape, `target_level_ms_ = underrun_optimizer_.GetOptimalDelayMs().value_or(kStartDelayMs)` -in `delay_manager.cc`, where the start delay is only what stands in until the optimizer has a value -and there is no fall bound at all. Running the seed through the bound instead, which is what the -first version did, left a 310 ms declaration sitting above a 20 ms path for tens of seconds on every -tune-in, and an unseeded 80 ms start above it for three. From the first observation half a second in -the target is a measurement, and rise and fall apply between measurements from there. - -`Sync` is then one term: `auto` is the measurement and a fixed delay is the number the viewer asked -for. `SyncTrack.advertised` has no reader left and goes. - -`playout-01.json` gains a `seeded` case: a 310 ms declaration starts the target at 320 ms -and the first resampled observation takes it to the 20 ms the steady trace settles on. A case -carrying `start_ms` is replayed from it in both languages. The 80 ms default is a seed on the same -rule, so nine of the fourteen existing cases lose their first seconds of walk-down too; every one of -them reaches the same settled target as before, and the arrival traces are untouched. - -The native `--delay` floor is untouched. That one is a caller asserting something the arrivals do -not say, which is a different thing from a publisher describing its own encoder. - -### The harness - -`test/audio-quality` plays a broadcast in headless Chromium over an impaired UDP path and counts what -a listener would have heard. The matrix is codec x profile x ring: 48 kHz Opus over fMP4 and -44.1 kHz AAC over MPEG-TS with ffmpeg's default PES packing, six profiles from `moq-shaper`, and both -the isolated and the postMessage ring, which is the production path. 24 rows. - -`rs/moq-shaper` is a new workspace crate, `publish = false`. It binds one socket, gives each observed -client 4-tuple its own upstream socket, and runs every datagram through one queue ordered by release -time with arrival order breaking ties, so a reorder is always a deliberate extra delay and never a -scheduler artifact. Each direction draws from its own seeded generator in a fixed order per datagram. -TCP on the listen port is pumped through untouched, because a relay serves `/certificate.sha256` over -HTTP on the same port and shaping a reliable transport would only measure how it retransmits. The -counters are as much the point as the impairment: a profile that silently treated nothing turns an -impaired run into an unimpaired pass, so a row whose active profile delayed nothing is void. - -Four things void a row rather than passing it quietly: the shaper's counters must show it treated -traffic; the session must have negotiated WebTransport rather than the TCP fallback that never -reaches the shaper; the document's isolation must match the ring the row asked for; and an -`AudioContext` whose `currentTime` stops tracking wall time voids the row, because that produces a -full set of plausible counters and reads as a flawless run. - -The metric schema in `src/schema.ts` is the deliverable that outlives this: units, clocks, stages as -exclusive spans with a named remainder, and an aggregation per metric, read by the page, the -analyzer, and the grader alike. Counters whose signals do not exist yet report null rather than zero, -and say which change would fill them in. - -The Safari lane is real Safari over safaridriver, not Playwright's WebKit, which is a different -network stack, media pipeline, and AudioWorklet scheduler. Three things follow from the engine and -each changes what the lane may claim: `@moq/net` refuses WebTransport on WebKit, so the session is a -WebSocket, which is TCP, which the shaper passes through, so those rows record `shaper: none` and -offer only the two profiles whose treatment is already nothing; Safari has to be frontmost, because -an unfocused window answers an Element Click with `{"value":null}` and the context sits in WebKit's -`interrupted` state over total silence; and `renderCapacity` is Chromium's alone, so `render_load` is -null and `worklet_cadence` stands in. - -## Evidence - -### The public relay, at auto, 120 s - -Headless Chromium, the production postMessage path, `cdn.moq.pro/demo` `bbb.hang`, unmuted, the real -`sites/watch` page built against this tree. - -| Build | Delay | Underruns | Short quanta | Concealed | Merges | Skips | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| estimator only | 200 ms | 36 | - | - | - | 0 | -| plus time stretch | 200 ms | 1 | 59 | - | - | 0 | -| plus concealment | 200 ms | 1 | 124 | 200 ms | 2 | 1 | - -The estimator-only build predates the engine and publishes no engine counters, so its short-quanta, -concealed and merge columns are absent rather than zero. - -### Local relay, bursty publisher - -The same page against a local relay publishing MPEG-TS with ffmpeg's default PES packing, which is -the arrival shape the public relay has. - -| Build | Delay held | Underruns | -| --- | ---: | ---: | -| estimator only | 700 ms | 6 | -| plus time stretch | 372 ms | 0 | -| plus concealment | 372 ms | 0 | - -### Concealment on versus off - -Back to back on the public relay, the only difference being `conceal` on the element, set before the -page connected. - -| conceal | Delay | Underruns | Short quanta | Concealed | Merges | -| --- | ---: | ---: | ---: | ---: | ---: | -| on | 200 ms | 3 | 70 | 360 ms | 3 | -| off | 200 ms | 5 | 151 | 0 | 0 | - -On the rare-tail traces from the first investigation, the 165, 220 and 111 ms gaps, concealment turns -130 silent quanta into 0. - -### Tune-in - -The estimator-only build settled between 0.9 and 1.9 s and stayed there for a minute on a loaded -machine. With both tune-in rules it settles within 5 s, and the ramp is gone from every lane. - -### Replay - -Recorded arrival traces replayed through both rings on a simulated clock. `lan-bbb` and -`relay-bbb-7frame` play with zero underruns and zero skipped samples after the target settles, -against 1 and 9/26 underruns at the round-trip target. `4k-webm` keeps one dry ring and one too-late -frame, which are holes in the recording itself. - -The A/V clock, replaying the recorded LAN trace through the real ring: video runs 7.1 ms ahead and -64.3 ms behind the audio a listener can hear, against 34.2 ms and 80.0 ms when it paced itself -against a wall clock. - -### The microphone traces - -Two new fixtures, trimmed the same way as the other three: `mic-local` is a browser publisher's -microphone over a local relay, `mic-remote` the same publisher through the public relay at a 45 ms -round trip. Both are 20 ms Opus, one frame per group, and both settle the estimator on its lowest -value, 20 ms, in both languages. - -Replayed at that settled target, which is the state a call reaches within a minute and stays in -(`fixed: 20`, counted after 1 s, identical on both rings): - -| trace | concealment | underruns | short quanta | level trough | -| --- | --- | ---: | ---: | ---: | -| `mic-local` | on | 1 -> 1 | 0 -> 0 | 0.0 -> 2.3 ms | -| `mic-local` | off | 2 -> 1 | 11 -> 7 | 0.0 -> 2.3 ms | -| `mic-remote` | on | 0 -> 0 | 0 -> 0 | 14.7 -> 19.7 ms | -| `mic-remote` | off | 1 -> 0 | 1 -> 0 | 0.0 -> 19.7 ms | - -The trough is the number the finding is about: before, the ordinary cadence took the ring to empty; -after, it rests a chunk higher and the floor is the target. `mic-local` keeps one underrun either -way, and it is not the rule: the trace carries one arrival 31 ms late, past the 95th percentile the -estimator reports by design, and past what 40 ms covers. Concealment carries it, which is why no -quantum reaches the device short. - -At the estimator's own target with its cold-start cushion still in the ring (the `auto` lane, counted -after 4 s) both traces are clean before and after: zero underruns, zero short quanta, zero skipped -samples. Twelve seconds is not long enough for a 20 ms ring to be pinned by an underrun, which is why -the fixed lens is the one that shows it. - -In the harness replay lane the same change moves the recordings that already had underruns: -`4k-webm` on the shared ring goes from 14.1 to 7.0 underrun episodes per minute and from 2.9% stalled -quanta to none. `lan-bbb` on the postMessage ring picks up one underrun in a ten second window where -it had none; that trace's arrivals run slower than its own media at the tail, so its buffer sits -within a few milliseconds of empty either way. Every replay budget was re-recorded from the measured -run and the lane is enforced. - -### The real microphone, headed - -Real Chromium with a window on the screen, the machine's own microphone through -`--use-fake-ui-for-media-stream` (the fake *device* is not used: its synthetic signal breaks the -encoder), one `publish.html` publishing camera and microphone and one `watch.html` watching it, both -against the local relay, sampled every 250 ms for 60 s and counted over the last 50. - -| build | ring | target | held | level min / p50 / max | underruns | A/V skew p50 | -| --- | --- | ---: | ---: | --- | ---: | ---: | -| before | shared | 40 ms | 60 ms | 22.2 / 69.1 / 114.7 ms | 0 | 1.8 ms | -| after | shared | 20 ms | 40 ms | 35.7 / 55.7 / 67.7 ms | 0 | -0.2 ms | -| before | postMessage | 40 ms | 60 ms | 53.7 / 77.0 / 97.0 ms | 0 | -14.6 ms | -| after | postMessage | 40 ms | 60 ms | 34.5 / 47.8 / 61.2 ms | 0 | -15.3 ms | - -Stated plainly: this path is clean enough that neither build stuttered in a minute. What the runs do -show is where the ring rests. Before, the shared ring ran down to 22 ms of unplayed audio against a -40 ms target, which is the state one late arrival turns into silence. After, its floor is 36 ms -against a 20 ms target, and the whole band is narrower: the engine is holding the level it was asked -to hold instead of drifting between the target and empty. - -### The three pages, live, after the cold start - -One 90 s headless Chromium run per cell, counted from 20 s in, unmuted at `delay="auto"`, re-measured -on a quiet machine at the tip. `demo/web` served plain is the production path and gets the -postMessage ring; the same build with COOP/COEP gets the shared one; the third is the copied moq.dev -site aliased at this checkout's `js/`. Every row's `AudioContext` opened at 44100 Hz, the Bluetooth -output device's own rate, so the chunk the ring holds on top of the target is 23.2 ms of AAC. - -| Page | Source | Ring | Target | Held | Level p50 / min | Underruns | Skips | Concealed | Accel | Expand | RTT | Skew p50 / p95 | -| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| demo, plain | `bbb-smooth.hang`, local | postMessage | 60 ms | 83.2 ms | 82.7 / 58.1 ms | 0 | 0 | 0 | 2 | 1 | 1 ms | -42.1 / 50.4 ms | -| demo, plain | `bbb.hang`, local | postMessage | 180 ms | 203.2 ms | 208.4 / 111.1 ms | 0 | 0 | 0 | 4 | 0 | 1 ms | -17.4 / 48 ms | -| demo, plain | `bbb.hang`, `cdn.moq.pro/demo` | postMessage | 200 ms | 223.2 ms | 216.9 / 70 ms | 0 | 0 | 0 | 32 | 34 | 39 ms | -18.7 / 33.6 ms | -| demo, isolated | `bbb-smooth.hang`, local | shared | 60 ms | 83.2 ms | 81.3 / 51.9 ms | 0 | 0 | 882 | 18 | 1 | 1 ms | -11.6 / 18.8 ms | -| demo, isolated | `bbb.hang`, local | shared | 180 ms | 203.2 ms | 200.4 / 86.2 ms | 0 | 0 | 0 | 1 | 1 | 1 ms | -0.8 / 7.4 ms | -| demo, isolated | `bbb.hang`, `cdn.moq.pro/demo` | shared | 220 ms | 243.2 ms | 242.1 / 104.3 ms | 0 | 0 | 0 | 11 | 16 | 44 ms | -20.2 / 40.6 ms | -| site | `demo/bbb-smooth.hang`, local | postMessage | 80 ms | 103.2 ms | 101.3 / 6.9 ms | 2 | 0 | 13230 | 22 | 1 | 1 ms | -14.4 / 40.8 ms | -| site, rerun | `demo/bbb-smooth.hang`, local | postMessage | 80 ms | 103.2 ms | 103 / 34.4 ms | 0 | 0 | 0 | 5 | 1 | 1 ms | -14.7 / 46.7 ms | -| site | `demo/bbb.hang`, local | postMessage | 180 ms | 203.2 ms | 200.2 / 102.7 ms | 0 | 0 | 0 | 3 | 0 | 1 ms | -32.6 / 42.4 ms | -| site | `bbb.hang`, cdn.moq.pro | postMessage | 240 ms | 263.2 ms | 238.5 / 110.7 ms | 0 | 0 | 0 | 28 | 31 | 46 ms | -19.2 / 47.1 ms | - -The two cdn rows from the demo pages are reached at `cdn.moq.pro/demo` rather than at the host alone, -because the public relay now closes a session opened at its root: the page opened one, subscribed to -the announce prefix `""`, was dropped and reconnect-looped, and no tile ever appeared. Both rows were -attempted twice against `https://cdn.moq.pro` and failed to start both times, so the path-qualified -substitutes are what the table carries. The site row needs no substitute because the site page -connects to `/` and so never opens a root session at all. - -The smooth publisher settles at 60 ms on both demo pages against the 302 ms its catalog advertises, -which is the whole point of the cold-start change: the declaration is where the estimate starts and -the measurement takes it from there. The bursty one settles at 180 ms, which is its real 140 ms flush -span plus a bucket. The public relay settles at 200 to 240 ms with a 39 to 46 ms round trip. Held is -the target plus the 23.2 ms chunk in every row, so the three cdn rows sit 23.2 ms above the 200 ms -relay rule by the chunk and by nothing else, and the two bursty LAN rows sit above the 150 ms LAN -rule because the source is the bursty publisher rather than because the estimator missed. - -Two rows are worth stating rather than smoothing over. The site page on the smooth local source -underran twice, at 26.3 s and 48.8 s, with the ring draining to 6.9 ms and 300 ms of concealment -behind it; the immediate rerun was clean, ring minimum 34.4 ms, and it is the only local page that -has produced an underrun at all. And the isolated smooth row conceals 882 samples, 20 ms, with no -underrun behind them. - -Every row negotiated WebTransport, and only the isolated page reports `crossOriginIsolated`, so each -one ran the ring it was meant to. `sync.out.delay` is the estimator's answer alone now, so what a -listener waits is that plus the chunk the ring holds. - -### Conferencing targets - -The thresholds a WebRTC conference is held to, applied to the microphone runs: one browser publisher -on the pinned USB device, one watcher, the local relay, 60 s counted from 20 s in. "Held" is the -estimator's target plus the chunk the ring keeps on top of it, which is what the listener waits. The -lead column is explained below. - -| case | ring | jitter target (20-100 ms) | held (LAN < 150 ms) | underruns after convergence | lead | -| --- | --- | ---: | ---: | ---: | ---: | -| Chromium, steady | postMessage | 40 ms | 60 ms | 0 | +1.1 ms | -| Chromium, steady | shared | 40 ms | 60 ms | 0 | +44.3 ms | -| Firefox, steady | postMessage | 40 ms | 60 ms | 0 | +5.6 ms | -| WebKit, steady | postMessage | 40 ms | 60 ms | 0 | +4.1 ms | -| Safari 26, steady | postMessage | 40 ms | 60 ms | 0 | -2.9 ms | -| Safari 26, mute and preset sequence | postMessage | 40 ms | 60 ms | 0 | +2.1 ms | -| WebKit, mute and preset sequence | postMessage | 40 ms | 60 ms | 0 | +5.7 ms | -| Chromium, mute and preset sequence | shared | 40 ms | 60 ms | 0 | +39.4 ms | -| `mic-local` fixture, both rings | | 20 ms | 40 ms | 0 | not measured: arrival-only trace | -| `mic-remote` fixture, both rings | | 20 ms | 40 ms | 0 | not measured: arrival-only trace | - -Every case is inside every threshold. The LAN total is 60 ms against a 150 ms ceiling on every engine -and both rings, so the question of why it might sit above does not arise. - -**What the skew metric can and cannot resolve.** A/V skew here is the painted video timestamp less -the audio playhead the player reports, sampled on a 250 ms grid. It has a floor of one frame plus one -display refresh, because the painted frame is the newest one due and it was painted at the last -vsync: 58 ms at 24 fps and 50 ms at 30 fps. It has a ceiling of one playhead reporting interval, -because the reported playhead is not extrapolated while the one video is paced against is: 14.5 ms of -worklet state messages at 44.1 kHz, 13.3 ms at 48 kHz, and one 50 ms poll on the shared ring. Every -settled sample on every row of this pass is inside that envelope to within 3 ms, and no postMessage -row ever paints ahead of the playhead by more than one state message, the largest positive sample -anywhere being +14.4 ms. Audio and video are within one frame on every row. - -So the pair to grade is signed, not absolute. The **lead** column above is the signed p95: -13.6 to -+5.7 ms on every row but the two shared-ring microphone rows, which read +39.4 and +44.3 ms and are -the 50 ms poll's frozen phase rather than a lead, since both `skew` and `videoLead` moved by the same -40 ms while the ring's measured depth did not move at all. The **lag**, the signed p05, is inside the -floor everywhere: -3.6 to -50.4 ms at 24 fps and -23.1 to -37.2 ms at 30 fps. - -The `Skew p95` column the earlier tables printed is the p95 of `|skew|` over that same distribution, -which is **41 to 49 ms by construction** (0.95 of one frame plus one refresh). Measured values of 30 -to 50 ms are therefore at or below the metric's own worst case, and grading them against one frame -grades the sampler rather than the player. The pre-fix "1 to 19 ms" column in the older browser table -was the *signed* p95 under the same name, so the two columns are not comparable and the distribution -did not move between them; the statistic that did move is `videoLead`, from 390 ms before finding 14 -to 64 to 91 ms after it. - -### The budget - -`budget-spares-an-arrived-flush` in `replay.test.ts` replays `relay-bbb-7frame` through a real -subscription and a real `Container.Consumer`. In steady state it now convicts nothing at any budget: -0 of 443 groups at the measured target, 0 with the headroom, and 0 at the 46 ms round-trip budget the -estimator replaced, which is a third of this publisher's 139 ms flush span. The measured target -clears the flush span by a wide margin either way, so the headroom is insurance against a path whose -target lands close to the flush rather than anything this recording needed. - -The same replay censored 91 of those groups when the head was judged by its timestamps alone. What -changed is what the budget rules on above the decoder: a group that has arrived is never convicted. -A finished head the delivery cursor has reached belongs to `next()`, which hands over whatever is -still queued and pops the group once it is spent; a finished head above the cursor is walked onto, -since what never arrived is the sequences below it; and what is left to convict is a group still -arriving with nothing to hand over. Every one of the 122 convictions that survived the walk read -`queued=0 cursor= closed`: a head the listener had already heard, still in the list -because `next()` had not popped it yet. Convicting one lost no audio, but it counted a skip, reported -the next delivery as discontinuous, and, because a Legacy frame carries no duration, judged the -contiguous successor a hole and re-anchored the reader. That was the watch-reload conviction. - -**A behaviour change for the maintainer to confirm**: `skipped` now counts only media nobody could -take. At `instant` (a zero budget) a reader that keeps up reports nothing at all, where it used to -count every group it had played. The two zero-budget cases in `consumer.test.ts` were updated to the -new semantics: one asserts that a reader keeping up plays the whole track and counts nothing, the -other that a group still arriving is counted, once each time it happens. - -### CPU - -`stretch.bench.test.ts` drives 60 s of quanta through the stubbed worklet forcing one operation per -100 ms. p999 of `process()` is 0.02 ms against a 0.67 ms budget, a quarter of the quantum, both while -stretching and while concealing. - -### Harness, before and after the estimator - -24 rows at 40 s, `upstream/dev` against the estimator-only tree, recorded in the run directory. The -shape is the same across the matrix: discarded audio falls sharply and the target inflates. - -| Row | Skipped ms/min | Target p95 | -| --- | --- | --- | -| opus, high-rtt, plain | 11491 -> 1935 | 156 -> 1760 ms | -| opus, bursty, plain | 6921 -> 369 | 186 -> 1540 ms | -| aac, bursty, plain | 5828 -> 85 | 395 -> 440 ms | -| opus, mild, plain | 4912 -> 0 | 63 -> 1980 ms | -| opus, step, isolated | 5504 -> 0 | 67 -> 240 ms | - -Two things inflate the targets in that column, and neither is the estimator. The first is the -tune-in defect: those runs are 40 s long, the main-thread block happens at tune-in, and the fall -bound at the time held the target up for the rest of the run. They are the measurement that led to -the two rules above. The second was not known until the budgets were being enforced: the shaper's -own jitter was reordering datagrams (finding 8), so on every jittered profile QUIC was -retransmitting and the target was tracking the video stream's congestion response. The numbers below -are re-measured on the fixed shaper and on 60 s rows. - -### The enforced budgets - -`budgets.json` was last re-recorded in `063cd3031`, after the cold-start change and the decoder -anchor, from two 60 second runs of the whole matrix: the worst of the two times 1.5, a hard 0 on the -counters the engine has to keep at zero wherever both runs measured zero, a share capped at 1 and a -convergence time capped at the row's own run. `b2099c477` then added the two `mic-firefox` replay -rows. - -A row is enforced when its two runs agreed within that same 1.5 on every graded metric, which is a -strict test: a metric one run measured as zero and the other did not is a disagreement, because the -ratio is unbounded. **Three Chromium rows clear it**, all three of them `fixed-250`, which is the -profile that adds a constant delay and nothing else. The other twenty-one keep the `recorded` -marker, which prints a breach and calls it out without failing the run, until the nightly runner -records its own. An earlier version of this report said ten of twenty-four rows were enforced with -120 checks; that was the previous recording, and the re-record on a tree whose targets settle within -a second rather than walking a declaration down for 40 s moved most of the rows out of agreement -with themselves. - -| Row | Enforced | underrun ep/min | skipped ms/min | silence | target p95 | converge s | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| opus, near-zero, isolated | | 1.7 | 989.1 | 0.183 | 240 | 84 | -| opus, near-zero, plain | | 0 | 0 | 0.389 | 240 | 46.5 | -| opus, mild, isolated | | 0 | 0 | 0.477 | 480 | 60.3 | -| opus, mild, plain | | 0 | 662.8 | 0.177 | 2160 | 73.5 | -| opus, bursty, isolated | | 0 | 1959.3 | 0.15 | 1020 | 87.8 | -| opus, bursty, plain | | 0 | 3295 | 0.564 | 2520 | 81.3 | -| opus, step, isolated | | 0 | 419.4 | 0.088 | 690 | 73.8 | -| opus, step, plain | | 0 | 4377.9 | 0.436 | 2880 | 88.2 | -| opus, high-rtt, isolated | | 1.7 | 70.5 | 0.007 | 480 | 85.2 | -| opus, high-rtt, plain | | 0 | 947.1 | 0.102 | 510 | 78.8 | -| opus, fixed-250, isolated | yes | 0 | 0 | 0.41 | 375 | 0 | -| opus, fixed-250, plain | | 0 | 0 | 0.422 | 375 | 0 | -| aac, near-zero, isolated | | 0 | 0 | 0.21 | 420 | 71.6 | -| aac, near-zero, plain | | 0 | 134.4 | 0.075 | 270 | 3 | -| aac, mild, isolated | | 0 | 351.5 | 0.402 | 990 | 60.8 | -| aac, mild, plain | | 0 | 2238 | 0.552 | 1710 | 87.8 | -| aac, bursty, isolated | | 0 | 2762.7 | 0.464 | 2640 | 87.8 | -| aac, bursty, plain | | 1.7 | 5172.9 | 0.075 | 1770 | 81 | -| aac, step, isolated | | 1.7 | 140.2 | 0.061 | 540 | 54.8 | -| aac, step, plain | | 1.7 | 818.2 | 0.177 | 480 | 60.4 | -| aac, high-rtt, isolated | | 0 | 85.9 | 0.401 | 330 | 11.6 | -| aac, high-rtt, plain | | 0 | 555.6 | 0.45 | 570 | 57.3 | -| aac, fixed-250, isolated | yes | 0 | 0 | 0.171 | 375 | 0 | -| aac, fixed-250, plain | yes | 0 | 0 | 0.191 | 375 | 0 | - -The matrix run at the branch tip passed all 36 enforced checks across the 24 rows and breached 17 -recorded ceilings. Those breaches are the reason the twenty-one are not enforced yet, rather than -pinned at a number that would fail on the weather. - -| Row | Metric | Measured | Ceiling | -| --- | --- | ---: | ---: | -| aac, bursty, isolated | underrun episodes/min | 1.1 | 0 | -| aac, bursty, isolated | underrun ms/min | 272.1 | 0 | -| aac, bursty, isolated | skipped ms/min | 3609.7 | 2762.7 | -| aac, bursty, plain | skipped groups/min | 21.8 | 9.8 | -| aac, high-rtt, isolated | target p95 | 340 | 330 | -| aac, high-rtt, isolated | converge s | 42.7 | 11.6 | -| aac, high-rtt, plain | skipped ms/min | 2895.3 | 555.6 | -| aac, near-zero, plain | converge s | 3.5 | 3 | -| opus, bursty, isolated | underrun episodes/min | 1.1 | 0 | -| opus, bursty, isolated | underrun ms/min | 272.5 | 0 | -| opus, bursty, plain | underrun episodes/min | 1.1 | 0 | -| opus, bursty, plain | underrun ms/min | 273.6 | 0 | -| opus, mild, isolated | stalled quanta share | 0.005 | 0 | -| opus, mild, isolated | skip-aheads/min | 1.1 | 0 | -| opus, mild, isolated | skipped ms/min | 44 | 0 | -| opus, step, isolated | underrun episodes/min | 1.1 | 0 | -| opus, step, isolated | underrun ms/min | 271 | 0 | - -Two observations about that list. `aac-high-rtt-isolated` converging in 42.7 s against an 11.6 s -ceiling is well outside the run-to-run spread the rest of the table shows, and it is open: nothing -here explains it. The rest is one shape repeated. Ten of the seventeen are a hard zero breached by a -single episode at 1.1 a minute and the 272 ms of concealment that episode cost, and a hard zero is -what the recording rule produces when both recorded runs happened to measure zero on a counter that -is not reliably zero. That is a recording artefact rather than a regression, and it is what the -`recorded` marker exists to absorb. - -One row voided on the same run: `opus-near-zero-isolated`, on the clock guard, with -`AudioContext.currentTime` drifting 4.92 percent from wall clock over 10 s. A void row is reported -rather than enforced on a recorded budget, and a drifting context is exactly the state that would -otherwise produce a full set of plausible counters. - -The replay lane is enforced in full, because it is deterministic: 148 checks across twelve rows, -every ceiling exactly what was measured, no void row. `lan-bbb` and `relay-bbb-7frame` hold zero -skips and zero discarded samples on both rings at an 80 ms and a 240 ms target. `4k-webm` keeps 7 -underrun episodes a minute and 1267.2 skipped ms, which are the holes in the recording. `mic-local` -and `mic-remote` are clean at 40 ms and 20 ms. `mic-firefox` is the receiver-stall recording from -finding 12, at a 300 ms target. - -**Residuals.** Seventeen rows carry a measured ceiling on a counter the engine is supposed to keep -at zero. Each is work that is left, not a bar that was cleared. - -| Row | skip-aheads/min | underrun episodes/min | -| --- | ---: | ---: | -| opus, near-zero, isolated | 13.0 | 1.7 | -| opus, mild, plain | 8.2 | | -| opus, bursty, isolated | 13.0 | | -| opus, bursty, plain | 41.0 | | -| opus, step, isolated | 4.9 | | -| opus, step, plain | 63.8 | | -| opus, high-rtt, isolated | 1.7 | 1.7 | -| opus, high-rtt, plain | 4.9 | | -| aac, near-zero, plain | 3.3 | | -| aac, mild, isolated | 4.9 | | -| aac, mild, plain | 24.6 | | -| aac, bursty, isolated | 34.3 | | -| aac, bursty, plain | 18.0 | 1.7 | -| aac, step, isolated | 3.3 | 1.7 | -| aac, step, plain | 13.0 | 1.7 | -| aac, high-rtt, isolated | 1.7 | | -| aac, high-rtt, plain | 11.4 | | - -Seven of the twenty-two entries are a ceiling of 1.7, which is one event in a minute measured once -and not the other time. The real remainder is the `bursty` and `step` skip-ahead ceilings, -`opus-step-plain`'s 63.8 worst of all: a 160 ms flush arriving as one burst still outruns what 15 ms -of stretch per 100 ms of output can absorb, so the band still fires. Bringing those to zero is the -next piece of work on the engine, not something this branch claims. - -### Before and after, on the fixed shaper - -`upstream/dev` at `246a4733f` (the base the before-run was taken on) against this branch, same -desktop, one row at a time. **The two sides -are not the same length**: the baseline was recorded in Stage 8a at 40 s a row and this branch's -rows are 60 s. The per-minute counters are comparable; `target p95` and `silence` are not strictly, -because the target has 50% more run to settle in and the silence share is taken over a longer -window. The gate is `budgets.json`, which is two 60 s runs of this tree against each other. - -| Row | underrun ep/min | skipped ms/min | silence | target p95 | -| --- | ---: | ---: | ---: | ---: | -| opus, near-zero, isolated | 0 -> 1.1 | 621.3 -> 587.6 | 0.093 -> 0.118 | 63 -> 100 | -| opus, near-zero, plain | 0 -> 0 | 0 -> 0 | 0.149 -> 0.245 | 63 -> 80 | -| opus, mild, isolated | 0 -> 0 | 0 -> 44.7 | 0.143 -> 0.345 | 63 -> 280 | -| opus, mild, plain | 8.6 -> 0 | 4911.5 -> 0 | 0.471 -> 0.1 | 63 -> 160 | -| opus, bursty, isolated | 8.6 -> 2.2 | 6158.2 -> 486.4 | 0.293 -> 0.1 | 67 -> 340 | -| opus, bursty, plain | 3.4 -> 1.1 | 6921.2 -> 1612.1 | 0.436 -> 0.382 | 185.8 -> 1720 | -| opus, step, isolated | 3.4 -> 0 | 5503.6 -> 0 | 0.107 -> 0.05 | 67 -> 300 | -| opus, step, plain | 0 -> 0 | 1111.7 -> 1481.1 | 0.286 -> 0.258 | 67 -> 1500 | -| opus, high-rtt, isolated | 0 -> 0 | 2013.8 -> 46 | 0.443 -> 0 | 142 -> 280 | -| opus, high-rtt, plain | 0 -> 1.1 | 11491.3 -> 44.3 | 0.307 -> 0.059 | 155.8 -> 180 | -| opus, fixed-250, isolated | 0 -> 0 | 0 -> 0 | 0.333 -> 0.259 | 297 -> 294 | -| opus, fixed-250, plain | 0 -> 0 | 0 -> 0 | 0 -> 0.281 | 297 -> 294 | -| aac, near-zero, isolated | 0 -> 0 | 0 -> 0 | 0 -> 0.136 | 395 -> 372 | -| aac, near-zero, plain | 0 -> 0 | 0 -> 0 | 0.064 -> 0.045 | 395 -> 372 | -| aac, mild, isolated | 0 -> 0 | 253 -> 0 | 0.489 -> 0.241 | 395 -> 372 | -| aac, mild, plain | 0 -> 0 | 715.9 -> 0 | 0.657 -> 0.314 | 395 -> 372 | -| aac, bursty, isolated | 0 -> 0 | 74.7 -> 3530.7 | 0.271 -> 0.309 | 395 -> 1000 | -| aac, bursty, plain | 0 -> 0 | 5827.9 -> 518.8 | 0.329 -> 0.005 | 395 -> 640 | -| aac, step, isolated | 0 -> 0 | 0 -> 0 | 0.121 -> 0.032 | 395 -> 372 | -| aac, step, plain | 0 -> 1.1 | 0 -> 0 | 0.007 -> 0.123 | 395 -> 372 | -| aac, high-rtt, isolated | 0 -> 0 | 0 -> 0 | 0.716 -> 0.267 | 507.5 -> 372 | -| aac, high-rtt, plain | 0 -> 0 | 0 -> 0 | 0.567 -> 0.294 | 475 -> 460 | -| aac, fixed-250, isolated | 0 -> 0 | 0 -> 0 | 0.213 -> 0.118 | 625 -> 622 | -| aac, fixed-250, plain | 0 -> 0 | 0 -> 0 | 0 -> 0.141 | 625 -> 622 | - -The shape is the one the branch is for. Where the baseline was discarding audio to hold a -round-trip-sized buffer it now holds a measured one and discards nothing: `opus-high-rtt-plain` -drops from 11.5 s of skipped audio a minute to 44 ms, `opus-mild-plain` from 4.9 s and 8.6 underrun -episodes to zero of both, `opus-step-isolated` from 5.5 s and 3.4 to zero, `aac-bursty-plain` from -5.8 s to 519 ms with the silence share going from 0.329 to 0.005. The targets rise to pay for it, -which is the trade: a buffer that can absorb a 160 ms flush cannot also be 63 ms deep. - -Two rows go the other way and are stated rather than explained away. `aac-bursty-isolated` skips -more than the baseline did (74.7 ms -> 3530.7 ms a minute) while its target climbs to 1000 ms: the -burst profile against the TS publisher's own multi-frame PES packing is the one arrival shape where -the estimator and the skip band still argue, and it is the same cell that would not agree with -itself across two runs. `opus-step-plain` skips 1481 ms against 1112 ms at a 1500 ms target. Both -are `recorded` rows and both are in the residual list above. - -### Browser coverage - -Chromium is not the only engine a viewer brings, and Firefox and Safari matter more than their share -here because the WebSocket fallback has an arrival shape of its own. So the same build was played in -four engines against one browser publisher on the real microphone, 60 s a row, sampled every 250 ms. -The reference is a WebRTC conference: a 20 to 100 ms jitter target, under 150 ms held on the LAN, -audio and video within a frame, and no underruns once converged. - -Engines: Chromium 153.0.8010.12, Firefox 155.0, Playwright WebKit 26.6, and real Safari 26 driven -through `safaridriver`. Held is the settled target plus the chunk the ring holds, which is what a -listener actually waits. Skew is the painted video timestamp minus the audio playhead, so a negative -number is video behind audio; read it against the resolution stated under "Conferencing targets" -above. `videoLead` is the newest received frame less the playhead, which is the statistic finding 14 -moved. - -These are the rows re-measured on the fixed publisher. The microphone is the shape a conference has, -and it is the shape both publisher findings live in, so it is what was re-run. The file-source rows -per engine are from the earlier build and are carried below exactly as they were measured. - -| Engine | Page | Ring | Transport | Target | Held | Level p50 / p95 / min | Underruns | Skips | Skew p50 / p95 | videoLead p50 | ctxRate | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| Chromium | plain | postMessage | webtransport | 40 ms | 60 ms | 60.1 / 80.1 / 30.1 ms | 0 | 0 | -19.2 / 33.9 ms | 75.5 ms | 48000 Hz | -| Chromium | isolated | shared | webtransport | 40 ms | 60 ms | 60.2 / 80.2 / 40.2 ms | 0 | 0 | +23.9 / 44.4 ms | 114.3 ms | 48000 Hz | -| Firefox | plain | postMessage | webtransport | 40 ms | 60 ms | 60.1 / 80.1 / 40.1 ms | 0 | 0 | -15.2 / 36.2 ms | 81.4 ms | 48000 Hz | -| WebKit | plain | postMessage | websocket | 40 ms | 60 ms | 60.1 / 80.1 / 20.1 ms | 0 | 0 | -11.2 / 30.1 ms | 64.4 ms | 48000 Hz | -| Safari 26 | plain | none | none | | | | | | | | | -| Safari 26, retry | plain | postMessage | websocket | 40 ms | 60 ms | 50.1 / 70.1 / 30.1 ms | 0 | 0 | -17.4 / 36.5 ms | 69.8 ms | 48000 Hz | - -Counters are the delta over the settled window, which starts 20 s in. The publisher is one Chromium -page on the real device with echo cancellation, auto gain, noise suppression and Opus DTX all off, -confirmed on the element's own track rather than on a second `getUserMedia`, and restarted before -every row. - -Three things the table says. The estimator lands on 40 ms on every engine and both rings, so they -agree with each other to the bucket. Held is 60 ms everywhere, a third of the LAN ceiling. And no row -underran or skipped, including the two that ran the WebSocket fallback. - -Firefox negotiated WebTransport rather than the WebSocket fallback, because the user-agent gate in -`js/net/src/connection/browser.ts` admits Firefox from 153.0 and this is 155.0. So the WebSocket lane -here is WebKit and Safari, not Firefox. Firefox no longer underruns after convergence on this tip; -the episodes an earlier pass recorded were its own content process blocking its read loop and the -estimator taking the block as path delay, which is finding 12. - -The shared-ring row's `+23.9 ms` skew p50 and its 114.3 ms `videoLead` are the 50 ms poll's reading -offset, not a deeper buffer: both statistics moved by the same 40 ms against the plain row while the -ring's held and level agree to 0.1 ms. That is the "Conferencing targets" resolution again. - -**The first real-Safari attempt produced no audio at all.** One Element Click was delivered, -`document.hasFocus()` was true, the `mic.hang` tile was found and its catalog arrived, video kept -painting for the whole minute, and the `AudioContext` read `suspended` at all twenty samples of the -five second post-click timeline and never reported a playhead within 60 s. The retry played, with the -context `suspended` at 34 ms and `running` from 288 ms onward. That is one run in eight on this -build, and it is finding 15: the click did start a context, and the next catalog frame closed it and -built a replacement that was born suspended. After `f846462e5` the same driver, the same one click, -ran 8 of 8 with the context `running` 4 to 12 ms after the click and a single context per run. The -two Playwright microphone rows were re-run on that commit as a control and are unchanged: Chromium -held 60 ms with 0 underruns and skew p50 -10.2 ms, Firefox held 60 ms with 0 underruns and skew p50 --16.8 ms. - -**The file sources, per engine, on the earlier build.** The same engines against `bbb-smooth.hang` on -the local relay and `bbb.hang` on the public one, 90 s a row, sampled every 250 ms. They were not -re-run after the two publisher fixes, because neither fix can reach a file publisher: both `bbb` -sources come off one container timeline and so carry no epoch offset at all. Chromium's rows for the -same two sources are in "The three pages, live" above. - -| Source | Engine | Transport | Target | Held | Underruns | Concealed | Skew p50 / p95 | -| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | -| `bbb-smooth.hang`, LAN | Firefox | webtransport | 100 ms | 123 ms | 1 | 40 ms | -24 / 3 ms | -| `bbb-smooth.hang`, LAN | WebKit | websocket | 80 ms | 103 ms | 0 | 0 | -21 / 2 ms | -| `bbb-smooth.hang`, LAN | Safari 26 | websocket | 60 ms | 83 ms | 0 | 0 | -22 / 3 ms | -| `bbb.hang`, cdn.moq.pro | Firefox | webtransport | 200 ms | 223 ms | 0 | 0 | -26 / 1 ms | -| `bbb.hang`, cdn.moq.pro | WebKit | websocket | 200 ms | 223 ms | 0 | 0 | -20 / -1 ms | -| `bbb.hang`, cdn.moq.pro | Safari 26 | websocket | 200 ms | 223 ms | 1 | 60 ms | -21 / 2 ms | - -The estimator lands in the 20 to 100 ms band on every engine for both LAN sources and the engines -agree to within one 20 ms bucket. The public relay settles at 200 ms on all three, 223 ms held once -the 23.2 ms AAC chunk is counted. The skew columns here are the **signed** p50 and p95, which is the -statistic the older tables printed under that name, so they are not comparable with the p95 of the -absolute value printed in the microphone table above; "Conferencing targets" says which is which and -why. - -### The mute and preset sequence, per engine - -The user's sequence on the microphone broadcast: ten seconds at auto, mute for three, unmute, then -2000 ms, 100 ms and auto presets for eight seconds each, five quick mute/unmute pairs, and ten -seconds idle. Skew is measured from the mute onwards, because the opening ten seconds are the cold -start rather than the toggle. - -| Engine | Worst skew | Where | Skew p50 | Underruns | -| --- | ---: | --- | ---: | ---: | -| Firefox | 3256 ms | 0.2 s after unmute | -14 ms | 0 | -| WebKit | 3076 ms | 0.1 s after unmute | 3 ms | 0 | -| Safari | not measured | the ring never started on that build | - | - | - -Real Safari on that build reached a tile and a catalog and no audio playhead, so there was nothing to -measure a skew against. What that was is the retraction below and finding 15: a click did start the -context, and the catalog frame that follows it closed that context and replaced it with a suspended -one. - -Both engines that ran it spike the same way at the same place: unmuting after three seconds of mute -leaves video roughly three seconds ahead of the audio playhead for about a second, then it recovers. -It is the muted interval reappearing as skew, and it is the same shape in an engine on WebTransport -and an engine on a WebSocket, so it is the player rather than the transport. It survived the fixes -the branch had at the time of this run, and the paragraph below is what it turned out to be. - -The mechanism is the port hop the postMessage ring reports its playhead over, which is the ring a -page gets unless it is cross-origin isolated. A mute flushes the ring, and the state message already -on the port when `reset` posts describes the ring the flush threw away, so its playhead is a whole -mute behind the one the reader resumes from. It was also the last thing the main thread heard until -the graph was connected again, because a disconnected worklet is never pulled and so never sends -another, which is the second the spike lasts. The worklet now echoes the timeline of the flush it -last applied and the main thread drops any message from an older one. The shared ring needs no -counter: a flushed ring is unanchored and simply publishes no playhead until the next insert -re-anchors it. A timestamp comparison would not do in either case, because the stale message carries -a position the ring really was at, and one the reader will never resume from does not look any -different from one it has not reached yet. `sync.replay.test.ts` replays the sequence over the -fallback transport with that message delivered, and the skew it measures goes from 2992 ms to 37 ms. - -The sequence was then run again on a served build of the tip, against one Chromium publisher on the -real microphone. The spikes are gone. What `19edd01ce` left behind was 332.9 ms in Firefox, and that -residual was not the player at all: it was the publisher's own audio and video epochs sitting a third -of a second apart, which a mute exposes because it takes the audio clock away and hands the picture to -the video arrivals. That is finding 14, fixed in `c53ecfe49`. What is left is one held interval: the -target plus the chunk the ring holds, 60 ms in these runs, and every worst reading is that interval -wide. - -| Engine | Ring | Worst skew | Where | Skew p50 | Underruns | -| --- | --- | ---: | --- | ---: | ---: | -| Safari 26 | postMessage | -41.1 ms | 1.3 s after `delay 100ms` | -13.9 ms | 0 | -| WebKit | postMessage | 64.3 ms | 0.2 s after the fifth quick unmute | -14.2 ms | 0 | -| Chromium | shared | 73.7 ms | 0.2 s after the fourth quick unmute | -6.9 ms | 0 | -| Firefox | postMessage | -46.8 ms | 3.0 s before the auto preset | -18.7 ms | 0 | - -The first three rows are the quiet re-measure, one row at a time with nothing else loading the -machine; the Firefox row is from the first pass on the same tip and was not re-run. Every row passes -the 200 ms toggle rule with room to spare, 41.1 to 73.7 ms, and none of them underran anywhere in the -sequence. Where an engine peaks after an unmute it peaks 0.2 s after it, which is the unmute -transient rather than a preset change. - -**Real Safari ran the sequence for the first time**, and it is the tightest of the four: worst skew --41.1 ms, p50 -13.9 ms and p95 30.4 ms from the mute onwards, zero underruns, one Element Click, the -WebSocket transport. Re-run on `f846462e5` it is unchanged: worst -43.6 ms, 1.6 s after the -`delay 100ms` marker, p50 -15.9 ms, zero underruns. - -**WebKit's idle-tail underrun did not repeat.** The previous WebKit toggle recorded a single underrun -8.3 s after the last unmute, in the idle tail. This run records none, so that was a one-off rather -than a property of the sequence. - -**Chromium on the shared ring records one skip, and it is the preset doing its job.** 1900 ms -skipped, at 24.0 s, 0.3 s after the `delay 100ms` marker: the ring discarding the 1.9 s it was -holding for the `2000ms` preset the sequence had just left. Counted against the zero-skips rule that -is a failure on a local row; by cause it is the only thing a drop from 2000 ms to 100 ms can mean. - -One limitation of the measurement, on all four rows. Each records six intervals with no -`audio.out.timestamp` at all, one of about 3.0 s and five of 0.5 to 0.76 s, lining up one for one -with the six mutes. A muted player publishes no playhead, so the skew metric is blind for the 6.0 to -6.8 s the tile is muted, which is by design (see `19edd01ce`) and not a gap the engine could fill. - -### Browser publishers - -| Publisher | Watcher | Result | -| --- | --- | --- | -| Firefox 155 | Chromium | published; target 20 ms, held 40 ms, level p50 40 ms, zero underruns, skew -19/-2 ms | -| WebKit 26.6 | Chromium | could not run: `no video permission`, `no audio permission` | - -The Firefox publisher is the tightest row in this whole pass: a 20 ms target over a 20 ms Opus -chunk, 40 ms held end to end on the LAN. Its track reports `echoCancellation`, `autoGainControl` and -`noiseSuppression` all false, so that is the raw device. - -WebKit could not be made a publisher here, and the reason is the harness rather than the engine. -Playwright's WebKit rejects `newContext({ permissions: ["microphone"] })` with `Unknown permission: -microphone`, and it has no `--use-fake-ui-for-media-stream` equivalent, so `getUserMedia` is denied -and the publish element logs `no video permission` / `no audio permission` and never announces. What -the engine does support was checked directly: `AudioEncoder` and `VideoEncoder` both exist and -`AudioEncoder.isConfigSupported({codec: "opus"})` is true, and the session itself reaches -`connected` over a WebSocket. `MediaStreamTrackProcessor` is absent, which -`js/publish/src/video/processor.ts` already has a fallback for. So WebKit publishing is untested -rather than broken. - -### Retracted: real Safari does get a browser publisher's catalog - -An earlier pass recorded this as the one hard engine defect: real Safari 26.6 on the WebSocket -transport never receiving the catalog of a broadcast published from a browser, while the two -ffmpeg broadcasts on the same session delivered theirs in under five seconds. That conclusion does -not hold. Re-tested against the same relay, the same page and the same headed Chromium publisher -(real microphone and camera, Opus at 16 kHz and `avc1.640028`), real Safari gets the catalog every -time, in about the same 140 ms as the ffmpeg broadcasts on the same session. - -| Run | Publisher age when Safari connected | Result | -| --- | --- | --- | -| 30 s, no interaction | 2 min | catalog at 143 ms, against 140 ms and 141 ms for the two ffmpeg broadcasts | -| 90 s, publisher started 30 s in | 0 | catalog within 5 s of the announcement, then stable | -| 60 s, alongside a non-browser subscriber | 5 min | both got it; the non-browser client at 4 ms | -| 60 s, tile clicked and muted/unmuted every 5 s | 25 min | catalog and video for the whole run | - -Subscriber churn does not degrade it either: 24 cycles of connect, subscribe to `catalog.json` and -leave, a quarter of them abandoning the session without closing anything, answered in 3 ms to 9 ms -every time. - -What does reproduce the reported symptom exactly, and reproduces it for any engine, is subscribing -to a broadcast whose publisher has died without closing its session. Killing the publisher's browser -process outright leaves the relay with a WebTransport session it has no way to know is gone, and for -the length of the QUIC idle timeout it keeps the broadcast announced and keeps accepting -subscriptions to it: - -```text - 1s announced=true catalog=TIMEOUT - ... - 35s announced=true catalog=TIMEOUT - 36s announced=false catalog=reset(StreamError: remote error: 33) -``` - -Thirty seconds of `subscribe ok` followed by silence, then a retraction and a loud RESET\_STREAM once -`DEFAULT_IDLE_TIMEOUT` in `rs/moq-tokio/src/quic.rs` expires. That reset carries stream code `0x33`, -which `js/net/src/error.ts` names `NotFound`. Inside that window the tile is on the -page, the session is `connected`, no group ever arrives, and nothing is written to the console: the -reported signature, line for line. The subscriber in the trace above is a Bun `@moq/net` client on -the WebSocket transport, so none of it is Safari. `rs/moq-relay/src/websocket.rs` says the same thing -from the other side, in the comment explaining why the WebSocket path needs its own keep-alive: -without one, "every broadcast it published stays announced for that entire window". - -The original evidence supports that reading rather than the engine one. `check2.json` from that pass -is a Chromium watcher on the native WebTransport transport showing the identical all-null shape, on -`demo/mic-chromium-mu0hvt72.hang`, recorded at a point where the publisher log shows that name had -already been replaced by `demo/mic-chromium-mu0j4p5m.hang`. And the Safari-versus-WebKit comparison -that the claim rested on was not simultaneous: WebKit passed on `mu0j4p5m` at 20:56, Safari failed on -it at 21:00 and 21:02, and no non-Safari watcher was run inside that window to tell a broken engine -apart from a publisher that had stopped serving. - -Nothing here is a defect in `js/net`, `js/watch`, `js/publish` or the relay's WebSocket path. What is -left is a diagnosability gap worth its own scope: a subscribe to an announced broadcast whose -publisher is gone is acknowledged and then silent for the whole idle-timeout window, which a viewer -cannot tell apart from a publisher that is merely slow to produce its first group. - -One softer Safari note from the same runs, fixed in two steps: the `moq-watch` element built its -`AudioContext` only once the catalog named an audio rendition, and `unlockOnGesture` was armed at -that moment, so the click that selected the tile had already passed. In Chromium and Firefox the -unconditional `resume()` succeeded anyway; in real Safari it did not, and a viewer had to click a -second time after the video appeared before they heard anything. `a1499011c` armed the listeners from -the moment audio is enabled, before any context exists, and `resume()` runs when one appears and on -every gesture until it is running. - -Measured with one Element Click on a tile for a broadcast that had not started yet, so the click -always preceded the catalog: 6 of 6 runs play on one click, counting a run only when the context -reached `running` and rendered output grew. Real Safari 26 is 4 of 4, twice on a 48kHz broadcast and -twice on a 16kHz one, with Chromium and Firefox 1 of 1 each, every run on a Bluetooth headset whose -own rate is 44100Hz and so matches neither. The muted-tile path measured the same way: 4 of 4 (Safari -twice, Chromium and Firefox once each, on a broadcast already publishing so the graph was pre-built -while muted, `suspended` before the click everywhere except Chromium, where its autoplay policy had -already started it, and `running` within 572ms after). - -**One conclusion drawn there is wrong, and correcting it is finding 15.** That pass read "a page's -activation carries to a context created later", from a context that appeared about 2.3 s after the -click and resumed anyway. It does not carry. WebKit honours a `resume()` outside a handler only while -the page's activation is still live, which is a few seconds; past that the context stays suspended -for good, video keeps painting and nothing is ever rendered. Both of that pass's measurements sat -inside that grace, which is why they passed, and a 2.3 s delay is not evidence of anything beyond it. -What did hold is the other half: one activation starts one context, which is why building a context -inside the gesture handler was tried and rejected (at 16kHz the primed 48kHz context was still open -and `running` when the graph rebuilt at the decoded rate, and the replacement stayed `suspended`, 0 -of 2 against 2 of 2 for arming alone). The same statement now reads: a gesture is reliably spent only -on the context that exists when it lands, so the graph must keep that context rather than rebuild -around it. - -### After the user's listening round, 2026-09-17 - -The user listened on a build of `f846462e5`, the last code commit below the eight above, and reported -nine things. The rows below are what each fix measured on the same quiet bench, one row at a time, -with the publisher restarted before every row. Every row is read out of its JSON on the same -250 + U(0, 50) ms grid as the tables above, except the unmute rows, which sample every 50 ms because -what they measure is shorter than a frame. Skew is the painted video timestamp minus the audio -playhead, and the statistics to grade are the signed lead and lag, for the reason under -"Conferencing targets": the p95 of the absolute value has a floor of one frame plus one display -refresh whatever the player does. - -**The unmute.** A tile playing at auto, muted for three seconds, then unmuted, sampled every 50 ms -around the unmute. The declared span is what a fresh consumer used to seed itself from, and the -target is what the estimator says; before these two commits the first was also the second. - -| Source and page | Declared span | Target before the mute | Target at the unmute | Ring fill | Spinner | Playhead back | Underruns | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| `bbb.hang`, plain page | 320 ms | 60 ms | 60 ms | 200 ms | 0 ms | 282 ms | 0 | -| `bbb.hang`, site page | 320 ms | 60 ms | 60 ms | 200 ms | 0 ms | 265 ms | 0 | -| `mic.hang`, plain page | 80 ms | 40 ms | 40 ms | 150 ms | 0 ms | 231 ms | 0 | -| `mic.hang`, plain page, re-run after the live edge | 80 ms | 40 ms | 40 ms | 150 ms | 0 ms | 231 ms | 0 | - -The spinner column is how long the buffering indicator was visible after the unmute, and it is zero -on every row: no sample after an unmute reports `interrupted`. The target is the same either side of -the mute on every row, which is the estimator being continued rather than reseeded. The ring fill is -the ring's own stalled interval, and it is now the measured target plus a chunk rather than the -declaration. - -**The mute and preset sequence, after each fix.** The user's sequence again, on the local relay. - -| Source | Engine | Tip | Worst skew | Where | Skew p50 | Underruns | Video stalls | -| --- | --- | --- | ---: | --- | ---: | ---: | ---: | -| `bbb.hang` | Chromium | the unmute fixes and the video rebuild | -48.0 ms | 0.4 s after `delay 100ms` | -20.4 ms | 0 | not measured | -| `bbb.hang` | Firefox | the unmute fixes and the video rebuild | -51.3 ms | 2.8 s after the fifth unmute | -26.8 ms | 0 | not measured | -| `bbb.hang` | Chromium | plus the live edge and the camera fix | -51.1 ms | 2.2 s after `delay auto` | -22.1 ms | 0 | 1, 1.5 s, at the `2000ms` preset | -| `mic.hang` | Chromium | the final tip | 51.9 ms | 0.2 s after unmute 2 of 5 | -17.0 ms | 0 | 1, 1.5 s, at the `2000ms` preset | -| `mic.hang` | Firefox | the final tip | -49.5 ms | 1.0 s before `delay auto` | -16.7 ms | 0 | 1, 1.5 s, at the `2000ms` preset | - -No row peaks on an unmute any more, and every worst reading is one held interval wide, which is the -envelope the earlier toggle table reports: 41 to 74 ms there, 48 to 52 ms here, against the 200 ms -toggle rule. The one video stall a row records starts 0.6 s after the `2000ms` preset and is the -renderer holding the two seconds it was asked for; no row records a decoder rebuild. - -**The re-subscribe, read off the relay.** Before, every re-subscription to a rendition that had -stopped encoding was completed at once; the excerpt is in finding 19. After, on the same sequence of -hides and shows, each subscription lives until the watcher itself cancels it: - -```text -11:51:41.020 subscribed started id=6 broadcast=toggle.hang track=video -11:51:50.193 subscribed cancelled id=6 9.2 s later, by the watcher -11:51:50.620 subscribed started id=7 -11:51:55.510 subscribed cancelled id=7 4.9 s -11:51:55.949 subscribed started id=8 -11:52:10.847 subscribed cancelled id=8 14.9 s -``` - -No `subscribed complete` anywhere in between. Through the whole mute and preset sequence one video -subscription lived 54.6 s (`11:56:44.428` to `11:57:39.045`), and the `complete` that ends it is the -publisher being stopped at the end of the row. - -**Hide and show video.** The publisher page with the USB camera, the camera button toggled twice -300 ms apart and again 5 s later. "Busy window" is an injected device release: this camera frees in -under 100 ms, which is not long enough to spend the retry budget, so the defect only appears when the -release is made to take longer than the budget. - -| Engine | Tip | Busy window | getUserMedia calls, refused | Camera back after the show | Preview painted | End state | -| --- | --- | --- | ---: | ---: | ---: | --- | -| Chromium | before | none | 6, 0 | 110 and 124 ms | 139 and 174 ms | live, rendition in the catalog | -| Chromium | after | none | 6, 0 | 104 and 112 ms | 155 and 139 ms | live, rendition in the catalog | -| Brave | before | none | 7, 0 | 106 to 128 ms | 132 to 167 ms | live, rendition in the catalog | -| Brave | before | 8 s | 12, 8 | never | never | no source, no video section in the catalog | -| Brave | after | 8 s | 41, 35 | 8.5 s and 8.1 s | within one sample of the first frame | live, rendition in the catalog | -| WebKit | after | none | 4, 0 | 47 and 54 ms | 75 and 78 ms | live, rendition in the catalog | -| WebKit | after | 8 s | 28, 24 | 8.2 s | within one sample | live, rendition in the catalog | -| Firefox | after | none | 4, 0 | 102 ms | never, see finding 25 | live, rendition in the catalog | - -The two `after` rows with the busy window are the proof: 35 and 24 refusals absorbed without ending -the capture, and the camera back within half a second of the device becoming free, against a `before` -row that gave up after 8 refusals and never asked again. - -**Real Safari, one Element Click per run.** The microphone broadcast, 20 s of settled window per run, -the WebSocket transport, Safari frontmost. Every run built exactly one `AudioContext`, at the click. - -| Run | Context at the first sample after the click | Contexts | Target | Held | Level p50 | Underruns | Skew p50 | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| 1 | `running` at 5 ms | 1 | 40 ms | 60 ms | 70.1 ms | 0 | -14.2 ms | -| 2 | `running` at 6 ms | 1 | 40 ms | 60 ms | 50.1 ms | 0 | -18.3 ms | -| 3 | `running` at 5 ms | 1 | 40 ms | 60 ms | 50.1 ms | 0 | -25.8 ms | -| 4 | `running` at 6 ms | 1 | 40 ms | 60 ms | 70.1 ms | 0 | -14.3 ms | -| 5 | `running` at 4 ms | 1 | 40 ms | 60 ms | 60.1 ms | 0 | -21.1 ms | -| 6 | `running` at 12 ms | 1 | 40 ms | 60 ms | 50.1 ms | 0 | -17.2 ms | -| 7 | `running` at 8 ms | 1 | 40 ms | 60 ms | 50.1 ms | 0 | -16.0 ms | -| 8 | `running` at 5 ms | 1 | 40 ms | 60 ms | 60.1 ms | 0 | -17.2 ms | - -Eight of eight, every row painting at 30 fps with zero video stalls and zero decoder rebuilds. The -autoplay warnings the same change removes, counted on a page load with the browser's own policy and -no gesture at all: - -| Page | Engine | Warnings at load | Contexts at load | Muted tiles holding a context | -| --- | --- | ---: | ---: | ---: | -| demo watch, two tiles, one muted | Chromium | 2 to 1 | 2 to 1 | 0 | -| demo watch, two tiles, one muted | Firefox | 4 to 2 | not recorded separately | 0 | -| the site page, player starts muted | both | 0 | 0 | 0 | - -The one warning Chromium keeps is the tile the demo page itself unmutes, which is an app enabling -audio before any gesture, and that is the policy doing its job rather than something to hide. - -**U5, the "Buffering (audio clock)" report, did not reproduce on a quiet bench.** The same self- -publish and watch in one Chromium, 40 s of settled window: - -| Target p50/p95 | Held | Level p50/p95/min | Underruns | Skips | Groups skipped | Concealed | Skew p50/worst | videoLead p50 | -| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| 40/40 ms | 60 ms | 60.2/70.1/40.2 ms | 0 | 0 | 0 | 0 | -14.6/-41.0 ms | 68.6 ms | - -The user's own observation was taken while agents were running headed encoder sessions on the same -machine, which is recorded rather than explained away: a loaded machine is the case the report came -from, and the matrix's long run is what will say whether it returns under load. - -### The cross-browser resilience matrix, 2026-09-17 - -49 rows, one publisher and one watcher each, the publisher restarted before every row, run serially -at `16e6242aa` with both served pages built from that tip. Each row is graded against the same -checks: a jitter target of 20 to 100 ms, held under 150 ms, zero underruns after a 10 s warmup, the -signed skew inside a frame plus a display refresh (-55 ms at 30 fps, -38 ms at 60), zero video -stalls after the first keyframe, painted at 80 percent of the nominal rate, and recovery inside -2.5 s where the row interrupts something. **24 rows passed and 25 failed**, and the 25 are eight -distinct causes rather than 25. - -Publisher across the top, watcher down the side. - -| Watcher \ Publisher | chromium | brave | firefox | real Safari | -| --- | --- | --- | --- | --- | -| chromium | pass 720p30, 1080p30, 1080p60, foreground and backgrounded | pass | fail: the picture 94 ms behind the sound | not run, nobody to click Allow | -| brave | pass | pass at 720p30 and 1080p30; 1080p60 foreground painted 47.2 of 60 | fail: 98 ms behind | not run | -| firefox | pass | pass | pass at 720p30 both ways and 1080p30 foreground; fail at 1080p30 backgrounded and both 1080p60 | not run | -| webkit | pass | pass | pass | not run | -| real Safari | pass | pass | pass | not run | - -Interactions, all with a chromium publisher: - -| Scenario | chromium | brave | firefox | webkit | real Safari | -| --- | --- | --- | --- | --- | --- | -| hide and show video | picture back in 1.96 s | 1.99 s | 1.95 s | 1.95 s | 2.00 s | -| mute, unmute, presets, five fast pairs | 0 underruns, one 1.45 s stall at the 2000 ms preset | 0 underruns, 1.50 s | 0 underruns, 1.47 s | 0 underruns, 1.51 s | 0 underruns, 1.51 s | - -The whole-system rows, chromium on both sides: the device switch failed on every publisher engine, -the relay restart left the tile with no picture, the impaired path failed on `mild` and `step`, and -the 30 minute run held 30 fps with two underruns. - -**The 25 failures, grouped.** Five product defects, three bench or grading artefacts, and one open -measurement. - -| Rows | What failed | Verdict | -| --- | --- | --- | -| `shaper-mild`, `shaper-step` | 7 and 12 video stalls, 9.9 s and 20.3 s, 1 and 4 decoder rebuilds, the picture 1.5 to 1.8 s behind | Defect: finding 26, fixed in `d2443eba9` | -| `relay-restart` | painted 0 fps for the last 66 s while audio recovered completely | Defect: finding 27, fixed in `804c13a20`, and a page-layout cause besides | -| `device-switch-chromium`, `-brave`, `-firefox` | no samples at all, or a watcher painting 0 fps, after the switch | Defect: finding 30, fixed in `dc6f8d38c` | -| `cross-firefox-to-chromium`, `cross-firefox-to-brave`, `self-firefox-1080p30-bg`, `self-firefox-1080p60-fg`, `self-firefox-1080p60-bg` | the picture 86 to 109 ms behind the sound, against a -55 ms envelope | Defect: finding 28, fixed in `69a89f17c` and `8b64a6aa6` | -| not a row: the native `moq` file publishers during the restart | both exited with `reconnect timed out after 10s: peer redirected immediately` | Defect: finding 31, fixed in `5be36750f` and `eec9016d9` | -| `hideshow-chromium`, `-brave`, `-firefox`, `-webkit`, `-safari` | `recovered <2.5s` over 0 events | Artefact: the recovery happened in every engine, 1.95 to 2.00 s, and the schedule put it before the graded window opened | -| `toggle-chromium`, `-brave`, `-firefox`, `-webkit`, `-safari` | one video stall of 1.45 to 1.51 s | Artefact: it opens 0.6 to 0.7 s after the `delay 2000ms` marker and is the renderer holding the two seconds it was asked for. `toggle-chromium` also flagged a spinner 1.94 s after an unmute, at the very end of the attribution window and 69 ms before that marker, which no other engine shows | -| `shaper-bursty`, `shaper-high-rtt` | `negotiated websocket transport, expected webtransport` | Artefact: the WebTransport handshake did not complete through the impaired port and the page fell back to TCP, so both rows were unimpaired rows wearing a profile's name. The driver caught it | -| `self-brave-1080p60-fg` | painted 47.2 fps against a 48.0 threshold | Artefact: the backgrounded twin of the same row paints 59.1 and chromium passes both | -| `long-run` | 2 underruns in 30 minutes | Open. Counters otherwise flat: 52079 painted of 52080 decoded, 0 stalls, 0 rebuilds, skew -36.1 to 4.1 ms throughout | - -**The re-run, 2026-09-17 11:00 to 12:08**, rows in `out10/` against `out9/` as the before. Target -and lag in milliseconds, `stalled` the total stalled time in milliseconds, `vskip` the video groups -the age budget or the transport threw away. - -| Row | | target | under | skew lag | fps | stalls | stalled | rebuilds | vskip | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| cross-firefox-to-chromium 720p30 | before | 40 | 0 | -94.2 | 29.9 | 0 | 0 | 0 | 2 | -| | after | 60 | 0 | -35.9 | 29.9 | 0 | 0 | 0 | 0 | -| cross-firefox-to-chromium 1080p30 | before | 40 | 0 | -98.0 | 30.1 | 0 | 0 | 0 | - | -| | after | 140 | 1 | -39.2 | 29.9 | 0 | 0 | 0 | 0 | -| shaper-mild | before | 140 | 0 | -1508.3 | 29.7 | 7 | 9875 | 1 | 7 | -| | after | 540 | 0 | -4092.2 | 29.9 | 5 | 14991 | 2 | 1 | -| shaper-step | before | 140 | 1 | -1834.5 | 29.5 | 12 | 20326 | 4 | 11 | -| | after | 220 | 0 | -39.1 | 30.0 | 1 | 116 | 0 | 0 | -| shaper-bursty | before | 20 | 0 | -35.7 | 30.0 | 0 | 0 | 0 | 0 | -| | after | 280 | 4 | -5370.7 | 0 | 1 | 58841 | 9 | 0 | -| shaper-high-rtt | before | 20 | 0 | -38.5 | 30.0 | 0 | 0 | 0 | 1 | -| | after | 120 | 0 | -36.8 | 30.0 | 0 | 0 | 0 | 0 | -| device-switch-chromium | before | - | - | - | - | - | - | - | - | -| | after | 100 | 1 | -52.5 | 26.4 | 0 | 0 | 0 | 0 | -| device-switch there and back | after | 20 | 0 | -45.9 | 30.0 | 1 | 26 | 0 | 0 | -| relay-restart | before | 20 | 0 | -34.3 | 0 | 1 | 66291 | 0 | -4 | -| | after, pass 1 | 80 | 0 | -37.2 | 0 | 1 | 66412 | 0 | 0 | -| | after, pass 3, `visible="always"` | 20 | 0 | -34.9 | 30.0 | 1 | 66450 | 0 | 0 | - -`device-switch-chromium` before produced no samples at all. Every shaper row still reads FAIL, -and three of them fail only against the LAN targets: the grader applies `target 20-100ms` and -`held <150ms` to every row, and a shaped path is supposed to widen past that. `shaper-high-rtt` at a -target of 120 and a held total of 140 on a 150 ms round trip is the estimator being right. - -**The bitrate confound, stated because it cuts both ways.** The chromium publisher's picture in the -re-run is 117 times heavier than in the earlier matrix. From the publisher heartbeats, bytes per -video frame: - -| | out9, the earlier matrix | out10, the re-run | -| --- | ---: | ---: | -| chromium publisher | about 67 B per frame, about 16 kbps | about 7.9 KB per frame, about 1.9 Mbps | -| firefox publisher | about 67 B per frame | about 67 B per frame | - -So no chromium-published video row is a like-for-like comparison. Where such a row improved it did -so while carrying 117 times the load, which makes it stronger; where one got worse, the bitrate is -the first thing to rule out and usually cannot be. The firefox rows are like for like, and the -67 B per frame they carry in both runs is finding 29. - -**The load caveat.** The user's other session runs a continuous headed Chromium public stream on -this machine, two 1080p60 encoders with a supervisor that restarts them, and it cannot be paused -from here. The load average over the re-run ran 2.5 to 10.7, and the per-row `uptime` is in -`out10/load.log`. Every timing number taken since mid-morning is an upper bound. Functional verdicts -(a subscription raised, a picture back, an announcement held) do not move with load; skews, stalls -and targets do. One row is contaminated outright and says so: the third relay restart overlapped a -`just fix` running `cargo clippy --fix`, which is the last section of finding 31. - -**Rows not re-run.** The self-publish, hide-and-show and toggle families, the other cross-engine -pairs, and the 30 minute long run. None of them was a watcher-side failure this work touches, and -the machine was never quiet enough for a 30 minute row. The matrix above remains their reference. - -**Two coverage gaps, both recorded rather than worked around.** Real Safari as a *publisher* was not -run: `safaridriver` can drive it, but its camera prompt needs a hand on the mouse and no user was -available to click Allow. Playwright's WebKit refuses the `getUserMedia` permission outright, so -there are no WebKit publisher rows either. Both engines ran as watchers in five rows each and passed -everything except the two artefacts above. - -**One harness defect worth a demo-page owner's eye.** All five hide-and-show rows first died with -` from subtree intercepts pointer events`: on `demo/web/src/publish.html` the preview -canvas sits over the publish controls, so a real mouse click cannot reach the camera button there -either. The driver falls back to `dispatchEvent`; a person cannot. That is a page layout question -rather than a `js/publish` one, and this run only establishes that a hit test refuses the click. - -### The quiet confirmation, 2026-09-17 - -Ten rows run again between 12:46 and 13:01 on a machine with nothing else on it, from the worktree -at `b80a2084a` with both served pages built from the code tip `eec9016d9`. One JSON, one publisher -log and one watcher log per row in `out11/`, plus `out11/load.log` with `uptime` taken before every -row. - -The point of the run is the machine. The re-run above was measured against a load average of 2.5 to -10.7 from a foreign Playwright Chromium stream, the user's Brave and two agents building Rust, and -said so. That stream is stopped. Load was 1.50 when the first row started, and the 2.5 to 4.1 -readings later in `load.log` are the rows' own two browsers, since each reading is taken while the -previous row's browsers are still exiting. No foreign browser ran: `pgrep` for Chrome for Testing, -Chrome, Chromium, Firefox, WebKit and safaridriver was empty before the run and empty again after -it. - -Target, held, skew and stalled in ms; `stalled` is the page's cumulative `video.out.stalled`, not -the row's own stall; `vskip` is the video groups the age budget or the transport threw away; `load` -is the one-minute average before the row. The loaded line under each quiet one is `out10` where that -run has the row, and the earlier matrix `out9` where it does not. - -| Row | run | target | under | skew lead | skew lag | fps | stalls | stalled | rebuilds | vskip | load | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| shaper-mild | quiet | 200 | 0 | 4.5 | -35.1 | 30.0 | 0 | 0 | 0 | 1 | 1.50 | -| | out10 loaded | 540 | 0 | 3.1 | -4092.2 | 29.9 | 5 | 14990.8 | 2 | 1 | 5.78 | -| shaper-step | quiet | 140 | 0 | 3.0 | -37.7 | 30.0 | 0 | 0 | 0 | 0 | 1.96 | -| | out10 loaded | 220 | 0 | 2.9 | -39.1 | 30.0 | 1 | 116.2 | 0 | 0 | 5.78 | -| shaper-high-rtt | quiet | 120 | 0 | 3.8 | -36.8 | 30.0 | 0 | 0 | 0 | 0 | 2.44 | -| | out10 loaded | 120 | 0 | 2.2 | -36.8 | 30.0 | 0 | 0 | 0 | 0 | 5.78 | -| cross-firefox-to-chromium 720p30 | quiet | 100 | 0 | 4.4 | -35.9 | 29.9 | 0 | 0 | 0 | 0 | 3.54 | -| | out10 loaded | 60 | 0 | 3.6 | -35.9 | 29.9 | 0 | 0 | 0 | 0 | 3.34 | -| self-chromium-1080p60-fg | quiet | 40 | 0 | 3.4 | -22.5 | 54.2 | 0 | 0 | 0 | 0 | 3.87 | -| | out9 earlier matrix | 40 | 0 | 4.2 | -20.0 | 57.4 | 0 | 0 | 0 | 0 | - | -| self-brave-1080p60-fg | quiet | 40 | 0 | 4.2 | -21.8 | 58.9 | 0 | 0 | 0 | 0 | 3.29 | -| | out9 earlier matrix | 40 | 0 | 5.6 | -24.0 | 47.2 | 0 | 0 | 0 | 0 | - | -| toggle-chromium | quiet | 40 | 0 | 1.6 | -38.6 | 29.9 | 1 | 1465.9 | 0 | 0 | 4.11 | -| | out9 earlier matrix | 60 | 0 | 6.5 | -38.9 | 29.8 | 1 | 1452.5 | 0 | 2 | - | -| toggle-firefox | quiet | 40 | 0 | 12.9 | -36.6 | 29.9 | 1 | 1517.0 | 0 | 0 | 3.14 | -| | out9 earlier matrix | 20 | 0 | 4.6 | -37.3 | 29.9 | 1 | 1474.0 | 0 | 1 | - | -| hideshow-chromium | quiet | 40 | 0 | 3.2 | -37.7 | 30.0 | 2 | 87.1 | 0 | 0 | 2.68 | -| | out9 earlier matrix | 20 | 0 | 3.3 | -38.2 | 30.1 | 0 | 0 | 0 | 6 | - | -| relay-restart | quiet | 80 | 0 | 3.0 | -38.1 | 30.0 | 1 | 66435.0 | 0 | 0 | 2.49 | -| | out10 loaded, pass 3 | 20 | 0 | 5.6 | -34.9 | 30.0 | 1 | 66450.3 | 0 | 0 | 4.35 | - -One line per row, against the same checks the matrix used: - -- **shaper-mild**: FAIL on the LAN targets alone (p50 200, held 220), every resilience target met. - The inconclusive row is settled: quiet, the impaired path costs latency and nothing else. Zero - stalls against five, zero rebuilds against two, and the skew lag back from -4092.2 ms to -35.1 ms. -- **shaper-step**: FAIL on the LAN targets alone (p50 140, held 160). Clean otherwise, and better - than loaded on every counter: no stall at all where loaded had one. -- **shaper-high-rtt**: FAIL on `target 20-100ms` alone (p50 120 on a 150 ms round trip, which is the - estimator being right); held 140 passes. Identical to loaded, which is the row saying the path, - not the machine, set its number. -- **cross-firefox-to-chromium 720p30**: PASS, every check. Not like for like with `out10`: see the - bitrate note below. -- **self-chromium-1080p60-fg**: PASS, every check, at 54.2 fps against the 48 fps floor. -- **self-brave-1080p60-fg**: PASS, every check, at 58.9 fps against the 47.2 that failed the floor - in the earlier matrix. -- **toggle-chromium**: FAIL on `0 video stalls` alone, one 1465.9 ms stall. No underruns, no spinner - on any unmute, and the skew and the target inside every envelope. -- **toggle-firefox**: FAIL on `0 video stalls` alone, one 1517.0 ms stall, the same shape and the - same cause as chromium's. -- **hideshow-chromium**: FAIL on `0 video stalls` alone, two stalls of 28.4 ms and 58.7 ms. - `recovered <2.5s` now passes with real evidence: worst 66.1 ms over two events. This row measured - nothing at all before, which is the harness note in the matrix section. -- **relay-restart**: FAIL on `0 video stalls` and `recovered <2.5s`, one stall from 54.5 s to - 69.7 s. The gap is the reconnect and the tile rebuild rather than the player: 15.2 s here against - 52.5 s under the clippy load of the third pass above. - -**The four rows that still fail are the grader, and each one has a problem line.** - -*A latency preset change stalls the picture for the length of the new target, and the grader counts -it as a video stall.* The `toggle` stall starts at 16.4 s in chromium and 16.8 s in firefox, within -700 ms of the `delay 2000ms` marker at 15.8 s and 16.1 s. Changing the preset to 2000 ms re-buffers, -and the picture is held to the new target while the buffer fills. The earlier matrix has the same -stall in the same place, 1452.5 ms and 1474.0 ms, on the old code and a loaded machine, so it is the -preset's own cost rather than a regression. - -*Hiding the camera stops the picture, which is a video stall by the grader's definition, so -`hideshow` cannot pass `0 video stalls` as written.* Its two stalls are the hide and the show -themselves, at 12.06 s and 22.49 s on the watcher's clock, which are the publisher's clicks at -15.5 s and 25.9 s on its own. They recover in 42 ms and 66 ms, and the check that matters, -`recovered <2.5s`, passes at 66.1 ms. - -*The picture cannot come back before the page rebuilds the tile, so `relay-restart` measures the -reconnect and the re-announcement rather than the player.* After the restart the picture returns at -`tile-swap`, 69.7 s, which is 15.2 s after the relay took its shutdown signal and is when the demo -page rebuilds its tile list. The player itself loses nothing across it: 0 underruns, 0 rebuilds, 0 -convictions and 30.0 fps afterwards. - -*A shaped path is supposed to widen past the LAN targets.* The three shaper rows fail only -`target 20-100ms` and `held <150ms`, which the grader applies to every row whatever the path under -it is. Per-row envelopes, one for a shaped path and one for a row whose whole subject is an -interruption, are a harness follow-up and are under "Open items and follow-ups"; nothing on the -player side is waiting on them. - -**The file publishers survived the restart on their own.** Both of them, with no intervention and no -`reconnect timed out`: - -```text -17:00:32.782 relay: shutdown signal received; draining sessions window=10s -17:00:32.79 both publishers reconnect, get app code=503, back off and retry - 5 refusals each across the 10 s drain, then 8 connection-refused - attempts each while the relay was down, backoff 0.58s -> 3.9s -17:00:55.363 the new relay serves its first session (conn id=0) -17:00:55.988 pub-demo-bbb connected; announce route=demo/bbb.hang/** at 17:00:55.990 -17:00:57.421 pub-bbb connected; announce route=bbb.hang/** at 17:00:57.423 -``` - -Both pids from `pids.txt` are the same processes as before the row, so neither was restarted by -hand, and the relay line was rewritten to the new pid. The 60 s give-up budget from `5be36750f` -covered a 24.6 s outage with room to spare, and the third-restart death in `out10` really was -`cargo run` recompiling under a concurrent `just fix` rather than the product. Worth noting for the -next restart row: the 503s are the relay's own drain window refusing new sessions for 10 s, which is -correct behaviour and reads as an error storm in the publisher logs. - -**The Firefox publisher was not blank this time.** The re-run recorded the Firefox publisher -emitting about 67 B per video frame in both earlier runs and read it as a blank picture through its -`MediaStreamTrackProcessor` polyfill. On the quiet machine it emits a real one. From the heartbeats -of the same row: - -| | video | audio | -| --- | ---: | ---: | -| out10 cross-firefox-to-chromium | 67 B per frame | 3 B per frame | -| out11 cross-firefox-to-chromium | 7.8 KB per frame | 133 B per frame | - -Three bytes per audio frame is not a quiet room, it is nothing being captured. The likeliest reading -is the one the device rows already suspected: another process was holding the camera and the -microphone, and Firefox got tracks that produced nothing. With the machine quiet, it captures. Two -consequences. Finding 29's blank-picture statement is observed once and not reproduced, and should -not be carried forward without a re-check. And this row is no longer like for like with `out10`: it -carries about 115 times the video bitrate, so a p50 of 100 against `out10`'s 60 is a heavier row -rather than a regression. Every other row in this run publishes from chromium or brave at 7.7 KB per -frame, 13.1 KB at 1080p60, matching `out10`, so those comparisons hold. - -**Rows not run.** The rest of the self-publish family, the other cross-engine pairs, the remaining -hide-and-show and toggle watchers, the device-change family, `shaper-bursty` and the 30 minute long -run. `shaper-bursty` is a capacity row against this publisher's bitrate rather than a bunching row, -and needs a wider profile or a lower publisher bitrate before it is worth running again. - -### Several watchers at once, 2026-09-17 - -Five watchers on one browser publisher of the USB camera and microphone at 720p30, 90 s a row, each -watcher running the toggle sequence offset four seconds from the one before it, six mutes and -unmutes each, and the publisher hiding and showing its camera twice, at about 20 s and about 41 s. -Two rows: one with three Chromium watchers plus Firefox plus WebKit, one with five Chromium. The -JSON, the publisher log and the row's own slice of the relay log are in `out12/`. Finding 34 is what -they are for. - -| Row | Watcher | Target | Held | Underruns | Skew lead / lag | Painted | Stalls | Rebuilds | -| --- | --- | ---: | ---: | ---: | --- | ---: | ---: | ---: | -| mixed | w1 chromium | 60 ms | 80 ms | 3 | 5.1 / -40.0 ms | 30.0 | 3 | 0 | -| mixed | w2 chromium | 40 ms | 60 ms | 2 | 3.9 / -37.7 ms | 29.9 | 3 | 0 | -| mixed | w3 chromium | 40 ms | 60 ms | 0 | 4.0 / -39.5 ms | 29.9 | 3 | 0 | -| mixed | w4 firefox | 80 ms | 100 ms | 8 | 8.8 / -42.4 ms | 29.6 | 5 | 0 | -| mixed | w5 webkit | 40 ms | 60 ms | 2 | 1.3 / -37.9 ms | 29.9 | 3 | 0 | -| five chromium | w1 | 40 ms | 60 ms | 1 | 6.0 / -36.0 ms | 29.9 | 4 | 0 | -| five chromium | w2 | 40 ms | 60 ms | 0 | -0.1 / -35.8 ms | 29.9 | 3 | 0 | -| five chromium | w3 | 40 ms | 60 ms | 0 | 1.1 / -36.5 ms | 30.0 | 3 | 0 | -| five chromium | w4 | 80 ms | 100 ms | 0 | 6.1 / -38.5 ms | 29.9 | 3 | 0 | -| five chromium | w5 | 40 ms | 60 ms | 0 | 7.0 / -39.2 ms | 30.0 | 3 | 0 | - -Every stall in both rows is accounted for by a named moment, the two camera hides and the toggle -sequence's `2000ms` preset, and every recovery is inside 2.5 s: worst 1477 ms in the mixed row and -1474 ms in the Chromium one, which is the preset's own cost again. No row records a decoder rebuild. -The skew envelope holds on every watcher in both rows. - -The underruns are the row's own load. Fifteen across the five mixed engines against one across five -Chromium watchers, on the same publisher and the same relay, is not an engine result: each watcher -decodes three broadcasts, because the plain page on 4400 subscribes to the announce prefix `""` and -the two file publishers are on the relay beside the probe. That is a harness property and it is in -the follow-ups. - -The publisher's own upstream leg, read out of the relay log, is the part the row exists to check: - -| | mixed | five chromium | -| --- | --- | --- | -| `subscribed complete` on the publisher's connection | 0 | 0 | -| `unannounce` anywhere in the row | 0 | 0 | -| upstream subscriptions to `catalog.json` | 1 | 1 | -| upstream subscriptions to `audio` | 1 | 1 | -| upstream subscriptions to `video` | 3, at the row's start and at each of the two hides | 3, the same | -| `subscribed complete` anywhere in the row | 9, all on the Firefox watcher's connection | 0 | - -Firefox closing its own subscriptions with `complete` where Chromium and WebKit reset them is -harmless and is recorded rather than changed: what finding 19 was about is a `complete` on the -*publisher's* leg, and there are none. - -### The first fill, before and after the trim - -One Chromium watcher of a browser publisher of the real microphone and camera through the local -relay, the toggle sequence, counted over the three seconds after each unmute. The trim is finding 33 -and the rows are in `sol/`. - -| Phase | Build | Accelerates | Stretched | Trimmed | Underruns | -| --- | --- | ---: | ---: | ---: | ---: | -| one unmute after a three second mute | before | 8 | 50.4 ms | not counted | 0 | -| | after | 0 | none | 80 ms | 0 | -| five rapid mute and unmute pairs | before | 15 | 144.1 ms | not counted | 0 | -| | after | 0 | none | 400 ms | 0 | - -Per unmute, the five rapid pairs before the trim cost 7, 7, 11, 8 and 5 accelerates and 74.3, 70.1, -102.2, 69.8 and 49.7 ms of compressed speech; after it, 0 accelerates each and 240, 240, 240, 160 -and 80 ms trimmed. The settled row either side is the control, and it does not move: target 20 ms, held -40 ms, zero underruns, zero skips, with the level p50 falling from 40.4 ms to 20.0 ms and the -accelerates over the settled window from 5 to 0, which is the ring resting on the level it holds -rather than above it. - -### A rendition that blinks, before and after - -The five-Chromium-watcher row again, run four times: once before the two commits of finding 35 and -three times after them. Ten hide windows per run, two hides by the publisher and five watchers, each -window read from 500 ms before the hide to three seconds after it. The runs are `out13-before/` -before, and `out14c/`, `out14/` and `out14d/` after. - -| Run | Hide windows | Video reading reaching the 80 ms guess | Underrun inside the window | -| --- | ---: | ---: | ---: | -| before | 10 | 10 | 3 | -| after, run 1 | 10 | 0 | 1 | -| after, run 2 | 10 | 0 | 1 | -| after, run 3 | 10 | 0 | 5 | - -**10 of 10 before and 0 of 30 after** is the fix: the estimator is no longer rebuilt at the -publisher's guess when the rendition blinks out of the catalog and back. The seven residual -underruns are the open part, and they are a different shape: each lands 0.90 to 1.45 s after the -hide, on the target the row had been holding (40, 40, 40, 40, 60, 20 and 40 ms), where every one of -the before-run's went with a target that had just stepped to the 80 ms guess. The second after-run -also carries a synchronised burst nothing here explains: all five watchers underran between 60.3 s -and 60.8 s, nowhere near a hide, under the row's own three-broadcast load. - -### The bench on the rebased build - -Both served pages rebuilt from the post-fixup rebase tip `91c5878bd`, the relay and both file -publishers relaunched from that build, the page servers left up, and the earlier logs rotated aside. -The machine was quiet: `pgrep` for Playwright browsers and safaridriver empty, load 3.4. The JSON is -in `out13-rebase/`. - -| Row | Verdict | What it read | -| --- | --- | --- | -| the copied moq.dev site on 4402 | pass | `player=[bbb.hang]`, unmute reaching `muted=false paused=false`, audio playhead 234250.4 ms with the context `running` | -| microphone, Chromium, 60 s | pass | target p50 20 ms, held 40 ms, 1 underrun (the cold-start conceal, 960 samples at 20 ms), 0 skips, 0 video stalls, 0 rebuilds, skew p50 -22.8 ms lead -3.7 lag -32.4, painted p50 30 fps min 24.6 | -| microphone toggle sequence, Chromium, 60 s | pass | 0 underruns, 0 skips, 0 video stalls, 0 rebuilds, skew p50 -24.7 ms worst -38.9 lag -35.4, painted p50 30 fps min 28.3, one 1509 ms recovery at the 2000 ms preset | - -The toggle row is the same shape as the pre-rebase `toggle-chromium` row or better: that one had a -1452 ms video stall and a -38.9 ms lag, this one has no stall and -35.4 ms, and its 1509 ms recovery -is the preset's own cost, which every toggle row on this branch records. The one known open finding -is unchanged by the rebase and still shows: one `skipping slow group: track=audio` warn per unmute, -which is finding 21's `moq-net` question, with zero underruns behind it. - -### The first ten seconds after an event, before and after - -One publisher event per row, on the real microphone, each row graded against its own last twenty -seconds, so every threshold is a multiple of something that row measured rather than a constant tied -to a 20 ms frame or a 48 kHz device. The Chromium watcher runs on the pinned USB microphone at -48 kHz, with the output context also at 48 kHz on every row. The ten rules the verdicts are read -under are finding 42 and are reproduced under "The listening bench". The rows are in -`bench/out15-before/`, `out15-after/`, `out15-after2/`, `out15-after3/` and `out15-after4/`, each -with its own generated `MUTE-2026-09-18.md`, and the tables below are those files' own verdict rows. - -**Before**, on `17d32eae4`, the tip these fixes start from: - -| Row | Preset | Browser | Verdict | Target | Chunk | Fill | Opening stretches | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | -| mute-basic-auto | auto | chromium | NOT GRADED | 20ms | 0ms | 20ms | - | -| mute-basic-100ms | 100ms | chromium | NOT GRADED | 100ms | 0ms | 100ms | - | -| pub-reload-auto | auto | chromium | PASS (3 not measured) | 20ms | 20ms | 40ms | - | -| pub-reload-100ms | 100ms | chromium | PASS (3 not measured) | 100ms | 20ms | 120ms | - | -| watch-reload-auto | auto | chromium | FAIL (1 checks) | 20ms | 20ms | 40ms | - | - -`3 rows graded, 2 passed, 1 failed, 2 not graded.` The two mute rows are not graded because 0 of 72 -and 0 of 73 tail samples carry an audio playhead at all: the row never recovered, which is finding -36\. The `watch-reload` failure is the conviction of finding 39. - -**After Fix 1** (finding 36), the same schedules plus the watcher reload inside a declared silence: - -| Row | Preset | Browser | Verdict | Target | Chunk | Fill | Opening stretches | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | -| mute-basic-auto | auto | chromium | PASS (2 not measured) | 20ms | 20ms | 40ms | 8 | -| mute-basic-100ms | 100ms | chromium | PASS (2 not measured) | 100ms | 20ms | 120ms | 1 | -| pub-reload-auto | auto | chromium | PASS (1 not measured) | 20ms | 20ms | 40ms | 11 | -| pub-reload-100ms | 100ms | chromium | PASS (1 not measured) | 100ms | 20ms | 120ms | 2 | -| watch-reload-auto | auto | chromium | FAIL (1 checks) | 20ms | 20ms | 40ms | 3 | -| watch-reload-silence-auto | auto | chromium | PASS (6 not measured) | 20ms | 20ms | 40ms | 5 | - -`6 rows graded, 5 passed, 1 failed, 0 not graded.` The mute rows are gradable for the first time and -they pass. The one failure is still the watcher-reload conviction. - -**After Fix 2** (findings 38 and 40), with the page's own selection and preset added as a row: - -| Row | Preset | Browser | Verdict | Target | Chunk | Fill | Opening stretches | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | -| mute-basic-auto | auto | chromium | PASS (2 not measured) | 20ms | 20ms | 40ms | 7 | -| mute-basic-100ms | 100ms | chromium | PASS (2 not measured) | 100ms | 20ms | 120ms | 2 | -| pub-reload-auto | auto | chromium | PASS (1 not measured) | 20ms | 20ms | 40ms | 12 | -| pub-reload-100ms | 100ms | chromium | PASS (1 not measured) | 100ms | 20ms | 120ms | 1 | -| watch-reload-auto | auto | chromium | FAIL (1 checks) | 20ms | 20ms | 40ms | 10 | -| watch-reload-silence-auto | auto | chromium | PASS (6 not measured) | 20ms | 20ms | 40ms | 8 | -| page-keeps-choice-100ms | 100ms | chromium | PASS (1 not measured) | 100ms | 20ms | 120ms | 1 | - -`7 rows graded, 6 passed, 1 failed, 0 not graded.` The conviction survives, and the narrowed guard -that was meant to stop it is proved unreachable on a cold tune-in: finding 39 has the timestamps. - -**After Fix 3** (finding 39), the final build of the player, `out15-after3/`: - -| Row | Preset | Browser | Verdict | Target | Chunk | Fill | Opening stretches | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | -| mute-basic-auto | auto | chromium | FAIL (1 checks) | 20ms | 20ms | 40ms | 1 | -| mute-basic-100ms | 100ms | chromium | PASS (2 not measured) | 100ms | 20ms | 120ms | 2 | -| pub-reload-auto | auto | chromium | PASS (1 not measured) | 20ms | 20ms | 40ms | 10 | -| pub-reload-100ms | 100ms | chromium | PASS (1 not measured) | 100ms | 20ms | 120ms | 5 | -| watch-reload-auto | auto | chromium | PASS (1 not measured) | 20ms | 20ms | 40ms | 6 | -| watch-reload-silence-auto | auto | chromium | PASS (6 not measured) | 20ms | 20ms | 40ms | 12 | -| page-keeps-choice-100ms | 100ms | chromium | PASS (1 not measured) | 100ms | 20ms | 120ms | 2 | - -`7 rows graded, 6 passed, 1 failed, 0 not graded.` The conviction is gone, with no warn line at all -where every build below printed one, and the failure has moved: `mute-basic-auto`'s unmute stretches -five times against a ceiling of three. That count was there before and the ring's own reset was -hiding it. It is reported, and it is in the open measurements. - -**After Fix 4** (finding 44), the three rows the 27-row regression set had failed on the overlay, -`out15-after4/`: - -| Row | Preset | Browser | Verdict | Target | Chunk | Fill | Opening stretches | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | -| hide-mute-auto | auto | chromium | FAIL (3 checks) | 20ms | 20ms | 40ms | 16 | -| hide-mute-100ms | 100ms | chromium | FAIL (4 checks) | 100ms | 20ms | 120ms | 2 | -| mute-basic-2000ms | 2000ms | chromium | FAIL (2 checks) | 2000ms | 20ms | 2020ms | 2 | - -`3 rows graded, 0 passed, 3 failed, 0 not graded.` The verdicts are worse-looking than the fix is, -and the reason is worth stating rather than smoothing: the check the fix targets, "the spinner is -hidden through the pause", **passes on both `hide-mute` rows for the first time**, 4578 ms and -4493 ms of overlay going to 0 ms. What still fails on them is the playhead rate and the missing -playhead through a hidden camera, which are the same ~110 ms resume gap the regression set found -everywhere, and on `mute-basic-2000ms` the 1420 ms of `Buffering (audio clock)` that a parked fresh -fill at a two second target produces by construction. Both are open measurements, neither is a -conviction and neither loses audio. - -### The regression set, 27 rows - -`mutematrix.py --set regression`, run for the first time at this tip, in `bench/out15-regression/`: -the eight schedules at `auto` and `100ms` in chromium, a burst of twenty mute and unmute pairs half -a second apart, `mute-basic` at `2000ms`, `mute-basic` and `pub-reload` at `auto` in brave, firefox -and webkit, and four error cases. The publisher is chromium on every row, because Playwright's -WebKit refuses a getUserMedia permission and cannot publish at all. - -`24 rows graded, 6 passed, 18 failed, 3 not graded.` - -What holds: - -- **Zero convictions and zero slow-group lines on every one of the 27 rows**, in all four engines. - That is finding 39 across the whole family rather than on one schedule. -- **`interrupted` is true on zero samples of zero rows.** Finding 36 holds everywhere the set - reaches, and it is what lets finding 44 attribute every remaining overlay to video. -- **The twenty rapid mute and unmute pairs work**, half a second apart, with the playhead back - inside half a second on each of them. -- The mute and reload family passes or fails only on the residuals below in brave, firefox and - webkit as well as chromium. - -What the 38 remaining failing checks are, and none of them is lost audio: - -- **A playhead gap of about 110 ms on every resume**, in the engine rows and the restart rows alike: - 19128 to 19230 ms on firefox, 21161 to 21268 ms and 21194 to 21314 ms on the two `hide-mute` rows, - and the same shape elsewhere. It is the fresh fill arriving at a deeper target, it is inside the - acceptance bar, and it is reported rather than fixed. -- **Stretch counts on resumes**, which rule 9 gives the tune-in allowance and which still exceed it - on a few rows. -- **58 of about 96 failing checks in the first pass were windows a second long**, from the burst of - twenty pairs. Rule 10 makes those NOT MEASURED, which is what took the count to 38. - -Three rows are **not graded**, and each says why: `device-bt-auto` and `device-bt-100ms` because -the driver reporting that no device matched the -Bluetooth headset's label, the headset simply not being connected, and `relay-during-mute-auto` because a blocking relay -restart moved the unmute marker inside the reference's own clear run. A fourth schedule, `instant`, -has no audio row at all by design: the preset disables audio download, so an `instant` audio row -measures only its own impossibility. - -**One thing the set found that is not a defect and is not explained.** Every tune-in in this run -took about 1.1 s where `out15-after3/` measured about 0.5 s, uniformly across rows: `pub-reload-auto` -reads `playerMs` 1112.9 ms here against 544.4 ms there. A fresh relay, fresh publishers and a fresh -browser reproduce the slower number rather than the faster one, `out15-drift/` reading `playerMs` -1142.6 ms on `pub-reload-auto` and 961 ms on the firefox unmute, so it is not accumulated bench -state. What differs between the two is run ordering: `out15-after3/` ran after about twenty-five -minutes of an idle machine. The idle-then-one-row test has since been run and the ordering -explanation did not survive it, which is finding 47: the difference is all in the stretch between the -tile swap and the first ring, it is bimodal across a boundary at 05:22 on 2026-09-18, and nothing on -the bench explains it. The honest statement is that the tune-in figure in this document is 0.33 s to -0.55 s on one side of that boundary and about 0.9 to 1.1 s on the other, and neither is a conviction, -a trim or an underrun. - -## Findings for the maintainer - -Things found while working that are separate from the fix. - -1. **`demo/web` pinned `delay="100ms"` on every tile.** That is the "100 ms chip on a fresh session" - from `watch.md`. Nothing was restoring anything, and no storage was involved. Fixed in - `47fbfb5c2`. - -2. **`ui/components/buffer-control.ts` sets a numeric delay on mousedown.** A single click on the bar - therefore leaves auto without the user asking for a value. Not changed here; it is a UI decision. - -3. **The released `latency` attribute falls back to a silent 100 ms** for an unparseable value, where - its `delay` sibling warns and falls back to the element default. Made consistent in `2d86e147a`. - -4. **`js/net` races a WebSocket against WebTransport after a 500 ms head start.** Any path that slows - the QUIC handshake past that head start silently falls back to TCP. On the harness's `bursty` - profile the shaper saw seven datagrams for a whole run, and the row measured a TCP session that - never touched it. The harness works around it by deleting both `WebSocket` and `WebSocketStream` - on the page before the module loads, which is a test hack. The real fix is an option on - `Connection`, and it is not in this branch. - -5. **`rs/moq-relay/tests/drills.rs` compiles zero tests today.** It is gated - `#![cfg(all(feature = "quinn", feature = "websocket"))]`, and `quinn` is not in `moq-relay`'s - default feature set, so `just test drill` runs nothing. It also still says `use moq_native::...`, - and `moq-native` is now a tombstone crate that refuses to build as a dependency, so the file would - not compile even with the feature on (34 API-rot errors when measured in session). A patch adding - the shaper's impaired second lane, which `transport-impairment-profile.md` asks for, is written - and was deliberately not landed on top of a test file that does not build. - -6. **Safari takes the WebSocket path**, so the shaper does not apply to it and the Safari lane grades - an unimpaired path. The lane records `shaper: none` rather than labelling an unimpaired run - `bursty`. - -7. **`conceal` needed an element attribute** to be settable from a page before connect, so - `` exists (`b23179839`). It is the only new attribute. - -8. **The shaper's own jitter was reordering datagrams, and it was grading the wrong thing.** Each - datagram drew its release time as `now + delay + jitter * gaussian`, independently of every other - one, so on any profile with non-zero jitter a later datagram was routinely released before an - earlier one. QUIC read that as loss, retransmitted, and backed off, so a profile named `mild` was - measuring congestion response. Real jitter is queueing delay on a FIFO path: it varies the delay - and preserves the order, and only the reorder draw is meant to overtake, which is what makes a - reorder a deliberate act the counters can attribute. Fixed in `f86598325`: each lane remembers - when the datagram in front of it leaves and clamps the next one to that, before any reorder delay. - On the 60 s opus plain rows, target p95 falls from 1960 ms to 420 ms on `mild` and from 1760 ms to - 360 ms on `high-rtt`. - - The mechanism is worth stating, because it is the player behaving correctly on a path that was - lying. The reordering cost the *video* stream, not the audio one: audio is small and fits a - datagram, video is a stream that QUIC throttled as it retransmitted. Under the reordering shaper - the measured audio spread stayed at 100 to 180 ms while the video spread ran at 1.6 to 2.0 s, and - `Sync` in auto takes the larger of the two across tracks, so the audio buffer was dragged to 1.5 - to 3 s by a number that had nothing to do with audio arrivals. Every jittered row's inflated - `target p95` in the recorded budgets was that, and every one of them came down when the shaper - stopped reordering. Whether one track's spread should be allowed to size another track's buffer is - a real question for the maintainer, separately from the shaper defect; this branch keeps the - `max()` across tracks that #3517 had. - -9. **The harness has to address the shaper by `127.0.0.1`, not by `localhost`.** Chromium resolves - `localhost` to `::1` first, and `moq-shaper` binds IPv4. With the WebSocket fallback still in the - page that produces a silent fallback to TCP straight past the shaper, which the production page - would do too; with the fallback removed, as the harness removes it, the page simply never - connects. Neither failure says what went wrong. Every URL the harness hands the page is literal - `127.0.0.1` for that reason. It is also a second argument for finding 4: a page that races a - WebSocket against WebTransport turns a name-resolution mismatch into a working but unimpaired - session, and there is no option on `Connection` to refuse that. - -10. **Publisher mute played as concealment, then room noise, for as long as the mute lasted.** A user - published from a real microphone, muted it, and the watcher carried on making sound. Two - mechanisms, one on each side. - - On the wire, a mute is invisible. The publish element's `muted` turns the audio encoder off, the - frames stop, and nothing says whether the next one is late or never coming. A capture of the - audio track across a 10 s mute, taken from the watcher's own connection, carried no marker of - any kind: group 590 at 14.81 s, then group 591 at 24.86 s, contiguous sequence, ten seconds of - presentation missing. The only thing that eventually stopped playback was the rendition leaving - the catalog, which is a different track, arrives on its own schedule, and on a loaded or remote - relay can lag the audio it describes by a lot. - - In the reader, concealment had no end. Commit 31 deviated from the pinned Chromium tree on - purpose and faded the concealment into the background level rather than to zero, on the reading - that ending an outage in digital silence was a regression. It is not: current WebRTC does end - there. `Expand::Process` scales each block by `mute_factor`, lowers it by the muting slope, and - pins it to zero outright past the third consecutive expansion, and - `BackgroundNoise::GenerateBackgroundNoise` ignores both the slope and the `TooManyExpands()` it - is handed and never raises its own mute factor, which `ChannelParameters::Reset` leaves at 0. On - a file or a tone the difference is inaudible, because the estimated background is the digital - floor. On a microphone it is the room, and the listener hears it forever. - - Both are fixed. Concealment now fades to silence and the ceiling is silence, in both engines. - The publisher writes the empty frame hang already defines as an audio endpoint when muting stops - the encoder, and declares a break before the first frame when it resumes; the watcher carries - the endpoint to the ring, which renders silence with no concealment, no underrun, and hands the - clock to wall time at once. - - Measured on this worktree's `demo/web`, headed Chromium, the real microphone, the local relay, - the shared-memory ring, 12 s of speech then 12 s paused then 12 s of speech. The pause is the - capture graph suspended rather than the element muted, so the subscription outlives it and the - engine's own behaviour is what is being measured: - - | Ring | | RMS p50 over the pause | peak | first silent 100 ms bucket | silent from then on | - | --- | --- | ---: | ---: | ---: | ---: | - | shared | before | 2.2e-4 | 1.6e-3 | never, over 12 s | no | - | shared | after | 0 | 4.3e-4 | 0.2 s | yes | - | postMessage | before | 3.1e-5 | 1.2e-4 | never, over 12 s | no | - | postMessage | after | 0 | 6.1e-5 | 0.2 s | yes | - - The postMessage pair ran in a quieter room, and it shows the mechanism plainly: over that pause - the before run's RMS *rose*, from 1.9e-5 while real audio was playing to 3.1e-5 once concealment - took over, because the comfort noise it settled on was louder than the room the microphone was - actually sending. - - The largest sample-to-sample step anywhere in the pause is 9.2e-5, so it fades rather than cuts, - and audio is back in the first 100 ms bucket after the resume. The engine still conceals the - outage (`concealed` grows by 11.4 s of samples in both runs, since nothing declared it): what - changed is where the concealment ends up. Both rings were measured, because the shared one is - what a cross-origin isolated page gets and the postMessage one is what production serves. - - With the element's own mute, and so the endpoint, the local relay delivers the catalog update - within ~100 ms and `` tears the whole audio graph down when the rendition leaves the - catalog, which is faster than the probe's resolution. What the run does show is the counter: - `audio.out.debug.concealed` over the same sequence went from 46,244 samples before to 0 after. - The endpoint's own path is covered by unit tests on both rings instead. - -11. **The re-anchor above handed the decoder a chunk it refuses, and the audio never came back.** - `DataError: Failed to execute 'decode' on 'AudioDecoder'` in roughly one run in three of - `chromium-opus-48000-mild-isolated`, always within a second of a rendition handover. This one is - the branch's own: `fdc9a2ae1` introduced it, and it does not pre-exist upstream. `upstream/dev`'s - audio decode loop has no `#reanchor` and no `Terminal.continues`, so the throwing call site is - not there at all; its only reset site is the container discontinuity in `#onNext`, which is - always followed by a group's first frame. The harness agrees: no run recorded before the hole fix - carries it (`aq-final-1` at 02:14, `aq-enforce` at 03:13), and every run after it carries exactly - one (`aq-int-1` at 16:20, `aq-int-2` at 16:45, `aq-int-enforce` at 17:12). `fdc9a2ae1` is dated - 13:02, between the two groups, and a sweep of every recorded run directory finds the string in - those three and nowhere else. - - The mechanism is one line. `Container.Consumer` marks only a group's *first* frame `keyframe: - true`, and deliberately so: `Cmaf.Format` never reports an audio keyframe, because packagers flag - every audio sample a sync sample and each one would otherwise open an epoch. WebCodecs is the - other way round: a decoder that was just configured, reset, or flushed accepts only a chunk - marked `key`. `#reanchor` resets and reconfigures the decoder and then the loop decodes the very - frame that reported the hole, and that frame is mid-group far more often than not, so Chromium - threw. The throw escaped `effect.spawn`, which ends the decode loop for the life of the session: - the ring drains, `stalled` latches true, and RMS sits at zero for the rest of the run. The - DataError is not a symptom of the audio dying, it is what kills it. - - Instrumented and reproduced deterministically at 240 s, where the publisher's catalog update at - about 183 s replaces the audio subscription: - - ``` - received catalog / subscribe close id=1 / subscribe start id=3 the handover - decode ts=182860000 key=true grp=79 idx=0 first frame of the run, forced key - decode ts=182903229 key=false grp=79 idx=1 43.2 ms later, no hole reported yet: - nothing had decoded, so there was no - frame duration to measure one against - reanchor at ts=182946438 key=false grp=79 43.2 ms again, now measurable: a hole - reanchored state=configured flush, reset, configure - decode ts=182946438 key=false ... DataError - ``` - - The 43.2 ms spacing is the handover joining a group already in flight under the subscription's - own max age, which is why the re-anchor was right to fire. What was wrong is the chunk it was - then handed. - - Fixed by `audio/anchor.ts`: a small `Anchor` that remembers the decoder was restarted and types - the chunk that reopens the run `key`, whatever the container called it. That is the same claim - the container already makes at every group boundary, and it is sound for exactly the reason - re-anchoring on an arbitrary frame is sound: every frame of Opus, AAC and MP3 is independently - decodable. Nothing is wrapped in a catch, so a decode that still fails surfaces the way it does - today. Both decode loops and both drain sites (`#reanchor`, `#declareEnd`) mark it, since a flush - alone ends the run the decoder will accept a chunk into. - - `anchor.test.ts` drives a mock decoder that refuses a non-key chunk the way Chromium does; four - of its five cases fail against the old `frame.keyframe ? "key" : "delta"`. Six 240 s harness runs - of the row after the fix, each of which reached the handover, record zero occurrences. - - One nuance worth passing on. The observed failure is this branch's, but the class of it is not - quite. The `#onNext` epoch reset exists on `upstream/dev` too, and it also resets, reconfigures, - and then decodes whatever the consumer hands back. That is normally a group's first frame, so it - is normally a key. The exception is `Consumer.#checkReset`: a publisher rewind resumes from the - earliest surviving group, and a survivor may already be part delivered, in which case the frame - after the discontinuity is mid-group. No run here has hit it, so this is a reading of the code - rather than a measurement, but the `Anchor` closes it wherever it is adopted. `Video.Decoder` has - no equivalent exposure: it configures once per run and never resets, and its frames are not all - independently decodable anyway, so it keeps the container's own flag. - -12. **Firefox's underruns after convergence are not the path.** Watching the same broadcast over the - same relay at the same moment, Chromium's worst reading gap was 46 ms against Firefox's 499 ms, - and a second Firefox round was clean at 32 ms. What Firefox records as a late arrival is its own - content process stalling for two to five hundred milliseconds at a time: the worklet keeps - rendering, `context.currentTime` advances 24 ms over 924 ms of wall clock, the painted video - timestamp barely moves, and the decoder's p99 of 402 ms is the length of the stall its callback - waited behind rather than any work the codec did. The damage lands in the estimator. - `Jitter.observe` already discards a receiver's own reading gap, dropping the arrival reference - when `idle` and `idle - progress` both exceed one resample interval, and in this trace it never - fired once: the worst gap was `idle - progress` of 499 ms against a 500 ms threshold, with six - more between 150 and 500 ms. Each one entered the delay measurement at full weight and stayed in - the histogram for about twenty-nine seconds of wall-clock memory, so the target climbed from - 40 ms to 780 ms and took about forty seconds to decay back, one bucket per sample. Replaying the - recorded arrival series through the real estimator, ring and stretcher on the simulated clock - reproduces it with no browser present: four underruns and an 820 ms peak from the Firefox trace, - none and 80 ms from the Chromium trace recorded beside it. The guard's shape is right and its - media term does its job. Only the threshold missed, and widening the target is the symptom - rather than the fix. Spacing alone cannot separate a 400 ms receiver block from a 400 ms flush, - so the fix observes the receiver directly: a 50 ms event-loop timer beside the consumer marks - the arrivals read in the tick after a block of 100 ms or more, and `Container.Jitter.observe` - takes that as an explicit input, in both languages, and drops the reference for it. The corpus - gains `receiver-stall` (target stays at 20 ms) and `receiver-stall-unflagged` (the same arrivals - reach 400 ms, the documented fallback), and the recorded Firefox window is `mic-firefox.json` in - the replay lane: the estimator peak falls from 1600 ms to 300 ms, the residual being blocks the - recording's 250 ms sampler could not resolve, and the underruns during the block itself remain, - because a receiver that stops executing for 674 ms cannot be buffered for without the 1.6 s of - latency the old target hid it with. - - Two rounds of 120 s each, one Chromium publisher on a pinned USB microphone feeding a Firefox - watcher and a Chromium watcher of the same broadcast at the same moment, both on the plain page. - Counters are deltas from 20 s in. - - | Round | Engine | Target p50/max | Level p50/p95/min | Underruns after 20 s | Concealed | A/V skew p50/p95 | Worst gap | - | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | - | 1 | Firefox | 40/780 ms | 70.9/751.7/0 ms | 1 | 1240 ms | 20.1/39.4 ms | 499 ms | - | 1 | Chromium | 40/40 ms | 68/81.3/41.3 ms | 0 | 0 | 21.7/41.6 ms | 46 ms | - | 2 | Firefox | 40/40 ms | 67.8/81.2/47.3 ms | 0 | 0 | 18.4/38 ms | 32 ms | - | 2 | Chromium | 40/40 ms | 71/84.8/44.1 ms | 0 | 0 | 20/40 ms | 14 ms | - - The microphone is a USB device at 48 kHz, 2 channels, with echo cancellation, noise suppression, - auto gain and DTX all off. This machine has no built-in microphone, so the publisher is pinned to - that device by `deviceId` and the run aborts if the live track is on anything else. - -13. **A browser publisher stopped seeding catalogs once its first viewer left.** The first watcher - played; every watcher after it got `subscribe ok` and no catalog frame, so no rendition was - chosen and the tile sat empty. The publisher logged `publish error: track=catalog.json - error=remote error: 1` per track as the first watcher went, which is `StreamCode.Cancel`, the - routine unsubscribe, and restarting the publisher was the only way back: in a call, where viewers - come and go, that makes a browser publisher single use. Reproduced in Chromium and in Firefox. - Upstream code, untouched by this branch. - - Three things line up. `js/net/src/broadcast.ts` `subscribe()` dedups on a cached track producer - it holds strongly, and drops the entry only when that producer closes, so repeat subscriptions - fan out from one request. Its own comment says the publishing side leaves `register` false, but - the publishing wire subscribes through a broadcast `Consumer` (`js/net/src/lite/publisher.ts` - `runSubscribe`), and `Consumer.subscribe` is the call that sets it. And nothing on the publishing - side closed that producer: `js/publish/src/broadcast.ts` served each subscription from a child - scope released on `track.closed`, which only its own cleanup would cause. The wire closes the - subscriber it read from, not the producer behind it. - - So the entry outlived the viewer, and the next SUBSCRIBE fanned out from a catalog track nobody - writes to between catalog updates. `Json.Snapshot.Producer` publishes the catalog with - `deltaRatio: 0`, one whole catalog per group and the group closed behind it, so the retained seed - ages out after `DEFAULT_MAX_AGE_MS`, 5000 ms in `js/net/src/track.ts`. Past that a subscriber - gets an accepted subscription and no frame, for as long as the publisher runs. - - What hides it is the relay: a watcher arriving within 30 s is served from the relay's own cached - track and never reaches the publisher at all. The trigger is any re-subscribe after the relay - releases that cache on its idle timeout, which is what makes it look intermittent. - - Fixed in `3f83de0fc`: the serving scope is released on `Promise.race([track.closed, - track.unused()])`, so a track that loses its last subscriber leaves the cache, and the next - subscription raises a fresh request the catalog producer seeds with the current catalog. Serving - is registered before the close cleanup too, so the snapshot producer drains while the track it - writes into is still open. Two tests in `js/publish/src/broadcast.test.ts` fail without it. On the - bench, three watchers of one microphone publisher with 50 s gaps between them all played. - - Two neighbours are recorded rather than fixed. `Rendition.track` documents a demand gate, - "producers should encode only while this is set", that no longer exists: the publisher encodes - with zero viewers. Making it real is larger than it looks, because it immediately exposes two - more, an audio encoder that does not re-anchor after a gated interval (40.7 s of A/V skew on the - first viewer after one) and a `Container.Legacy.Producer.close()` that throws `group is closed` - once its track has gone. Those two are fixed here after all, in `b366db8bb`; the gate itself - belongs in its own quest rather than in this branch. - - The other shape this fix could take is in `js/net`: hold the publishing side's dedup weakly, the - way `rs/moq-net/src/model/broadcast.rs` holds its tracks in a `WeakCache`, so an unused track - leaves the cache on its own. It was not taken because the subscribing side leans on the strong - entry: it re-checks demand level-triggered, so a tile unmuted a moment later resumes the same - subscription instead of opening another. Which layer should own it is the maintainer's call. - - One half of that is now answered, in finding 19. The same release was tried for the media tracks - and it is wrong there: the publishing wire subscribes through a broadcast `Consumer`, which sets - `register = true`, so a repeat subscription already fans out from the live producer, and - releasing the scope ends the track on the wire, which is the defect finding 19 is about. The - catalog track keeps this fix, because its producer writes only when the catalog changes and a - subscriber arriving in between has to be seeded by a fresh request; a media track is being - written continuously and needs nothing. - -14. **A browser publish put its two tracks on two different epochs, and sound led picture by a third - of a second.** `js/publish/src/audio/capture.ts` handed the worklet `zero: performance.now() * - 1000` when the node was constructed, and the worklet stamped each quantum `sampleCount / - sampleRate + zero`. A capture graph renders a couple of quanta as soon as it is built, then its - clock stalls about 245 ms while the microphone opens, then runs in real time, so sample 0 carried - the wall time of the node's construction rather than of the audio it describes, backdated by the - priming and the stall together. Video has no such gap: `js/publish/src/video/processor.ts` - `Epoch.stamp` anchors on the first frame's arrival, and a probe inside the page reads it within a - millisecond of the wall clock. Upstream code, untouched by this branch. - - Measured with that probe on both captures: audio lagged the wall clock by 235 to 272 ms across - the probe runs, against 0 to -0.3 ms for video, so the epoch offset is the audio lag. The saved - probe files read 235.0, 238.9 and 249.3 ms; 272 ms is the run quoted in `c53ecfe49`'s own commit - message. The same thing read from the watcher's side is - the video lead, `sync.out.timestamp` minus `audio.out.timestamp`: 352 to 464 ms across five - publisher runs, and 0 on the file publishers, whose two tracks come off one container timeline. - - Every viewer of a browser publish heard sound that far ahead of the picture, and the skew metric - could not see it, because video is painted at the audio playhead. It is also the whole unmute - transient: during a mute nothing nominates a clock, so `Sync.received()` skips video to `maxAge` - off the live edge (measured at `syncTs - 33`, the 20 ms delay plus one frame), and the unmute - hands the clock back and walks the picture the offset's worth back down. That is the 332.9 ms - Firefox still showed after `19edd01ce`. - - Fixed in `c53ecfe49`: the worklet posts `currentFrame`, its position on the context's own sample - clock, and the main thread stamps `zero + frame / sampleRate` with `zero = Time.Micro.now() - - context.currentTime` read as one pair. The pairing is re-derived whenever an observation runs - more than 20 ms ahead of the anchor, which is a device opening, a suspend or the machine - sleeping, and which the framer then sees as the discontinuity it is. A context that cannot say - where it is in time fails the capture stream rather than shipping a silent offset. - `getOutputTimestamp()` was measured and rejected: it reports zeros across the priming quanta and - then sits 24 ms later than `currentTime`, because it describes output rather than capture. - - After: the probe reads 5.0, 5.1 and -8.2 ms over three publisher runs, and the watcher's video - lead in steady state is 68 to 91 ms (68.6, 73.2, 83.2, 90.5), which is the playout hold rather - than an epoch. The worst - skew over the mute sequence falls from 333 ms to -46.8 ms in Firefox, -38 ms in Chromium and - 60.2 ms in WebKit. The new case in `js/watch/src/sync.replay.test.ts` replays the sequence with a - video arrival feed and a configurable publisher offset: 36.7 ms when the two clocks agree, - 344.7 ms with them 350 ms apart. - - What is left is the camera pipeline's own latency, which sits inside the video arrival anchor and - cannot be measured from inside the page. An external reference, a clap or a flashing screen - recorded off the two ends, is what would resolve it, and it is a follow-up rather than something - this branch settles. - -15. **Real Safari played a broadcast's video in silence about one run in eight, and the click had - already been spent.** On the demo page the click landed on the tile, the tile unmuted, video - painted for the whole minute, and the `AudioContext` read `suspended` at every sample and never - reported a playhead. Measured through `safaridriver` with exactly one Element Click per run - against a microphone broadcast: 7 of 8 runs played, 1 did not. - - Two things spent the activation, and each alone is enough. The graph was rebuilt whenever - anything in the decoder config changed, and a browser publisher's Opus rendition gains its - `description` in a *later* catalog frame, so the context the click had just started was closed - and replaced about 30 ms later by one born suspended. And the unlock was armed only while audio - was enabled, so the click that unmutes a tile armed its listeners a microtask after its own - gesture had already passed. Neither should have worked at all. Both did, most of the time, - because WebKit honours a `resume()` outside a handler while the page's activation is still live, - which is a few seconds; once that lapses the context stays suspended for good. The retraction - above records the earlier reading of that grace as a page activation carrying to a context - created later, and corrects it. - - Fixed in `f846462e5`. The context is built by an effect keyed on the rate it runs at and nothing - else, so a later catalog frame rebuilds the worklet and ring *under* a context that is already - playing. A decoded rate that differs from the catalog's is still a new context, because a - context's rate is fixed for its lifetime and that is the one case worth paying for. The unlock is - armed for the decoder's lifetime rather than as a consequence of the click, so a gesture is spent - on the context that exists when it lands, and it bails where there is no `document` instead of - assuming a browser. `js/watch/src/audio/decoder.test.ts` counts the contexts a decoder builds: - one case clicks on a muted tile and expects the context it started to be running, the other feeds - a catalog frame that only adds the codec description and expects the same context, still running, - still the only one built. Both fail without the fix. - - After: 8 of 8 real-Safari runs reach `running` 4 to 12 ms after the one click and hold a playhead, - with a single context per run. Chromium and Firefox microphone rows and the real-Safari mute and - preset sequence are unchanged, and are in the tables above. - - **One behaviour change to decide on, because it is visible on the demo page.** A gesture anywhere - on the page now starts the context of *every* tile that has one, since the unlock is no longer - gated on a tile being unmuted. On the five-tile demo page that is five `AudioContext`s started by - the first click. They render nothing while their tiles are muted or disconnected, so the cost is - five idle contexts rather than five sounds. Three shapes: - - - **Keep it.** Simplest, and one gesture reliably unlocks every tile, including the one the - viewer unmutes a minute later. The cost is the idle contexts. - - **Start only the tile that was clicked, and arm the rest.** Needs the gesture's target, which - means the unlock stops being a document-level listener and starts knowing about elements. - - **Build contexts lazily behind one shared silent unlock context.** A different shape for the - whole audio graph, and the largest change of the three. - - Recommended: keep it, and say so in the element's docs. An idle `AudioContext` is cheap next to a - viewer who hears nothing and does not know why, and the two alternatives both buy the saving with - a coupling the current design does not have. - - **Answered in finding 23**, and not by the recommendation above. The second shape turned out to - cost nothing: a tile builds its context inside the gesture that lands on it, so a muted tile - holds none at all, and real Safari still plays 8 of 8 on one click. The judgement item is - retired. - -16. **CodeRabbit over the whole branch: 22 findings, 18 fixed, 4 rejected.** The review exceeded the - free plan's 150-file cap, so it ran per top-level directory instead. The eighteen real ones - became `d8d78ead0`, `220b09934`, `1b90331be`, `6efdb3c73`, `c3571e072` and this docs commit. The - four rejected: a suggested `max_age` clamp on the native target that - `playout::delay::Constraints::ceiling()` already applies one layer down; a claim that - `playwright@^1.63.0` is unpublished, which `bun.lock` and two sibling harnesses contradict; and - two on the same line asking for `runs-on: ubuntu-24.04` where the comment above it explains that - `ubuntu-latest` is deliberate parity with `smoke.yml` and `wasm.yml`. - -17. **Upstream's `moq-auth` lease changed what an empty `public` means.** `#3688` made `auth.public` - a path pattern rather than a prefix, so the harness relay's `public = ""` went from granting - everything to granting nothing, and every Chromium row would have been voided on a rejected - session. `499d7806c` gives it the `"**"` upstream gave the demo and smoke configs. Worth knowing - for any other config in the wild that still carries the empty string. - -18. **A video track that stopped once stopped for good, and the player called it "recovering".** A - user published from their own browser and watched their own tile: audio played for the whole - session, the picture froze, the stats panel read one burst of video and then 0 kbps and 0 fps, - and the badge said `stalled: recovering` while nothing was recovering. The relay log of that - session names it. The watcher subscribed to `track=video`, cancelled it, and did that six times - before stopping for good, then never asked for video again for the rest of the session, while it - re-subscribed to `audio` after every publisher reconnect. - - Nothing in the player could bring video back. The `VideoDecoder` error callback called - `effect.close()`, which closed the `DecoderTrack`'s whole effect and its subscription with it, - while `#active` kept pointing at the dead track, so `#runPending` never built a replacement: the - relay saw the subscription cancelled and never saw another. The 500 ms `stalled` flag only ever - labelled the result, and `demo/web`'s "recovering" was a string beside it. A hole in delivery fed - a delta to a codec with no reference for it, which is what raises the error for a container that - carries keyframes inside a group, so the two are one chain. - - Fixed in `b366db8bb`. The decoder records why a track died, and the parent rebuilds it at the - live edge: a 1 s retry, a stall of 5 s doubling to a 60 s ceiling so a hopeless rendition costs - one subscription a minute rather than one every five seconds, and the ceiling reset on the first - painted picture. After a hole, deltas are withheld until the next keyframe. - - The publisher had the same shape one layer down. `Fanout` closed on a source that ended or failed - and told nobody, so a capture pipeline that stopped while the camera track was still live left - the preview holding its last frame, the capture rate at zero and the encoders quiet, with the - element still announcing a live broadcast. `Fanout.ended` and `Capture.out.stopped` now say why, - `Capture` rebuilds on a track that can still deliver, and the resolved dimensions stay with the - source across the gap instead of blanking and dropping the rendition out of the catalog, which - would take every viewer's subscription with it. That is option A of the three the catalog had. - - Two more landed alongside, both found by trying the demand gate finding 13 describes: the audio - encoder declares a break after a gated interval, so the first frame after it is not read as - continuing the last one, and `Container.Legacy.Producer` no longer flushes an endpoint into a - group its closed track has already torn down. And a subscriber leaving resets the stream with - `Cancel`, which is the normal end of a subscription, so all three sites log it at debug rather - than as a publish error. - - **The lip-sync hypothesis is false.** Late video is painted late, not discarded: the run that - forced skipping at 1080p60 skipped 7 video groups with zero stalled samples, zero underruns and a - painted-minus-audio median of -7.7 ms over 302 samples. **And the capture stop never reproduced - on the bench**: this USB camera sustains 1080p60 with its track live throughout, 60 fps end to - end, painted skew p50 8 ms and p95 32 ms after warmup, zero stalls, zero skipped groups and zero - underruns over 60 s of self-publish. What the user's relay log named instead is the defect above, - and the one below. - -19. **A video rendition's track was finished on the wire whenever it stopped encoding, and the relay - answered every later subscription from the finished track.** After the rebuild above, or a hide - and show, the watcher still never got its picture back. The relay logged `subscribed complete` - 2 to 20 ms after `subscribed started` for a watcher re-subscribing every five seconds, with no - upstream subscription raised in between, while the publisher kept capturing and kept the - rendition in its catalog: - - ```text - 10:31:08.929 subscribed started id=7 broadcast=toggle.hang track=video - 10:31:08.932 subscribed complete id=7 broadcast=toggle.hang track=video - 10:31:14.251 subscribed started id=8 - 10:31:14.256 subscribed complete id=8 - 10:31:19.783 subscribed started id=9 - 10:31:19.803 subscribed complete id=9 - ``` - - The container producer is built per encode run, and `Container.Legacy.Producer.close()` closes - the track under it, so hiding video, a capture that went away or a reconfigure ended the *track* - rather than the group. A clean close FINs the subscribe stream in `js/net`, - `rs/moq-net/src/lite/subscriber.rs` reads that FIN as `ServeEnd::Finished`, and a finished track - is finished for good: every later subscription is served from it and completed at once. - - Fixed in `002ae00e1`: the encoder stops at the group with `producer.cut()`, the way the audio - encoder already declares its stop, and the track producer belongs to the Broadcast, which closes - it when the rendition is superseded or unregistered. An encoder error still closes the track with - the error, which resets the stream rather than finishing it, so the peer can come back. - - This also answers the dedup question finding 13 left open, and closes it the other way. The - publishing wire subscribes through a broadcast `Consumer`, which sets `register = true`, so a - repeat subscription fans out from the live producer and needs no fresh request. The `unused()` - release for media tracks is therefore dropped for good: it ends the track on the wire, which is - this same defect, and it measured 5421 ms of worst skew on the mute sequence when it was tried. - - Three test files across two packages: `js/net` that a repeat peer subscription fans out from the - live track rather than raising a fresh request, the video encoder that a rendition hidden and - shown keeps one live subscription and resumes on a keyframe and that a subscription opened after - the stop is served, and the audio encoder that a subscription dropped and reopened three seconds - later lands on the same live track at the live timestamp. On the bench, three re-subscriptions to - one video rendition lived 9.2 s, 4.9 s and 14.9 s, each ended by the watcher's own cancel and - none by a `complete`, and one video subscription lived 54.6 s through the whole mute sequence and - was completed only when the publisher stopped. - - **Worth the maintainer's eye.** A relay latching a finished track is protocol-correct: a FIN says - the track is over. It is also unforgiving for any publisher that finishes a track it means to - resume, and the failure is silent on both sides: the subscriber gets an accepted subscription - that completes immediately, and the publisher keeps capturing and advertising. Whether `js/net` - should make that distinction harder to get wrong, or whether "stop the group, never the track" is - simply the rule a publisher has to know, is a call worth making once rather than per publisher. - -20. **An unmute started the arrival estimate over, and the spinner could not tell a fill from an - underrun.** A muted tile downloads no audio, so unmuting subscribes again and built a second - `Container.Consumer` seeded from the rendition's declared flush span: 320 ms on the file - publisher and the 80 ms floor on a browser microphone. The ring then parks the playhead until it - holds that plus a chunk, and the buffering indicator keyed on the ring's raw `stalled` flag, - which is one flag for three different states: never played, deliberately parked, and ran dry - mid-playback. The viewer saw a spinner over video that never stopped, on every unmute. - - Two fixes. `c90eec220` makes the estimator live as long as the rendition rather than as long as a - subscription: `ConsumerProps.jitter` widens to `Time.Milli | Jitter`, so the replacement consumer - is handed the estimator that was already measuring and continues it through its existing - `reanchor()`, which drops the arrival reference and keeps the distribution, exactly as a - discontinuity does. NetEq keeps its delay manager across a pause for the same reason - (`modules/audio_coding/neteq/delay_manager.cc`). The measured spread is published for the - rendition's lifetime too, because clearing it with the subscription made `Sync` read the delay as - collapsing and then deepening across the mute, which is the decoder's own cue to park a second - time. `86baf84e7` adds `Audio.Decoder.out.interrupted` beside `out.stalled`, a stall entered - after the ring had played, and the indicator keys its audio arm on that and its video arm on - video stalling while the player is not paused. - - Measured on both pages and on the microphone: the spinner is visible for 0 ms on an unmute, the - ring fills against the measured 60 ms or 40 ms rather than the declared 320 ms or 80 ms, the - target is the same either side of the mute, the playhead is back 231 to 282 ms after the unmute, - and no toggle row peaks on an unmute any more. The tables are under "After the user's listening - round" below. - - `rs/moq-audio` has the same estimator and the same seeding rule and needs no change: its engine - is built once per `decode::Consumer` and it has no notion of a resubscribe, so there is nothing - to mirror and the conformance corpus is untouched. - -21. **A replacement subscription replayed the window its predecessor had filled.** Repeat - subscriptions to one track share a single upstream, so an unmute, a rendition swap or a - reconnect landed on that cache and walked the whole retained window at decode speed before it - reached live media. On the bench an unmute convicted a group on the way back in - (`skipping slow group: track=audio`) and discarded the rest a frame at a time, three seconds - being the mute. - - `97f32ea66` moves a media subscription's local read cursor to the live edge, which is what - `rs/moq-audio/src/decode/consumer.rs` does for `Start::Latest` and what the JS side never had. - Deliberately the local cursor and not the subscription's own group start: that field is a request - to the publisher, aggregated across every live subscriber, so naming a stale cached sequence - there asks the publisher to rewind the track for everyone reading it. Buffered playback keeps - `oldest`, since media written ahead of the playhead is the point there. - - Tune-in no longer convicts a group. What is left is one conviction per unmute, and it is a - question for `moq-net` rather than a patch here: a group from before the mute is still delivered - alongside the live edge and convicted by the 40 ms budget - (`sync[audio]: 9 late frame(s), max 3.1s behind`), although `Subscription::max_age` should have - skipped it (`rs/moq-net/src/model/track.rs:1494-1496`). Either freshness is judged by a group's - newest frame rather than its oldest, or a new subscription on an already-served track does not - re-apply the budget. One line of instrumentation settles which: the convicted group's first and - last timestamps, and whether it was open or closed when it was judged. The cost today is one warn - line per unmute and zero underruns. - -22. **Hiding video and showing it again left the preview black, because a busy device spent the - retry budget.** Hiding stops the camera, which starts handing the device back to the OS, and a - browser finishes that some time after `stop()` returns, so the capture the re-show starts can be - refused with `NotReadableError`. Every refusal spent retry budget, and `Retry`'s `LIMIT = 3` with - a 250 ms to 1 s backoff is 0.9 to 1.8 s of budget: four refusals inside that window ended the - capture for good, `out.source` stayed undefined, the preview painted black, the video rendition - dropped out of the catalog, and nothing ever asked for the camera again. - - `3539e4ef2` makes a busy device a wait rather than a verdict. `NotReadableError` and `AbortError` - back off without spending budget, so the capture keeps asking for as long as it is wanted, and - every other reason spends it exactly as before. The next attempt awaits the previous capture's - release, which nothing ordered before. The reason is kept rather than swallowed, so `Camera`, - `Microphone` and the element can say why there is no picture: `MoqPublish.errors` and red camera - and microphone buttons with the reason in the tooltip, instead of a black canvas speaking for it. - Two more alongside: the codec probe races `effect.cancel`, so showing video again while a probe - was still out no longer holds the encoder at its old answer with no rendition in the catalog - meanwhile, and the mirrors wiring a selected source into the capture inputs bind to the run that - made them rather than to the element, so switching source no longer leaves one behind per switch. - - **It is not reproducible unaided on this bench**: this camera frees in under 100 ms, so the first - frame after a show arrives 47 to 128 ms later and the preview paints within another 30 ms, in - Chromium, Brave, WebKit and on the `preview="encoded"` path, before the fix and after it. - Firefox delivers the frame in the same time and paints nothing, which is finding 25 and predates - this branch. It was proven with an injected 8 s busy - window in Brave and in WebKit, which is longer than the whole retry budget: before, the camera - was never asked for again, the preview stayed black and the catalog had no video section 15 s - later; after, the camera comes back within half a second of the device being free. The table is - below. - - **A judgement item.** A busy device is now retried for as long as video is switched on, at most - once a second. That is deliberate, since the alternative is the black preview above, and it is - the one place on this branch where a failure is retried without a bound. The bound that exists is - the user's own switch: turning video off stops it. - -23. **The audio context was built at load for every tile, and building it on the gesture instead is - both quieter and more reliable.** This retires the judgement item finding 15 left open, with a - measurement rather than an argument. Every tile used to build an `AudioContext` as soon as the - catalog named a rate, so the demo page's muted tiles each cost a "The AudioContext was not - allowed to start" warning and a suspended context with its own render thread, for audio nobody - had asked to hear. - - `16e6242aa` builds a tile's context when it has a reason to: inside the gesture handler for a - tile that has none, and when the app turns audio on for a tile that is not muted. The gesture - path builds and resumes synchronously, because an effect the gesture schedules runs a microtask - later and WebKit only starts a context resumed outside a handler while the page's activation - lasts. The rate the decoder turns out to emit still replaces the context, since a context's rate - is fixed for its lifetime, and the replacement is armed again rather than assumed started. - - Measured, in browsers with their default autoplay policy. Real Safari 26 through `safaridriver` - with exactly one Element Click on a muted tile: 8 of 8 runs build one context at the click, read - `running` at the first sample after it, 4 to 12 ms later, and hold a playhead with zero - underruns. The demo watch page with two tiles, one muted: Chromium falls from 2 warnings and 2 - contexts to 1 and 1, the one that is left being the tile the page itself unmutes; Firefox falls - from 4 warnings to 2. Muted tiles hold no context at all, and the site page, whose player starts - muted, logs nothing and holds nothing. The nightly - Chromium lane still produces a playhead and its counters (`fixed-250 opus plain`, not voided), - which is the lane that drives a browser with no gesture at all. - - Gesture first, with the broadcast arriving long after the click, fails inside and outside this - change alike, so it is not a regression. With the catalog 7.1 s after the click, the context is - built at the click and runs, the catalog's rate replaces it, and the replacement stays suspended - with no playhead; the shape this replaces has no context for the first five seconds, builds one - at 7.9 s and ends in WebKit's `interrupted` state without ever running. Inside the activation - (catalog 2.6 s after the click) it plays either way. - - The recorded costs: an app that enables audio before any gesture still warns, because that is - what the policy is for, and a video-only tile holds an idle context once the first gesture has - landed. - -24. **The ring fallback was a wall of warnings.** A page that is not cross-origin isolated cannot - have a `SharedArrayBuffer`, so the postMessage ring is the path it is meant to run on. Every - player on the page said so at warning level, and the worklet added a second line from inside each - one. Cross-origin isolation is a property of the document, so `6e04673c9` answers once per page, - at info level, with the isolation hint kept for a page that wants the shared-memory path, and - drops the worklet's two lines, which said the same thing from the other side of the port. - -25. **The canvas preview never paints in Playwright Firefox, and it predates this branch.** In the - hide and show rows, Firefox delivers a camera frame 102 ms after the show, 819 frames reach the - main thread over the run, the camera source is live at the end and the catalog carries the - rendition (`avc1.640028 1280x720`), and both readbacks of the preview canvas read 0 at every - sample. Firefox has no `MediaStreamTrackProcessor` and takes the polyfill path in - `js/publish/src/video/processor.ts`, which is the difference between it and the three engines - that paint. Nothing on this branch touches it, and it is recorded here so the matrix's Firefox - self-publish rows are not read as a regression. Whether a watcher of that publisher sees the - picture is what those rows will say. **Answered in finding 29**, from the wire rather than from - the canvas: it does not. - -26. **The age budget convicted a group for being long, not for being late, and it cost the picture - once per GOP.** Under path jitter the picture collapsed while the sound held: seven stalls - totalling 9.9 s in 90 s on the shaper's `mild` profile, twelve totalling 20.3 s on `step`, four - decoder rebuilds, and the picture 1.5 to 1.8 s behind the audio playhead at the worst. - - `Container.Consumer` measured a group's age as the span from its oldest undelivered frame to the - newest frame the track had reached, so the verdict was a function of the group's length. An - audio group holds one frame, so that span reads as lateness and the rule looked right. A 2 s - video GOP whose tail was merely late was convicted the moment its successor opened: the rest of - the GOP was thrown away, the decoder then had to wait for the next keyframe, and the picture was - out until one arrived. Once per GOP, for as long as the path stayed slow. - - The wire half of the same budget already had the right rule. `Subscription::max_age` measures a - group by how far it could still reach, which its successor's first timestamp bounds, against the - newest frame the track has reached (`is_stale`, `rs/moq-net/src/model/track.rs`). The consumer - now uses that rule verbatim, which is the point: the two halves of one budget cannot be allowed - to disagree. - - One thing landed with it. The video decoder built a fresh arrival estimator per subscription and - cleared `out.spread` while a track was being rebuilt. Every reason to rebuild is a path that has - just proved it delivers late, so starting the measurement over at the publisher's declaration - collapsed the shared delay to whatever audio had measured and handed the replacement - subscription a budget the picture could never meet. The estimator is now one per rendition and - outlives any one subscription, exactly as the audio decoder already keeps its own. - - Fixed in `d2443eba9`, with the consumer's cases in `js/hang` and the decoder's in `js/watch`. - - Measured on the `step` profile, counting the `skipping slow group: track=video` lines the - watcher logged: **30 convictions before, 0 after**. The picture went from 12 stalls totalling - 20.3 s, 4 decoder rebuilds, one underrun and 9.68 s concealed to **one 116 ms stall**, no - rebuilds, no underruns and 60 ms concealed, with the skew lag falling from -1834 ms to -39 ms. - On `mild` the convictions fell from 14 to 2. It did all of that while carrying **117 times the - video bitrate** of the run it is compared against, which is the confound recorded under the - matrix above and which here only makes the result stronger: the earlier matrix's chromium - publisher was sending about 67 bytes per frame and the re-run about 7.9 KB. `high-rtt` (75 ms - each way plus 30 ms of jitter, every one of 67863 downstream datagrams delayed, the queue - peaking at 577) is now clean outright: no stalls, no rebuilds, no convictions, 30.0 fps, nothing - concealed. - -27. **The video download gate was armed once, on a canvas that was not in a document yet, and never - re-armed.** A tile that was rebuilt, or whose node the page moved, played audio and never asked - for video again. The relay-restart row reproduced it: after the restart the watcher - re-subscribed to `catalog.json`, `meta.json` and `audio`, and never to `video`, for the rest of - the row. The device-switch rows carry the same signature once the publisher's announcement - flaps. - - The first reading was that the video decoder does not rebuild its subscription when the session - under it is replaced. It does. `208755120` drives a fake broadcast whose consumer is swapped the - way a reconnect swaps it, and shows video raising a second request on the new session. The test - is on the branch so the next reader of that row does not spend the time again. - - The gate is one layer up: `Renderer`'s `IntersectionObserver` on the ``, armed by an - effect keyed on the canvas and the configured distance. Neither changes when the element's node - leaves a document and comes back, so nothing re-evaluated it. A custom element may not touch its - children in its constructor, so `createElement`, `appendChild(canvas)`, then insert is the - ordinary order, and that arms the gate on a canvas that is not in a document yet. A page that - re-appends its tiles to reorder them is one disconnect and one connect in the same task, which a - boolean input would coalesce away, so the gate kept whatever it had last decided. With `visible` - false the decoder's `enabled` is false, so `#runPending` never opens a subscription and - `#runBuffering` never even labels the tile stalled: no picture, no spinner, nothing in the log. - - Fixed in `804c13a20`: the element republishes the canvas on every connect, which forces the - notification because the object is unchanged and its place in the document is not, and drops it - on disconnect so a removed tile stops downloading. The test drives a fake observer. - - **And the relay-restart row is not what it was read as.** Three passes, each with more - instrumentation. Pass 1 reproduced it exactly, and the instrumented sampler named the layer: at - 66.6 s the page replaced the `` element, and the fresh tile reported - `videoEnabled: false` and `rendererVisible: false` for the remaining 50 s, with `paused: false`, - a rendition selected and no source error. Pass 2 added canvas geometry and settled it: - - ```text - before the swap: canvas y=420 h=413 in a 720-high viewport, scrollY=638, rendererVisible=true - after the swap: canvas y=-871 h=413, scrollY=1423, rendererVisible=false - ``` - - The canvas is 871 pixels above the top of the screen, connected, laid out at 734x413, with the - document not hidden. The relay restart makes the demo page rebuild its tile list; the list - re-sorts while the page keeps its scroll, so the `me.hang` tile lands off screen and - `` correctly stops downloading video at its default `visible="20%"` while audio keeps - playing. Pass 3 pinned every tile to `visible="always"` through an init script, so the row - measures the reconnect and nothing else, and the picture comes back: painted rises 1617 to 2036, - **30.0 fps, 0 underruns, 0 convictions, 0 rebuilds**, the target back at 20 ms and the skew lag - at -34.9 ms, where the row before it read 0 fps. - - So the verdict on this row, and on the user's U1 in this form: **the player recovers a relay - restart completely**, and what the matrix recorded as lost video was the demo page re-sorting its - tiles and leaving this one 871 px above the viewport, where the default `visible="20%"` correctly - stops downloading. `804c13a20` is still right and still needed, and its test proves the gate was - never re-armed on a reconnect at all, but it cannot make an off-screen canvas download and should - not. The row still grades FAIL on `0 video stalls` and `recovered <2.5s`, because the picture - cannot return before the page rebuilds its tile: 12 s in pass 1, 52 s in pass 3 under a - concurrent clippy. That gap is the reconnect and the re-announcement, not the player. - - **A question for the maintainer, an API one rather than a bug.** A tile that is not downloading - because it is off screen is indistinguishable from outside from one that is playing. With - `enabled` false the decoder's buffering watchdog bails before it labels the tile stalled, so - `video.out.stalled` keeps whatever value it last held, there is no error, and nothing reports - "not downloading". The demo's badge says `stalled: recovering` and cannot know better. The - information does exist publicly, in `renderer.out.visible`, so there are two shapes: leave the - join to the page, which already holds both halves, or give the decoder an explicit "not - downloading, and why" output. Recommended: leave it to the page unless a second consumer wants - it, since the alternative adds a public signal to a surface that is already wide, and the page - is the only thing that knows why it moved the tile. - -28. **Nothing measured one track being uniformly later than the other, so a Firefox publisher's - picture ran about 100 ms behind its sound.** Rows `cross-firefox-to-chromium` (-94.2 ms), - `cross-firefox-to-brave` (-97.8), `self-firefox-1080p30-bg` (-98.0), `self-firefox-1080p60-fg` - (-86.4) and `self-firefox-1080p60-bg` (-109.4), against an envelope of a frame plus a refresh. - It is not the watcher: the same publisher is inside the envelope at WebKit (-38.3) and at real - Safari (-36.1), and every Chromium or Brave publisher is inside it at every watcher. A capture - probe accounts for about 22 ms of it, Firefox stamping its video 10 ms behind its audio where - Chromium stamps it 12 ms ahead; the rest is the video path simply delivering later. - - The watcher paints a frame when the playhead reaches its timestamp, so a picture that has not - arrived by then is painted as soon as it decodes, which is late. Nothing in the player could see - it, because every existing measurement compares a track against *itself*: each arrival is - measured against that track's own fastest recent arrival, so a track uniformly later than the - other reads a spread of zero and the shared delay cannot tell them apart. - - `69a89f17c` adds the missing quantity, as **option B** of the shapes that were on the table. - `Sync` measures each track's arrival floor, the windowed minimum of arrival minus timestamp over - two rotating 2 s windows, and publishes their difference as `Sync.out.offset`: how much later the - picture arrives than the sound for the same media timestamp. It covers both halves of the cause - at once, the path and a publisher that stamps its two timelines differently, because both move a - frame's arrival relative to its timestamp. The rotating windows are so that a muted or hidden - track drops out rather than holding the sound deep for the rest of the session; the term is - cleared on a rewind, as the estimator clears its own reference, and quantised to a whole bucket, - with less than a bucket reading zero. - - Four decisions inside it, each of which was tried the other way: - - - **One-directional.** Audio is the clock and video is painted when the playhead reaches its - timestamp, so a picture that arrives early is already held for free and needs no term. Only a - late picture needs one. Delaying the earlier track is what WebRTC does, in - `modules/video_coding/stream_synchronization.cc`. An absolute term regressed the 350 ms epoch - case that finding 14 fixed, which is why the sign is deliberate rather than incidental. - - **Capped at 200 ms** (`8b64a6aa6`). Uncapped, the term reached 2 s during a jittery tune-in - and held there for 25 s, and it was not wrong about the world: the video track was still - replaying the span between the last keyframe and the live edge while the shaper's queue built, - so every arrival honestly looked that late. Holding the sound to match is the wrong trade. A - picture two seconds behind is out of sync whatever the sound does, and the hold just makes - everything late. 200 ms covers the window a viewer notices, ITU-R BT.1359 putting that at - 45 ms of picture-ahead and 125 ms of picture-behind, with margin over every publisher offset - the matrix measured, the worst of them 109 ms. **The cap is 100 ms after `50b0a72b9`**, and - the term also gained the 45 ms tolerance and a one-bucket-a-second ramp there; finding 32 is - why, and it is the same reasoning taken one step further. - - **Spent in two places that have to move together**: the audio ring holds - `delay + offset + chunk`, and the age budget reaches back over `delay + offset + buffer`, - because a budget sized without the term convicts the very group the deeper hold is waiting - for. - - **`Sync.out.delay` is untouched.** It stays the estimator's answer, which is what the corpus - describes and what both languages are held to. This is a property of a *pair* of tracks and - `rs/moq-audio` has only one, so the native side needs no change and the conformance corpus is - unmoved. - - The rule is written down in a new section of `doc/concept/playout.md`, beside the estimator it - sits next to, with the cap and its reasoning. - - Measured: `cross-firefox-to-chromium` moves from -94.2 ms to **-35.9 ms** at 720p30, inside the - envelope, and the 1080p30 pair from -98.0 ms to **-39.2 ms**. The offset contributed 160 ms of - hold at 720p30. On the capped re-run the term takes only 0 and 200, which is the cap doing - exactly what it was added for. - -29. **The Firefox publisher's picture was blank on the wire, at about 67 bytes per frame, and it was - observed once and not reproduced on the quiet run.** Finding - 25 recorded that the preview canvas never paints in Playwright Firefox and left open what a - watcher of that publisher sees. The matrix answered it without a canvas: across both runs on the - loaded machine the Firefox publisher's heartbeats give about **67 bytes per video frame**, - roughly 16 kbps at - 720p30, against about 7.9 KB per frame from the Chromium publisher on the same bench in the - re-run. That reads as an encoder being handed blank pictures rather than a codec being - efficient, and it is - the same fact the preview readback reports from the other side. Firefox has no - `MediaStreamTrackProcessor` and takes the polyfill path in `js/publish/src/video/processor.ts`, - which is the one difference between it and the three engines that paint. - - **It did not reproduce on a quiet machine.** The same row in `out11/` carries 7.8 KB per video - frame and 133 B per audio frame, against 67 B and 3 B in `out10/`. Three bytes per audio frame - is not a quiet room, it is nothing being captured, so the likeliest reading is the one the - device rows already suspected: another process held the camera and the microphone, and Firefox - got tracks that produced nothing. The blank-picture statement therefore stands as observed once - and not reproduced, and it should not be carried forward without a re-check. - - It predates this branch and nothing here touches it, but it bounds two things that are in here. - A Firefox self-publish row taken on a contended machine is not a picture test. And the - cross-engine rows behind finding 28 - measure the timing of a stream that carried no picture: the timing is still real, since when a - frame arrives relative to its timestamp does not depend on what is in it, but no row on this - branch says a Firefox publisher's video *looks* right to anybody. Pinning it down belongs to - whoever owns the polyfill path, and it is in the follow-ups. - -30. **A device replacement un-announced the whole broadcast, and every subscriber was dropped for the - tenth of a second it took to open the next device.** A device change stops one capture and opens - another, so the element holds no live track at all for as long as the browser takes to answer, - measured here at 100 to 500 ms. `announce="source"` recomputed "any track live" on every change - and read that gap as an empty broadcast: the relay dropped every subscription, the catalog with - it, and answered the next request with `unroutable` until the re-announce landed. One switch of - both sources in Chromium flapped the announcement twice in a second and cost the watcher its - catalog, its audio and its video, and the whole broadcast went down because a *second microphone* - could not be opened. - - Fixed in `dc6f8d38c`: the mode latches. It still waits for the first live track, so a broadcast - with no permission is never advertised, and from then on it stays announced until the selected - source changes. A track that stops is one source being switched or re-acquired, not the end of - the broadcast, and the other renditions keep serving through it. A device that cannot be opened - still fails loudly on its own track, through `MoqPublish.errors` and the red button, and drops - its own rendition from the catalog. That is the shape the rule should have had: a failure scoped - to the thing that failed. - - Measured on the bench, publisher and watcher both browsers. Before: two `unannounce - route=me.hang` lines, four subscriptions ending `err=dropped`, and one `track info error ... - err=unroutable` in a single row. After: zero, zero and zero, in Chromium, Brave and Firefox - alike, each keeping its catalog subscription for the whole row. The switch to the OBS Virtual - Camera and back, at 20 s and 35 s: `announce: true` across both legs, the catalog back at - `avc1.640028` within about 200 ms each time, and on the watcher 0 underruns, 0 convictions, 0 - rebuilds, 30.0 fps and one 26 ms stall with a 29.6 ms recovery, with the painted counter rising - through both switches and no plateau. - - **A correction to the earlier reading of these rows.** The matrix's device-switch rows ran - against a UGREEN FineCam 4K that handed every opener a track which ended immediately, and an - earlier note said the camera was simply switched off. That is retracted: the user switched it on - and a probe holding it by `deviceId` still saw both its camera and its microphone report - `readyState: "ended"` at the 1000 ms poll, twice, with 2.5 s and 1.5 s of release time after the - warm-up so the probe was not racing itself. The camera does report settings (640x480 at 30) - before it ends. The likeliest explanation is not hardware: the user's own Brave has had a video - capture utility process alive since 04:21:31 that day - (`utility-sub-type=video_capture.mojom.VideoCaptureService`), which Chromium spawns on demand - when a page starts capturing, so Brave has most likely been holding a camera for about seven - hours, spanning both runs, and a device already captured by another page can hand a second - opener a track that ends at once. That is a hypothesis and is recorded as one: the service can - also linger after capture stops, so its presence does not prove this device is held now, and it - does not on its own explain the UGREEN microphone ending too. The fix stands either way, which - is the point of scoping the failure to the track. - -31. **The native client could not survive the relay's own graceful restart, and three defects lined - up to make certain of it.** During the matrix's relay-restart row both native `moq` file - publishers died, 10 s into a 20 s restart, with `reconnect timed out after 10s: peer redirected - immediately`. The browser in the same row reconnected fine, which is how this was nearly - dismissed as a native-only curiosity. It is not: that is the browser scraping through on timing. - - Three defects, one per layer: - - - **The relay kept accepting during its own drain.** On SIGTERM it drains for its whole window - and waved every new session away with a GOAWAY naming no URI. - - **The client read that wave-off as a redirect.** A GOAWAY with an empty URI is not a redirect - at all, and the client took each one as the replacement, retired the predecessor that was - still carrying its groups 56 ms into the handover window it should have ridden, and called it - `peer redirected immediately`. - - **The give-up window equalled the drain window.** Both were 10 s, so the budget was spent on an - endpoint that had already said it was leaving, before a replacement could exist. - - Fixed in `5be36750f`, and in `eec9016d9` for the bindings. A relay that has started draining - refuses a new session with `503` on every transport, the code a client retries, through a new - `moq_relay::Shutdown::draining()` the listeners consult. A GOAWAY naming no URI is reported as - the drain it is, logged `peer is draining` rather than as a redirect nobody asked for, and a - replacement waved away before it ever served never displaces a predecessor that is still serving. - The give-up window moves from 10 s to 60 s in Rust, in JS and in the FFI. - - **That last one is a documented default change, and it is a maintainer judgement item.** The - reasoning is plain: the shipped client could not survive the shipped relay's own graceful - restart, because the relay drains for 10 s before its process even exits, so a 10 s give-up - budget is spent before the relay has finished leaving, let alone come back. The browser only - scraped through by riding the drained session to the force-close and reconnecting on its last - in-flight attempt, which is luck rather than design. Past a restart the window is still - deliberately short, so a failure that has not cleared is still reported rather than retried in - silence, and a loop nobody watches still wants `timeout: 0`. If the number should be smaller, or - should be derived from the peer's own drain deadline rather than picked, say which; the defect - under it is fixed either way. - - Four Rust tests and one in JS, each failing without the fix: - `a_draining_relay_refuses_a_new_session` in `rs/moq-relay/tests/shutdown_signal.rs`, and in - `rs/moq-tokio/tests/reconnect.rs` - `an_immediately_drained_replacement_keeps_the_serving_session`, - `a_peer_draining_every_session_names_the_drain` and - `a_peer_away_longer_than_a_relay_restart_is_reconnected_to`, with the JS twin of the last one in - `js/net/src/connection/reload.test.ts`. - - Measured on the bench, both file publishers through one relay restart at 15:42:03: - - ```text - 15:42:03.458 received goaway uri= the drain begins - 15:42:03.468 session error err=app code=503 the replacement is refused, not accepted - ... nine refusals across both publishers, backing off 0.8 s to 4.9 s - 15:42:13.461 old session did not drain in time; closing 10.0 s of serving kept - 15:42:20.347 listening addr=[::]:4443 kind="quic" the new relay binds - 15:42:20.886 announce route=demo/bbb.hang/** - 15:42:22.570 announce route=bbb.hang/** - ``` - - Nine `relay shutting down; refusing a new session` lines on the relay side, the serving session - kept for the full 10.0 s of the drain instead of being retired 56 ms in, both broadcasts - re-announced within 0.5 s and 2.2 s of the new relay binding, and **zero `reconnect timed out` - anywhere in the window**. - - **The honest part.** A later, third restart in the same session did kill both publishers, at - `reconnect timed out after 60s: ... tcp connect error: Connection refused`. That was a - scheduling error of this bench's own making, not a product one: a concurrent `just fix` running - `cargo clippy --fix` had invalidated the build cache, so `cargo run` spent 48.8 s recompiling - `moq-relay`, the relay did not finish listening until 15:53:10.869, and the publishers gave up - at 15:53:09.139. They **missed it by 1.7 seconds**. What the run does show is that the failure - mode changed from a 10 s budget spent on a redirect that never happened to an honest 60 s budget - and a truthful reason. The lesson for the next matrix is written down: never run a Rust gate - during a restart row. The quiet run settles it: one restart, both publishers alive on their own - pids, 5 refusals with `503` across the drain, 8 refused retries while the relay was down, and - both broadcasts re-announced 0.6 s and 2.1 s after the new relay served, a 24.6 s outage inside - a 60 s budget. It is in the Evidence section. - - **One of those tests measured the paused clock rather than the outage, and it is fixed in - `b80a2084a`, in the test.** `a_peer_away_longer_than_a_relay_restart_is_reconnected_to` failed at - `eec9016d9`. It mixed `tokio::time::pause()` with a real socket, and a paused clock jumps - straight to the next deadline whenever the runtime waits on that socket, so the dial the peer's - return had to complete raced a deadline in virtual time that could pass while the handshake was - still in flight. The test read that as the client giving up. It passed in the run that landed it - only because the whole suite was running, which changed how the polls and the jumps interleaved; - alone it failed every time. The outage still runs on the paused clock, so 20 s of it costs - nothing and the shipped give-up default is still what is under test, and what the outage proves - is now read off that same clock: after 20 s the loop has not stopped, which the window this - default used to carry could not manage and which no dial's progress can affect. The clock is - resumed before the peer returns, because the reconnect is real socket work. The pattern is pause - for the wait, resume for the work. `rs/CLAUDE.md` says a time-dependent async test calls - `tokio::time::pause()` first, which holds while the timed window has no real I/O in it; that is - a clause worth adding to the house rule, and it is in the follow-ups rather than edited there. - -32. **Two terms restart when a viewer tunes in or unmutes, and both used to land on the ring in one - step.** The user heard the first seconds of their own self-publish held and then hurried, which - is what a term arriving whole sounds like from the other end of it. - - The first is the ring's own target. A browser publisher declares one frame duration, 20 ms of - Opus, and the cold start is `max(80, declared)`, so the estimator holds NetEq's 80 ms guess - until the first measurement replaces it, and on a local relay that first measurement is the - 20 ms the publisher had already declared. Spent at once, the ring is 60 ms deeper than it holds - and the reader compresses the difference away: measured on a self-publish through the local - relay, eleven pitch periods dropped a second for the half second it took. The ring now sheds a - fall one bucket a second, inside the stretch bound. A fall past that bound still lands at once, - because past it the reader skips rather than stretches, and pacing a skip only delays it. (The - commit message says the publisher declares no flush span, which is wrong in the same way finding - 35's video question is: it declares one, and the floor swallows it.) - - The second is the cross-track hold of finding 28. At tune-in the video arrival floor is set by - the camera's warm-up frames, which arrive far later than their timestamps, so the term stood at - its ceiling until both 2 s windows had rotated past them. It is now three things it was not: it - subtracts the lead a viewer cannot notice, 45 ms of sound-ahead from ITU-R BT.1359, so a hold is - spent only on a picture that is actually late; it caps at 100 ms rather than 200 ms, because past - that a call would rather have the latency than the sync; and it moves one bucket a second like - the target beside it. The sound's window is also carried across a mute rather than re-derived - from a cold one on the unmute, the way the estimator already carries its own measurement. Worth - stating plainly: **the term measured 0 ms on every row of this pass**, which is what a - same-machine publisher and watcher should read. The tolerance and the cap are what the matrix - measured elsewhere, and the ramp is what the tune-in needed. - - Measured on a self-publish through the local relay, Chromium watcher: the compression in the - first four seconds fell from 37 ms to 16 ms and from 40 ms to 0.8 ms across two pairs of rows, - and the five rapid mute and unmute cycles from 14 accelerates and 125 ms of bent speech to 6 and - 69 ms. Zero underruns throughout. - - **What this commit did not fix**, stated because the next one is why. A single unmute after a - three second mute was unchanged at 7 accelerates: that excess is the backlog the resubscription - admits, not a term moving, and it is finding 33. - -33. **Playout started on the oldest sample it had rather than on the level it holds, and spent the - next seconds compressing the surplus away.** A viewer unmuting is served the live edge, and the - relay then follows it with every group it still has inside the age budget, whose headroom is a - stretch bound wide on purpose. All of it decodes before the reader's next block, so the ring - started playing 65 ms deep against the 40 ms it holds. Measured on a self-publish through the - local relay, Chromium watcher: **8 accelerates and 50.4 ms of compressed speech in the three - seconds after one unmute, and 15 and 144.1 ms across five rapid mute and unmute pairs**. That is - the stutter a listener hears after every unmute, and none of it was audio anyone was waiting for. - - Nothing has been played at that moment, so where the playhead starts is still the player's to - choose. It now starts at the newest audio less the level the ring holds rather than at the oldest - sample buffered, and what it steps over is counted as `trimmed`, its own term beside the reader's - `skipped` and the writer's `discarded`, because it is the one drop no listener can hear and no - time stretch had to close. NetEq reaches its target the same way at the start of a stream, by the - position playout begins at rather than by accelerating into it - (`modules/audio_coding/neteq/decision_logic.cc` and `delay_manager.cc`). - - **The first fill only**, armed by a re-anchor and disarmed the moment the reader takes a sample. - From there a surplus is on its way to being heard, and a publisher's flush burst in particular - inflates the ring and drains again before the next one, so the reader's own time stretch stays - what closes it. A surplus under one chunk is left alone, because a fill lands a chunk at a time. - Both rings take the rule and so does the native engine (`91c5878bd`), where `Buffer::flush` keeps - its multiple of the target underneath because that one is protecting a cushion already being - played out of, while the trim has no cushion to protect and lands on the hold exactly. - - | Phase | | accelerates | stretched | trimmed | underruns | - | --- | --- | ---: | ---: | ---: | ---: | - | one unmute after a three second mute | before | 8 | 50.4 ms | not counted | 0 | - | | after | 0 | none | 80 ms | 0 | - | five rapid mute and unmute pairs | before | 15 | 144.1 ms | not counted | 0 | - | | after | 0 | none | 400 ms | 0 | - - The playhead is continuous across each unmute either way. The settled microphone row is - unchanged, which is the control: target 20 ms, held 40 ms, level p50 20 ms, zero underruns, and - zero accelerates where the before-run had five. - - The new counter is `Playout.Snapshot.trimmed` in the browser and `Stats.trimmed` natively, and - the invariant grows its term with it: - `READ == output - concealed + stretched + queued + skipped + trimmed`. The player's stats panel - gains a Trimmed row beside Skipped. - - One adaptation in the native tests is worth naming, because it reads like a regression and is - not. `playout_hands_back_one_block_at_a_time` fed the engine a `LEAD` of 5 packets before the - first pull. With the trim in, a lead past the level playout holds is where the playhead starts - rather than audio anyone waits for, so the engine drops it and the test's own arithmetic no - longer holds. `LEAD` is 4, which is exactly what fills the 80 ms cold-start target plus the - packet being played, and the case asserts the same thing it always did. - -34. **Several watchers on one browser publisher, which is the shape a call has, and the publisher's - upstream leg never notices them.** The user asked for this directly, and it is a stability - question rather than a defect report: the fixes behind findings 13, 19 and 21 each moved when a - track gains or loses a subscriber, and nothing on the branch had exercised more than one watcher - at a time. - - `7168d1a8b` is the unit twin: one browser-style publisher with a video and an audio rendition - through fake encoders, against three watchers joining and leaving the two media tracks at - staggered moments. A mutes audio for three seconds while B keeps it, C hides its video tile, A - unmutes, B closes the tab, and D arrives after all of that. Each watcher subscribes through its - own `net.consume()` handle, which is the shape the wire has: repeat subscriptions to a track - share one producer, so one watcher leaving must not end the track under the ones that stayed. It - asserts every live subscriber keeps reaching the same live edge on both tracks, that each - newcomer is seeded with a catalog carrying both renditions, that no subscription is ever closed - under a watcher that still wants it, and that the broadcast is announced exactly once. It is a - stability pin rather than a guard on one fix: reverting either of the single-watcher fixes leaves - it green, because the publisher never stops encoding here and no track loses its last subscriber. - It is sensitive to the multi-watcher hazard itself, and dropping the dedup fast path in - `net.subscribe` fails it at the first drain, with B reading nothing while A reads the group. - - On the bench, five watchers on one browser publisher, 90 s, six mutes and unmutes each and two - camera hides by the publisher. **The publisher's upstream leg is clean**: zero - `subscribed complete`, zero unannounce, the catalog and audio subscribed once each for the whole - row, and video released and re-raised exactly at the two hides and at nothing else. The same - holds with five Chromium watchers and with three Chromium plus Firefox plus WebKit. The tables - are in the Evidence section. - - The underruns those rows show are the row's own load rather than the player: each watcher decodes - three broadcasts, because the plain page on 4400 subscribes to the announce prefix `""` and the - two file publishers are on the same relay. Five engines cost 15 underruns across the five - watchers; five Chromium watchers of the same publisher cost 1. Two harness follow-ups come out of - that and are in "Open items": the multi-watcher page needs a broadcast prefix, or the file - publishers need to be down for the row; and the grader needs per-row envelopes, because a - five-watcher row is not a single-watcher row with more browsers in it. - - One engine difference, harmless and recorded: **Firefox closes its own subscriptions with - `complete`** where Chromium and WebKit reset them. The nine `subscribed complete` lines in the - mixed row are all on the Firefox watcher's connection and none on the publisher's, which is the - distinction that matters, since a `complete` on the publisher's leg is what finding 19 was about. - -35. **A rendition that blinks rebuilt its arrival estimate from the publisher's guess, and every - ring on the page was resized to pay for it.** A publisher hiding its camera takes the rendition - out of the catalog for a few hundred milliseconds and puts the same one back. The estimator lived - in the effect keyed on that rendition, so it went with it, and the replacement started over at - the publisher's declaration, which for video is the estimator's own 80 ms guess. `Sync` holds - every track to the widest reading, so one track's guess deepens the buffer every other track has - already measured: the audio ring is resized mid-playback, parks to refill, and the listener hears - the gap. - - Measured on the bench with five Chromium watchers on one browser publisher: at a 300 ms camera - hide the video reading went 20 ms, then none, then 80 ms inside one sample grid, the shared delay - followed it, and three of five watchers took an underrun and a spinner. The audio estimator never - moved, which is what separated this from the unmute it happened to land beside. - - `088752023` moves the estimator out of the effect and keys it by the rendition compared by value, - so a rendition that leaves and comes back unchanged resumes what it measured. A rendition whose - catalog entry changes is a different path making a different claim, and still starts over. Both - decoders take the rule, because audio has the same defect when a microphone is hidden. - - `cd43ff0a8` is the other half, and it was only visible once the first half was in. The estimator - survived the gap, but the track still stops publishing a reading while it is away, so the - widest-across-tracks delay fell to whatever the remaining tracks measured and went straight back - up when the track returned. The audio ring paid for both, resized down a bucket and then parked - to refill it: one underrun per camera hide on two of five watchers, with the target unchanged on - either side of it. A track's last reading now stays in the comparison for two seconds after the - track stops publishing one, which is the window the cross-track arrival floor beside it already - uses and for the same reason: wide enough that an ordinary cadence refreshes it many times over, - narrow enough that a track that has really gone stops holding the buffer open. The window starts - where the track goes and not where its reading last moved, because a steady path republishes the - same number and notifies nothing. `doc/concept/playout.md` carries both rules. - - **Measured: the video reading reached 80 ms in 10 of 10 hide windows before, and in 0 of 30 - after**, across three five-watcher runs. The table is in the Evidence section. - - **The first reading of these rows was tested and refuted.** It was that a resubscription's burst - of cached groups is read by the estimator as arrival spread, and that an arrival below the live - edge known at subscribe time is replay rather than - path jitter and should not be observed at all. It already is not: `Container.Jitter.observe` - publishes and returns for any arrival that is not strictly newer than the newest it has admitted, - which is the reordered-arrival rule `doc/concept/playout.md` pins, and the estimator now survives - the resubscribe (finding 20) so it still holds that watermark. A cached group delivered alongside - the live edge carries an older timestamp than the watermark by definition, so it never enters a - delay measurement. There was nothing there to fix and no commit was written for it. What moved - the audio target across a hide was the *video* estimator being rebuilt beside it, which is what - the two commits above are. - - **Two questions for the maintainer, both about the same rule.** A video rendition's declaration - never reaches the estimator as itself. `renditionJitter` reads the catalog's `jitter`, and a - browser publisher writes one frame duration there, 34 ms at 30 fps, because it flushes each frame - as it encodes it. The cold start is `max(80, declared)`, NetEq's `kStartDelayMs` being the floor, - so every video estimator starts at 80 ms whatever the publisher said and however well the path - has been delivering. Either the floor should not apply to a declaration that is below it, which - is a change to the rule `doc/concept/playout.md` pins and both languages replay, or the video - estimator should seed from what the shared delay already holds. And more broadly: should a track - that has measured nothing yet be allowed to raise the shared delay at all? The `max()` across - tracks is #3517's and this branch kept it; a track carrying only a prior arguably has no vote - until it has an observation. The follow-up under "Whether one track's spread should size another - track's buffer" is the same question from the other side. - - **Two residuals, both open and neither explained here.** Across the three runs after the fix, - seven of thirty hide windows still take one underrun, 0.90 to 1.45 s after the hide, with the - target flat on either side of it and the spinner showing on two of them. And one run has a - synchronised burst: all five watchers underran within half a second of each other at 60 s, under - the row's own three-broadcast load, which is the load reading rather than a hide. Both are in - "Open measurements". - -36. **A publisher's mute declared an endpoint the watcher never reopened, so the tile sat on - Buffering until the page was reloaded.** This is the defect the user reported as symptom 2, and - it is the rebase's doing rather than the fix's: the writer survived the rebase and the reader - that answered it did not. - - The branch's `3f6bc1cdd` makes a browser publisher write an endpoint frame alone in its group - when the microphone mutes (`js/publish/src/audio/encoder.ts:436-462`), an empty group when it - resumes (`:411-417`), and the same pair when the demand gate closes and opens (`:476-484`). The - hang draft says a media-less group is the discontinuity and that an empty group means nothing - (`drafts/draft-lcurley-moq-hang.md:549-551`), which is what upstream `9931cfa8b` settled. Before - the rebase the empty group was what raised the discontinuity in the JS consumer, and that is the - line the rebase dropped, leaving a publisher writing a marker that no longer reopened anything. - - Underneath it the two languages disagreed about **when** a marker group raises the playhead. - Rust delivers `FrameEnd(T)` and bumps the playhead when the group closes - (`rs/moq-mux/src/container/consumer.rs:415-418`), so the result that carries `end` is one result - and the result that reopens the timeline is the next one. JS bumped on the marker frame itself - (`js/hang/src/container/consumer.ts:581`) and returned `end` in that same result, so - `Terminal.update` (`js/watch/src/audio/terminal.ts:55-67`) reset the endpoint and immediately - re-applied it to the run that followed, `span` (`:121-129`) trimmed every resumed sample to a - zero-length span, and `#emit` closed them silently - (`js/watch/src/audio/decoder.ts:858-861`). `#onNext` (`:984-989`) then reset the ring on the - marker's own playhead event before `#declareEnd` could play the tail out, and `ring.end()` - could not release a stall on a ring with nothing in it - (`js/watch/src/audio/ring-buffer.ts:437-442`). `interrupted` stayed true, the spinner stayed up, - and every later conviction that might have reopened the timeline was itself a playhead event - that reset the ring again. - - The fix is the Rust order, in five places. The group-close result carries the playhead event, so - a marker group's `discontinuity + 1` and `continuous: false` arrive after the endpoint rather - than with it, and a marker whose group is still receiving media raises nothing - (`js/hang/src/container/consumer.ts`, the group-done branch). The publisher stops writing the - break group entirely: the endpoint alone in its group is the discontinuity, so `#paused` and its - writes are deleted from `js/publish/src/audio/encoder.ts`. The sentence the draft gained in - `3f6bc1cdd`, "a publisher resuming after one MUST declare a discontinuity before its first - frame", is deleted from `drafts/draft-lcurley-moq-hang.md:574` and `doc/concept/hang.md:133-137` - and replaced by what the marker already means. Both rings release the stall on `end()` - unconditionally (`ring-buffer.ts:437-442`, `shared-ring-buffer.ts:741-749`) and re-anchor on the - first write after a played-out endpoint (`ring-buffer.ts:240-244`, - `shared-ring-buffer.ts:324-327`), which is a fresh fill rather than a continuation. And - `Interruption` is gone: the ring itself reports `fresh` on `Playout.Snapshot`, and - `interrupted = enabled && stalled && !fresh` at `js/watch/src/audio/decoder.ts:404-417`, so a - ring that is filling for the first time is never reported as an interruption. - `js/watch/src/audio/playout/interruption.ts` and its test are deleted. - - Measured on the bench, `mute-basic` on the real microphone at both presets, mute at about 15.3 s - and unmute three seconds later, `out15-before/` against `out15-after/`: - - | Row | before | after | - | --- | ---: | ---: | - | `mute-basic-auto` `interrupted` over the mute window | 6686 ms | **0 ms** | - | `mute-basic-auto` `interrupted` over the unmute window | 9770 ms | **0 ms** | - | `mute-basic-100ms` `interrupted`, mute then unmute | 6724 / 9724 ms | **0 / 0 ms** | - | spinner, every one of those four windows | the same numbers again | **0 ms** | - | playhead after the unmute | never returns | **+462 ms** (auto), **+441 ms** (100ms) | - | trailing silence at the row's end | 38.7 s | **none** | - | `concealed` delta at `100ms` | -960 | **0** | - - The two before rows are **not graded at all**, because 0 of 72 and 0 of 73 tail samples carry an - audio playhead: a row that never recovers has no steady state to grade against, and that is the - finding. The after rows do: reference target 20 ms, chunk 20 ms, level p50 30.1 ms, and the - final sample of the auto row reads `interrupted false`, `stalled false`, `spinner false`, - `dbg.buffered 1445`, `ctxState "running"`. `out15-after2/` reproduces it on the next build, - playhead back 450.6 ms and 449.3 ms after the unmute with every counter flat. - - On the final tip, `out15-after3/`, the same pair reads the playhead back **453.6 ms** after the - unmute at `auto` and **454.7 ms** at `100ms`, with `interrupted` and the spinner still 0 ms on - every window, zero convictions, zero underruns, zero trims and zero concealment. That row's one - remaining failing check is the unmute's own stretch count, `accelerates + expands = 5` against a - ceiling of 3. It is the real cost of refilling from empty, and it shows up only now because the - ring no longer resets across that window and hides it; it is reported rather than argued away, - and it is in the open measurements. - - Two details are worth keeping because they are the fix working rather than residue. Through the - declared pause the playhead is **present and frozen**, 3234 ms held against 113 ms of nothing at - all in `out15-after` and 3219 ms in `out15-after2`, which is the endpoint being played out and - then held rather than the timeline disappearing. And the `countersReset` flag on the auto mute - window, `accelerates` delta -2, is the fresh-fill re-anchor on the first write after that - played-out endpoint: the counters live on the ring, so the reset reads here as a delta that went - backwards, and the flag exists to say the arithmetic across that window is not a difference. - - A watcher that reloads **inside** the declared silence recovers too, which is the other half of - symptom 3. `watch-reload-silence-auto` builds no ring at all while the publisher is muted, - `dbg` null on every sample from 17460 to 21526 ms, and then the playhead returns **441 ms after - the unmute** with `interrupted` 0 ms, spinner 0 ms, zero convictions and zero underruns. - - Tests, each failing without the change: in `js/hang`, `consumer.test.ts` "Consumer reports a - marker group as a playhead event" moved to the Rust order, "delivers an endpoint, a break and - the resumed media without a conviction" and "does not raise the playhead for a marker whose - group still receives media"; in `js/watch`, `terminal.test.ts` "an endpoint delivered before its - playhead event trims the flush and not the resumed run", `decoder.test.ts` "a publisher's mute - and unmute play the resumed audio", and in both rings "a declared endpoint releases a stall on - an empty ring" and "media after a played-out endpoint is a fresh fill"; in `js/publish`, - `encoder.test.ts` "writes only media when it resumes" in place of the two break assertions; in - Rust, `a_marker_group_closes_the_run_before_it` in `rs/moq-mux`. The replay lane gains an - `endpoint` arrival (`js/watch/src/audio/replay.ts:30`, skipped natively at - `rs/moq-audio/src/playout/replay.rs:20-36`), a derived fixture `fixtures/mic-local-mute.json` - with a three second cut, two `budgets.json` rows, and "plays through a declared pause without an - underrun or a skip". - - **What the native side does through a declared pause is different, and it is not fixed here.** - `rs/moq-audio`'s playout engine conceals through the pause and counts every block it conceals as - an underrun: **49 underruns in 500 ms at 10 ms blocks**, because the marker never reaches the - decode consumer at all. The resumed run then plays correctly, so nothing is lost; the number is - wrong rather than the audio. `a_declared_endpoint_then_resumed_media_decodes` in - `rs/moq-audio/src/decode/consumer.rs` keeps asserting zero and is marked `#[ignore]` with that - measurement in the reason, so the day the marker reaches the consumer the test fails loudly - instead of quietly passing. Surfacing the marker there, and an `Engine::end()` to go with it, is - a follow-up quest and is in the open items. - - The wire impact is one group less per resume: a publisher no longer writes an empty group when - it comes back. The public API impact is none, and `Interruption` was internal. - - `63ee6639c` is the whole of it. - -37. **The old ring counted a declared pause as media somebody lost.** Found while the tests for - finding 36 were being written, and it explains two numbers that had been read as properties of - the harness. - - A ring that was told the stream had ended, and then given media again, treated the gap between - them as samples it had skipped, so `skipped` grew by the length of the pause. That is the - invariant `READ == output - concealed + stretched + queued + skipped + trimmed` being balanced - with the wrong term: nothing was lost, the publisher said it was stopping. Two places had been - absorbing it. The replay lane carried a **144000** sample skip through the recorded pause, which - is three seconds at 48 kHz and had been budgeted as though it were real, and the render - worklet's own test balanced its arithmetic against the same miscount. With the fresh-fill - re-anchor of finding 36 in place the pause is a re-anchor rather than a loss, both numbers go to - zero, and the two tests that had been pinning the old behaviour were rewritten to the new one. - - It matters beyond the tests because `skipped` is the counter a listener's complaint is graded - against. A declared pause that counts as lost media makes every mute look like a defect, and - makes a real defect during a mute invisible inside it. - - `63ee6639c`, with finding 36. - -38. **A publisher reload is two seconds of a page with no tile and half a second of a player.** The - user reported the reload as a cut, and it is, but four fifths of it happens before any audio - code runs. - - On the demo page a re-announced broadcast does not reuse the element it had. The old - `` is removed when the broadcast unannounces and a brand new one is built when it - comes back (`demo/web/src/index.ts:95-171, 226-246`), so a publisher reload, a watcher reload - and a fresh publish are the same event seen three ways: a cold tune-in on an element that has - never seen this broadcast. The bench now reports the two halves separately, `noTileMs` for the - stretch where the page has no element to sample and `playerMs` from the element existing to the - first advancing playhead, and only `playerMs` is graded. - - Measured on the bench, `pub-reload` at both presets, `out15-after2/`: - - | | `pub-reload-auto` | `pub-reload-100ms` | - | --- | ---: | ---: | - | `noTileMs`, the page waiting for the re-announce | **1902 ms** | **1877 ms** | - | `playerMs`, the tune-in itself | **572.3 ms** | **557.4 ms** | - | `groupsSkipped` / slow-group notes | 0 / none | 0 / none | - | `trimmed` / `discarded` / `skipped` / `concealed` | 0 / 0 / 0 / 0 | 0 / 0 / 0 / 0 | - | `underruns` | 0 | 0 | - | context after the swap | running, no second click | running, no second click | - - The before rows are the same shape. Two quantities, and they are not the same one: `startupMs`, - the whole stretch with no advancing playhead, is **2565 ms** at `auto` and **2480 ms** at - `100ms`, while the **spinner** is up for **2027 ms** and **2033 ms**, shorter because for the - first part of it there is no tile to put a spinner on. Both were taken before the - `noTileMs` / `playerMs` split existed. Fix 1 does not move them, which is right: this was never - the endpoint path. On the final tip, `out15-after3/`, the split reads `noTileMs` 1875 ms and - 1919 ms against `playerMs` 544.4 ms and 438.6 ms, the same shape again. - - **The tune-in itself is clean, and that refutes two of the ranked sites.** The first ring depth - on these rows is 80 ms, which is the seed rather than a race with the worklet load, so \[D] does - not fire; no group is convicted inside the first three seconds with the budget above the walked - target, so \[E] does not fire; and the latency re-anchor never parks, so \[G] and \[H] do not fire - on this path either. Those four stay unmeasured rather than fixed, and they are in the open - items. - - One real page defect came out of the rows. The rebuilt tile was created with `delay="auto"` - whatever the viewer had chosen, so a publisher reload silently put a listener at a fixed 100 ms - back on the auto preset, and the page also re-picked `list[0]` instead of the broadcast the - viewer was watching (`demo/web/src/index.ts:273-278`). The page now carries its own `delay` onto - a new tile and remembers the active broadcast across an unannounce. - - Measured on the bench, `page-keeps-choice-100ms`, all three broadcasts listed so that the - self-publish is not `list[0]`, the bench's own tile pin off, and the preset set through the - page's control rather than an attribute. The driver set the element's `delay` **property** and - left the attribute at `auto`: - - ```text - delay via the page's own control: {"delay":"100ms","delayAttr":"auto"} - ``` - - After the reload and the tile swap at 17604 ms the rebuilt tile carries `delayAttr: "100ms"` on - all **197** post-swap samples, which only the page can have put there. Over that window: - `delayAtMark` and `delayAtEnd` both 100 ms, the reference target 100 ms, `mutedProp` false on - every post-swap sample, the playhead advancing from 2283 ms to 41717 ms, **one click** for the - whole row, `groupsSkipped` 0, `underruns` 0, `trimmed` 0 ms, `lost` 0 ms, `interrupted` 0 ms, - `noTileMs` 1962 and `playerMs` 541.5. The row passes again on the final tip, `noTileMs` 1902 and - `playerMs` 552.5. - - Keeping the element across an unannounce, rather than rebuilding it, is the larger change and is - a follow-up rather than this branch's; it is what would take the two seconds out. - - `1c2605450`. - -39. **The age budget convicted a group the reader had already played, and on legacy audio the - conviction fabricated a hole that reset the ring.** This is the last of the four symptoms, and - the one that was misread twice before the warn line was widened enough to read it. - - `#checkMaxAge` in `js/hang/src/container/consumer.ts` judges the head of the queue by how far it - could still reach, and dropped it when that reach fell behind the live edge. It did that without - asking whether the reader had already had it. Measured on the real recording, **all 172 - convictions are `queued=0 cursor= closed`**: a finished group the - consumer had already delivered and popped from, still sitting at the head until `next()` takes - it away. Convicting one counts a skip that lost nothing, and on legacy audio, where frames carry - no duration, its contiguous successor then reads as a hole, and a hole is a playhead event, and - a playhead event resets the ring and the sync. Rust never sees it: the mux consumer pops a spent - group before the age check runs (`rs/moq-mux/src/container/consumer.rs:327-348`). - - The rule is now one rule, and it is Rust's. A finished head the cursor has **reached** is never - convicted, with `<=` rather than `<` because the cursor is advanced past a completed group - before `next()` pops it. A finished head **above** the cursor is walked onto rather than dropped, - with a playhead event only where it is not contiguous with what came before. Only a group that - is still arriving, or an empty one above the cursor, can be convicted. The warn line - (`js/hang/src/container/consumer.ts:417-419`, twin at - `rs/moq-mux/src/container/consumer.rs:381`) now prints the convicted group's first and last - timestamps, whether it is open or closed, its reach, the live edge, the budget, and now `queued=` - and `cursor=`, which is what made the diagnosis readable at all. - - The narrowed guard that preceded it is worth recording because it could not work. It exempted a - finished head with frames queued only while the cursor sat on it, and on a cold tune-in the - cursor does not exist yet: on `watch-reload-auto` the conviction is logged at **15597.2 ms** and - that tile's first playhead arrives at **15677.1 ms**, so `#active` was still undefined when the - decision was taken. An exemption that keys on the cursor is unreachable by construction on - exactly the event that needs it. - - Measured on the replay lane, the profile that exists for this. Four numbers, and each counts a - different thing, so each is named: - - - **91**, what the profile counted when it was called `budget-censors-the-tail` and the head was - judged by its timestamps alone. That is the number the previous revision of this document - carried. - - **115**, the same recording re-counted on `relay-bbb-7frame` at the 46 ms round-trip budget - once the JS-against-Rust cursor-walk divergence had been named. Every one of those 115 is JS - dropping a head Rust would have walked onto. - - **122**, the convictions that survived the cursor walk, once the widened warn line showed what - was actually being convicted: a spent head the reader had already played. - - **0**, at every budget under the rule above. The profile is renamed - `budget-spares-an-arrived-flush`, because a profile named for censoring a tail that is no - longer censored is a lie in the filename. - - Measured on the bench, `watch-reload-auto` at `auto`, the build before this rule - (`out15-after2/`): **1** group convicted, **80 ms** trimmed and **80 ms** discarded against a - ceiling of 60 ms, the row's only failing verdict in the set. The warn line for it reads - - ```text - skipping slow group: track=audio 758 -> 759 - first=18060666 last=18060666 closed reach=18080666 live=18300666 budget=215000 - ``` - - a closed group holding one frame, convicted by 5 ms. On the final tip, `out15-after3/`, the same - row records **0** convictions and **no slow-group warn line at all**, where each of the three - builds below it printed exactly one. Nothing a listener could hear is lost: `trimmed` 100 ms and - `discarded` 80 ms are both media **before the start point**, which rule 8 reports beside the - grade rather than charging to the tune-in, `lost` is 0 ms, and `playerMs` is **328.8 ms** flat. - The 27-row regression set says the same thing across the family: **zero convictions and zero - slow-group lines on every row of it**, in chromium, brave, firefox and webkit alike, with - `watch-reload` passing at both presets. - - Tests: `consumer.test.ts` at `:503` and `:566` move to the Rust semantics, and the Rust twin is - not needed because the mux consumer already pops the spent group. The Rust rule is the - specification and the JS is now written against it. - - **One behaviour change to confirm, and it is why this is a maintainer item.** `skipped` at the - `instant` preset now counts only media that was actually lost. It used to count a group the - reader had already played, so any harness, budget or dashboard that treated the old number as a - ceiling will see it fall. Nothing is being hidden: the drop is the count that was wrong going - away. "The budget" under "What changed, per stage" states it beside the replay numbers. - - `1c2605450`. - -40. **A republished broadcast on an element that keeps its tile is a tune-in, and nothing treated it - as one.** Not the demo page, which rebuilds the element and so cannot see this, but any page that - pins a `` and lets the broadcast change underneath it, which is the shape an - application actually wants. - - The arrival estimator is keyed by content identity, so when a publisher dies and a new one - publishes the same broadcast the estimator carries the dead publisher's reading into the live - one. The new publisher's timestamps start near zero, the sync still holds the old reference, and - every frame is discarded as being far in the past until something raises a discontinuity and - resets it. From outside it looks like a tile that is connected and silent: `discarded` climbing - while `audioTs` sits frozen. - - `#runDecoder` in `js/watch` now resets the ring and the sync when the **broadcast consumer** - identity changes, and the video decoder does the same. A rendition swap deliberately does not: - that is the case finding 35 exists for, where the estimator must survive the swap, and the two - rules pull in opposite directions on purpose. A new negative-control test guards the second - half, because the finding 35 tests did not: they prove the estimator survives a rendition - change, and passed happily while a broadcast change was being treated the same way. - - This is the one fix in this pass with no bench row of its own. The demo page cannot produce the - event, and the 27-row regression set has no pinned-element schedule either, so it adds nothing - here: the evidence is the two tests and the reading above. The schedule that would measure it, - one pinned `` with the publisher killed and the same broadcast republished under it, - is in the open items. - - `1c2605450`. - -41. **The relay unit this repository packages has never been able to start the relay.** - `packaging/moq-relay/moq-relay.service` passes the configuration file as `--file`, and the relay - takes it positionally (`rs/moq-relay/src/config.rs:106-109`). The binary refuses the flag - outright, `unexpected argument '--file' found`, so the packaged unit fails at startup on any - build that has this argument parser. It was found by deploying the unit unmodified onto a clean - Debian 13 x64 host and watching it fail; it is a one-line fix, and the documentation already - describes the positional form, so nothing else moves. - - **`upstream/dev` carries the same line**, so this applies upstream unchanged and does not depend - on anything else on this branch. - - Worth a decision rather than just a fix: `moq-relay` takes its config positionally and - `moq-bench` takes the same thing as `--file`. One of the two spellings should win. This branch - only makes the unit match the binary it starts. - - `1b0d8d831`. - -42. **A grader that applies one rule to every window grades the wrong thing four times out of - seven.** This is a bench finding rather than a product one, and it is recorded because the rows - in this section are only worth reading if the rules behind them are stated. - - Ten rules decide what each event window is answerable for, and every one of them exists because - the rule it replaced was charging a window for something it did not own. They are reproduced in - full at the head of every generated table and under "The listening bench": - - 1. `noTile`, the stretch where the page has no element to sample, is reported and never graded. - It is the demo waiting for a re-announce, not the player. - 2. `player`, the element existing to the first advancing playhead, is what a tune-in is held to, - with a ceiling of one second. - 3. The `player` clock stops through a declared pause. A watcher that reloads into a muted - publisher is waiting for the publisher, and wall time says otherwise. - 4. Between a `mute` and its `unmute` the playhead-rate and missing-playhead checks do not apply. - A publisher that has said it is stopping sends nothing to advance a playhead with, so the - playhead stopping is the declared behaviour, and the unmute's own window grades what follows. - 5. On a tune-in the missing-playhead check opens at that tune-in's first playhead. The stretch - before it is the tune-in, which rule 2 already grades. - 6. A counter delta taken where the row had no ring at the mark, and any field a row was measured - before the bench recorded, are **NOT MEASURED**. Neither a pass nor a failure, and neither - ever counts against a verdict. - 7. A window grades only the region no later event owns. A ten second window can contain whole - other events, and what those cost belongs to their own windows, which hold them to the right - budget. The tables still report the whole ten seconds, because that is what happened; the - grade uses the part before the next marker and says so when the two differ. - 8. On a tune-in, what counts as lost is what a listener could hear missing, which is `skipped` - plus `concealed`. Playout starts on the level the ring holds, so media older than that start - point was never going to be played and dropping it is not a cut, which is finding 33. - `trimmed` and `discarded` are reported beside the grade as loss before the start point. - 9. A resume after a declared pause, an `unmute`, a `watch-unmute` or a `show`, refills from - empty exactly as a tune-in does and gets the tune-in stretch allowance rather than the - steady-state one, which would leave converging no room at all. - 10. A check whose window is too short to hold it is **NOT MEASURED**. The playhead rate needs a - whole second after the fill to average over, and "target back at the reference" asks where - the target ended up, which means the end of a full window. A burst of mute pairs half a - second apart cuts every window to a second, and asking either question there measures the - burst's spacing rather than the player. - - Rules 1 to 6 were applied with `--report-only` to the same JSON, so no row was re-run for them. - Rule 7 was found on `watch-reload-silence-auto`, whose `mute` window failed - `stretching at the row's own rate` at `accelerates+expands=4` against a ceiling of 2: that - window is ten seconds long, it swallows the watcher reload and the unmute that follow it, and it - was being charged for a tune-in while held to a steady-state budget. The `watch-reload` window - covers the same region, gets the tune-in allowance, and passed with the same 4. Applying rule 7 - moved two verdict lines, both on that row, `PASS (4 not measured)` to `PASS (6 not measured)` in - `out15-after` and `FAIL (1 check)` to `PASS (6 not measured)` in `out15-after2`, and nothing - else. `watch-reload-auto` has a single marker, so its window is never narrowed and its - conviction is graded exactly as before. - - Rules 8, 9 and 10 came out of the 27-row regression set, and 10 is the largest of them: **58 of - about 96 failing checks in that set were windows a second long**, which is what a burst of - twenty mute pairs half a second apart produces, and asking a per-second question inside one - measures the schedule rather than the player. With all ten applied the same set reads 6 passed, - 18 failed and 3 not graded over 27 rows, with 38 failing checks left, and those are mostly the - resume playhead gap and the stretch counts rather than anything convicted or lost. - - The honest limit of the whole instrument is unchanged and belongs beside the rules: **the - sampler cannot hear**. Every number above is a counter, and the user's ear is the confirmation. - - This one, with the docs. - -43. **A temporary Debian 13 x64 VPS was stood up as a second test host, and the deployment found - three defects the Mac bench never could.** It runs the full gate chain on x64, serves the demo - pages, the site copy and two file publishers over a real certificate, and is the remote relay a - local browser dials over a real wide-area path. It is temporary and it is torn down when this - work is finished; the concrete addresses, keys and dates are in the private bench archive rather - than here. - - What it carries: the relay and the CLI built from this branch; a second relay built with the - `quinn`, `quiche` and `qlog` features, which the default feature set does not include, so the - congestion campaign can select a backend and write packet-level traces; the demo pages and the - site copy built against this branch's `js/`; the two `bbb.hang` publishers; and a gate runner - that drives `just check`, `just test default`, nextest over the touched crates, `just js test`, - the replay lane, `just drafts check` and `just test smoke-full` inside the repository's own nix - shell, serially, overnight. Seven systemd units, enabled and proven across two reboots. QUIC is - confirmed from a public address on the real certificate, and the WebSocket fallback is - distinguishable in the relay log, which is what makes a Safari row readable. - - Three things it found, none of which a local bench can: - - - **The packaged relay unit's `--file`**, which is finding 41. It is only visible to somebody - who installs the unit the repository ships. - - **`LogsDirectory` does not exist yet when `append:` wants it.** A unit that sends its output to - a file under a directory systemd is creating for it fails at start, because the redirection is - resolved before the directory is made. The ordering is the bug, the workaround is to create - the directory, and it is worth knowing before anybody copies the unit. - - **The nix build wants more memory than a small host has.** A single `rustc` inside the dev - shell can exceed a 4 GB box and lives on swap while it does, so every service in the demo - stack carries an `OOMScoreAdjust` drop-in and the kernel kills a build rather than the relay a - listener is on. A nix build also belongs to the client that asked for it: killing the - `nix develop` that started one cancels it, and two clients on one derivation doubled the - resident set and turned a five minute build into forty-five. - - One smaller fact for anybody scripting against the CLI: `moq-cli` has no flag to override the - TLS server name, so a publisher on the same host dials the public name rather than loopback. - - The gate chain was still running at the time of writing and its results are not in this - document. What the box has already proven is the deployment: the units, the certificate, the - reboots, the transport and the pages. - - `1b0d8d831`, for the packaging one-liner; the rest is infrastructure in the bench archive and not on - the branch. - -44. **The video watchdog called a rendition that had left the catalog a stall, so hiding the camera - put a Buffering overlay over the tile for the whole length of a mute.** Found by the 27-row - regression set rather than reported, and it is the one defect in this pass that audio never - touched. - - `#runBuffering` in `js/watch/src/video/decoder.ts` reports a stall when it has no rendition to - decode. A publisher that hides its camera takes the rendition out of the catalog and leaves it - out, which is not a stall: it is a publisher that has stopped sending pictures on purpose, and - the tile has nothing to wait for. The overlay went up anyway and stayed up for as long as the - camera was hidden, over audio that was playing perfectly. It is the same event finding 35 is - about, read by a different consumer. - - The attribution is what makes it clean. Across all 27 regression rows `interrupted` is true on - **zero samples**, and `spinner === videoStalled && !paused` on **every** sample of every row, so - the overlay in these rows is video's alone and audio never contributed to it. - - Measured on the bench, `hide-mute` at both presets, the camera hidden and then the microphone - muted for three seconds, the spinner counted over the `mute` window after the fill: - - | Row | regression set, before | `out15-after4/`, after | - | --- | ---: | ---: | - | `hide-mute-auto`, spinner over the mute | **4578 ms** | **0 ms** | - | `hide-mute-100ms`, spinner over the mute | **4493 ms** | **0 ms** | - | titles seen on it | `Buffering (audio clock)`, `Buffering` | none | - | spinner anywhere else in the row | the hide and the unmute windows too | one span at the opening tune-in | - - A three second mute was showing four and a half seconds of overlay, which is the hide's stretch - and the mute's run together. After the fix each row has exactly one spinner span left and it is - the row's own opening tune-in. That last span is one sample on the 250 ms grid rather than a - measured duration, so it is reported as a span and not as a number. - - `mute-basic-2000ms` is unchanged by construction and is reported rather than fixed: 1420 ms of - `Buffering (audio clock)` through a mute at the 2000 ms preset, which is the parked fresh fill - holding video while it fills a two second target. That is a deeper question than a watchdog - predicate, whether the audio clock should be nominated at all while a ring is parked for a fresh - fill, and it is in the open items rather than in this commit. - - The test is "a rendition that left the catalog is not a stall" in - `js/watch/src/video/decoder.test.ts`, and it fails without the change. `js/watch` is 420 tests. - - `1c2605450`, with the reload work, because it is the same function's neighbourhood and the two are - not separately measurable on the same rows. - -45. **The first seconds of a fresh camera-and-microphone publish are still cut on a real path, and - it heals after about twenty seconds.** Reported by the user on 2026-09-18 from a remote listen in - Brave at the `auto` preset, against the temporary Debian 13 x64 host of finding 43 over a - wide-area path with a 55 ms round trip. It is the one symptom of the four they reported that this - pass has not closed, and it is the next session's first job. - - What the console says is that nothing convicts. At warn level the page prints only - `skipping covered group`, which is benign and is finding 46's second half, and the demo page's - own `meta.json` error, which is finding 46 itself. No conviction line, no slow-group line, no - underrun. Unresolved; no commit. - - The trace below is code reading on the branch's own tree, not measurement. It is here because it - names the sites a row would have to discriminate between, and every one of them is reachable on - a fresh publish over a jittery path and on no row in this document. - - - **The estimator's young-histogram window is arithmetic.** `js/hang/src/container/jitter.ts:278` - computes the forget factor as `1 - 2/(adds + 1)`, so the n-th observation carries weight 2/n - and outweighs a 5 percent tail on its own until n reaches 40. At one observation per 500 ms - that is the first 39 observations, about 19.5 s, and for the whole of that window the published - p95 is simply the last interval's maximum rounded up to a bucket. - - **The 80 ms seed is replaced outright by a quiet first interval.** The cold start is - `max(80, declared)` and a browser publisher declares one frame duration, so every cold start is - 80 ms. The first measurement is typically 20 ms, because at tune-in a relay backlog arrives - nearly simultaneously and each later arrival becomes the new arrival floor, and it replaces the - seed whole. Every later interval then measures its rise against 20 rather than 80, which turns - ordinary 40 to 70 ms jitter into a rise of more than two buckets. - - **The video estimator drives the audio ring.** `js/watch/src/sync.ts:338` takes the max across - tracks. Video is seeded at 80 ms for the same reason and is fed 2 s keyframes on a cold - congestion window, so on a fresh publish the widest reading is rarely audio's own. - - **A rise of more than two buckets parks the ring.** `js/watch/src/audio/decoder.ts:519-534` - calls `ring.stall()` 150 ms later. The block in flight is dropped, the reader conceals until - the hold is refilled, `interrupted` goes true, the picture freezes with it, and `underruns` - stays flat. That last part is the discriminator: a park is not an underrun, and no budget in - this document counts one. - - **A smaller rise is gated out of stretching.** `js/watch/src/audio/playout/decision.ts:169` - refuses to expand once the level is below half the hold, so the ring plays on under its new - target until it runs dry and takes the underrun instead. - - **The native engine has no park at all.** `rs/moq-audio/src/playout/engine.rs:348-351` moves - the target and lets `Expand` walk into the deeper hold. The park is a browser-only addition. - - The publisher side is ruled out by reading: audio takes a reservation at priority 80 and ignores - its bandwidth grant, so no early or absent bandwidth estimate can starve it or delay it. - - The archive agrees with the twenty seconds without covering the case. The audio-quality matrix's - high-RTT rows measure convergence at 57 to 85 s, and every one of them is a file publisher graded - with the tune-in excluded from the counters, so nothing there grades the first seconds of a fresh - camera-and-microphone publish at all. - - Fix candidates, none applied and each a separate change: park only past `STRETCH_BOUND`, and - delete the `POSTPONE` gate with it, in both languages; skip a track that has measured nothing - from the max; hold the cross-track offset until each track has a full window; keep the seed as a - floor, which is a maintainer question because it contradicts the rule both languages replay; and - let the age budget follow the walked depth rather than the target. - - Measurement staged and not run: `bench/mutematrix.py --set wan`, five rows including the user's - exact shape, with the publisher-side sampler beside it, and a temporary `[playout]` tracing plan - behind that. The whole of it is written up in the private `initial-audio-cut-handoff.md` in the - bench archive. - - No commit. - -46. **The demo publisher advertises `meta.json` before it registers the track, so the demo watcher's - metadata read fails with code 0.** Found in the console of the same remote listen, beside finding - 45 and not on the audio path. - - `demo/web/src/publish.ts:389-419` announces the broadcast and then registers `meta.json` on it, - and `demo/web/src/index.ts:444-477` subscribes to that track as soon as it sees the announce. A - request for a track the broadcast does not hold yet is rejected at - `js/publish/src/broadcast.ts:269-272` with a bare `Error`, which reaches the watcher as Internal - `0x0` rather than NotFound `0x33`, and the watcher does not retry. The page loses its metadata - for the life of that broadcast. - - Two ways to close it, and the choice is in the open items: refuse an unknown track with - `NotFound`, so a watcher can tell "not yet" from "the publisher broke", or register every track - before the broadcast is advertised so the window does not exist. - - The other line the same console prints is benign. `skipping covered group` comes from - `#tryDurationSkip` in `js/hang/src/container/consumer.ts:493-513` when a group is dropped because - its successor has already started. Nothing is lost and nothing is counted; what is wrong with it - is the warning level, which is the open item carried from the previous pass. - - No commit. - -47. **The tune-in on this Mac takes about 0.9 s where it took 0.33 s before 05:22 on 2026-09-18, for - a reason nothing on the bench explains.** The factor-of-two tune-in figure the regression set - found is now localised and still unexplained, which is worth more than the guess it replaced. - - The split puts all of the difference in one place. `noTileMs` is unchanged and `playerMs` is not, - and inside `playerMs` it is the stretch from the tile swap to the first non-null - `audio.out.debug`, which is the element existing and the ring not yet existing. It reads about - 330 ms in `out15-after/`, `out15-after2/` and `out15-after3/` and about 970 ms in - `out15-regression/`, `out15-drift/` and `out15-idle/`: bimodal, not drifting. - - What was ruled out, in `out15-idle/`, `out15-bisect/` and `out15-after5/`: machine load, the - relay, the publishers, the page servers and the served build, all identical across the boundary; - the sampler and driver edits made between 05:22 and 05:30 local, bisected by running the same - `--set after` sequence with `--no-stats`, with `--no-burst` and with the page server restarted, - none of which moves the number; and the output device, a USB headset constant at 48 kHz with a - 16 ms output latency on every row. The identical sequence now reads 876 and 877 ms where it read - 328 to 347 ms before 05:22. - - One structural fact narrows it. The audio context and the ring appear in the same sample in every - row, so the delay is before the context is created rather than in building the ring, which puts - it in connect, announce or catalog. - - Two things would settle it and neither was run: reboot the machine and take one row, and - instrument the element's connect-to-catalog timing so the stretch has parts. Until then the - honest statement is that the six fast readings are historical and about 880 ms is what this - machine produces now. The 1 s `player` bar fails every reload row because of it, and it is left - failing rather than moved. - - No commit. - -## Public API and wire impact - -No wire format change anywhere. One wire *semantics* addition: an audio endpoint bounds the source -media rather than the track, so a publisher that pauses declares one, alone in its group, and that -marker group is itself the discontinuity: the run before it closes when the group closes, and the -next group opens a run the endpoint does not trim. -`drafts/draft-lcurley-moq-hang.md` and `doc/concept/hang.md` say so, and finding 36 deletes the -sentence that used to ask a resuming publisher to declare a second discontinuity of its own. The -bytes are the empty frame the draft already defines. The one wire *impact* of the final pass is one -group fewer per resume, because a publisher coming back from a mute no longer writes an empty group -after its marker. All of the below is `dev` material. - -The eight commits of 2026-09-17 change no wire format and no message. The one thing they change on -the wire at all is that a video rendition's track is no longer finished when it stops encoding -(`002ae00e1`), which is a lifetime rather than a byte. - -The eight commits above `c2e248ec5` change no wire format and no message either. What they change is -one new output, one corrected doc comment, two behaviours, and a default that moves in five places -at once. No draft is touched, and no `moq-net` or `hang` byte moves. - -The six commits of findings 32 to 35 change no wire format, no message and no track lifetime. What -they add is one counter in each language and nothing else; the rest is behaviour inside the ring, the -estimator's lifetime and one internal constant. - -The four commits of findings 36 to 44 change no wire format and no message either. Their wire impact -is one group fewer per resume, above; their public API impact is none. `Interruption` was removed -from `js/watch`, and it was internal, though anything reading `out.debug` saw it: the ring reports -`fresh` on `Playout.Snapshot` and `interrupted` is `stalled && !fresh`. The behaviour change worth -confirming is `skipped` at the `instant` preset, which is finding 39 and is stated under "The -budget". - -**Carried by the rebase, not introduced here.** The rebase onto `61da0d247` adopted three upstream -renames that touch surface this branch also touches, and they belong to upstream rather than to this -work: `MoqBackoff.{initial,max,timeout}_ms` became `{initial,max,timeout}_us` in microseconds -(`3e1e736a5`), the `@moq/net` backoff fields became branded `Time.Milli` (`948cf2ce5`), and -`moq_relay::AuthConfig` became `moq_relay::auth::Config` (`61da0d247`). Every number this branch -changes is stated below in upstream's units, so the FFI give-up default reads `timeout_us` of -`60000000` rather than `timeout_ms` of `60000`. Upstream's "timelines only move forward" -(`9931cfa8b`) also removed the rewind path this branch's estimator re-anchored on; the re-anchor now -hangs off the playhead event instead, which is the same signal under a different name and no surface -of its own. The rebase section above has all of it. - -**`@moq/net`** - -- New `Expired extends StreamError`, re-exported as `Moq.Group.Expired` beside `Lagged`. A peer's - `DELIVERY_TIMEOUT` reset now decodes to it rather than to a plain `StreamError`: same code, same - base class. -- `ReloadDelay.timeout` defaults to **60000 ms rather than 10000** (`5be36750f`). A caller who set - it is unaffected; a caller who did not now rides a relay restart instead of surfacing an error - part way through one. `0` still means unlimited, and giving up still does not dispose the loop. - See finding 31 for why the number moved and what the maintainer is being asked. - -**`@moq/hang`** - -- `Container.Jitter` is now a class, not a namespace. `Container.Jitter.BUCKET` and - `Container.Jitter.CEILING` are static readonly members rather than module exports. -- `Container.Jitter`'s constructor takes an optional `JitterProps` with a `start`, the publisher's - declared flush span the cold-start target is taken from. `JitterProps` is a new export. -- `Container.Jitter.observe(timestamp, now, observation)` takes a `JitterObservation` of - `{reordered?, stalled?}` where it used to take a bare `reordered` boolean. That is a breaking - change to a published method signature, which is the one reason this slice belongs on `dev` rather - than `main`. `JitterObservation` is a new export. -- `js/hang/src/container/stall.ts` is new: a `Stall` monitor, one 50 ms event-loop timer per - document, that tells a consumer whether this receiver was blocked. It is **not** exported from - `@moq/hang/container`; `Container.Consumer` owns the only instance. -- New `ConsumerProps.jitter`, which passes that declaration through from the rendition config. -- `c90eec220` widens `ConsumerProps.jitter` to `Time.Milli | Jitter`: a duration is still the - publisher's declaration, and a `Jitter` is an estimate already measuring that the consumer - continues through `reanchor()`. Additive, one prop with one meaning; every existing caller compiles - untouched. -- `ConsumerProps.maxAge` keeps its type and its name, and its *meaning* is corrected - (`d2443eba9`). It documented the span from the oldest buffered frame to the newest; it now - documents the rule the wire budget has always used, a group measured by how far it could still - present, which its successor's first timestamp bounds, against the newest frame the track has - reached. A caller passing the same number gets a different and better verdict on a long group, - which is the fix in finding 26. No signature moves. -- New `Container.Consumer.spread` and `Container.Consumer.skipped`. - -**`@moq/watch`** - -- `SyncInput` is reduced to `{delay, buffer}`; it loses `probe`, `audio`, `video`, `audioSpread` and - `videoSpread`. -- `SyncTrack.advertised` is removed. The rendition's declared flush span reaches the estimator as its - cold start instead, so there is nothing left here to read it. -- A numeric `delay` is now the whole delay. It used to have the advertised flush span added to it, so - a viewer asking for 100 ms on a source declaring 300 waited 400. -- `Sync.track()` is new, and `Sync.out.clock` names whichever track is driving playback. -- `Clock`, `SyncTrack` and `SyncClockTrack` are new exports. -- `Audio.Decoder.out` gains `spread`, `underruns`, `skipped` and `debug`. -- `AudioBuffer.timestamp` is `Getter` rather than `Getter`: a - ring that has been flushed and not yet re-anchored has no playhead, and zero is not that. - `audio.out.timestamp` keeps its type and is simply undefined for longer. The `timeline` field on - the postMessage ring's state and reset messages, which is what lets the main thread drop a message - composed before a flush, is internal to that transport and not exported. -- `unlockOnGesture` takes a `Getter` rather than a context, so the gesture - listeners are armed before any context exists. Internal to `js/watch`, not exported. -- `f846462e5` changes nothing on the surface. The `AudioContext` moves into a private effect keyed on - a new private `#rate` computed, so the worklet and ring rebuild under it instead of replacing it, - and `unlockOnGesture` is armed for the decoder's lifetime and returns without arming anything where - there is no `document`. Behaviour did change on a page with several tiles: a gesture anywhere - started the context of every tile that had one, not only of an unmuted tile. `16e6242aa` below - settles that: a muted tile has no context to start. -- `86baf84e7` adds `Audio.Decoder.out.interrupted`, a stall entered after the ring had played. - `out.stalled` keeps its meaning and its reader in the stats panel; the buffering indicator is what - moves onto the new one. -- `97f32ea66` gives `subscribeMedia` an optional `start` and exports `MediaStart` beside it. Both are - `@internal` to `@moq/watch`. Nothing about the request changes, only which groups this reader - delivers. -- `b366db8bb` adds no surface. The video decoder's rebuild, its backoff and the keyframe it waits for - after a hole are private, and its only public trace is the warning it prints, which is the - follow-up below about a rebuild counter. -- `16e6242aa` adds no surface either. A tile's `AudioContext` is built inside the first gesture, or - when the app turns audio on for an unmuted tile, rather than as soon as the catalog names a rate. - That is visible to a page only as the warnings it no longer prints and the contexts it no longer - holds; see finding 23. -- `Sync.out.offset` is new and additive (`69a89f17c`): how much later the picture arrives than the - sound for the same media timestamp, one-directional, bounded at 200 ms by `8b64a6aa6` and at - 100 ms after `50b0a72b9`. `Sync.out.delay` and `out.jitter` keep their meanings exactly; - `out.maxAge` is now derived from the offset as well. `` gains no attribute. The player's - "total buffer" row and the live badge report the third term. See findings 28 and 32. -- `Video.Decoder.out.spread` is unchanged in type and gains a lifetime (`d2443eba9`): it is - published per rendition and stays published across a subscription being rebuilt, where it used to - be cleared with the subscription. A reader that treated an absent spread as "no measurement yet" - sees one fewer gap. -- `804c13a20` adds no surface. `` gains no attribute and the renderer's inputs are - unchanged; the element republishes its canvas on every connect and drops it on disconnect, so the - download gate is re-evaluated when a tile is rebuilt or moved. See finding 27. -- `Video.Decoder.out` gains `skipped`. -- `DecoderInput.conceal` is new, with the element attribute ``. -- `audioMaxAge` and `maxAgeHeadroom` are new in the audio config module. -- Internal to the audio ring, not exported: `AudioBuffer.end()` on both transports, `RingView.ended` - on the reader surface, and an `ENDED` control slot, which takes the shared ring from 18 to 19. -- `Playout.Snapshot.trimmed` is new and additive (`e437fef5a`): samples the writer dropped off the - first fill on a timeline, before anything had been played. It is its own term rather than part of - `skipped` or `discarded` because it is the one drop no listener can hear and no time stretch had to - close, and the ring's invariant grows with it: - `READ == output - concealed + stretched + queued + skipped + trimmed`. It reaches a page through - `audio.out.debug`, which already carried the rest of the snapshot, so nothing else moves. The - player's stats panel gains a Trimmed row. See finding 33. -- `Estimator` is new in `js/watch/src/media.ts` and is `@internal` (`088752023`): the arrival - estimator one rendition keeps, handed back whenever that rendition is playing. It is not exported - from `@moq/watch`; both decoders own one. See finding 35. -- `50b0a72b9` and `cd43ff0a8` add no surface. `Sync.out.offset` keeps its type and its meaning and - changes value: it subtracts a 45 ms tolerance, caps at 100 ms rather than 200, ramps a bucket a - second, and carries the sound's arrival window across a mute. A new private `SPREAD_WINDOW` in - `js/watch/src/sync.ts`, 2 s, is how long a departed track's last reading stays in the - widest-across-tracks comparison; it is the same window the cross-track arrival floor beside it - already used. `doc/concept/playout.md` carries both rules. See findings 32 and 35. - -**`@moq/publish`** - -- No API change. `Audio.Encoder` writes an endpoint when it stops encoding and a break when it - resumes, which is behaviour rather than surface. -- `3f83de0fc` adds nothing and removes nothing: a served catalog track is now released when its last - subscriber leaves as well as when it closes. Same messages, same groups; what changes is that a - later subscriber is seeded instead of being left with an accepted subscription and no frame. -- `c53ecfe49` is also surface-neutral. The port message the capture worklet posts is a `Quantum` - rather than an `AudioFrame`, both internal to `js/publish/src/audio` and neither exported. The - audio timestamps a publisher sends change value, by the epoch error they carried, and the framing - does not. -- `b366db8bb` adds `Fanout.ended`, why a source stopped (`null` for a clean end, an `Error` for a - failure, `undefined` while it is delivering), and `Video.Capture.out.stopped`, the same answer for - a capture pipeline. Both additive. -- `3539e4ef2` adds `Camera.out.error` and `Microphone.out.error`, the reason a source has no track, - `MoqPublish.errors` for the element that holds them, and the `.control--error` class the camera and - microphone buttons take when there is one. All additive. -- `002ae00e1` adds and removes nothing. A video rendition's track is cut at the group rather than - closed when it stops encoding, so the same groups and frames go out with the subscription left open - instead of finished. That is behaviour, not format; it is the difference between a peer that can - come back and one that cannot. -- `dc6f8d38c` adds and removes nothing either, and changes what `announce="source"` means. - The mode still waits for the first live track, and then it **latches**: the broadcast stays - announced until the selected source changes, rather than un-announcing itself whenever no track is - live. A device being switched or re-acquired therefore keeps its subscribers, and a device that - cannot be opened fails only its own track. `doc/lib/js/publish.md` says so in the attribute table. - See finding 30. - -**`rs/moq-audio`** - -- `decode::Config::delay`, `decode::Config::conceal`, `decode::Config::DELAY_MAX`. `Config` is - `#[non_exhaustive]`, so these are additive for callers who build through `new`. -- `decode::Consumer::delay` and `decode::Consumer::playhead`. -- The native estimator's `Jitter::observe(timestamp, now, Observation)` takes the same - `{reordered, stalled}` pair as the browser's. Both the method and - `playout::delay::Observation` are `pub(crate)`, so this is parity rather than public surface, and - nothing sets `stalled` natively: a tokio task that stops polling stops reading the socket with it. -- The catalog's `jitter` seeds the native estimator's cold start, which is behaviour rather than - surface. The `delay` floor is untouched: a caller who knows something the arrivals do not say is a - different thing from a publisher describing its own encoder. -- `Consumer::read` returns one 10 ms block rather than a decoder packet when `delay` is set. -- `playout::engine::Stats::trimmed` is new (`91c5878bd`), the twin of `Playout.Snapshot.trimmed`: - frames dropped off the front before playout had played anything. `Stats` is `pub(crate)`, so this - is parity rather than public surface, and what a native player reports to a viewer and through - which type stays a decision for whoever builds that panel. See finding 33. - -**`rs/moq-relay`** - -- `Shutdown::draining()` is new and additive: whether the drain has already begun. The listeners - consult it, and a relay that has started draining now refuses a new session with `503` on every - transport (QUIC, WebTransport and WebSocket) instead of accepting one and waving it away with an - empty-URI GOAWAY. That is a behaviour change visible to any client that redials during a restart, - and it is the one the 503 exists for. `doc/bin/relay/index.md` states it beside `trigger.start()`, - and `doc/bin/relay/config.md` records that an empty redirect URI is a drain rather than a - redirect. See finding 31. - -**`rs/moq-tokio`** - -- `Backoff::timeout` defaults to **60s rather than 10s**, which moves `--backoff-timeout`, the - `MOQ_BACKOFF_TIMEOUT` environment variable and the `connect.backoff.timeout` setting with it. A - caller who set the value is unaffected; `0` still means unlimited. -- A GOAWAY naming no URI is no longer treated or logged as a redirect. The client reports `peer is - draining` and redials the same address with backoff, and a replacement session waved away before - it ever served does not displace a predecessor that is still serving. Behaviour, not surface. - -**`rs/moq-ffi` and the bindings** - -- `MoqBackoff.timeout_us` defaults to **60000000 rather than 10000000** (`eec9016d9`), because the record - mirrors `moq_tokio::Backoff` and its doc claimed a default it no longer matched. The Go wrapper - resolves its own unset fields before the FFI call, since a Go zero means "retry forever" on the - wire, so it carries the number too. The Dart bindings are generated but checked in, and only that - one line is regenerated. The Swift, Kotlin and Python bindings are generated into gitignored paths - at build time and pick up the default and the doc line on the next build. -- **Judgement item**: the FFI default follows the native one rather than staying put, so a binding - caller who never set `timeout_us` now gets 60 s. - -**`moq play`** - -- `--delay` is a floor rather than the delay, and is capped at 2 s, the estimator's range, rather than - the speaker ring's 10 s. That is the `!` break on this branch. - -**New crate** - -- `rs/moq-shaper`, `publish = false`. `Shaper::{bind, local_addr, report, run}`, `Config`, `Counters`, - `Report`, `Profile::{parse, names}`, `Direction`, `Burst`, `ByteRate`, `Step`. - -**New private workspace member** - -- `@moq/audio-quality`, not published. - -**Timestamp values, not format** - -- The MPEG-TS importer now carries the fractional remainder of a sample duration between frames, so - the timestamp values it emits change, and so does the timescale they are expressed in. The framing, - the fields, and their encoding do not. - -## Departures from the quests - -Each of these is deliberate and each is isolated so it can be dropped. - -**Concealment is core here.** `watch-audio-time-stretch.md` keeps an underrun a ramped gap and says -no packet loss concealment. The user asked for NetEq's expand and merge. It is its own commit -(`b7b3806b7`), it is switchable with `conceal` on the audio decoder, and turning it off restores the -ramp byte for byte. The stretch commit before it passes its own budgets alone. - -**The audio headroom is this fork's addition.** The quests only require observing above the budget, -which commits 9 and 10 already achieve, and `native.md` treats `max_age` as an independent ceiling. -Commit 11 is droppable with no other change. - -**No frame term in the estimator.** `Container.Jitter` reports the bucket's upper edge and nothing -else. The consumer's own granularity, the render quantum in the browser and the sink period natively, -belongs to the ring above the estimator. That is what removes the timestamp-derived spacing that -produced #3517's 14.56 s. - -**The fall bound is not NetEq's.** NetEq falls one bucket per step. This falls a sixth of the -remaining distance, one bucket at the floor, and `doc/concept/playout.md` argues why: the quantile is -a bucket index and comes down in handfuls of buckets at a time, which a fixed step cannot follow, and -the time to undo an overshoot should not grow with the overshoot. - -**`kPostponeDecodingLevel` is re-pointed.** NetEq uses the half-target hysteresis to postpone -decoding. Here it gates expansion, so it does not fight a refill: the ring's re-stall already holds -until the full target is back. The deviation is documented at the site. - -**No frame-buffer smart flushing in the browser.** The trough-based skip band is stricter than a -partial flush toward target would be, and adding both would give the same surplus two owners. The -native engine has the frame buffer and does flush. - -**The skip band rules on a trough.** A flush that drains again within the window is played rather -than discarded, so the band is not evaluated on the instantaneous level the quests describe. - -## The rebase onto upstream dev, 2026-09-18 - -The branch sat on `61da0d247` from the rebase of the day before. `upstream/dev` moved twenty-five -commits past it to `712ffd810`, so the branch was replayed commit by commit again before the -listening round could be taken against the tree it would land in. All 125 commits replayed, none was -dropped as already upstream, and all 125 keep their `Co-Authored-By` trailer and still carry no -`Claude-Session` one. The pre-rebase tip is kept locally on -`debug-findings-solution-pre-rebase-20260918`. The full log is in the bench archive as -`REBASE-CONFLICTS-2026-09-18.md`; this is what it says. - -**What upstream moved.** Five of the twenty-five are marked breaking: -`5caf4bae7 feat!: name every rate estimate estimated_*_rate`; -`a1ed0d8a2 feat(net)!: register the four placeholder stream codes`; -`5e03b9b3d feat(net)!: fold the session placeholders into PROTOCOL_VIOLATION`; -`010f0f80f feat(auth)!: the lease reports what it ended with, and the relay Lease owns the recheck`; -and `712ffd810 feat(net)!: reprice a PUBLISH_NAMESPACE with REQUEST_UPDATE`, which is the new base. -Two more are not marked breaking and had to be followed anyway: -`a9ac34be2 fix(hang): refuse a malformed text catalog section in JS too`, which renamed the group -expiry message, and `d518b61b4 feat(native): default the QUIC backend to noq`, which rewrote a line -of `doc/bin/cli.md` beside the branch's own. Those seven are the ones that could reach this branch; -the rest are tests, quests and merges. - -**One commit conflicted**, `e02cd2ac2 fix(net): give the latency budget verdict a stream error type`, -in three files, with `a1ed0d8a2` and `a9ac34be2` on the other side of each. Everything else replayed -clean, including the six files the pre-rebase survey had flagged as touched on both sides, which all -auto-merged. - -| File | Resolution | -| --- | --- | -| `js/net/src/group.ts`, the export doc block | Upstream moved `FrameTooLarge` out of the reserved range to its own assigned code `0x38` and taught `fromTransport` to decode it, so its sentence became "All three carry a moq-lite stream code, and a peer's reset with one decodes back into the same class." The branch adds a fourth, `Expired`. Resolved as upstream's sentence with the branch's count, "All four ...", and the branch's caveat that reserved codes stay opaque deleted, because upstream made it false | -| `js/net/src/group.ts`, `Consumer.#expire` | Upstream renamed the terminal message from "group exceeded the subscription latency budget" to "... max age budget"; the branch replaces the bare `Error` with `new Expired()`. Resolved as the branch's `new Expired()`, with upstream's rename carried into the `Expired` message instead, which is what its three assertions in `js/net/src/track.test.ts` read | -| `js/net/src/error.ts` | Auto-merged, then corrected for upstream's rename: the message is now `"expired: group exceeded the subscription max age budget"` and the doc line says "max age budget", matching the term the rest of the tree already uses. Without it the three upstream `toThrow("max age budget")` assertions would fail | -| `js/net/src/error.test.ts` | Both sides kept: the branch's three assertions that `DeliveryTimeout` decodes to `Expired` first, because they continue the run of `fromTransport` round trips above them, and upstream's loop over the four codes that were sent before assignment last | - -**The semantic sweep**, run before building rather than after, because a rename that does not -conflict is the one that bites: - -- `5caf4bae7` renames the FFI and C rate surface, `send_rate_bps` to `estimated_send_rate_bps` and - its `recv` twin, with the Dart, gst and OBS spellings behind them. No branch code uses any of the - eight old spellings. The branch's own readers are on the browser WebTransport names - `estimatedSendRate` and `estimatedRecvRate`, which upstream did not touch, and - `moq_net::ConnectionStats::estimated_send_rate` was already spelled that way. -- `a1ed0d8a2` moves `StreamCode.FrameTooLarge` from `0x25` to `0x38` and leaves - `StreamCode.DeliveryTimeout` at `0x2`, so the branch's `Expired` mapping needed no renumbering. -- `5e03b9b3d` deletes three `SessionError` variants and the Go and Dart constants for them; the - branch names none of the three. -- `010f0f80f` reshapes `moq_relay::auth`; the branch's only touch in that file family is the draining - refusal above the `admit` call, which merged intact. - -**One fixup**, made as a `fixup!` and squashed with `git rebase --autosquash`, which left the tree -byte identical to the one the gates ran against and the count at 125: `just fix` rewrote fifteen -lines of this file, escaping a line-leading `36.` that the formatter reads as an ordered-list marker -and the bracketed site labels it reads as link references, and re-indenting a continuation paragraph -under its list item. That is not upstream's doing. The previous docs commit landed after the gate run -below it, so its prose had never been through the formatter. No source change was required by any of -the twenty-five upstream commits. - -**The gates on the rebased tree**, at the post-fixup tip `f4b1e0081`, 125 commits above -`upstream/dev` `712ffd810`, run one at a time under the bench's cargo lock in the repository's own -nix shell. Every one of them exited 0: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `bun install --frozen-lockfile` | 0 | 791 installs across 885 packages, no change; neither lock needed regenerating | -| `bun test` in `js/net` | 0 | 823 passed, 45 files | -| `bun test` in `js/hang` | 0 | 252 passed, 21 files | -| `bun test` in `js/watch` | 0 | 420 passed, 32 files | -| `bun test` in `js/publish` | 0 | 156 passed, 22 files | -| `cargo nextest run` over the seven touched crates | 0 | 1787 passed, 4 skipped, against 1779 before the rebase; upstream added the eight | -| `just fix upstream/dev` | 0 | One file changed, the formatting fixup above | -| `just check upstream/dev` | 0 | The full chain including the `_flake` step, the packagers and the release-asset check | -| `just test default upstream/dev` | 0 | 4708 Rust passed, 9 skipped, plus every JS package and Python | -| `just test audio-quality --runtime replay --enforce` | 0 | 172 enforced checks across 14 rows, the same count as before the rebase | -| `just drafts check` | 0 | 12 drafts, e2ee vectors generated | -| `just test smoke-full` | 0 | 32 of 32 cross-language publish and subscribe pairs | - -**Two red herrings, neither of them the rebase.** The first `just test default` exited 100 on -`moq-tokio quinn::tests::fixed_addresses_verify_hostname_fail_over_and_share_endpoint`, which failed -at 0.164 s with both loopback addresses timing out. `rs/moq-tokio/src/quinn.rs` is untouched by all -twenty-five upstream commits and by this branch, the test passes three times out of three in -isolation, and the re-run of the whole gate passed all 4708. It is the loopback-contention flake -`quest/m2/flaky-timing-tests.md` already tracks, made likelier by the bench relay and publishers -still running for the listening round. The second is `bun test` over the whole `js/` tree in one -process, which reports six failures and one error; the identical six appear on the pre-rebase branch, -every file passes on its own and per package, and it is `mock.module` leaking across files in a -single bun process rather than a rebase regression. No gate runs it that way: `just test default` -runs bun per package. - -**The fork's `dev` was reset in the same round.** The fork's `dev` had the Sep 12 WIP checkpoint -merged into it, so the pull request showed twenty-five conflicts in files upstream had never touched, -none of which the branch itself had any part in. On the user's decision the fork's `dev` was -force-pushed to upstream's `712ffd810` with a lease, and its old value was tagged locally before the -push so nothing is unreachable. - -## The rebase onto upstream dev, 2026-09-17 - -The branch sat on `877a561d8` from the previous rebase. `upstream/dev` moved forty-five commits past -it, so the branch was replayed commit by commit onto `61da0d247` before any of it could be read -against the tree it would land in. All 116 commits replayed, none was dropped as already upstream, -and 114 of them keep their `Claude-Session` trailer exactly as before, the two without it being the -maintainer's own cherry-picks from #3517, which never had one. The full log is in the bench archive -as `REBASE-CONFLICTS-2026-09-17.md`; this is what it says. - -**What upstream moved.** Fifteen of the forty-five are breaking. The ones that reached this branch: -`9931cfa8b feat(hang)!: timelines only move forward`, which deleted the rewind machinery; -`948cf2ce5 feat(js)!: mirror Rust names in @moq/net and @moq/pattern`, which brands the backoff -fields `Time.Milli`; `3e1e736a5 feat(ffi)!: use microseconds and matching origin verbs`, which -renames every `MoqBackoff` field; `ff0bf5f38 feat(net)!: scope origins with pattern unions` and -`7d3f2840c feat(net)!: one name per announce, request, and origin config concept`, which turn an -announce prefix into a `Path.Pattern` scope; `a1a58a477 feat(net)!: name stats counter edges started -and ended`; and `61da0d247 feat(relay)!: public modules and typed cluster peer config`, which -replaces `moq_relay::AuthConfig` with an `auth` module. The other breaking ones -(`dbe0eb058 feat(hang)!` one continuous broadcast clock, `ff97e2b4c feat(json)!`, -`6504e9777 feat(ffi)!` dropping `MoqCancel`, `90581451f feat(tokio)!`, `b46920a5f feat(tokio)!` -renaming `ConnectionStatsReader`, `e1b632337 feat!` refusing released spellings, -`ab7f40b37 feat!` borrowing publisher finish, `cd9aad115 feat(net)!`) touch nothing this branch -holds; a sweep for every identifier they removed found no use on the branch. - -**Eight conflict shapes, across nine commits**, each in a file both sides had edited: - -| Commit | File | Resolution | -| --- | --- | --- | -| `e4b7e9eff` | `js/hang/src/container/consumer.ts` | Upstream's `#checkMalformed` and `#abortIfRewound` taken whole. The branch's `reanchor()` on a rewind has no home left, because a rewind now aborts the track; the consumer's other `reanchor()` is kept and `#spread.observe()` still runs after upstream's guard. The comment naming `#checkReset` was corrected to name the abort | -| `4c488b08a` | `js/hang/src/container/consumer.test.ts` | Upstream rewrote the rewind tests as abort tests and added its own `nextFrame()` helper with the same body as the branch's. Upstream's side taken, the branch's duplicate helper dropped, the branch's arrival tests kept on the one remaining helper | -| `c0f623941` | `doc/lib/js/net.md` | Upstream's reworded Discovery bullet verbatim, the branch's `Expired` clause folded into upstream's Subscriptions sentence | -| `47f6b3cba` | `rs/moq-audio/src/decode/consumer.rs` | The branch's playout methods kept, upstream's doc comment on `apply_discontinuity` kept, and only `playout.engine.reanchor()` kept from the branch's body change. Upstream deliberately dropped `decoder.reset()` and `resampler.reset()` there in favour of `reapply_delay()`, and that stands | -| `e044c97a4`, `c90eec220` | `js/hang/src/container/consumer.test.ts` | The same duplicate-helper shape twice more: the new test kept, the re-added helper dropped | -| `d2443eba9` | `js/hang/src/container/consumer.test.ts` | The branch's new test inserted above upstream's renamed `test("Consumer zero-budget skip keeps a contiguous marker")` | -| `5be36750f` | `js/net/src/connection/reload.ts`, `rs/moq-relay/tests/shutdown_signal.rs`, `js/net/src/connection/reload.test.ts` | Upstream's `Time.Milli` typing with the branch's value, `Time.Milli(60000)`; `use moq_relay::{Config, Relay, auth}` plus the branch's `Hop` import; both new tests kept, upstream's first | -| `eec9016d9` | `rs/moq-ffi/src/session.rs`, `dart/moq_ffi/lib/src/moq.dart`, `go/wrapper/backoff_internal_test.go` | Upstream's microsecond field names throughout with the branch's value: `#[uniffi(default = 60000000)]`, `this.timeoutUs = 60000000`, `TimeoutUs: 60_000_000` | - -Neither `Cargo.lock` nor `bun.lock` conflicted, and neither needed regenerating: both `just fix` and -`just check` run `--locked` and passed, and `bun install` reported no changes. - -**Six fixups, each found by a gate and fixed at the source**, then squashed back into the commit that -owned them with `git rebase --autosquash`, which left the tree byte identical to the one the gates -had run against and the count at 116: - -1. `moq_net::stats::Presence::sessions_closed` became `sessions_ended` (`a1a58a477`), in - `rs/moq-tokio/src/connection.rs`. The only branch use; the other hits are upstream's own serde - shims and the relay's Prometheus names. -2. The `delay` literal in the new relay-restart test needed branded `Time.Milli` values. -3. `origin.announced(Path.empty())` became `origin.announced()`, since the call meant everything and - the default scope is now `Path.Pattern.all()`. -4. The re-anchor test, below. -5. The replay harness's group numbering, below. -6. Three `MoqBackoff.timeout_ms` / `60000` lines in this document, corrected to `timeout_us` / - `60000000`. - -**Two semantic adaptations to `9931cfa8b`**, which are the only places where the branch's intent had -to be re-expressed rather than merged. Neither changes what is asserted. - -- `test("Consumer re-anchors the arrival reference when the timeline rewinds")` drove the estimator's - re-anchor by rewinding the media timeline. Under "timelines only move forward" that is a malformed - track and the consumer aborts, so the test threw rather than asserting. The branch's intent - survives, because `#spread.reanchor()` now sits in `#markPlayhead()`, which a playhead marker still - reaches, so the case is rewritten as - `test("Consumer re-anchors the arrival reference on a playhead event")`: group 0, a marker group, - then a group 20 s further forward, which is how a publisher restarts a source under the new rule. - It asserts the same thing. That also cleared a third failure: the aborting test never reached its - `consumer.close()`, so it leaked a `Consumer` holding the shared `Stall` monitor and - `stall.test.ts`'s holder count never returned to zero. -- The replay harness in `js/watch/src/audio/replay.test.ts` numbered each replayed group with - `sequence++`, in arrival order. `relay-bbb-7frame.json` holds seven reordered arrivals, so those - became higher-numbered groups carrying older timestamps, which is a rewind under the new rule. - Fixed at the source: the group sequence is now the arrival's rank on the media timeline, which is - what the wire does, so a reordered arrival is what it really is, a low-numbered group that arrived - late. The measurement keeps its shape: 0 of 443 groups convicted at the measured target, 0 with the - headroom, and 148 at the round-trip control against the 91 the old numbering produced, so the - control still holds. - -**Name mappings adopted**, recorded because each is a decision to follow upstream rather than to -carry a shim: - -| Old | New | Where it bit | -| --- | --- | --- | -| `Consumer.#checkReset` and the `Reset` rewind machinery | `#checkMalformed` plus `#abortIfRewound`; a rewind aborts | `js/hang/src/container/consumer.ts` | -| `decoder.reset()` and `resampler.reset()` on a discontinuity | `decoder.reapply_delay()` alone | `rs/moq-audio/src/decode/consumer.rs` | -| `moq_relay::AuthConfig` | `moq_relay::auth::Config` | `rs/moq-relay/tests/shutdown_signal.rs` | -| `MoqBackoff.{initial,max,timeout}_ms` | `{initial,max,timeout}_us`, in microseconds | `rs/moq-ffi`, `dart/moq_ffi`, `go/wrapper` | -| plain-number backoff defaults | `Time.Milli(...)` | `js/net/src/connection/reload.ts` | -| `origin.announced(prefix)` | `origin.announced(scope)`, a prefix-shaped `Path.Pattern` | `doc/lib/js/net.md` | -| `moq_net::stats::Presence::sessions_closed` | `sessions_ended` | `rs/moq-tokio/src/connection.rs` | - -**The gates on the rebased tree**, at the post-fixup tip `91c5878bd`, 116 commits above -`61da0d247`. Every one of them exited 0: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `just fix upstream/dev` | 0 | No tracked change | -| `just check upstream/dev` | 0 | Every package the branch touches, scoped as CI scopes it | -| `just test default upstream/dev` | 0 | 4675 Rust tests (8 skipped), 2068 Bun tests across fourteen packages, 64 Python | -| `cargo nextest run` over the touched crates | 0 | 882 passed, 3 skipped | -| `just test audio-quality --runtime replay --enforce` | 0 | 12 rows, 148 enforced checks, 0 void | -| `just test smoke-full` | 0 | 32 of 32 cross-language publish/subscribe pairs | -| `just drafts check` | 0 | 12 drafts | - -**The bench was rebuilt from the rebased tip and re-checked**, three rows, all pass. Its table is in -the Evidence section under "The bench on the rebased build". - -## Verification and limitations - -### What was run - -- `just fix`, `just check`, `just test` at every stage, scoped to the branch the way CI scopes them. -- The 17-case corpus, replayed in Bun and in nextest, exact integer equality on both sides. -- `bun test` over `js/hang/src/container`, `js/watch/src/audio`, and the sync tests. -- `just rs test -p moq-audio`. -- `just test audio-quality` over the full 24-row matrix, twice: once against `upstream/dev` and once - against this tree. -- `just test audio-quality --runtime safari`, one 60 s run per row on macOS 26.6 Safari 26: eight - rows graded, no voids, both rings. -- `just test audio-quality --runtime replay --enforce`, which is a second and grades twelve rows - against ceilings that are exactly what was measured. -- A CodeRabbit CLI triage per stage, each recorded. The findings that were real became their own - commits: `1a6c1fe45`, `bd27266cc`, `d5a0b834a`, `3a02a1afa`, and part of `bec43e540`. -- A CodeRabbit CLI pass over the whole branch against `upstream/dev`, run per top-level directory - because the branch is past the free plan's 150-file cap: 22 findings, 18 fixed, 4 rejected. See - finding 16. -- A quiet re-measure at the tip, one row at a time with nothing else loading the machine: the three - pages against the local and public relays, the microphone row per engine on a fixed browser - publisher, and the mute and preset sequence in real Safari, WebKit and Chromium. Every number in - the tables above is read out of that run's row JSON rather than off a printed line. -- Eight real-Safari runs on `f846462e5` and eleven on the commit below it, one Element Click each, - counting a run only when the context reached `running` and a playhead appeared. -- Listening rounds by the user after the estimator, after the stretch, and after concealment. -- The A/V desync the user reported after all of that, reproduced and re-measured in headless - Chromium on a fresh `demo/web` build of this tree against the local relay: the user's own sequence - of mutes and latency presets, sampled every 250 ms, on both rings and with concealment both ways. - Worst painted-video-minus-audio skew 3418 ms before and 50 ms after; the regression test reruns the - same sequence with each half of the fix removed. -- Each of the five re-landed fixes has a test that fails without it and passes with it. -- One pre-existing flaky `console.error` spy test was fixed on the way past, in `d862f4ab5`. -- A listening round by the user on a build of `f846462e5`, which produced the nine reports the eight - commits of 2026-09-17 answer. -- The rows behind findings 18 to 25, all on the quiet bench with the publisher restarted before each - one: the unmute on both pages and on the microphone, the mute and preset sequence per engine after - each fix, hide and show in Chromium, Brave, Firefox and WebKit with and without an injected 8 s - busy device, eight real-Safari runs with one Element Click each, a page load per engine counting - the autoplay warnings with the browser's own policy, and the relay's own log read either side of - the re-subscribe fix. -- Tests with each of those fixes, in `js/hang`, `js/net`, `js/publish` and `js/watch`; the counts are - in the two gate passes below. -- **The cross-browser resilience matrix**, 49 rows at `16e6242aa`, 08:01 to 10:01, one browser at a - time with the publisher restarted before every row: five engines publishing and watching, hide and - show, mute and unmute, a device change, four shaper profiles, a relay restart and a 30 minute run. - 24 passed, 25 failed, and the failures are the eight causes the Evidence section groups. -- **The re-run of the affected rows**, 10:59 to 12:08, after the four watcher-side fixes and the - publisher's announce latch: the two Firefox cross-engine pairs, all four shaper profiles with the - WebSocket fallback denied, the device switch and a there-and-back switch, and the relay restart - three times over with the sampler instrumented further on each pass. Its table is in the Evidence - section, with the bitrate confound and the load caveat that bound it. -- **The native reconnect on a real relay restart**, read out of the two file publishers' logs and - the relay's own: nine refusals with `503`, the serving session kept for the full 10.0 s drain, and - both broadcasts re-announced within 0.5 s and 2.2 s of the new relay binding. Finding 31. -- **The quiet confirmation**, ten of those rows run again from 12:46 to 13:01 on a machine with - nothing else on it, with both served pages built from the code tip: the three shaper profiles, the - Firefox cross-engine pair, 1080p60 self-publish in chromium and in brave, the toggle sequence in - two engines, hide and show, and a relay restart. Every resilience target is met in every row, and - the four rows that still read FAIL fail against the grader's single envelope rather than against - the player. Its table is the last one in the Evidence section. -- **A listening round by the user**, on 2026-09-17, on the bench's own pages with a real microphone - and camera and a headset, which reported it clean. Nothing was recorded from it, so it confirms the - rows above rather than adding a number to them. It is the round `e6be6a775` records. -- **A dead-code and stale-comment sweep over the whole diff at the end**, which dropped five exports - nothing imports and one ring test the surviving case already brackets, and corrected every comment - a later commit on this branch had made false, for a net 9 lines and no behaviour change, with - `just fix`, `just check` and `just test default` re-run green on that tree. -- **The two transients the same listener reported next**, findings 32 and 33, each measured on the - quiet bench with the publisher restarted before every row: the first four seconds of a self-publish - and the five rapid mute and unmute pairs, before and after each of the two commits, read off the - ring's own counters rather than off the target. -- **The multi-watcher rows the user asked for**, findings 34 and 35: five watchers on one browser - publisher, 90 s a row, 30 mutes and unmutes and two camera hides, once with three Chromium plus - Firefox plus WebKit and four times with five Chromium, with the publisher's own upstream leg read - out of the relay log each time. The unit twin in `js/publish/src/watchers.test.ts` runs three - watchers joining and leaving both media tracks with a fourth arriving after them. -- **The rebase onto `upstream/dev` `61da0d247`**, commit by commit, with every conflict resolved - against upstream's side and recorded, six fixups found by the gates and squashed back into the - commits that owned them, and the whole gate set re-run on the rebased tree. The section above has - it. - -### Gates on the final tree - -Seven passes, each stated with the tip it ran on, one command at a time. - -At `29af6fb9d`, 72 commits above `upstream/dev`: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `just check upstream/dev` | 0 | Every package the branch touches, scoped as CI scopes it | -| `just check-all` | 0 | Every package, every language, including the bindings and the docs site | -| `just test all` | 0 | 4569 Rust tests (8 skipped), 1942 Bun tests across fifteen packages, 63 Python | -| `just test smoke-full` | 0 | 32 of 32 cross-language publish/subscribe pairs | -| `just drafts check` | 0 | 10 drafts, including the hang draft this branch changes | -| `just test audio-quality --runtime replay --enforce` | 0 | 10 rows, 124 enforced checks | -| privacy grep over the branch's added lines | 0 hits | No home path, name, address, token, or session id | -| CodeRabbit CLI, per directory | 22 findings | 18 fixed, 4 rejected with reasons; see finding 16 | - -At the tip `116d4ad88`, 81 commits above `upstream/dev`: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `just fix upstream/dev` | 0 | No tracked change | -| `just check upstream/dev` | 0 | Every package the branch touches, scoped as CI scopes it | -| `just test default upstream/dev` | 0 | 4573 Rust tests (8 skipped), 1956 Bun tests across fifteen packages, 63 Python | -| `cargo nextest run -p moq-audio -p moq-shaper` | 0 | 200 tests | -| `just test audio-quality --runtime replay --enforce` | 0 | 12 rows, 148 enforced checks, 0 void | -| `just test audio-quality --enforce` | 0 | 24 Chromium rows at 60 s, 36 enforced checks, 0 enforced breaches | - -At the tip `f846462e5`, 85 commits above `upstream/dev`, after the three fixes behind findings 13, 14 -and 15: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `just fix upstream/dev` | 0 | No tracked change beyond the three delivery documents | -| `just check upstream/dev` | 0 | Every package the branch touches, scoped as CI scopes it | -| `just test default upstream/dev` | 0 | 4573 Rust tests (8 skipped), 1965 Bun tests across fifteen packages, 63 Python | -| `just test audio-quality --runtime replay --enforce` | 0 | 12 rows, 148 enforced checks, 0 void | -| `bun test` in `js/publish` | 0 | 141 tests across 20 files | -| `bun test` in `js/watch` | 0 | 366 tests across 31 files | - -The Chromium matrix was not re-run at this tip: the three fixes above it are one `js/watch` effect and -two `js/publish` capture changes, and the matrix has no browser publisher in it, so no row's inputs -moved. The replay lane, which does cover the player, is re-run above and is clean. - -The Chromium matrix at `116d4ad88` breached 17 recorded ceilings and voided one recorded row, -`opus-near-zero-isolated`, on `AudioContext.currentTime` drifting 4.92 percent from wall clock over -10 s. Both are in the enforced-budgets section above, with every measured value against its ceiling. - -**`check-all`, `test all` and `smoke-full` were last run thirteen commits below that tip.** Those -thirteen commits touch `js/watch`, `js/hang`, `js/publish`, `rs/moq-shaper`, `rs/moq-audio`, the -harness, the nightly workflow and the delivery documents. Every one of those is inside the scope the -`check` and `test` at the tip cover, so nothing in them is unchecked; what is not re-run is the rest -of the workspace, which they do not touch, and the cross-language interop lane. - -At `6e04673c9`, with the video recovery work that became `b366db8bb` already in the tree: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `just fix upstream/dev` | 0 | No tracked change | -| `just check-all` | 0 | Every package, every language, including the bindings and the docs site. The round before it failed on two TypeScript errors in the new `js/watch/src/video/decoder.test.ts`, which were fixed before the commit | -| `just js test` | 0 | 804 `@moq/net`, 230 `@moq/hang`, 373 `@moq/watch`, 145 `@moq/publish`, every package exiting 0 | -| `demo/web` and the copied site rebuilt | 0 | So the listening rows that follow run the tip rather than an older build | - -At the code tip `16e6242aa`, 96 commits above `upstream/dev`, covering `97f32ea66`, `3539e4ef2`, -`002ae00e1` and `16e6242aa`: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `just fix upstream/dev` | 0 | No tracked change | -| `just check upstream/dev` | 0 | Every package the branch touches, scoped as CI scopes it | -| `just js test` | 0 | 805 `@moq/net`, 230 `@moq/hang`, 383 `@moq/watch`, 153 `@moq/publish` | -| `just test audio-quality`, one row | 0 | `chromium-opus-48000-fixed-250-plain`: zero underruns, target p95 250 ms, not void. Its 16 skip-aheads a minute are over a recorded ceiling of zero, reported rather than enforced, which is the `recorded` marker doing its job | -| `demo/web` and the copied site rebuilt | 0 | Both served pages come from the final tip | - -**What was not re-run above `462637ef7`**: the deterministic replay lane, the 24-row Chromium matrix, -`just test default`, `check-all` at that tip, and `smoke-full`. The eight commits of the morning -touch `js/hang`, `js/net`, `js/publish`, `js/watch` and one paragraph of `doc/concept/playout.md`, -and no Rust, no wire and nothing `moq play` reaches, so the cross-language lane had nothing new to -cover. The replay lane and the Chromium matrix do cover the player, and they are the gap: what -stands in their place is `just js test` at both passes and the bench rows above, which are -instrument readings on one machine rather than a graded run. - -At the code tip `eec9016d9`, 105 commits above `upstream/dev`, covering `d2443eba9`, `208755120`, -`804c13a20`, `dc6f8d38c`, `69a89f17c`, `8b64a6aa6`, `5be36750f` and `eec9016d9`: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `just fix upstream/dev` | 0 | No tracked change | -| `just check` | 0 | Every package, every language: the root orchestration moved, so it checked everything rather than the branch's scope | -| `just js test` | 0 | 806 `@moq/net`, 231 `@moq/hang`, 391 `@moq/watch`, 155 `@moq/publish`, and every other JS package, each exiting 0 | -| `just test default upstream/dev` | **100** | 4365 of 4366 Rust tests passed and **one failed**, in the test rather than in the client | - -**That one failure was the test's own, and it is fixed in `b80a2084a`.** -`moq-tokio::reconnect a_peer_away_longer_than_a_relay_restart_is_reconnected_to` panicked at -`rs/moq-tokio/tests/reconnect.rs:251` with `the client gave up on a peer that was away for 20s: -Elapsed(())`, in a scoped `cargo nextest run -p moq-tokio` as well as in the full run. It mixed a -paused clock with a real socket, and a paused clock jumps straight to the next deadline whenever the -runtime waits on that socket, so it was timing the jumps rather than the outage. The outage now runs -paused and the reconnect runs on the real clock. Finding 31 has the detail, and the behaviour the -case guards is what the bench measured on two real restarts. - -At the tip `b80a2084a`, 107 commits above `upstream/dev`, with that fix in: - -| Gate | Exit | What it covered | -| --- | ---: | --- | -| `just fix upstream/dev` | 0 | No tracked change | -| `just check upstream/dev` | 0 | Every package the branch touches, scoped as CI scopes it | -| `just test default upstream/dev` | 0 | 4577 Rust tests passed (8 skipped), 2011 Bun tests across sixteen packages, 63 Python | -| `cargo nextest run -p moq-tokio -p moq-relay -p moq-ffi` | 0 | 687 passed, 3 skipped: the reconnect, the shutdown and the FFI cases of finding 31 | -| `just test audio-quality --runtime replay --enforce` | 0 | 12 rows, 148 enforced checks, 0 void | -| `just js test` | 0 | 806 `@moq/net`, 231 `@moq/hang`, 391 `@moq/watch`, 155 `@moq/publish`, every JS package exiting 0 | - -The Chromium harness matrix and `smoke-full` were not re-run above `c2e248ec5`. `5be36750f`, -`eec9016d9` and `b80a2084a` are the first Rust on this branch since, and they touch the relay's -shutdown path, the client's reconnect loop and one test rather than anything on the wire, so -`smoke-full` has no new pair to cover; the `moq-relay` and `moq-tokio` tests are what covers them, -and the replay lane in the table above is what covers the player. - -At the tip `cd43ff0a8`, 118 commits above `upstream/dev` `61da0d247`. The first seven rows ran on the -rebased tree at `91c5878bd`, which is the rebase's own post-fixup tip and the widest pass this branch -has had; the rest ran at `cd43ff0a8` itself, which is two `js/watch` commits above it: - -| Gate | Tip | Exit | What it covered | -| --- | --- | ---: | --- | -| `just fix upstream/dev` | `91c5878bd` | 0 | No tracked change | -| `just check upstream/dev` | `91c5878bd` | 0 | Every package the branch touches, scoped as CI scopes it | -| `just test default upstream/dev` | `91c5878bd` | 0 | 4675 Rust tests (8 skipped), 2068 Bun tests across fourteen packages, 64 Python | -| `cargo nextest run` over the touched crates | `91c5878bd` | 0 | 882 passed, 3 skipped | -| `just test audio-quality --runtime replay --enforce` | `91c5878bd` | 0 | 12 rows, 148 enforced checks, 0 void | -| `just test smoke-full` | `91c5878bd` | 0 | 32 of 32 cross-language publish/subscribe pairs | -| `just drafts check` | `91c5878bd` | 0 | 12 drafts, including the hang draft this branch changes | -| `bun test` in `js/watch` | `cd43ff0a8` | 0 | 411 tests | -| `bun test` in `js/hang` | `cd43ff0a8` | 0 | 245 tests | -| `cargo nextest run -p moq-audio` | `cd43ff0a8` | 0 | 177 tests, the corpus and the native engine including the trim | -| `just fix upstream/dev` | `cd43ff0a8` | 0 | No tracked change | -| `just check upstream/dev` | `cd43ff0a8` | 0 | Every package the branch touches, scoped as CI scopes it | - -**What was not re-run at `cd43ff0a8`**: `just test default`, the replay lane, `smoke-full`, the -drafts and the 24-row Chromium matrix. The two commits above `91c5878bd` are `js/watch` only, they -touch no Rust, no wire and nothing `moq play` reaches, and both carry their own cases in the `bun -test` rows above. The replay lane and the Chromium matrix do cover the player and they are the gap, -as they were at the previous tip; what stands in their place is those two `bun test` runs and the -multi-watcher bench rows, which are instrument readings on one machine rather than a graded run. - -One caveat on the CPU bench in `stretch.bench.test.ts`: its ratio assertion -(`max(stretch) / max(normal) < 30`) fails about one run in fifteen on this machine and passes the -rest, with the absolute budget (0.67 ms) never close. It is a wall-clock ratio on a shared machine -rather than a regression, and it is the one flake left in the suite. - -**The gates at this tip are not in this table.** The five code commits above `17d32eae4` were gated -together, in the same order and with the same commands, and the run is recorded in -`bench/gates-2026-09-18/` rather than here: every gate at this tip exited 0, which is `just fix` with -no tracked change, `just check` including the flake step, `just test default` at 4685 Rust tests -passed and 9 skipped, 2088 Bun and 64 Python, `cargo nextest` over the seven touched crates at 1779 -passed and 4 skipped, `just test audio-quality --runtime replay --enforce` at 172 enforced checks -across 14 rows with the two new rows at their measured ceilings, `just drafts check`, -`just test smoke-full` at 32 of 32, the privacy and em-dash greps at 0 hits, and both pages rebuilt -at 06:48. Beside them stand the per-package -`bun test` counts quoted in findings 36, 39 and 44, the Rust runs quoted in finding 36, and the bench -rows in "The first ten seconds after an event", which are instrument readings on one machine rather -than a graded run. A second chain over the same commands is running on a temporary Debian 13 x64 -host, which is finding 43, and it is confirmation rather than the gate this document reports. - -### The budgets, and what the harness will and will not enforce - -`test/audio-quality` re-records its budgets from two 60 second runs of the whole matrix: the worst of -the two times 1.5, a hard 0 on the counters the engine has to keep at zero wherever both runs -measured zero, a share capped at 1, and a convergence time capped at the row's own run. A row is -enforced only when its two runs agreed within that same 1.5 on every graded metric, and a metric one -run measured as zero and the other did not counts as a disagreement, because the ratio is unbounded. -An enforced row is therefore one that played the same way twice. **Three of twenty-four are enforced -today**, all three of them `fixed-250`, the profile that adds a constant delay and nothing else. Ten -cleared it on the previous recording, and the re-record on a tree whose targets settle within a -second rather than walking a declaration down for 40 s moved most of the rows out of agreement with -themselves; that is the honest answer rather than a better one. The replay lane is enforced in full, -148 checks across twelve rows, because it is deterministic. Every row, every breach against its -ceiling, and every residual is in "The enforced budgets" above. - -### What was not run, or not verified - -- **No iOS device.** Desktop Safari over safaridriver is the closest proxy this branch has. iOS is - not measured, and nothing here should be read as a claim about it. -- **The nightly job has never run on the nightly runner.** `78a4010c9` adds the `audio-quality` job - to `.github/workflows/nightly.yml` and it runs `--enforce`, but every number in `budgets.json` was - measured on one desktop. The first nightly run is what says whether these ceilings hold on the - runner's hardware, and the rows that are still `recorded` are the ones waiting for it. -- **Local measurements were taken on a loaded machine.** Load average reached 13 to 23 during several - passes, and everything driven by the local relay shares a CPU with the relay and two ffmpeg - publishers. One control re-run of the same bytes on the same build gave 1360 ms where an earlier - pass gave 372 ms. Treat every local number as an upper bound. The public-relay rows are remote, - single-tile, and held steady across every pass they appear in. -- **Safari budgets were removed** after one cell exceeded its own ceilings on the next run. They were - recorded at the worst observed times 1.5 and were not stable enough to keep. The Safari lane runs - and prints; it does not grade. -- **`cargo semver-checks` was not run.** -- **The `4k-webm` fixture has real holes**, so it keeps 1 to 2 underruns, now concealed. That is the - recording, not the player. -- **`just test smoke-full` is the cross-language gate** for the `moq play` change and should be run - before any of the native commits are adopted. It was run on the rebased tree: 32/32. -- **The `mild`, `bursty` and `step` Chromium rows do not reproduce themselves.** Two 60 s runs of the - same tree disagree by 2.2 against 36 skip-aheads a minute and by 460 against 1700 ms of target p95, - which is why twenty-one of twenty-four rows keep the `recorded` marker. Part of that was the cold - start: an `auto` row spent 20 to 40 s of its window walking the declaration down, and a target that - is still moving is one the ring skips against. That is fixed above, in the estimator and in - `doc/concept/playout.md` rather than in the budgets: the first measurement replaces the seed - outright, so an `auto` row is on its measured target within a second. `budgets.json` was - re-recorded against that in `063cd3031`, and the rows still disagree with themselves: three clear - the test now where ten did before, which is the honest answer rather than a better one. -- **The listening round happened one tip below the rebase, and the rebased tip has not been heard.** - On 2026-09-17 the user listened and watched on the bench built from the code tip that `e6be6a775` - records, in their own browser with a headset, a real microphone and a real camera, on both served - pages: the plain one on 4400 and the copied site on 4402. Everything was reported fine: no stutter, - no spinner on an unmute, video and audio together, and hide and show and self-publish working. They - then reported two transients that round had not covered, the first seconds of their own - self-publish and the stutter after an unmute, and both are fixed and measured as findings 32 and - 33; two more came out of the multi-watcher round, findings 34 and 35. **Nobody has listened to - `cd43ff0a8`.** It is gated, and the bench rows on the rebased build and the multi-watcher rows are - what stands in for an ear. It is one listener, one machine and one session in any case, and nothing - was recorded from the round, so it confirms the measured rows rather than replacing them; every - number in this document is still an instrument reading. -- **The underruns in the multi-watcher rows under load are not explained.** Seven of thirty hide - windows after the fix still take one, and one run has a synchronised burst across all five watchers - at 60 s. The rows carry three decoded broadcasts per watcher because the page has no prefix, so the - load is the first thing to rule out and the harness cannot rule it out as it stands. Finding 35 has - both, and they are in "Open measurements". -- **The cross-browser resilience matrix has run**, and what it did not cover is named rather than - implied: real Safari as a *publisher* (its camera prompt needs a hand on the mouse), Playwright - WebKit as a publisher (it refuses the `getUserMedia` permission outright), and the self-publish, - hide-and-show, toggle and 30 minute families in the re-run, none of which was a watcher-side - failure the fixes touch. -- **The `mild` shaper profile was not a usable before-and-after, and the quiet run settles it.** Its - third run carried a third more - traffic and double the queue of the second, on a machine the foreign stream never left alone: - 47103 downstream datagrams and a queue peaking at 41 in the old run, 62744 and 90 in the re-run. - Convictions did fall from 14 to 2, which is the thing the fix claims, and the stall and target - columns said nothing either way until it was re-run quiet. It has been: zero stalls against five, - zero rebuilds against two, and the skew lag back from -4092.2 ms to -35.1 ms. `bursty` is not a - jitter row at all any - more: the profile paces about 0.42 Mbps against a 1.9 Mbps publisher, so it is a capacity cap four - times below the stream, and the row measures what the player does when the path cannot carry the - picture. Both are recorded in the matrix section as bench properties rather than as player - findings. -- **The two cdn rows from the demo pages could not be run as written.** `https://cdn.moq.pro` with no - path is what `demo/web` points at, and the public relay closes a session opened at its root, so - both rows failed to start twice and are recorded that way. `cdn.moq.pro/demo` stands in on both - pages, and the site page reaches the same relay on its own path. Whether the relay should refuse a - root session, or the demo page should stop opening one, is not this branch's to settle. -- **`test/audio-quality/clients/js` has no unit test target.** The grader, the beacon, the probe and - the Safari lane are exercised only by running the harness end to end, which is why the six - fail-quietly defects in `1b90331be` were found by a reviewer rather than by a test. Giving that - package a `bun test` target is a follow-up this branch did not take. -- **`aac-high-rtt-isolated` converging in 42.7 s against an 11.6 s ceiling is unexplained.** It is - the one breach in the matrix run that is outside ordinary run-to-run spread, and it is open. -- **The intermittent `DataError: Failed to execute 'decode' on 'AudioDecoder'` is diagnosed and - fixed.** See finding 11. It did not predate the hole fix: the earlier note that it appears in the - stage 8b logs is wrong, and a sweep of every recorded run directory finds it only in the three runs - taken after `fdc9a2ae1`. It was reproduced deterministically at 240 s, where the publisher's catalog - update near 183 s replaces the audio subscription, and six 240 s runs after the fix, each reaching - that handover, record zero. - -## How to run - -```bash -# the whole branch, scoped the way CI scopes it -nix develop --command just fix -nix develop --command just check -nix develop --command just test - -# the algorithm and its corpus, both languages -nix develop --command just rs test -p moq-audio -bun test js/hang/src/container/jitter.test.ts js/hang/src/container/jitter.vectors.test.ts - -# the browser engine and the replays -bun test js/watch/src/audio - -# regenerate the corpus; without --write it checks freshness instead -bun js/hang/src/container/jitter.vectors.ts --write - -# the harness: the full matrix, about half an hour -nix develop --command just test audio-quality - -# list the rows, run one, or enforce the budgets -nix develop --command just test audio-quality --list -nix develop --command just test audio-quality --codecs opus --profiles bursty --rings plain -nix develop --command just test audio-quality --enforce - -# the recorded traces through the same player, no relay, shaper, or browser: a second -nix develop --command just test audio-quality --runtime replay --enforce - -# real Safari, local only, serialised, never in CI -nix develop --command just test audio-quality --runtime safari - -# compare two run directories -bun test/audio-quality/clients/js/compare.ts --before --after - -# cross-language interop, needed for the moq play change -nix develop --command just test smoke-full -``` - -A failing run keeps its directory, with each process's log, its generated relay config, and its -endpoints, and prints where it is. - -### The listening bench - -The measurements above that are not the harness were taken on a small bench built out of the -repository's own recipes. It is written down here because the tables are worth nothing if they cannot -be reproduced. Every URL handed to a browser is literal `127.0.0.1`: Chromium resolves `localhost` to -`::1` first, which is finding 9. - -**The relay.** From `demo/relay`, `cargo run --bin moq-relay -- localhost.toml`, which is that -directory's default recipe. QUIC and HTTP on 4443, and the config already grants anonymous access -(`public = "**"`, which is what finding 17 is about). - -**The publisher.** From `demo/pub`, the smooth source is the `ts` recipe: - -```bash -just ffmpeg-ts media/bbb.mp4 | moq --connect http://localhost:4443 --broadcast bbb.hang import ts -``` - -`ffmpeg-ts` already passes `-pes_payload_size 0`, and that flag is the whole difference between the -two sources the tables name: with it, one AAC frame per PES and a 140 ms flush span, which is the -smooth row; without it, ffmpeg's default packing of roughly seven frames per PES and the 302 to -372 ms the catalog then declares, which is the bursty row and the arrival shape the public relay has. -The copied site's local rows need the broadcast published under a `demo/` prefix, because that page -connects to `/` rather than to the relay root. - -**The pages.** `bun install`, then from `demo/web`: - -```bash -VITE_RELAY_URL=http://localhost:4443 bun --bun vite build -``` - -which writes `demo/web/src/dist`. Serve that build twice from any static server: once plain, which is -the production postMessage ring, and once with - -``` -Cross-Origin-Opener-Policy: same-origin -Cross-Origin-Embedder-Policy: require-corp -``` - -which is the cross-origin isolated page and the shared-memory ring. `vite dev` and `vite preview` -cannot serve the plain page: `demo/web/vite.config.ts` applies `crossOriginIsolation()` from -`js/common/vite-plugin-isolate.ts` to both, so the vite server is always the shared ring. Only the -isolated page should report `crossOriginIsolated`, and checking that per page is what keeps a row -from grading the ring it did not run. - -**The copied moq.dev site**, for the third page, is built with its package aliases pointed at this -checkout's `js/` rather than at a published release, and its watch page takes its source and relay -from the query string: `/watch/?project=&name=&relay=`. The -`?project=` is not optional, and it is also why the site row never hit the root-session failure that -the two cdn rows from `demo/web` did. - -**The drivers.** Four engines and one publisher, all driven headlessly except where an engine refuses -to be: - -- Playwright Chromium, Firefox and WebKit, from the versions the smoke harness already pins, opening - the served page, clicking the tile once to spend the gesture, and sampling from there. -- Real Safari through `safaridriver`, which is a different network stack, media pipeline and - AudioWorklet scheduler from Playwright's WebKit and is the only way to measure Safari at all. Two - rules it does not forgive: **exactly one Element Click per run**, because a second one hides - precisely the defect finding 15 is about, and **Safari frontmost**, because an unfocused window - answers an Element Click with `{"value":null}` and the context then sits in WebKit's `interrupted` - state over total silence. -- The publisher is a headed Chromium page on `publish.html` with `--use-fake-ui-for-media-stream`, - and never the fake device, whose synthetic signal breaks the encoder. The page has no device - picker, so the device is chosen from inside it: `enumerateDevices` after the permission grant, the - USB capture device picked **by label**, handed to the element's microphone source, and then the - element's own live track read back. The run aborts unless that track is the device asked for and - its `sampleRate` is 48000. Echo cancellation, auto gain, noise suppression and Opus DTX are all - off, confirmed on the element's own track rather than on a second `getUserMedia`, and the publisher - is restarted before every row. -- The mute and preset sequence, which is the toggle table above: ten seconds at auto, mute for three, - unmute, then the `2000ms`, `100ms` and `auto` presets for eight seconds each, five quick - mute/unmute pairs, and ten seconds idle. Skew is measured from the mute onwards, because the - opening ten seconds are the cold start rather than the toggle. - -**The sampler.** Every 250 ms it reads the active tile's `` signals the way -`test/audio-quality/clients/js/src/probe.ts` does (the target `sync.out.delay`, the ring level, -underruns, concealed, skipped, short quanta, the negotiated transport, `crossOriginIsolated` and the -round trip), and additionally the painted video timestamp against the audio playhead, which the -harness does not measure. Counters are deltas from 20 s in. - -That last pair has a resolution, and every skew number above is read against it. The metric has a -floor of one frame plus one display refresh, 58 ms at 24 fps and 50 ms at 30 fps, because the painted -frame is the newest one due and it was painted at the last vsync. It has a ceiling of one playhead -reporting interval, 14.5 ms of worklet state messages at 44.1 kHz, 13.3 ms at 48 kHz and one 50 ms -poll on the shared ring, because the reported playhead is not extrapolated while the one video is -paced against is. The pair to grade is therefore the signed lead (p95) and the signed lag (p05), not -the p95 of the absolute value, which over that distribution is 41 to 49 ms whatever the player does. - -**The driver scripts themselves are not on the branch.** They were written as bench scaffolding -outside the repository and they are not in a state worth committing as they stand. The judgement -worth recording rather than acting on tonight is that they have a natural home: -`test/audio-quality/clients/js` already carries `driver.ts`, `safari.ts`, `webdriver.ts`, `sink.ts` -and `replay.ts`, which do most of this against the harness page, so folding the headed publisher, the -per-engine matrix and the toggle sequence in beside them is the right shape. That is a follow-up, -listed below, and it is the same follow-up as giving that package a `bun test` target. - -**The rules the event rows are graded under.** The tables in "The first ten seconds after an event" -grade a window rather than a row, and ten rules decide what each window is answerable for. They are -finding 42, they are reproduced at the head of every generated `MUTE-2026-09-18.md`, and they are -here because a verdict is worth nothing without them. - -1. `noTile`, the stretch where the page has no `` to sample, is reported and never - graded. That is the demo waiting for a broadcast to be re-announced, not the player. -2. `player`, the tile being there to the first advancing playhead, is what a tune-in is held to, - with a ceiling of one second. -3. The `player` clock stops through a declared pause. A watcher that reloads into a muted publisher - is waiting for the publisher rather than filling, and wall time says otherwise. -4. Between a `mute` and its `unmute` the playhead-rate and missing-playhead checks do not apply. A - publisher that has said it is stopping sends nothing to advance a playhead with, so the playhead - stopping is the declared behaviour, and the unmute's own window grades everything after it. -5. On a tune-in the missing-playhead check opens at that tune-in's first playhead. The stretch - before it is the tune-in itself, which rule 2 already grades. -6. A counter delta taken where the row had no ring at the mark, and any field a row was measured - before the bench recorded, are NOT MEASURED. Neither is a pass nor a failure, and neither ever - counts against a verdict. -7. A window grades only the region no later event owns. A ten second window can contain whole other - events, and what those cost belongs to their own windows, which hold them to the right budget. - The tables still report the whole ten seconds, because that is what happened; the grade uses the - part before the next marker, and says so when the two differ. -8. On a tune-in, what counts as lost is what a listener could hear missing, which is `skipped` plus - `concealed`. Playout starts on the level the ring holds, so media older than that start point was - never going to be played and dropping it is not a cut, which is finding 33. `trimmed` and - `discarded` are reported beside the grade as loss before the start point, and on a tune-in - `trimmed` is reported rather than graded for the same reason. -9. A resume after a declared pause, an `unmute`, a `watch-unmute` or a `show`, refills from empty - exactly as a tune-in does and gets the tune-in stretch allowance rather than the steady-state one, - which would leave converging no room at all. -10. A check whose window is too short to hold it is NOT MEASURED. The playhead rate needs a whole - second after the fill to average over, and "target back at the reference" asks where the target - ended up, which means the end of a full window. A burst of mute pairs half a second apart cuts - every one of its windows to a second, and asking either question there measures the burst's - spacing rather than the player. - -The stretch budget is the steady-state rate over ten seconds, except on a tune-in, where it is the -larger of that and what this row's own opening tune-in cost, because a converged reference stretches -nothing and would leave converging no allowance at all. - -## Open items and follow-ups - -### The state of the branch - -`debug-findings-solution` on the fork `fperex/moq`, base `dev` at `712ffd810`, opened as draft pull -request #3 there and nowhere else. The last change to the player is `1c2605450`; the commit above the -five code commits of findings 36 to 44 is the docs commit `f4b1e0081`, and this document's own commit -sits above that. The measurement behind findings 36 to 44 is posted on that pull request as a -comment, , whose source is -`bench/pr-comment-findings-2026-09-18.md` in the bench archive. The branch was rebased from -`61da0d247` on 2026-09-18 and from `877a561d8` on 2026-09-17, so every hash on it has been rewritten -twice and the pull request needs a force push each time; rows 117 to 125 of the commit table above -carry the 2026-09-18 hashes and the rows below them still carry the 2026-09-17 ones. The pre-rebase -tips are kept locally on `debug-findings-solution-pre-rebase-20260918` and -`debug-findings-solution-pre-rebase-20260917`, and an earlier state of the work is archived on -`debug-findings-solution-wip-20260912`. The fork's own `dev` was reset to upstream's `712ffd810` in -the same round, which is in the 2026-09-18 rebase section. - -Two things that produced numbers above are deliberately not on the branch: the full run matrix the -quiet re-measure wrote, and the bench driver scripts. Every number from the matrix that matters is -folded into this document, which is the thing to read; the drivers are described in full under "The -listening bench" and have a home proposed below. - -### Quests worth opening - -- **The publisher demand gate.** `Rendition.track` documented a gate, "producers should encode only - while this is set", that no longer exists: a browser publisher encodes with zero viewers. The - comment is corrected in `c55af8566` rather than the code. Two of the three defects that surfaced - when the gate was tried are fixed here, in `b366db8bb`: the audio encoder now declares a break - after a gated interval, and `Container.Legacy.Producer` no longer flushes into a group its closed - track has torn down. What is left is the gate itself, and it is still larger than it looks, - because a rendition that stops encoding must stop at the group and not at the track, which is - finding 19. Findings 13, 18 and 19 have the detail. -- **The camera's own pipeline latency.** Finding 14 fixed the publisher's two epochs, and what is - left sits inside the video arrival anchor and cannot be measured from inside the page. An external - reference is what would resolve it: a clap, or a flashing screen, recorded off both ends at once. -- **The multi-tile audio context decision is answered, not open.** Finding 15 left three shapes and a - recommendation. `16e6242aa` takes the second one: a tile builds its context inside the gesture, or - when the app turns audio on, so a muted tile holds none at all. Real Safari is 8 of 8 on one click - and the warnings at load fall from 2 to 1 in Chromium and 4 to 2 in Firefox. Finding 23 has the - measurement and the two costs that remain. -- **Which layer should stop a publisher finishing a track it means to resume.** Finding 19 fixes it - in `js/publish` by cutting the group rather than closing the track. The relay is right to latch a - finished track, and the failure it produces is silent on both sides, so the question is whether - `js/net` should make the distinction harder to get wrong or whether "stop the group, never the - track" is simply a rule a publisher has to know. -- **The unmute conviction, in `moq-net`.** One group from before a mute is delivered alongside the - live edge and convicted by the 40 ms budget, although `Subscription::max_age` should have skipped - it. Finding 21 names the two candidate explanations and the one line of instrumentation that picks - between them. It costs one warn line per unmute today and no underruns. -- **A public rebuild counter on `video.out`.** The decoder now rebuilds a track that stopped - producing, and the only trace of it outside the module is the warning it prints, which is what the - bench drivers count. `Video.Decoder`'s generation and the `DecoderTrack.failed` signal behind it - are both private; a counter beside `video.out.skipped` would let a harness grade recovery rather - than parse a log line. -- **The publisher preview on the `MediaStreamTrackProcessor` polyfill path.** Findings 25 and 29: in - Playwright Firefox the frames arrive, the canvas never paints, and the encoder sends about 67 - bytes per frame, so the picture is blank on the wire as well as on the preview. Pre-existing, and - the reason no row on this branch says a Firefox publisher's video looks right to anybody. -- **A `bun test` target for `test/audio-quality/clients/js`**, and the bench drivers folded in beside - `driver.ts`, `safari.ts` and `webdriver.ts` while it happens. The grader, the beacon, the probe and - the Safari lane are exercised only by running the harness end to end, which is why the six - fail-quietly defects in `1b90331be` were found by a reviewer rather than by a test. -- **The replay harness's `floorMs`.** `js/watch/src/audio/replay.ts` holds its measured target above - a floor the player itself does not have: `Sync` held the advertised jitter as a floor before the - cold-start fix and does not now, and the harness kept the term, so the replay budgets are recorded - against a rule the player no longer follows. Either drop it and re-record those budgets, or keep - it and say plainly that it is the harness's own term, which is what its comment now does. -- **Whether one track's spread should size another track's buffer.** `Sync` in auto takes the `max()` - across tracks, which #3517 had and this branch kept. The shaper defect (finding 8) is what made it - visible: the reordering hurt the video stream, the video spread ran at 1.6 to 2.0 s while audio - stayed at 100 to 180 ms, and the audio buffer was dragged to 1.5 to 3 s by a number that never came - from an audio arrival. The shaper is fixed; the question is not. -- **Which layer owns the publish-side dedup.** Finding 13 is fixed in `js/publish` by releasing the - serving scope when the track loses its last subscriber. The other shape is a weak cache in - `js/net`, the way `rs/moq-net/src/model/broadcast.rs` holds its tracks, and it was not taken - because the subscribing side leans on the strong entry to resume a tile unmuted a moment later. -- **A diagnosability gap, from the retraction.** A subscribe to an announced broadcast whose - publisher is gone is acknowledged and then silent for the whole QUIC idle-timeout window, which a - viewer cannot tell apart from a publisher that is merely slow to produce its first group. Nothing - is broken; there is simply no way to tell. -- **An option on `Connection` to refuse the WebSocket race**, which is finding 4 and, with finding 9, - is two ways the same silent fallback hides an impaired or misaddressed UDP path. The harness works - around it by deleting `WebSocket` and `WebSocketStream` on the page, which is a test hack. -- **`rs/moq-relay/tests/drills.rs`**, which compiles zero tests today (finding 5). The shaper's - impaired second lane is written as a patch and was deliberately not landed on top of a test file - that does not build. -- **Per-row envelopes in the resilience grader.** It applies the LAN targets, `target 20-100ms` and - `held <150ms`, to every row, including shaped paths where a wider target is the estimator being - right, and it counts any interruption as a video stall, including the hide, the show, and a - latency preset change that re-buffers on purpose. Four of the ten quiet rows fail on that alone - and on nothing else. What each row is owed is an envelope of its own: a shaped row graded against - its profile, and an interruption row graded on recovery rather than on stall count. Bench tooling, - not on this branch. -- **The paused-clock rule in `rs/CLAUDE.md`.** "Time-dependent async tests call `tokio::time::pause()` - first" holds while the timed window has no real I/O in it. With a real socket inside it, a paused - clock jumps straight to the next deadline whenever the runtime waits on that socket, and the test - measures the jumps instead, which is what `b80a2084a` fixes in finding 31's reconnect case. The - pattern that works is pause for the wait and resume for the work, and it is worth a clause in the - house rule. Not edited there, because that file is a prompt and takes its own pass. - -The questions the afternoon's fixes leave for the maintainer, each with what it costs to answer the -other way: - -- **How a tile should report that it is not downloading.** Finding 27. Off screen and playing look - the same from outside: `video.out.stalled` keeps its last value, there is no error, and the demo's - badge guesses. `renderer.out.visible` already carries the fact, so the page can join them. - Recommended: leave it to the page. The alternative is an explicit "not downloading, and why" - output on the decoder, which is a new public signal on a wide surface. -- **The 60 s reconnect give-up default.** Finding 31. It moved from 10 s because the shipped client - could not survive the shipped relay's own graceful restart, and the relay drains for 10 s before - its process exits. It could instead be derived from the peer's drain deadline, or left at 10 s - with the three defects under it fixed and the restart simply lost. The number is the maintainer's; - the defects are fixed either way. -- **The FFI default following the native one.** Finding 31, `eec9016d9`. A binding caller who never - set `timeout_us` now gets 60 s. The alternative is to leave the bindings at 10 s and let their doc - stop claiming to mirror `moq_tokio::Backoff`, which is worse in a different way. -- **The announce latch.** Finding 30. `announce="source"` now latches after the first live track and - holds until the selected source changes. That is a behaviour change to a published attribute: a - page that relied on an un-announce when every track stopped no longer gets one. The alternative - shapes are a debounce, which is a timeout standing in for a fact, and a separate - `announce="latched"` value, which is a second way to spell the sensible default. -- **The cap on the cross-track hold, now 100 ms, and the 45 ms it subtracts.** Findings 28 and 32. - Both are judgements about what lip sync is worth, taken from ITU-R BT.1359: 45 ms of sound-ahead a - viewer cannot notice, and a cap past which a call would rather have the latency than the sync. The - cap started at 200 ms and came down when the tune-in showed it standing at its ceiling on the - camera's warm-up frames. A deployment with a consistently later video path would want more and a - conferencing one might want less. Both are constants in `js/watch/src/sync.ts` and both are - documented at the site. -- **Whether a video rendition's declaration should reach the estimator as itself.** Finding 35. A - browser publisher declares one frame duration, 34 ms at 30 fps, and the cold start is - `max(80, declared)`, so every video estimator starts at NetEq's 80 ms guess whatever the publisher - said. Either the floor should not apply below itself, which changes the rule - `doc/concept/playout.md` pins and both languages replay, or the video estimator should seed from - what the shared delay already holds. The cost of leaving it is the one finding 35 measured: a - rendition rebuilt for any reason raises the shared delay for every track until it measures again. -- **Whether a track carrying only a prior may raise the shared delay at all.** Finding 35, and the - same question as the `max()` follow-up below from the other side. `Sync` holds every track to the - widest reading, and a track that has not observed anything has only its declaration to offer. -- **A public rebuild counter on `video.out`**, which the bench drivers still get by counting a warn - line. Unchanged from below, and finding 27's question is the same surface. -- **`#tryDurationSkip`'s warning on the happy path.** `js/hang/src/container/consumer.ts` prints - `skipping covered group` at `console.warn` every time a group is dropped because its successor has - already started, which after finding 26 is an ordinary event on any jittery path rather than a - fault. It belongs at debug, the way the relay's `Cancel` was moved in `b366db8bb`, and it was left - alone here only to keep that commit about the budget. -- **The `Stall` monitor is a module singleton, and `stall.test.ts` knows it.** - `js/hang/src/container/stall.ts` keeps one `current` for the whole document, which is right for a - page and awkward for a suite: the acquire and release cases share one instance and depend on the - order they run in and on nothing else in the file holding a handle. A reset hook for tests, or an - injectable registry, would make them independent. -- **The `bursty` shaper profile cannot test bunching at a realistic bitrate.** It paces about - 0.42 Mbps, and a browser publisher on this bench sends about 1.9 Mbps, so the row is a capacity - cap rather than a jitter shape. Either the profile needs a wider window or the row needs a lower - publisher bitrate; as it stands it measures a different thing from the one it is named for. -- **The multi-watcher page needs a prefix, and the grader needs per-row envelopes.** Finding 34. The - plain page on 4400 subscribes to the announce prefix `""`, so every watcher in a five-watcher row - decodes the two file broadcasts beside the probe as well: five engines cost 15 underruns and five - Chromium watchers cost 1, on the same publisher over the same relay, which is a load reading rather - than an engine one. Either the page takes a broadcast prefix for the row or the file publishers - come down for it. And a five-watcher row is not a single-watcher row with more browsers in it: it - wants its own envelope, the way a shaped row and an interruption row already do. Both are bench - tooling and neither is on this branch. -- **What `skipped` means at the `instant` preset.** Finding 39. It used to include a finished group - the reader had already played and popped from, which is a count of nothing, and on legacy audio - the conviction that produced it also fabricated a hole that reset the ring. It now counts only - media actually lost, so the number falls on any path that was carrying spent heads. That is a - behaviour change on a counter other people grade against, and it is stated here rather than buried - in a commit: if a harness, a budget or a dashboard treats the old value as its ceiling, the - ceiling was measuring the wrong thing and wants re-recording. Recommended: keep the new meaning, - because a counter that says media was lost when none was makes a real loss during a mute - invisible. -- **The JS and Rust container consumers agreed on the budget and now agree on the marker; one gap is - left.** Finding 36 and finding 39 closed two divergences: a marker group now raises the playhead - when the group closes in both languages, as Rust always did - (`rs/moq-mux/src/container/consumer.rs:415-418`), and neither language convicts a finished head - the cursor has reached, which Rust got for free by popping the spent group first - (`rs/moq-mux/src/container/consumer.rs:327-348`). What is still open is on the native side: the - declared endpoint never reaches `rs/moq-audio`'s decode consumer at all, so the engine conceals - through the pause and counts every concealed block as an underrun, 49 of them in 500 ms at 10 ms - blocks. The resumed audio is correct; the counter is not. - `a_declared_endpoint_then_resumed_media_decodes` in `rs/moq-audio/src/decode/consumer.rs` is - `#[ignore]` with that number in its reason, so it fails loudly the day the marker arrives. Two - pieces of work: surface the marker to the decode consumer, and give the engine an `Engine::end()` - so a caller can declare the same pause the browser can. Recommended as one quest, because the - second is what makes the first testable. -- **Whether a target rise should park the ring at all, when the native engine does not.** Finding 45. - `js/watch/src/audio/decoder.ts:519-534` stalls the ring whenever the target rises by more than two - buckets, and `rs/moq-audio/src/playout/engine.rs:348-351` just moves the target and lets `Expand` - walk into the deeper hold. One of the two is wrong, and the browser one is the addition. - Recommended: park only past `STRETCH_BOUND`, which is the bound the rest of the walk already - respects, and take the `POSTPONE` gate out with it in both languages, so that a rise too small to - park is not also too small to stretch through. The cost of leaving it is a cut with `underruns` - flat, which no budget in this document catches. -- **Whether a track that has measured nothing may raise the shared delay.** Finding 45, and the same - question finding 35 asks from the other side. On a fresh publish the widest reading in `Sync`'s - `max()` is the video estimator's, seeded at 80 ms and fed 2 s keyframes on a cold congestion - window, and it sets the audio ring's depth before audio has observed anything of its own. -- **Whether the cold-start seed should be a floor rather than a starting point.** Finding 45. The - first measurement replaces the 80 ms seed whole, typically with the 20 ms a quiet tune-in interval - reads, and every later rise is then measured against 20. Holding the seed as a floor while the - newest observation still outweighs the quantile tail, which is the first 39 adds, contradicts the - rule `doc/concept/playout.md` pins and both languages replay, so it is the maintainer's. -- **Whether the cross-track offset should move before each track has a full window.** Finding 45. - `js/watch/src/sync.ts` emits an arrival floor from the very first frame and walks the offset one - 20 ms bucket per second to a 100 ms cap, so a camera's warm-up frames can add 100 ms to the audio - ring's depth over the first five seconds, on top of whatever the estimator is doing. -- **What a publisher should say about a track it has not registered yet.** Finding 46. The demo page - advertises the broadcast and then registers `meta.json` on it, and a request that lands in between - is refused with a bare `Error`, which reaches the watcher as Internal `0x0`. Either `js/publish` - should refuse an unknown track with `NotFound`, so a watcher can tell "not yet" from "the publisher - broke", or a broadcast should carry its tracks before it is advertised. Recommended: both, the - error type because it is the one a watcher can act on, and the ordering because it removes the - window. -- **Every tune-in figure in this document is one of two numbers.** Finding 47. It is 0.33 to 0.55 s - or about 0.9 to 1.1 s depending on which side of one hour on one machine it was taken, the delay - sits before the audio context is created, and nothing on the bench explains the boundary. Nothing - here is the maintainer's to decide; it is stated so that no tune-in figure in this document is read - as a property of the player. -- **Whether the audio clock should be nominated while a ring is parked for a fresh fill.** Finding - 44's residue. The watchdog no longer calls a hidden camera a stall, but at a deep preset the - parked fresh fill still holds video for the length of the fill: 1420 ms of - `Buffering (audio clock)` through a three second mute at the `2000ms` preset, where `auto` and - `100ms` are now zero. The overlay is telling the truth about the clock and the wrong thing to the - viewer. The option is to stop nominating the audio clock while the ring is parked for a first - fill, which is a rule about what the A/V clock means rather than a predicate in a watchdog, and it - is the maintainer's to settle. -- **`moq-relay` takes its config positionally and `moq-bench` takes it as `--file`.** Finding 41. - The packaged unit said `--file` and could never have started the relay; that is fixed to match the - binary. Which spelling should win across the workspace is the maintainer's, and it is a small - breaking change either way: teaching `moq-relay` to accept `--file` as well is additive and leaves - two spellings, and moving `moq-bench` to positional is a break on a test tool. Recommended: - positional everywhere, since that is what the relay's documentation already says. -- **`skipping covered group` is still a warning on the happy path.** Carried from the previous pass - and now measured rather than argued: `#tryDurationSkip` in `js/hang/src/container/consumer.ts` - prints it at `console.warn` every time a group is dropped because its successor has already - started, and the `watch-reload-auto` row alone prints it twice in forty seconds on an unimpaired - LAN. After finding 26 that is an ordinary event on any jittery path. It belongs at debug, the way - the relay's `Cancel` was moved in `b366db8bb`. It was left alone again here to keep the budget - commits about the budget. Fixed in `b92e6e1ab`: it prints at `console.debug` now. -- **The demo page should keep its element across an unannounce.** Finding 38. The page now carries - its `delay` onto a rebuilt tile and remembers which broadcast the viewer was watching, which fixes - the two things a reload silently changed. What it does not fix is the rebuild itself: 1902 ms of - the 2474 ms a listener hears as a cut is the page having no `` at all between the - unannounce and the re-announce, against 572 ms for the tune-in that follows. Keeping the element - and letting it wait for the broadcast to come back is what would take that two seconds out. It is - a `demo/web` change with a real question underneath it, which is how long a tile should hold a - broadcast that has gone away before it admits it is gone, and it is deliberately not in this pass. -- **A watcher other than Chromium stops painting pictures across an unmute.** Found by the - regression set and not chased: on the brave, firefox and webkit rows `painted` stops advancing for - 580, 584 and 555 ms, starting about 220 ms after the unmute, where the chromium row's longest flat - run over the same stretch is 23 ms. The publisher was chromium on every one of those rows, so on - this evidence it is watcher-side, not publisher-side as an earlier revision of this document said. - It is not the watchdog of finding 44, and it wants its own row before anything is claimed about - it. -- **The congestion-control campaign's follow-ups.** The campaign is planned test work and is - described in the bench archive; five things it wants are not measurable with what the tree exposes - today, and each is a small, separable addition: - - **Which track the relay shed.** `moq_relay_stale_{groups,frames,bytes}_total` is per tier, so a - row can prove something was shed and not that audio was spared. Proposal: a `track` label behind - a flag, or one debug line per expiry carrying track, sequence and code. - - **Retransmitted video against fresh audio.** qlog carries stream ids, not tracks, so the - ordering the design promises can be graded at the transport level and not per track. Proposal: a - trace line when a group's stream opens, naming the track. - - **Downstream loss and congestion window on a grid.** The relay's `/metrics` has no RTT, cwnd or - loss at all, so qlog is the only packet-level source and it is per connection and offline. - Proposal: `moq_relay_quic_*` gauges under `[internal]`. - - **Browser transport RTT.** Absent everywhere; the PROBE stream's own `rtt` and Chromium's - `getStats()` are the only readings, and a WebSocket row has neither. - - **A link with tail drop.** `rs/moq-shaper`'s rate bucket delays an over-budget datagram and - never drops it, so every cap profile is a bufferbloat link and CUBIC never sees loss on one. The - shaper's queue with a `dropped` count, added as test tooling in this pass, is what closes it; - the profiles that use it are not written yet. - Also unverified rather than open: whether the browser publisher's priority reaches QUIC - `sendOrder`, and what rate the audio encoder is actually granted. - -### Open measurements - -- **The first seconds of a fresh camera-and-microphone publish, on a real path.** Finding 45, and the - open item this document ends on. The user hears it and the counters do not: every row here is a LAN - row or a file publisher, and the audio-quality matrix excludes the tune-in from its counters, so - nothing measures the case at all. `bench/mutematrix.py --set wan` is staged, five rows including - the user's exact shape, with a publisher-side sampler beside it and a `[playout]` tracing plan - behind that. It is the next session's first job. -- **The demo page's `meta.json` read fails and nothing retries it.** Finding 46. Seen in a console - rather than measured, and no row counts it, because it is not on the audio path. -- **The tune-in on this machine takes about 0.9 s where it took 0.33 s.** Finding 47. Localised to - the stretch between the tile swap and the first ring, bimodal across a boundary at 05:22 on - 2026-09-18, with the bench, the build, the devices and the load all cleared. A reboot and one row - is the next experiment, and instrumenting the connect-to-catalog timing is the one after it. -- **A listening round on the fixed tip, on the LAN bench.** The tip has been heard remotely, in Brave - over a wide-area path, which is where finding 45 comes from. Neither `cd43ff0a8` and the six - commits of findings 32 to 35 under it, nor the code commits of findings 36 to 44 above it, has been - heard on the LAN bench with the headset and the real camera, which is the round every earlier - finding here was confirmed by. -- **The residual underrun after a camera hide, 0.90 to 1.45 s later.** Finding 35. Seven of thirty - hide windows after the fix still take one, on the target the row was already holding rather than on - a step to the 80 ms guess, which is what the two commits removed. Whatever it is, it is not the - estimator being reseeded. It is 0.9 to 1.5 s after the hide, which is about where the re-acquired - camera's first group lands, so the first thing to rule out is the hide itself and the second is the - row's own load. -- **The synchronised burst at 60 s in one five-watcher run.** All five watchers underran between - 60.3 s and 60.8 s, nowhere near a hide or an unmute. Five browsers each decoding three broadcasts - on one machine is the obvious reading and nothing here confirms it; the row needs the prefix above - before it can say anything else. -- **Real Safari and Playwright WebKit as publishers.** Neither was run: Safari's camera prompt needs - a hand on the mouse and WebKit refuses the permission outright. Both ran as watchers and passed. - A real-Safari publisher row needs one click from a person and is worth having. -- **The 30 minute long run on a quiet machine.** The only one measured ran under the foreign stream - and kept 2 underruns in 30 minutes with every other counter flat, 52079 painted of 52080 decoded. - The quiet run had no time for it, and it is the row most worth repeating now that the machine is - free. -- **The twenty-one `recorded` harness rows.** They print a breach and do not fail a run. What would - move them is the nightly runner recording its own budgets on its own hardware; every ceiling in - `budgets.json` today was measured on one desktop. -- **`aac-high-rtt-isolated` converging in 42.7 s against an 11.6 s ceiling.** It is the one matrix - breach outside ordinary run-to-run spread, and nothing here explains it. -- **The hard-zero ceilings breached by a single episode.** Ten of the seventeen breaches are a 1.1 a minute - episode and the 272 ms of concealment it cost, against a ceiling of zero that the recording - rule produced because both recorded runs happened to measure zero on a counter that is not reliably - zero. That is a recording artefact rather than a regression, and it is what the `recorded` marker - exists to absorb. -- **The clock-drift void on `opus-near-zero-isolated`**, `AudioContext.currentTime` drifting 4.92 - percent from wall clock over 10 s. The guard did its job; why the context drifted is open. -- **The `bursty` and `step` skip-ahead residuals**, worst at 63.8 a minute on `opus-step-plain`. A - 160 ms flush arriving as one burst still outruns what 15 ms of stretch per 100 ms of output can - absorb, so the band still fires. That is the next piece of work on the engine, and this branch does - not claim it. -- **The sampler's own resolution.** Skew is read on a 250 ms grid, and the signed lead and lag are - what it can resolve; the p95 of the absolute value is 41 to 49 ms by construction and grading - against it grades the sampler. A finer instrument, or a painted-frame callback rather than a poll, - is what would settle the residual tens of milliseconds either way. -- **WebKit's idle-tail underrun did not repeat.** One underrun 8.3 s after the last unmute, in the - idle tail, on the previous toggle pass; none in the re-measure. Recorded as a one-off rather than a - property of the sequence, and worth watching rather than chasing. -- **The two cdn rows from the demo pages cannot be run as written.** `demo/web` points at - `https://cdn.moq.pro` with no path and the public relay closes a session opened at its root, so the - page subscribes to the announce prefix `""`, is dropped, and reconnect-loops without ever showing a - tile. `cdn.moq.pro/demo` stands in on both rows. Whether the relay should refuse a root session, or - the demo page should stop opening one, is not this branch's to settle. -- **The `wasm32` debug artifacts** and **the `demo/web` tile arming residual**, both carried from an - earlier pass, neither reproducing on this tip; the arming residual measured fine. -- **One flake is left in the suite**, the CPU bench's wall-clock ratio assertion in - `stretch.bench.test.ts`, about one run in fifteen, with the absolute budget never close. -- **The unmute's own stretch count, five against a ceiling of three.** Finding 36. On the final tip - `mute-basic-auto`'s unmute window stretches five times where the rule allows three. It is the real - cost of refilling a ring from empty and it became visible only when the ring stopped resetting - across that window, so it is a number that was always being paid and never counted. It is the one - failing check left in the seven-row set. -- **A playhead gap of about 110 ms on every resume.** Finding 44's table and the regression set: - 19128 to 19230 ms on the firefox row, 21161 to 21268 ms and 21194 to 21314 ms on the two - `hide-mute` rows, the same shape in the restart rows. It is the fresh fill landing at a deeper - target, it is inside the acceptance bar, and nothing here explains why it is so consistent. -- **The tune-in figure differs by a factor of two between two runs**, which was read as run ordering - and is now finding 47: the ordering explanation did not survive the idle row. - `playerMs` reads 0.44 to 0.55 s in `out15-after3/` and about 1.1 s in `out15-regression/`, - uniformly across rows. A fresh relay, fresh publishers and a fresh browser reproduce the slower - number (`out15-drift/`: 1142.6 ms on `pub-reload-auto`, 961 ms on the firefox unmute), so it is not - accumulated bench state; what `out15-after3/` had that the others did not is about twenty-five - minutes of an idle machine before it. An idle-then-one-row test is what would settle it. -- **A watcher other than Chromium stops painting pictures for 555 to 584 ms across an unmute.** - Brave, firefox and webkit rows, chromium not; the gap opens about 220 ms after the unmute and - chromium's longest flat run over the same stretch is 23 ms. The publisher was chromium on every - engine row, so it is watcher-side rather than publisher-side. Unmeasured beyond the observation, - and it has no row of its own. -- **The pinned-element broadcast change has no bench row.** Finding 40 is guarded by two tests and - nothing else, because the demo page rebuilds its element and so cannot produce the event, and the - regression set has no pinned-element schedule. A schedule that pins one ``, kills the - publisher and republishes the same broadcast under it is what would measure it. -- **Every row so far is one microphone and one output device.** The USB microphone at 48 kHz, with - the output context also at 48 kHz on every row. The 44.1 kHz output case is untested, and the - 16 kHz Bluetooth headset microphone is untested because the headset was not connected for the - regression run, which is why `device-bt-auto` and `device-bt-100ms` are its two not-graded rows. - The sample-rate rebuild a 16 kHz microphone forces is therefore also unmeasured, and the rule each - fix is written to is stated in buckets, frame durations and the stretch bound rather than in - milliseconds so that it can be. -- **`--same-browser` and `--via button` have never been exercised.** The first is the user's own - single-context shape, the second drives the publisher's mute through the page's own control rather - than the element property. -- **The x64 gate chain was still running when this was written.** `just check`, `just test default`, - nextest over the touched crates, `just js test`, the replay lane, the drafts and `smoke-full`, in - order, on a temporary Debian 13 x64 host, in the repository's own nix shell. The Mac's gates are - the ones this document reports; the x64 pass is confirmation and is not folded in here. -- **The four ranked tune-in sites that never fired.** \[D] a first ring depth racing the worklet - load, \[E] the budget dropping to the measured target while the ring walks down, \[G] and \[H] a - target rise past two buckets parking the ring. None of them appears on any row measured here: the - first depth is the 80 ms seed, no group is convicted inside the first three seconds with the - budget above the target, and the latency re-anchor never parks. They are unmeasured rather than - absent, and the changes proposed for them are deliberately not applied. Finding 45 puts \[G] and - \[H] at the top of that ranking for a fresh publish over a wide-area path, which is the one shape - no row here reaches. - -### The listening state, plainly - -**The user listened on 2026-09-18, remotely, in Brave, over a wide-area path**, on the pages served -from the temporary x64 host of finding 43 rather than on the LAN bench. **The mute and reload family -is fixed to the ear**: the mute that never recovered, the publisher reload and the watcher reload are -gone as symptoms, which is what findings 36 and 38 to 40 predicted. **What is still there is the -first seconds of a fresh camera-and-microphone publish**, cut on the real path and healing after some -seconds of listening. That is finding 45, it is the open item this document ends on, and it is the -next session's first job. - -The rest of this section is the state before that round, kept because it is what each tip below was -measured against. - -**Two tips below this one had been heard, and this one had not.** On 2026-09-17 the user listened -and watched on the bench built from the code tip that `e6be6a775` records, in their own browser with -a headset, a real microphone and a real camera, on the plain page on 4400 and on the copied site page -on 4402, and reported everything fine: no stutter, no spinner on an unmute, video and audio together, -and hide and show and self-publish working. **The same listener then reported four transients that -round had not covered**, and every one of them is what this pass is about: the first seconds of a -fresh publish, a publisher page reload, a watcher page reload, and a publisher-side mute and unmute -that left the tile on Buffering and stayed there. - -Findings 32 and 33 answered the first two of the earlier reports and were never heard either. -Findings 34 and 35 came out of the multi-watcher round the user asked for next. **Nobody has listened -to `1c2605450`** before the remote round above, and nobody listened to the rebase below it. The mute defect, which is the loudest of -the four and the one that never recovered on its own, is fixed and measured here as finding 36, and -the three others are findings 38, 39 and 40. Finding 44, the Buffering overlay a hidden camera left -over the tile, was found by the bench rather than reported and nobody has heard that either. - -What stands in for an ear at this tip is the bench: the before rows on the tip these fixes start -from, the after rows on each fix, the 27-row regression set across four engines, and the graded -tables behind findings 36 to 44, in which every number is read out of a row's JSON rather than off a -printed line. That is a real instrument and it is not a listener: **the sampler cannot hear**. It -counts underruns, convictions, trims, concealment and the playhead, and a defect that is audible -without moving one of those counters would pass every row in this document. - -It is one listener, one machine and one session in any case, and nothing is recorded from a listening -round, so the measured rows are the evidence and the ear is the confirmation. The bench the round -runs on is the relay, the plain page, the copied site page, one smooth `bbb.hang` publisher on the -fixed recipe, and a browser publisher of the USB camera and microphone for the self-publish half. The -same pages are also served from a temporary Debian 13 x64 host over a real certificate, so the round -can be taken over a wide-area path as well as on the LAN. - -## Attribution and licensing - -**WebRTC.** `modules/audio_coding/neteq` in the Chromium tree is the specification for everything in -`js/watch/src/audio/playout/` and `rs/moq-audio/src/playout/`. It was read, cited by file name in each -module header, and reimplemented in `f64`. No source was copied. The constants and the algorithm -shapes are theirs; the fixed point is not carried over, because it exists to target DSPs. - -**videocall-rs.** The `neteq` crate there is MIT OR Apache-2.0 and was read as a possible starting -point for the Rust side. Nothing was adapted from it. Its time stretch removes the longest low-energy -run rather than a pitch period, its expand emits quiet noise rather than concealment, and its buffer -level filter adds a jump detector NetEq does not have, so it is not the port it looks like. The -reasoning is in the module header of `8eb68753c`. - -**PR #3517.** The two cherry-picked commits keep their author. The arrival observation point, the -`spread` plumbing, the ring slack, the re-stall, `stall()`, the underrun counter, the worklet ramp, -and the stats and buffering-indicator wiring are all from there and are kept. - -**The harness.** `sink.ts`, `compare.ts`, `src/probe.ts` and `analyze.ts` are adapted from this fork's -own earlier work on branch `debug/rt-audio`, against 130 raw ndjson traces published as release -`rt-audio-traces-2026-09-06`. Each module says so, and `test/audio-quality/README.md` has the -attribution section. What changed in upstreaming: the CDP driver became Playwright on the smoke -harness, the patched-in probes became public signals, which is what lets the same page measure a -published build, and the arrival impairment moved from a shell script driving the OS to `moq-shaper` -with a seed and its own counters. diff --git a/js/hang/src/container/stall.test.ts b/js/hang/src/container/stall.test.ts index 459044f216..df25267265 100644 --- a/js/hang/src/container/stall.test.ts +++ b/js/hang/src/container/stall.test.ts @@ -316,11 +316,7 @@ describe("the event loop monitor", () => { }); }); -// review consumer-sync-video F12. A hidden tab rations the tick to one a second while an -// audio-only track delivers a PES of several frames every 200ms: the loop is fine, the arrivals -// are simply further apart than the gap threshold, and the tick is no longer there to vouch for -// the loop in between. Flagging each of them drops the estimator's reference on every arrival, -// so its target decays to one bucket; its own idle rule would wait 500ms for that. +// Hidden tabs can throttle the timer below the cadence of healthy audio arrivals. it("a rationed tick does not flag arrivals spaced under the idle threshold", () => { const timer = fake(); const stall = new Stall(timer); diff --git a/js/json/src/window/window.test.ts b/js/json/src/window/window.test.ts index 9e24b40cc4..048ed69b11 100644 --- a/js/json/src/window/window.test.ts +++ b/js/json/src/window/window.test.ts @@ -419,12 +419,7 @@ test("an uncommitted edit leaves the window unchanged", () => { expect(encoder.window).toEqual([2, 3]); }); -// F7 (review 2026-09-27): the same max-age verdict, reset by a JS publisher and by a Rust one, -// as the JS subscriber decodes each RESET_STREAM (js/net fromTransport). The Rust publisher sends -// Old (0x34), which reads back as Stream(Old). The JS publisher sends the code of its `Expired` -// verdict, which reads back as `Expired`. Both are a group the publisher gave up on, so both must -// be a gap the window resyncs from, not a fatal error. -test("F7: an expired group is a gap whichever publisher reset it", async () => { +test("an expired group is a gap whichever publisher reset it", async () => { for (const [publisher, error] of [ ["rust (Error::Old, 0x34)", new NetError.Stream(StreamCode.Old)], ["js (Expired)", new NetError.Expired()], diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index d5b13085f7..a3d1c2c5a0 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -1509,11 +1509,8 @@ async function settleMicrotasks() { for (let i = 0; i < 200; i++) await Promise.resolve(); } -// F6 (review 2026-09-27): every group waiting for its FIN acknowledgement holds one guard, and -// each guard subscribes to the track's expiry signals. In a dev build @moq/signals throws at the -// 100th subscriber of one signal ("may be leaking"). A publisher with 100+ groups in flight -// (congestion, one group per audio frame) must not fail a group, or leave a promise unobserved, -// because of that cap. +// The development subscriber cap is 100. Pending FIN acknowledgements must not +// retain expiry listeners after the group's data has been sent. test("120 groups waiting for their FIN do not trip the dev subscriber cap", async () => { const N = 120; const pair = createMockTransportPair(ALPN_05); diff --git a/justfile b/justfile index dee586e408..b5cd2fd191 100644 --- a/justfile +++ b/justfile @@ -52,22 +52,9 @@ bench-runtime $ROUNDS="3" $WORKERS="": #!/usr/bin/env bash exec just --justfile bench/justfile runtime "$ROUNDS" "$WORKERS" -# A linked worktree's Git metadata does not live under its own root: the -# per-worktree directory is `--git-dir` and everything shared (objects, remote -# refs, the branch namespace) is under `--git-common-dir`, which for an agent -# checkout is inside the main repository. Write access to the source tree -# therefore says nothing about whether this checkout can fetch, branch, or -# rebase; the answer is a property of those two directories, and finding out by -# running `git fetch` and reading the error is the slow way. -# -# `check` reports; `setup` fetches, points the branch at its base, and records -# the SHA it fetched under the per-worktree Git directory, where `check` reads it -# back to say how stale the recorded base has become. Neither ever resets, -# rebases, cleans, or checks anything out: a dirty tree is someone's work in -# progress, and adopting a checkout must not be able to destroy it. -# -# BASE follows the same rule as the rest of the repo (see `_base`): `main` -# unless the branch's upstream says otherwise. +# `check` reports shared and per-worktree Git access. `setup` fetches the base, +# records its commit, and sets the upstream without changing checked-out files. +# BASE uses `_base`, which defaults to main unless the upstream selects another base. # Report a worktree's base, Git metadata access, and state; `setup` also fetches. worktree ACTION="check" $BASE="": @@ -87,9 +74,7 @@ worktree ACTION="check" $BASE="": common_dir=$(cd "$(git rev-parse --git-common-dir)" && pwd) branch=$(git branch --show-current || true) - # A written probe rather than `[[ -w ]]`: the directory can be readable and - # nominally writable while the sandbox, a read-only mount, or an ACL refuses - # the create, and it is the create that fetch and branch creation need. + # A write probe catches sandbox and ACL restrictions that `[[ -w ]]` misses. access() { local dir="$1" probe [[ -d "$dir" ]] || { echo missing; return; } @@ -103,8 +88,7 @@ worktree ACTION="check" $BASE="": fi } - # A directory git will create on demand is only as writable as its nearest - # existing parent, so probe upward instead of reporting it missing. + # Git creates missing directories, so test their nearest existing parent. access_or_parent() { local dir="$1" parent while [[ ! -d "$dir" ]]; do @@ -117,8 +101,7 @@ worktree ACTION="check" $BASE="": base=$(just _base "$BASE") stamp="$git_dir/moq-base" - # Whichever remote provides the base, not always origin. A base whose first - # segment names no remote (a local branch, a tag) leaves origin. + # Local refs have no remote prefix and use origin. remote="${base%%/*}" git remote | grep -qx "$remote" || remote=origin @@ -126,16 +109,9 @@ worktree ACTION="check" $BASE="": heads=$(access "$common_dir/refs/heads") worktree_meta=$(access "$git_dir") worktree=$(access "$root") - # Tracking is `branch..remote`/`.merge` in the repository config, which - # lives in the common directory alongside the `config.lock` the write needs, - # not in the branch's ref. Probing refs/heads for it would refuse on a - # writable config and, worse, proceed on a read-only one. + # Setting the upstream needs config.lock in the shared Git directory. config=$(access "$common_dir") - # A fetch writes three places, not one: objects, the remote-tracking refs it - # updates, and FETCH_HEAD in the per-worktree directory. The refs are the - # narrow one: `refs/remotes/` is where `.lock` is created, so - # probing `refs/remotes` stops a directory short and passes a split that git - # then fails on. + # Probe the selected remote's directory, where fetch creates branch locks. tracking=$(access_or_parent "$common_dir/refs/remotes/$remote") echo "worktree: $root" @@ -164,20 +140,10 @@ worktree ACTION="check" $BASE="": echo " grant write access to the main repository's Git directory, not just this worktree" >&2 exit 1 fi - # The remote resolved above, not always origin: recording a stamp against a - # ref nobody refreshed is worse than recording none, and `just check` would - # scope the branch against a base that has since moved. git fetch --quiet "$remote" - # Repointing an upstream someone chose would silently change what `just - # check` scopes against, so only three cases write it: no upstream, an - # upstream `_base` discards anyway (the branch's own remote copy, which - # `git push -u` leaves behind and which says nothing about what the branch - # merges into), and a base the caller named on the command line. + # Preserve an intentional upstream unless BASE overrides it. Tracking this + # branch's own remote copy does not identify a base for diff-scoped checks. upstream=$(git rev-parse --abbrev-ref '@{upstream}' 2> /dev/null || true) - # Two things stop the write, and they fail identically: a detached HEAD has - # no branch to hang an upstream on, and the config it lands in may be - # read-only. Skipping either silently would report a setup that recorded - # the caller's base while `just check` still scoped against origin/main. blocker="" if [[ -z "$branch" ]]; then blocker="HEAD is detached" @@ -188,22 +154,15 @@ worktree ACTION="check" $BASE="": if [[ -z "$blocker" ]]; then git branch --set-upstream-to "$base" "$branch" elif [[ "$base" == origin/main ]]; then - # Nothing is lost: with no upstream `_base` falls back to - # origin/main, which is what this would have written. + # `_base` already falls back to origin/main without an upstream. echo "warning: cannot record the upstream; $blocker" >&2 else - # The upstream is the only place this choice survives, so a setup - # that could not write it did not do what it was asked. echo "error: cannot set the upstream to $base; $blocker" >&2 echo " the branch would keep scoping against origin/main" >&2 exit 1 fi fi - # Resolved, written, then renamed into place. A redirect straight into the - # stamp truncates it before git runs, so a failure there would leave an - # empty file that reads back as a recorded base that never existed. Each - # setup needs its own temporary file so concurrent runs cannot rename or - # overwrite each other's in-progress stamp. + # A unique temporary file keeps failed or concurrent writes from corrupting the stamp. if [[ "$worktree_meta" == write ]]; then stamp_tmp=$(mktemp "$git_dir/.moq-base.XXXXXXXX") if ! git rev-parse "$base" > "$stamp_tmp"; then @@ -227,9 +186,7 @@ worktree ACTION="check" $BASE="": recorded="" [[ -f "$stamp" ]] && recorded=$(cat "$stamp") - # A stamp that no longer names a commit is worse than none: reporting it would - # abort here on the `rev-parse --short` rather than say what to do about it. - # An interrupted setup, or a base garbage-collected out of the repository. + # A recorded commit may have been garbage-collected. if [[ -z "$recorded" ]]; then echo "recorded: (none; run 'just worktree setup')" elif ! git rev-parse --verify --quiet "$recorded^{commit}" > /dev/null; then From 057e01bbf6461e7708452ddda35b569a557be49b Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:31:01 -0400 Subject: [PATCH 034/127] fix(watch): exclude isolated outages from video cadence Estimate recovery from the second-longest of four recent picture intervals. Preserve recurring sparse gaps without retaining one network outage for the source lifetime. Co-Authored-By: Codex --- js/watch/src/video/decoder.test.ts | 70 ++++++++++++++++++++++++++++++ js/watch/src/video/decoder.ts | 22 +++++----- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 8e260432cb..ba1e9f072a 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -637,3 +637,73 @@ test("a new rendition does not inherit a sparse rendition's recovery window", as fx.close(); } }); + +for (const resumed of [1, 3]) { + test(`a recovered 30 fps source keeps its recovery window after ${resumed} frame(s) resume`, async () => { + const fx = fixture(); + const paint = async (timestamp: number) => { + fx.sync.track("audio").clock.set({ + timestamp: Time.Micro(timestamp), + reference: Time.Milli.now(), + rate: 0, + }); + fx.served[0].encode(payload(16), Time.Micro(timestamp), true); + await settle(); + built.at(-1)?.emit(timestamp); + await settle(); + }; + try { + expect(await fx.subscriptions(1)).toBe(1); + for (let i = 0; i < 3; i++) { + await paint(i * 33_333); + await advance(34); + } + await advance(30_000); + for (let i = 0; i < resumed; i++) { + await paint(30_100_000 + i * 33_333); + await advance(34); + } + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(30_100_000 + (resumed - 1) * 33_333); + const before = built.length; + await advance(6_000); + expect(built).toHaveLength(before + 1); + } finally { + fx.close(); + } + }); +} + +test("alternating sparse intervals stop rebuilding once both gaps repeat", async () => { + const fx = fixture(); + let timestamp = 0; + const paint = async () => { + fx.sync.track("audio").clock.set({ + timestamp: Time.Micro(timestamp), + reference: Time.Milli.now(), + rate: 0, + }); + fx.served[0].encode(payload(16), Time.Micro(timestamp), true); + await settle(); + built.at(-1)?.emit(timestamp); + await settle(); + }; + const cycle = async () => { + for (const interval of [2_000, 6_000]) { + await advance(interval); + timestamp += interval * 1_000; + await paint(); + } + }; + try { + expect(await fx.subscriptions(1)).toBe(1); + await paint(); + await cycle(); + await cycle(); + const before = built.length; + for (let i = 0; i < 4; i++) await cycle(); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(48_000_000); + expect(built).toHaveLength(before); + } finally { + fx.close(); + } +}); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 34f58eea69..4de5f754e8 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -150,11 +150,9 @@ export class Decoder { // set. #recover = RECOVER; - // The longest wall-clock wait seen between two successive new pictures, and the last new one. - // A rendition that only sends a frame when its content changes (a static screen share) is - // healthy through silences longer than RECOVER; once one such gap has been seen, the recovery - // window stretches to cover it, so the watchdog stops rebuilding a subscription that is fine. - #cadence = Time.Milli.zero; + // Four intervals retain both long gaps of an alternating cadence; exclude the single longest + // when estimating recovery so an isolated outage does not teach a slower frame rate. + #intervals: Time.Milli[] = []; #painted?: { at: Time.Milli; timestamp: number }; #cadenceSource?: { broadcast: Moq.Broadcast.Consumer; track: string }; @@ -332,7 +330,7 @@ export class Decoder { effect.cleanup(() => active.close()); if (this.#cadenceSource?.broadcast !== active.broadcast || this.#cadenceSource.track !== active.track) { this.#cadenceSource = { broadcast: active.broadcast, track: active.track }; - this.#cadence = Time.Milli.zero; + this.#intervals.length = 0; this.#painted = undefined; } @@ -399,7 +397,10 @@ export class Decoder { const now = Time.Milli.now(); const painted = this.#painted; if (!painted || frame.timestamp > painted.timestamp) { - if (painted) this.#cadence = Time.Milli.max(this.#cadence, Time.Milli.sub(now, painted.at)); + if (painted) { + this.#intervals.push(Time.Milli.sub(now, painted.at)); + if (this.#intervals.length > 4) this.#intervals.shift(); + } this.#painted = { at: now, timestamp: frame.timestamp }; } @@ -417,9 +418,10 @@ export class Decoder { if (!effect.get(this.#active)) return; if (!effect.get(this.#out.stalled)) return; - // Twice the longest gap a healthy source has already shown, so a sparse rendition is not - // rebuilt at every silence; still bounded by the ceiling for a subscription that really died. - const after = Time.Milli(Math.min(RECOVER_MAX, Math.max(this.#recover, 2 * this.#cadence))); + // With only one interval, allow the first sparse gap before there is a cadence to compare. + const intervals = [...this.#intervals].sort((a, b) => b - a); + const cadence = intervals[1] ?? intervals[0] ?? Time.Milli.zero; + const after = Time.Milli(Math.min(RECOVER_MAX, Math.max(this.#recover, 2 * cadence))); effect.timer(() => { this.#recover = Time.Milli(Math.min(RECOVER_MAX, after * 2)); this.#rebuild(`no frame for ${after}ms`); From d836924c6a0b8e8c3f2a4a9c0013fbd01be2b069 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:29:01 -0400 Subject: [PATCH 035/127] perf(audio): remove allocation and backlog scans from playout Reuse a fixed noise-analysis window, stop the short-run lookup once later audio is found, and compile regression counters only in tests. Keep channel and backlog benchmarks beside the crate, including a zero-allocation check. Co-Authored-By: Codex --- Cargo.lock | 1 + bench/README.md | 9 +++ rs/moq-audio/Cargo.toml | 5 ++ rs/moq-audio/benches/playout.rs | 101 +++++++++++++++++++++++++++ rs/moq-audio/src/decode/consumer.rs | 14 +--- rs/moq-audio/src/playout/buffer.rs | 9 ++- rs/moq-audio/src/playout/engine.rs | 80 ++++++++++++--------- rs/moq-audio/src/playout/noise.rs | 16 ++--- rs/moq-mux/src/container/consumer.rs | 5 -- rs/moq-relay/tests/runtime_uring.rs | 3 - 10 files changed, 180 insertions(+), 63 deletions(-) create mode 100644 rs/moq-audio/benches/playout.rs diff --git a/Cargo.lock b/Cargo.lock index e4cf9dc097..c61ba5550d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4187,6 +4187,7 @@ dependencies = [ "block2 0.6.2", "bytes", "cpal", + "criterion", "dispatch2", "fixed-resample", "hang", diff --git a/bench/README.md b/bench/README.md index 9b3b3968aa..eac20a13a6 100644 --- a/bench/README.md +++ b/bench/README.md @@ -21,6 +21,15 @@ Compare the current tree with another revision: nix develop --command just bench origin/main ``` +Measure native audio noise updates and the check for audio held beyond a hole: + +```bash +nix develop --command cargo bench --locked -p moq-audio --bench playout +``` + +The cases sweep 1, 2, and 6 channels and 1, 10, 100, and 1,000 queued packets. +The benchmark also rejects heap allocation during a warmed noise update. + Compare one multi-threaded Tokio runtime with the same number of independent Tokio/epoll and io\_uring workers: diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index 8806afd1f1..62f93eabdb 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -135,7 +135,12 @@ objc2-core-media = { workspace = true, features = ["CMSampleBuffer", "CMBlockBuf objc2-foundation = { workspace = true, optional = true } objc2-screen-capture-kit = { workspace = true, features = ["SCStream", "SCShareableContent"], optional = true } +[[bench]] +name = "playout" +harness = false + [dev-dependencies] +criterion = { workspace = true } # Reads the playout conformance corpus, which is JSON generated by the browser # estimator. Tests only: nothing in the library parses JSON. serde_json = { workspace = true } diff --git a/rs/moq-audio/benches/playout.rs b/rs/moq-audio/benches/playout.rs new file mode 100644 index 0000000000..158989bf71 --- /dev/null +++ b/rs/moq-audio/benches/playout.rs @@ -0,0 +1,101 @@ +//! Native playout costs across channel counts and queued packets. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::hint::black_box; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; + +// Compile the implementation here to keep the DSP modules private. +#[path = "../src/playout/buffer.rs"] +#[allow(dead_code)] +mod buffer; +#[cfg(test)] +#[path = "../src/playout/fixture.rs"] +#[allow(dead_code)] +mod fixture; +#[path = "../src/playout/noise.rs"] +#[allow(dead_code, unused_imports)] +mod noise; + +struct Counter; + +static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); + +#[global_allocator] +static ALLOCATOR: Counter = Counter; + +unsafe impl GlobalAlloc for Counter { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + unsafe { System.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + unsafe { System.realloc(ptr, layout, new_size) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } + } +} + +fn frames(rate: u32, duration: Duration) -> usize { + (f64::from(rate) * duration.as_secs_f64()).round() as usize +} + +fn noise_update(c: &mut Criterion) { + let mut group = c.benchmark_group("playout_noise_channels"); + for channels in [1, 2, 6] { + let mut seed = 42u32; + let pcm: Vec = (0..960 * channels) + .map(|_| { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + (seed as f32 / u32::MAX as f32 - 0.5) * 0.006 + }) + .collect(); + let mut estimate = noise::Noise::new(channels); + estimate.update(&pcm); + let allocations = ALLOCATIONS.load(Ordering::Relaxed); + estimate.update(black_box(&pcm)); + assert_eq!( + ALLOCATIONS.load(Ordering::Relaxed), + allocations, + "noise update allocated" + ); + group.bench_function(BenchmarkId::from_parameter(channels), |b| { + b.iter(|| { + estimate.update(black_box(&pcm)); + black_box(estimate.energy(0)); + }); + }); + } + group.finish(); +} + +fn buffered_after_hole(c: &mut Criterion) { + let mut group = c.benchmark_group("playout_backlog_packets"); + for packets in [1, 10, 100, 1000] { + let mut buffer = buffer::Buffer::new(48_000, 1); + buffer.insert(Duration::ZERO, &[0.25; 168]); + for i in 1..packets { + buffer.insert(Duration::from_millis(i * 20), &[0.5; 960]); + } + let ready = buffer.ready(); + assert_eq!(buffer.has_after(ready), packets > 1); + group.bench_function(BenchmarkId::from_parameter(packets), |b| { + b.iter(|| black_box(black_box(&buffer).has_after(black_box(ready)))); + }); + } + group.finish(); +} + +criterion_group!(benches, noise_update, buffered_after_hole); +criterion_main!(benches); diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 4fdea2e022..c1f267a88f 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1668,7 +1668,7 @@ mod tests { producer.cut(None).unwrap(); } - /// N1: a finite Opus track has to end. The pre-skip leaves every packet boundary 168 + /// A finite Opus track has to end. The pre-skip leaves every packet boundary 168 /// frames off the 10 ms block, and live pacing (one packet per two pulls) keeps the /// first fill under the trim that would realign it, so the last 168 frames are less /// than a block. `read` returns `None` only once playout has drained. @@ -1728,7 +1728,7 @@ mod tests { panic!("a finished track waited for audio that cannot refill it"); } - /// N2: one skipped group is a hole in the timeline, not a new timeline. Eleven packets + /// A skipped group leaves the held audio on the same timeline. Eleven packets /// (0 to 220 ms) arrive and playout starts on them; group 11 (220 ms) is never /// published, and groups 12 onward arrive together, far enough past it that the /// container gives up on 11 at once. The audio playout already held in front of the @@ -1743,7 +1743,6 @@ mod tests { } let first = consumer.read().await.unwrap().expect("first block"); let held = consumer.playout.as_ref().unwrap().engine.stats(); - let budget = consumer.playout.as_ref().unwrap().budget; // Group 11 is lost. 12 to 22 land together (the burst after a stall), which puts the // newest (440 ms) a budget past the hole, so the container gives up on 11 at once. It @@ -1755,19 +1754,10 @@ mod tests { } let mut timestamps = vec![first.timestamp.as_micros()]; - let mut silent = 0; for _ in 0..30 { let frame = consumer.read().await.unwrap().expect("a block"); - let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap(); - if pcm.iter().all(|sample| *sample == 0.0) { - silent += 1; - } timestamps.push(frame.timestamp.as_micros()); } - let stats = consumer.playout.as_ref().unwrap().engine.stats(); - eprintln!("held before the hole: {held:?}, budget {budget:?}"); - eprintln!("block timestamps (us): {timestamps:?}"); - eprintln!("all-zero blocks: {silent}, stats after: {stats:?}"); // Opus pre-skip moves decoded audio 6.5 ms earlier, so the run before the hole // ends at 213.5 ms. A block starting anywhere in its last 25 ms is that audio played. diff --git a/rs/moq-audio/src/playout/buffer.rs b/rs/moq-audio/src/playout/buffer.rs index fc0b26ea89..b77b45640a 100644 --- a/rs/moq-audio/src/playout/buffer.rs +++ b/rs/moq-audio/src/playout/buffer.rs @@ -151,9 +151,12 @@ impl Buffer { ready } - /// Frames held in all, including any past a hole. - pub(crate) fn held(&self) -> usize { - self.packets.iter().map(|packet| packet.pcm.len() / self.channels).sum() + /// Whether any frames are held beyond `count`, including across a hole. + pub(crate) fn has_after(&self, count: usize) -> bool { + self.packets + .iter() + .try_fold(count, |left, packet| left.checked_sub(packet.pcm.len() / self.channels)) + .is_none() } /// Media time just past the newest sample held, or `None` when the buffer is empty. diff --git a/rs/moq-audio/src/playout/engine.rs b/rs/moq-audio/src/playout/engine.rs index 2e462e8bb7..c27ac9f4e7 100644 --- a/rs/moq-audio/src/playout/engine.rs +++ b/rs/moq-audio/src/playout/engine.rs @@ -51,11 +51,8 @@ pub(crate) struct Config { pub(crate) conceal: bool, } -/// What the engine has been doing. -/// -/// Counted always and read by the tests; what a player reports to a viewer, and -/// through which type, is a decision for whoever builds that panel. -#[allow(dead_code)] +/// Playout counters used by the regression tests. +#[cfg(test)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct Stats { /// Blocks the engine had to invent because nothing was held. @@ -130,6 +127,7 @@ pub(crate) struct Engine { produced: Vec, concealed: Vec, + #[cfg(test)] stats: Stats, } @@ -173,6 +171,7 @@ impl Engine { scratch: Vec::new(), produced: Vec::new(), concealed: Vec::new(), + #[cfg(test)] stats: Stats::default(), }) } @@ -218,7 +217,10 @@ impl Engine { // ours to choose. let trimmed = self.trim(); if trimmed > 0 { - self.stats.trimmed += trimmed as u64; + #[cfg(test)] + { + self.stats.trimmed += trimmed as u64; + } // Playout has not begun, so the playhead is simply where it is going to begin. self.playhead = self.buffer.front(); } @@ -295,7 +297,10 @@ impl Engine { // The loop above cannot leave the buffer short, so this is the counter // that says so rather than a path with behaviour. out.fill(0.0); - self.stats.short += 1; + #[cfg(test)] + { + self.stats.short += 1; + } return; } @@ -402,7 +407,7 @@ impl Engine { // A hole cannot extend its preceding fragment, and an endpoint cannot refill // a stalled buffer. Drain those runs while preserving each splice's media time. - if self.ended || (ready > 0 && ready < self.block && self.buffer.held() > ready) { + if self.ended || (ready > 0 && ready < self.block && self.buffer.has_after(ready)) { if ready == 0 { self.pause(); } else if !contiguous { @@ -466,7 +471,10 @@ impl Engine { let shift = match self.stretch.accelerate(&input, &self.noise, fast, &mut produced) { Stretched::Applied { frames, .. } => { - self.stats.accelerates += 1; + #[cfg(test)] + { + self.stats.accelerates += 1; + } frames as i64 } Stretched::Skipped => { @@ -496,7 +504,10 @@ impl Engine { let shift = match self.stretch.preemptive_expand(&input, 0, &self.noise, &mut produced) { Stretched::Applied { frames, .. } => { - self.stats.accelerates += 1; + #[cfg(test)] + { + self.stats.accelerates += 1; + } -(frames as i64) } Stretched::Skipped => { @@ -524,8 +535,9 @@ impl Engine { // Nothing to play: invent a block, or ramp into silence if the caller asked for // concealment to stay off. - fn conceal(&mut self, underrun: bool) { - if underrun { + fn conceal(&mut self, _underrun: bool) { + #[cfg(test)] + if _underrun { self.stats.underruns += 1; } @@ -543,7 +555,10 @@ impl Engine { } } - self.stats.expands += 1; + #[cfg(test)] + { + self.stats.expands += 1; + } self.commit(&produced, Duration::ZERO, 0, true); self.produced = produced; } @@ -677,13 +692,12 @@ impl Engine { } } - fn record_skip(&mut self, dropped: usize) { - if dropped == 0 { - return; + fn record_skip(&mut self, _dropped: usize) { + #[cfg(test)] + if _dropped > 0 { + self.stats.skips += 1; + self.stats.skipped += _dropped as u64; } - - self.stats.skips += 1; - self.stats.skipped += dropped as u64; } } @@ -1100,20 +1114,22 @@ mod tests { #[test] fn a_finished_run_keeps_the_playhead_across_a_hole() { - let mut engine = Engine::new(config(1, false)).unwrap(); - engine.insert(Duration::ZERO, 0.0, &[0.25; 960]); - engine.insert(Duration::from_millis(40), 40.0, &[0.5; 960]); - engine.end(); - let mut out = vec![0.0; engine.block()]; - for _ in 0..10 { - if engine.drained() { - break; + for conceal in [false, true] { + let mut engine = Engine::new(config(1, conceal)).unwrap(); + engine.insert(Duration::ZERO, 0.0, &[0.25; 960]); + engine.insert(Duration::from_millis(40), 40.0, &[0.5; 960]); + engine.end(); + let mut out = vec![0.0; engine.block()]; + for _ in 0..10 { + if engine.drained() { + break; + } + engine.pull(&mut out); } - engine.pull(&mut out); + assert!(engine.drained()); + assert_eq!(engine.playhead(), Some(Duration::from_millis(60))); + assert_eq!(engine.stats().underruns, 0); } - assert!(engine.drained()); - assert_eq!(engine.playhead(), Some(Duration::from_millis(60))); - assert_eq!(engine.stats().underruns, 0); } #[test] @@ -1143,7 +1159,7 @@ mod tests { ) } - /// N1: a run shorter than one block at the front of the buffer, with a hole behind + /// A run shorter than one block at the front of the buffer, with a hole behind /// it, never grows into a block. Live pacing (one 20 ms packet per two pulls) keeps /// the first fill under the trim, so the play cursor stays on multiples of 480 and /// the run before the hole ends 168 frames short of a block. The audio behind the diff --git a/rs/moq-audio/src/playout/noise.rs b/rs/moq-audio/src/playout/noise.rs index 5f71b48126..159e456d0e 100644 --- a/rs/moq-audio/src/playout/noise.rs +++ b/rs/moq-audio/src/playout/noise.rs @@ -82,14 +82,14 @@ impl Noise { return; } + let mut window = [0.0; WINDOW]; for (index, channel) in self.channels.iter_mut().enumerate() { - let window: Vec = pcm[(total - take) * channels + index..] - .iter() - .step_by(channels) - .copied() - .collect(); + let samples = pcm[(total - take) * channels + index..].iter().step_by(channels); + for (sample, value) in window[..take].iter_mut().zip(samples) { + *sample = *value; + } - if channel.update(&window) { + if channel.update(&window[..take]) { self.initialised = true; } } @@ -130,7 +130,8 @@ impl Channel { /// Returns whether this window replaced the estimate. fn update(&mut self, window: &[f32]) -> bool { - let energy = window.iter().map(|x| x * x).sum::() / window.len() as f32; + let r0: f32 = window.iter().map(|x| x * x).sum(); + let energy = r0 / window.len() as f32; if energy >= self.threshold { // Loud window. Open the search up a little and remember the peak, so the @@ -142,7 +143,6 @@ impl Channel { return false; } - let r0: f32 = window.iter().map(|x| x * x).sum(); // Digital silence carries no room level and must not lower the search threshold. if r0 <= 0.0 { return false; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index b51e8ec9a4..6ab7240a6a 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -2775,11 +2775,6 @@ mod tests { assert!(matches!(event, Some(Event::FrameEnd(end)) if end == ts(15_000))); } - // review consumer-sync-video F16: the skip's log-only `open` field registers the read loop's - // waiter on the group it is about to drain. Counted as wake calls when that group's FIN lands - // afterwards. The same poll registers the waiter on group 0 twice elsewhere before it decides to - // skip, while group 0 is still what it waits on (two wakes with the log line's registration - // removed); a third is the log line's. #[test] fn a_skip_registers_no_extra_waker_on_the_group_it_drains() { use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/rs/moq-relay/tests/runtime_uring.rs b/rs/moq-relay/tests/runtime_uring.rs index 971faf1155..91ddb3eabb 100644 --- a/rs/moq-relay/tests/runtime_uring.rs +++ b/rs/moq-relay/tests/runtime_uring.rs @@ -511,9 +511,6 @@ async fn spawn_auth_server(policy: moq_auth::serve::Policy) -> url::Url { url } -/// F4 (review 2026-09-27): a draining relay refuses a new session on the io_uring workers too, -/// instead of admitting it and sending a GOAWAY on arrival, as `a_draining_relay_refuses_a_new_session` -/// (tests/shutdown_signal.rs) checks for the tokio listeners. #[tokio::test] async fn a_draining_uring_relay_refuses_a_new_session() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); From f0dc165dbef5c3e628be45ce3deb756a092a359e Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:31:46 -0400 Subject: [PATCH 036/127] ci: measure native audio playout nightly Run channel and backlog sweeps with the zero-allocation check. Format the updated relay drain documentation. Co-Authored-By: Codex --- .github/workflows/nightly.yml | 6 ++++++ doc/bin/relay/index.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7b03884a4d..26adb8ff15 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -96,6 +96,12 @@ jobs: nix develop --command cargo bench --locked -p moq-shaper --bench forward -- --warm-up-time 1 --measurement-time 2 --sample-size 10 + - name: Audio playout benchmark + if: ${{ !cancelled() }} + run: >- + nix develop --command cargo bench --locked -p moq-audio --bench playout -- + --warm-up-time 1 --measurement-time 2 --sample-size 30 --noplot + # The OBS plugin and libmoq's C fixtures are the only things that link # libmoq.a from outside cargo, so a stale `rs/libmoq/native-libs/*.txt` # breaks them and nothing else. obs.yml lists the inputs that can do that, diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index ffa967be9e..83143afb9d 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -80,7 +80,7 @@ The accessors borrow and `run` consumes the relay, so clone `cluster`, tasks before calling it. `trigger.start()` drains every session with a GOAWAY and `run` returns once the drain window elapses, with the listeners released and the workers joined. A relay that has started draining refuses new sessions -with `503` on every transport under both tokio and io_uring, so a client redialing during a restart backs off +with `503` on every transport under both tokio and io\_uring, so a client redialing during a restart backs off and keeps the session it is still being served on, instead of being handed one that is waved away on arrival. Build routes from `web().routes()` (or `internal().routes()`): `with_web` replaces the router, so `Router::new()` From 55daa91b84ca9a6e80a406fa379758e20b35a5a4 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:34:22 -0400 Subject: [PATCH 037/127] style: finish scope guard and test cleanup Co-Authored-By: Codex --- js/justfile | 6 +++--- js/watch/src/sync.test.ts | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/js/justfile b/js/justfile index 7d999cd98e..419ea53407 100644 --- a/js/justfile +++ b/js/justfile @@ -75,11 +75,11 @@ _scope-test: set -euo pipefail missing=() while IFS= read -r workspace; do - grep -qE '{{ scope }}' <<< "$workspace/package.json" || missing+=("$workspace") + grep -qE '{{ scope }}' <<< "$workspace/package.json" || missing+=("$workspace") done < <(jq -r '.workspaces[]' ../package.json) if ((${#missing[@]})); then - echo "js: workspaces outside the JS scope: ${missing[*]}" >&2 - exit 1 + echo "js: workspaces outside the JS scope: ${missing[*]}" >&2 + exit 1 fi # Auto-fix lint and formatting issues. Skips on the same terms as `check`. diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index eed240765e..7ffcc181bd 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -875,7 +875,6 @@ describe("Sync wakes a waiting frame only when its deadline moves", () => { } it("samples exactly on the extrapolated line do not wake it", async () => { - // The review's literal case: the playhead advances exactly as extrapolated. expect(await timersWhileSampling(() => 0)).toBeLessThanOrEqual(2); }); From a7dd859c6264536ab37330710aefea3e9568f3b3 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:39:57 -0400 Subject: [PATCH 038/127] style(audio): keep benchmark manifest tables sorted Co-Authored-By: Codex --- rs/moq-audio/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index 62f93eabdb..ab8e23153f 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -135,13 +135,13 @@ objc2-core-media = { workspace = true, features = ["CMSampleBuffer", "CMBlockBuf objc2-foundation = { workspace = true, optional = true } objc2-screen-capture-kit = { workspace = true, features = ["SCStream", "SCShareableContent"], optional = true } -[[bench]] -name = "playout" -harness = false - [dev-dependencies] criterion = { workspace = true } # Reads the playout conformance corpus, which is JSON generated by the browser # estimator. Tests only: nothing in the library parses JSON. serde_json = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util"] } + +[[bench]] +name = "playout" +harness = false From 7ab0463d5acafb53b0c7595819897ef55b034def Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:51:51 -0400 Subject: [PATCH 039/127] perf(net): skip redundant anchors for an unbounded resumed track Let the underlying cursor supply expiry for a logical track with one unbounded segment and no outer constraint. Clear shared and inner anchors once when returning to that shape, preserving held groups across cap and route changes. Cover both reset paths with behavior tests and add a nightly delivery benchmark over retained routes, cache depth, and subscriber count. Co-Authored-By: Codex --- .github/workflows/nightly.yml | 6 ++ bench/README.md | 9 +++ rs/moq-net/Cargo.toml | 4 + rs/moq-net/benches/resume.rs | 137 +++++++++++++++++++++++++++++++++ rs/moq-net/src/model/resume.rs | 102 +++++++++++++++++++++++- 5 files changed, 255 insertions(+), 3 deletions(-) create mode 100644 rs/moq-net/benches/resume.rs diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 26adb8ff15..244c4449d3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -96,6 +96,12 @@ jobs: nix develop --command cargo bench --locked -p moq-shaper --bench forward -- --warm-up-time 1 --measurement-time 2 --sample-size 10 + - name: Resumed track delivery benchmark + if: ${{ !cancelled() }} + run: >- + nix develop --command cargo bench --locked -p moq-net --bench resume -- + --warm-up-time 1 --measurement-time 2 --sample-size 10 --noplot + - name: Audio playout benchmark if: ${{ !cancelled() }} run: >- diff --git a/bench/README.md b/bench/README.md index eac20a13a6..7916285cf7 100644 --- a/bench/README.md +++ b/bench/README.md @@ -30,6 +30,15 @@ nix develop --command cargo bench --locked -p moq-audio --bench playout The cases sweep 1, 2, and 6 channels and 1, 10, 100, and 1,000 queued packets. The benchmark also rejects heap allocation during a warmed noise update. +Measure resumed-track delivery through 1, 2, and 3 retained routes: + +```bash +nix develop --command cargo bench --locked -p moq-net --bench resume +``` + +The cases sweep 1, 10, and 100 cached groups per route and concurrent subscribers. +Route setup and subscription creation are excluded from the delivery measurement. + Compare one multi-threaded Tokio runtime with the same number of independent Tokio/epoll and io\_uring workers: diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index 66b697784e..20c58cbaba 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -77,6 +77,10 @@ harness = false name = "track" harness = false +[[bench]] +name = "resume" +harness = false + [[bench]] name = "priority" harness = false diff --git a/rs/moq-net/benches/resume.rs b/rs/moq-net/benches/resume.rs new file mode 100644 index 0000000000..ad40b788de --- /dev/null +++ b/rs/moq-net/benches/resume.rs @@ -0,0 +1,137 @@ +//! Cached group delivery through a logical track, across cache depth, fanout, and route segments. + +use std::hint::black_box; +use std::task::Poll; +use std::time::{Duration, Instant}; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_net::{Timestamp, broadcast, cache, group, origin, track}; + +const SETUP_TIMEOUT: Duration = Duration::from_secs(10); + +struct Source { + _origin: origin::Producer, + _broadcasts: Vec, + _tracks: Vec, + _broadcast: broadcast::Consumer, + _reader: track::Subscriber, + track: track::Consumer, +} + +fn replay() -> track::Subscription { + track::Subscription::default() + .with_start(track::Position::group(0)) + .with_max_age(Duration::from_secs(5)) +} + +async fn source(groups: u64, segments: u64) -> Source { + let mut config = origin::Config::default(); + config.pool = cache::Pool::new(cache::Config::default().with_expiry(None)); + let (origin, driver) = origin::Producer::new(config); + tokio::spawn(moq_net::time::run(driver)); + let consumer = origin.consume(); + let mut broadcasts = Vec::new(); + let mut tracks = Vec::new(); + let mut resolved = None; + let mut resolved_broadcast = None; + let mut reader: Option = None; + + for segment in 0..segments { + let broadcast = origin.create_broadcast("bench/live").unwrap(); + let track = broadcast.create_track("data", None).unwrap(); + for sequence in segment * groups..(segment + 1) * groups { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence).unwrap(), b"frame".as_ref()) + .unwrap(); + group.finish().unwrap(); + } + broadcast.announce(origin::Route::default()).unwrap(); + if reader.is_none() { + let broadcast = consumer.request_broadcast("bench/live").await.unwrap(); + let track = broadcast.track("data").unwrap(); + reader = Some(track.subscribe(replay()).await.unwrap()); + resolved = Some(track); + resolved_broadcast = Some(broadcast); + } + // Reading each route's groups proves the front applied the takeover before measurement. + for sequence in segment * groups..(segment + 1) * groups { + assert_eq!( + reader.as_mut().unwrap().recv_group().await.unwrap().unwrap().sequence, + sequence + ); + } + broadcasts.push(broadcast); + tracks.push(track); + } + + Source { + _origin: origin, + _broadcasts: broadcasts, + _tracks: tracks, + _broadcast: resolved_broadcast.unwrap(), + _reader: reader.unwrap(), + track: resolved.unwrap(), + } +} + +fn delivery(c: &mut Criterion) { + let mut group = c.benchmark_group("resume_delivery"); + for segments in [1, 2, 3] { + for groups in [1, 10, 100] { + for subscribers in [1, 10, 100] { + let id = + BenchmarkId::from_parameter(format!("{segments}segments_{groups}groups_{subscribers}subscribers")); + group.throughput(Throughput::Elements(segments * groups * subscribers)); + group.bench_function(id, |b| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let source = runtime.block_on(async { + tokio::time::timeout(SETUP_TIMEOUT, source(groups, segments)) + .await + .expect("origin route setup timed out") + }); + let waiter = kio::Waiter::noop(); + b.iter_custom(|iterations| { + runtime.block_on(async { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + let mut cursors = tokio::time::timeout(SETUP_TIMEOUT, async { + let mut cursors = Vec::with_capacity(subscribers as usize); + for _ in 0..subscribers { + cursors.push(source.track.subscribe(replay()).await.unwrap()); + } + cursors + }) + .await + .expect("cached subscriptions timed out"); + let start = Instant::now(); + for sequence in 0..segments * groups { + for cursor in &mut cursors { + let Poll::Ready(Ok(Some(received))) = cursor.poll_recv_group(&waiter) else { + panic!("cached group must be ready"); + }; + assert_eq!(received.sequence, sequence); + black_box(received); + } + } + for cursor in &mut cursors { + assert!(cursor.poll_recv_group(&waiter).is_pending()); + } + elapsed += start.elapsed(); + } + elapsed + }) + }); + }); + } + } + } + + group.finish(); +} + +criterion_group!(benches, delivery); +criterion_main!(benches); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index e2227a17f1..0059fbf891 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -630,6 +630,7 @@ impl Consumer { end_sequence: None, outer: Anchor::default(), drift_anchor: kio::Producer::new(Anchor::default()), + identity_anchor: false, } } @@ -1495,8 +1496,10 @@ pub struct Subscriber { outer: Anchor, /// The logical drift anchor ([`Self::end_sequence`] and [`Self::outer`] combined, /// with the newest edge across the segments), shared with groups that outlive this - /// cursor poll. Refreshed on every [`Self::poll_sync`]. + /// cursor poll. Reconciled on each poll; a sole unbounded source uses its own edge. drift_anchor: kio::Producer, + /// The sole unbounded segment's anchors are clear; its own track supplies expiry. + identity_anchor: bool, } impl Subscriber { @@ -1602,6 +1605,7 @@ impl Subscriber { /// Apply a producer snapshot: move boundaries on known segments and subscribe /// to new ones. fn apply(&mut self, snapshot: Snapshot) { + self.identity_anchor = false; let Snapshot { epoch, finished, @@ -1737,9 +1741,28 @@ impl Subscriber { /// The cap is the tightest any reader of this subscriber imposes: its own /// [`Self::end_at`] and whatever a wrapping splice pushed down. The edge is the /// newest across the producer's segments within that cap, or a wrapping splice's if - /// newer. Resolved on every sync, since each segment is a separate track and the - /// logical edge moves whenever any of them grows. + /// newer. A sole unbounded segment uses its own track's edge; other shapes resolve + /// on every sync, since each segment can grow independently. fn refresh_anchor(&mut self) { + if let [seg] = self.segments.as_mut_slice() + && seg.start.is_none() + && seg.end.is_none() + && self.end_sequence.is_none() + && self.outer == Anchor::default() + { + if !self.identity_anchor { + if let Ok(mut current) = self.drift_anchor.write() { + *current = Anchor::default(); + } + seg.anchor = Anchor::default(); + if let Some(sub) = seg.stale_sub_mut() { + sub.set_anchor(Anchor::default()); + } + self.identity_anchor = true; + } + return; + } + self.identity_anchor = false; let outer = self.outer.clone().capped(self.end_sequence); let state = self.state.read(); let edge = state @@ -1767,6 +1790,7 @@ impl Subscriber { /// [`track::Subscriber::set_anchor`], for this subscriber nested as a segment of /// another splice. pub(crate) fn set_anchor(&mut self, anchor: Anchor) { + self.identity_anchor = false; self.outer = anchor; self.refresh_anchor(); } @@ -2318,6 +2342,7 @@ impl Subscriber { /// on the inner segment cursors, so a capped group parks here and a rising cap /// re-offers it. pub fn end_at(&mut self, end: impl Into) { + self.identity_anchor = false; self.end_sequence = end.into().exclusive(); // The cap bounds each segment's drift anchor as well as this reader's own // delivery: a segment must not measure against groups this cap hides. @@ -2432,6 +2457,77 @@ mod test { assert!(sub.recv_group().now_or_never().is_none(), "should have blocked"); } + #[tokio::test] + async fn removing_an_outer_edge_keeps_a_held_group_within_its_own_budget() { + let (mut track, consumer) = track_pair("source"); + let (mut outside, outside_consumer) = track_pair("outside"); + let mut producer = Producer::new(); + producer.switch(&consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_millis(100))); + let mut open = track.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(Timestamp::ZERO, b"head".to_vec()).unwrap(); + write_group_at(&mut track, 1, "next", Duration::from_millis(20)); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + + write_group_at(&mut outside, 10, "far ahead", Duration::from_secs(1)); + sub.set_anchor(Anchor { + edge: outside_consumer.live_edge(None), + ..Anchor::default() + }); + sub.set_anchor(Anchor::default()); + assert_eq!(read(&mut held), b"head"); + assert!(held.read_frame().now_or_never().is_none()); + + // The held group follows a route change without polling the subscriber again. + let (mut next, next_consumer) = track_pair("replacement"); + let mut continuation = next.create_group(group::Info { sequence: 0 }).unwrap(); + continuation + .write_frame(Timestamp::ZERO, b"duplicate".to_vec()) + .unwrap(); + continuation + .write_frame(Timestamp::from_millis(10).unwrap(), b"tail".to_vec()) + .unwrap(); + write_group_at(&mut next, 1, "next", Duration::from_millis(20)); + producer + .switch(&next_consumer, Position { group: 0, frame: 1 }) + .unwrap(); + assert_eq!(read(&mut held), b"tail"); + assert!(held.read_frame().now_or_never().is_none()); + } + + #[tokio::test] + async fn removing_a_cap_restores_held_group_expiry() { + for outer in [false, true] { + let (mut track, consumer) = track_pair("source"); + let mut producer = Producer::new(); + producer.switch(&consumer, None).unwrap(); + let mut sub = producer.consume().subscribe(None); + if outer { + sub.set_anchor(Anchor { + cap: Some(1), + ..Anchor::default() + }); + } else { + sub.end_at(..1); + } + let mut open = track.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(Timestamp::ZERO, b"held".to_vec()).unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + write_group_at(&mut track, 1, "next", Duration::from_secs(1)); + write_group_at(&mut track, 2, "new", Duration::from_secs(2)); + assert_eq!(read(&mut held), b"held"); + assert!(held.read_frame().now_or_never().is_none()); + if outer { + sub.set_anchor(Anchor::default()); + } else { + sub.end_at(..); + } + assert!(matches!(held.read_frame().now_or_never(), Some(Ok(None)))); + } + } + /// A waker that counts its wakes, for asserting a pending poll left a live /// registration behind. struct CountWaker(std::sync::atomic::AtomicUsize); From 10716882b7ce6d35519bbb4f6a56b554bcbe8472 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:54:53 -0400 Subject: [PATCH 040/127] test(relay): isolate process-wide shutdown signals Hold a mutex across both relay integration tests so parallel libtest runs cannot deliver one test SIGINT to the other relay. Co-Authored-By: Codex --- rs/moq-relay/tests/shutdown_signal.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rs/moq-relay/tests/shutdown_signal.rs b/rs/moq-relay/tests/shutdown_signal.rs index 4c7335a4ed..1e50d62e4d 100644 --- a/rs/moq-relay/tests/shutdown_signal.rs +++ b/rs/moq-relay/tests/shutdown_signal.rs @@ -14,7 +14,7 @@ #![cfg(unix)] -use std::{net::TcpListener, time::Duration}; +use std::{net::TcpListener, sync::Mutex, time::Duration}; use moq_relay::{Config, Relay, auth}; @@ -23,8 +23,12 @@ use moq_relay::{Config, Relay, auth}; /// one second before exiting. const DRAIN_TIMEOUT: Duration = Duration::from_secs(3); +// SIGINT reaches every relay in the process when libtest runs these concurrently. +static SIGNAL_LOCK: Mutex<()> = Mutex::new(()); + #[test] fn sigint_drains_sessions_before_exiting() { + let _signal = SIGNAL_LOCK.lock().unwrap_or_else(|err| err.into_inner()); // Same reason as the cluster tests: under `--all-features` a `Connection` // carries every transport backend, and holding one across awaits overflows // libtest's 2 MiB per-test stack in an unoptimized build. @@ -116,6 +120,7 @@ async fn sigint_drains_sessions_before_exiting_inner() { /// the window, which is how a relay restart killed every native publisher on it. #[test] fn a_draining_relay_refuses_a_new_session() { + let _signal = SIGNAL_LOCK.lock().unwrap_or_else(|err| err.into_inner()); // Same reason as above: a `Connection` carrying every transport backend // overflows libtest's per-test stack in an unoptimized build. std::thread::Builder::new() From 97d712ea8b0dfcda4573609d092e889b90cd1aa2 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:16:08 -0400 Subject: [PATCH 041/127] fix(watch): keep stale GOPs out of video decode Co-Authored-By: GPT-6 --- js/watch/src/video/decoder.test.ts | 173 ++++++++++++++++++++++++++++- js/watch/src/video/decoder.ts | 7 ++ 2 files changed, 174 insertions(+), 6 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index ba1e9f072a..4820e3deba 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, expect, jest, test } from "bun:test"; +import { afterEach, beforeEach, expect, jest, spyOn, test } from "bun:test"; import { Container } from "@moq/hang"; import * as Catalog from "@moq/hang/catalog"; import * as Moq from "@moq/net"; @@ -15,6 +15,7 @@ import type { Source } from "./source"; /** One `VideoDecoder` the code under test built, with the chunks it was handed. */ type Built = { chunks: string[]; + timestamps: number[]; /** Emit a decoded picture, which is what keeps the tile out of the stalled state. */ emit(timestamp: number): void; /** Raise a codec error, the way a decoder does when it cannot decode what it was fed. */ @@ -59,6 +60,7 @@ beforeEach(() => { constructor(init: { output: (frame: unknown) => void; error: (error: Error) => void }) { this.#entry = { chunks: [], + timestamps: [], emit: (timestamp: number) => init.output(new FakeVideoFrame(timestamp)), fail: (error: Error) => init.error(error), }; @@ -69,8 +71,9 @@ beforeEach(() => { this.state = "configured"; } - decode(chunk: { type: string }): void { + decode(chunk: { type: string; timestamp: number }): void { this.#entry.chunks.push(chunk.type); + this.#entry.timestamps.push(chunk.timestamp); } close(): void { @@ -123,6 +126,62 @@ async function settle(rounds = 10): Promise { const TRACK = "video"; const payload = (n: number) => new Uint8Array(n).fill(1); +const CMAF_CONFIG = (() => { + const base = Catalog.VideoConfigSchema.parse({ + codec: "avc1.640028", + container: { kind: "legacy" }, + description: "01640028", + codedWidth: 16, + codedHeight: 16, + }); + const init = Container.Cmaf.createVideoInitSegment(base); + return Catalog.VideoConfigSchema.parse({ + ...base, + container: { kind: "cmaf", init: btoa(String.fromCharCode(...init)) }, + }); +})(); + +type TestContainer = "legacy" | "cmaf"; + +function testConfig(container: TestContainer): Catalog.VideoConfig { + return container === "cmaf" + ? CMAF_CONFIG + : Catalog.VideoConfigSchema.parse({ codec: "avc1.640028", container: { kind: "legacy" } }); +} + +function writeTestFrame( + container: TestContainer, + group: Moq.Group.Producer, + timestamp: Time.Micro, + keyframe: boolean, + sequence: number, +): void { + let data: Uint8Array; + if (container === "cmaf") { + data = Container.Cmaf.encodeDataSegment({ + kind: "video", + data: payload(1), + timestamp, + duration: 33_000, + keyframe, + sequence, + }); + } else { + const header = Moq.Varint.encode(timestamp); + data = new Uint8Array(header.length + 1); + data.set(header); + data[header.length] = 1; + } + group.writeFrame({ payload: data, timestamp: Time.Timestamp.now() }); +} + +function finishTestGroup(container: TestContainer, group: Moq.Group.Producer, end: Time.Micro): void { + if (container === "legacy") { + group.writeFrame({ payload: Moq.Varint.encode(end), timestamp: Time.Timestamp.now() }); + } + group.close(); +} + /** * Keep the newest decoder producing pictures until the returned function is called. * @@ -167,16 +226,14 @@ class ServedTrack extends Moq.Track.Producer { } /** A live broadcast, a `Decoder` reading it, and every subscription it has raised. */ -function fixture() { +function fixture(config = testConfig("legacy")) { const broadcast = new Moq.Broadcast.Producer(); const consumer = broadcast.consume(); const source = { in: { broadcast: new Signal({ relativeBroadcast: () => consumer } as unknown as Broadcast) }, out: { track: new Signal(TRACK), - config: new Signal( - Catalog.VideoConfigSchema.parse({ codec: "avc1.640028", container: { kind: "legacy" } }), - ), + config: new Signal(config), catalog: new Signal(undefined), }, } as unknown as Source; @@ -250,6 +307,110 @@ test("video advances to a buffered keyframe when the audio playhead reaches it", } }); +for (const container of ["legacy", "cmaf"] as const) { + test(`${container} video never submits an older GOP after live media`, async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const fx = fixture(testConfig(container)); + const live = new Moq.Group.Producer(10); + const old = new Moq.Group.Producer(5); + const forward = new Moq.Group.Producer(11); + try { + expect(await fx.subscriptions(1)).toBe(1); + fx.sync.track("audio").clock.set({ + timestamp: Time.Micro(10_000_000), + reference: Time.Milli.now(), + rate: 0, + }); + + fx.track.writeGroup(live); + writeTestFrame(container, live, Time.Micro(10_000_000), true, 0); + writeTestFrame(container, live, Time.Micro(10_033_000), false, 1); + await settle(); + + // Warm-cache groups can arrive below the group already submitted to WebCodecs. + fx.track.writeGroup(old); + writeTestFrame(container, old, Time.Micro(5_000_000), true, 2); + await settle(); + + old.close(); + finishTestGroup(container, live, Time.Micro(10_066_000)); + + fx.track.writeGroup(forward); + writeTestFrame(container, forward, Time.Micro(10_066_000), true, 3); + writeTestFrame(container, forward, Time.Micro(10_099_000), false, 4); + forward.close(); + await settle(); + + expect(built[0].timestamps).not.toContain(5_000_000); + expect(built[0].timestamps.slice(-2)).toEqual([10_066_000, 10_099_000]); + expect(built[0].chunks.slice(-2)).toEqual(["key", "delta"]); + } finally { + old.close(); + live.close(); + forward.close(); + fx.close(); + warn.mockRestore(); + } + }); +} + +test("a declared marker still resets video when its group arrives behind live media", async () => { + const fx = fixture(); + const live = new Moq.Group.Producer(10); + const marker = new Moq.Group.Producer(6); + try { + expect(await fx.subscriptions(1)).toBe(1); + fx.track.writeGroup(live); + writeTestFrame("legacy", live, Time.Micro(10_000_000), true, 0); + writeTestFrame("legacy", live, Time.Micro(10_033_000), false, 1); + await settle(); + expect(fx.sync.out.reference.peek()).toBeDefined(); + + fx.track.writeGroup(marker); + marker.writeFrame({ payload: Moq.Varint.encode(Time.Micro(5_033_000)), timestamp: Time.Timestamp.now() }); + marker.close(); + await settle(); + + expect(fx.sync.out.reference.peek()).toBeUndefined(); + } finally { + live.close(); + marker.close(); + fx.close(); + } +}); + +test("a forward discontinuity waits for the next keyframe", async () => { + type Next = NonNullable>>; + const frame = ( + group: number, + timestamp: number, + keyframe: boolean, + discontinuity: number, + continuous: boolean, + ): Next => ({ + group, + discontinuity, + continuous, + frame: { payload: payload(1), timestamp: Time.Micro(timestamp), keyframe }, + }); + const results: Next[] = [ + frame(10, 10_000_000, true, 0, false), + frame(10, 10_033_000, false, 0, true), + frame(11, 10_066_000, false, 1, false), + frame(11, 10_099_000, true, 1, true), + frame(11, 10_132_000, false, 1, true), + ]; + const read = spyOn(Container.Consumer.prototype, "next").mockImplementation(async () => results.shift()); + const fx = fixture(); + try { + await settle(); + expect(built[0].timestamps).toEqual([10_000_000, 10_033_000, 10_099_000, 10_132_000]); + } finally { + fx.close(); + read.mockRestore(); + } +}); + test("late backlog cannot replace the picture shown before clock catch-up", async () => { const fx = fixture(); try { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 4de5f754e8..e13157a7fd 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -631,6 +631,7 @@ class DecoderTrack { let previous: Time.Micro | undefined; // Nothing has been decoded yet, so the first thing fed to the codec has to be a keyframe. let keyframeNeeded = true; + let decodedGroup: number | undefined; effect.spawn(async () => { for (;;) { @@ -645,6 +646,8 @@ class DecoderTrack { const { frame } = next; if (!frame) continue; // The group is done + // Groups can arrive newest-first, but a stateful video codec cannot rewind to an older GOP. + if (decodedGroup !== undefined && next.group < decodedGroup) continue; if (!next.continuous) keyframeNeeded = true; if (keyframeNeeded) { @@ -688,6 +691,7 @@ class DecoderTrack { previous = frame.timestamp; decoder.decode(chunk); + decodedGroup = next.group; } }); } @@ -725,6 +729,7 @@ class DecoderTrack { let previous: Time.Micro | undefined; // See `#runLegacy`: nothing has been decoded yet, so the codec needs a keyframe first. let keyframeNeeded = true; + let decodedGroup: number | undefined; effect.spawn(async () => { for (;;) { @@ -739,6 +744,7 @@ class DecoderTrack { const { frame } = next; if (!frame) continue; + if (decodedGroup !== undefined && next.group < decodedGroup) continue; // A hole in delivery makes every following delta undecodable. See `#runLegacy`. if (!next.continuous) keyframeNeeded = true; @@ -776,6 +782,7 @@ class DecoderTrack { timestamp: frame.timestamp, }), ); + decodedGroup = next.group; } }); } From 14bd9dd25ce0c138416e93d50231e4946533ff28 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:36:09 -0400 Subject: [PATCH 042/127] fix(hang): keep historical loss behind the playhead Co-Authored-By: GPT-6 --- js/hang/src/container/consumer.test.ts | 214 ++++++++++++++++++++++++- js/hang/src/container/consumer.ts | 51 ++++-- js/watch/src/video/decoder.test.ts | 50 ++++++ 3 files changed, 299 insertions(+), 16 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 7d4c00e408..aa2e31a8e7 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -2117,7 +2117,8 @@ test("Consumer measures a slow group against the next group that has a frame", a const frames = await drainFrames(consumer, 300); expect(frames.map((f) => f.timestamp as number)).toEqual([300_000, 333_000, 366_000, 400_000, 433_000, 466_000]); // Two groups were given up on, and the span they would have carried is really missing, so the - // reader re-anchors exactly once. + // reader re-anchors exactly once. The later groups were buffered but not returned when the + // verdict landed, so they cannot make the loss historical. expect(consumer.skipped.peek()).toBe(2); expect(consumer.discontinuity).toBe(1); @@ -2126,6 +2127,217 @@ test("Consumer measures a slow group against the next group that has a frame", a consumer.close(); }); +test("Consumer does not move the playhead for age loss behind returned media", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("video"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxAge: 100 as Time.Milli }); + const live = new Group.Producer(10); + const old = new Group.Producer(5); + const expired = new Group.Producer(4); + try { + track.writeGroup(live); + live.writeFrame({ payload: encodeLegacy(10_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + live.writeFrame({ payload: encodeLegacy(10_033_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + expect((await consumer.next())?.group).toBe(10); + expect((await consumer.next())?.group).toBe(10); + + track.writeGroup(old); + old.writeFrame({ payload: encodeLegacy(5_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + expect((await consumer.next())?.group).toBe(5); + + track.writeGroup(expired); + expired.writeFrame({ payload: encodeLegacy(4_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + expect(consumer.skipped.peek()).toBe(1); + + old.close(); + await settle(); + const oldDone = await consumer.next(); + expect(oldDone?.group).toBe(5); + expect(oldDone?.frame).toBeUndefined(); + expect(oldDone?.discontinuity).toBe(0); + + live.writeFrame({ payload: encodeLegacy(10_066_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + const resumed = await nextFrame(consumer); + expect(resumed?.group).toBe(10); + expect(resumed?.discontinuity).toBe(0); + expect(resumed?.continuous).toBe(true); + } finally { + expired.close(); + old.close(); + live.close(); + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); + +test("Consumer does not break continuity for a historical group reset", async () => { + const track = new Track.Producer("video"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxAge: 100 as Time.Milli }); + const live = new Group.Producer(10); + const old = new Group.Producer(5); + try { + track.writeGroup(live); + live.writeFrame({ payload: encodeLegacy(10_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + expect((await nextFrame(consumer))?.group).toBe(10); + + track.writeGroup(old); + old.writeFrame({ payload: encodeLegacy(5_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + expect((await nextFrame(consumer))?.group).toBe(5); + old.close(new NetError.Stream(StreamCode.Old)); + await settle(); + expect((await consumer.next())?.discontinuity).toBe(0); + + live.writeFrame({ payload: encodeLegacy(10_033_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + const resumed = await nextFrame(consumer); + expect(resumed?.group).toBe(10); + expect(resumed?.discontinuity).toBe(0); + expect(resumed?.continuous).toBe(true); + } finally { + old.close(); + live.close(); + consumer.close(); + track.close(); + } +}); + +test("Consumer keeps a historical marker across intervening old media", async () => { + const track = new Track.Producer("video"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxAge: 100 as Time.Milli }); + const live = new Group.Producer(10); + const marker = new Group.Producer(6); + const old = new Group.Producer(5); + try { + track.writeGroup(live); + live.writeFrame({ payload: encodeLegacy(10_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + expect((await nextFrame(consumer))?.group).toBe(10); + + track.writeGroup(marker); + marker.writeFrame({ + payload: encodeLegacyFrame(10_000_000 as Time.Micro, new Uint8Array()), + timestamp: Time.Timestamp.now(), + }); + await settle(); + const endpoint = await consumer.next(); + expect(endpoint?.group).toBe(6); + expect(endpoint?.end).toBe(10_000_000 as Time.Micro); + + track.writeGroup(old); + old.writeFrame({ payload: encodeLegacy(5_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + old.close(); + await settle(); + expect((await nextFrame(consumer))?.group).toBe(5); + expect((await consumer.next())?.group).toBe(5); + + marker.close(); + await settle(); + const reset = await consumer.next(); + expect(reset?.group).toBe(6); + expect(reset?.frame).toBeUndefined(); + expect(reset?.discontinuity).toBe(1); + } finally { + old.close(); + marker.close(); + live.close(); + consumer.close(); + track.close(); + } +}); + +test("Consumer does not let a historical marker rewind the delivery cursor", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("video"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxAge: 100 as Time.Milli }); + const live = new Group.Producer(10); + const marker = new Group.Producer(6); + const old = new Group.Producer(7); + try { + track.writeGroup(live); + live.writeFrame({ payload: encodeLegacy(10_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + live.writeFrame({ payload: encodeLegacy(10_033_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + expect((await nextFrame(consumer))?.group).toBe(10); + expect((await nextFrame(consumer))?.group).toBe(10); + + track.writeGroup(marker); + marker.writeFrame({ + payload: encodeLegacyFrame(6_000_000 as Time.Micro, new Uint8Array()), + timestamp: Time.Timestamp.now(), + }); + await settle(); + expect((await consumer.next())?.end).toBe(6_000_000 as Time.Micro); + + track.writeGroup(old); + old.writeFrame({ payload: encodeLegacy(7_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + const resumed = await nextFrame(consumer); + expect(resumed?.group).toBe(7); + expect(resumed?.discontinuity).toBe(1); + old.close(); + await settle(); + expect((await consumer.next())?.group).toBe(7); + + live.writeFrame({ payload: encodeLegacy(10_066_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + const tail = await Promise.race([nextFrame(consumer), settle(100).then(() => "stalled" as const)]); + expect(tail).not.toBe("stalled"); + expect(tail).toBeDefined(); + if (!tail || tail === "stalled") return; + expect(tail.group).toBe(10); + expect(tail.discontinuity).toBe(1); + } finally { + old.close(); + marker.close(); + live.close(); + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); + +test("Consumer does not treat an endpoint as returned media", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const track = new Track.Producer("video"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxAge: 100 as Time.Milli }); + const endpoint = new Group.Producer(10); + const missing = new Group.Producer(1); + const resumed = new Group.Producer(2); + try { + track.writeGroup(endpoint); + endpoint.writeFrame({ + payload: encodeLegacyFrame(10_000_000 as Time.Micro, new Uint8Array()), + timestamp: Time.Timestamp.now(), + }); + await settle(); + expect((await consumer.next())?.end).toBe(10_000_000 as Time.Micro); + + track.writeGroup(missing); + track.writeGroup(resumed); + resumed.writeFrame({ payload: encodeLegacy(2_000_000 as Time.Micro), timestamp: Time.Timestamp.now() }); + await settle(); + + const next = await nextFrame(consumer); + expect(next?.group).toBe(2); + expect(next?.discontinuity).toBe(1); + expect(next?.continuous).toBe(false); + } finally { + missing.close(); + resumed.close(); + endpoint.close(); + consumer.close(); + track.close(); + warn.mockRestore(); + } +}); + // Same starvation, except the middle group closes with nothing in it. A group that held nothing // says nothing about the timeline, so closing it changes who reaches the verdict and nothing else. test("Consumer measures a slow group past a successor that closed empty", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 0ae50ec413..4733fb3454 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -99,8 +99,11 @@ export class Consumer { // Group of the last frame next() returned, so it can report whether the following result // continues that frame's timeline. Undefined until the first delivery and after a playhead event. #deliveredGroup?: number; - // Set whenever the consumer throws content away: a group that aged past `maxAge`, - // a group truncated by a decode error. Reported (and + // Highest group whose media next() returned. Unlike #liveEdge this excludes endpoints, and unlike + // #deliveredGroup it never moves backward when cached groups arrive newest-first. + #highestDeliveredGroup?: number; + // Set when forward delivery loses content or a declared marker ends its epoch. Loss strictly + // behind media already returned is historical and leaves the forward path intact. Reported (and // cleared) on the first frame delivered from the next group, which is where the missing span // sits. Only the consumer can know this, which is why next() reports it instead of leaving // callers to guess from group numbers. @@ -331,7 +334,7 @@ export class Consumer { } finally { group.done = true; - if (group.consumer.sequence === this.#active) { + if (!this.#isHistorical(group) && group.consumer.sequence === this.#active) { this.#recordPresented(group); // Advance to the next buffered group's actual sequence, but ONLY if it continues this @@ -380,6 +383,10 @@ export class Consumer { return sequence === this.#deliveredGroup || !this.#gap; } + #isHistorical(group: Group): boolean { + return this.#highestDeliveredGroup !== undefined && group.consumer.sequence < this.#highestDeliveredGroup; + } + #checkMaxAge() { if (this.#active === undefined) return; @@ -452,13 +459,16 @@ export class Consumer { // `rs/moq-mux`'s consumer cannot reach either verdict: its read arm returns a buffered // frame, and closes out a spent group as `GroupEnd`, before the budget is consulted. if (first.done && first.frames.length === 0 && paused) { + const marker = !first.empty && !first.media; + const historical = this.#isHistorical(first); + const affectsPlayhead = marker || !historical; this.#groups.shift(); - if (first.consumer.sequence === cursor) { + if (!historical && first.consumer.sequence === cursor) { this.#recordPresented(first); this.#active = continues(first, this.#groups[0]) ? this.#groups[0].consumer.sequence : cursor + 1; } - if (first.truncated) this.#gap = true; - if (!first.empty && !first.media) hole = true; + if (first.truncated && affectsPlayhead) this.#gap = true; + if (marker) hole = true; first.consumer.close(); walked = true; continue; @@ -480,8 +490,13 @@ export class Consumer { break; } + const marker = !first.empty && !first.media; + // Losing content strictly behind media already handed to the caller cannot break or + // rewind the forward playhead. A declared marker still ends its epoch wherever it arrives. + const historical = this.#isHistorical(first); + const affectsPlayhead = marker || !historical; this.#groups.shift(); - this.#active = this.#groups[0]?.consumer.sequence; + if (!historical) this.#active = this.#groups[0]?.consumer.sequence; // Everything the verdict was reached on, since the same line has to answer whether the // group was actually late or merely long: what it still held, how much of that nobody // had read, where delivery stood, whether more was coming, and the three numbers the @@ -498,14 +513,13 @@ export class Consumer { // group that held nothing says nothing about the timeline, so reading the immediate // successor here would call a starved group a hole and re-anchor the reader over media // that turned out to be contiguous. - const marker = !first.empty && !first.media; - if (marker || !ptsContiguous(first.end ?? this.#presentedEnd, reach)) { + if (affectsPlayhead && (marker || !ptsContiguous(first.end ?? this.#presentedEnd, reach))) { hole = true; } first.consumer.close(); first.frames.length = 0; skipped = true; - this.#gap = true; + if (affectsPlayhead) this.#gap = true; // The local half of the same verdict the wire budget reaches, so it lands in the // same counter. #tryDurationSkip does not: it only drops a group the next one // already covers, so nothing is lost there. @@ -605,7 +619,9 @@ export class Consumer { * numbers, which are not required to be sequential: adjacency neither proves the timeline is * unbroken nor catches a group dropped on the way past. * - * It reports what this consumer dropped plus marker groups the publisher declared. + * It reports drops that can affect the forward playhead plus marker groups the publisher + * declared. An undeclared loss strictly behind media already returned only increments + * {@link skipped}. * An unmarked forward timestamp jump still reads as continuous because nothing on the wire says * the missing span will never arrive. * After buffered groups drain, a finished track returns undefined and an aborted track throws. @@ -691,6 +707,7 @@ export class Consumer { const continuous = this.#continuesDelivery(seq); if (seq !== this.#deliveredGroup) this.#gap = false; this.#deliveredGroup = seq; + this.#highestDeliveredGroup = Math.max(this.#highestDeliveredGroup ?? seq, seq); const live = this.#liveEdge; if (live === undefined || frame.timestamp > live.timestamp) { @@ -709,24 +726,28 @@ export class Consumer { // group gains a frame, so waiting here is woken, and #checkMaxAge bounds // how long a stalled head can hold delivery up. if (this.#groups[0].done) { - if (this.#groups[0].consumer.sequence === this.#active) { + const head = this.#groups[0]; + const marker = !head.empty && !head.media; + const historical = this.#isHistorical(head); + const affectsPlayhead = marker || !historical; + if (!historical && head.consumer.sequence === this.#active) { // The cursor moves past this group here rather than in #runGroup's finally // block whenever the group finished before it became active, so this is the // site that has to record its presentation end. Advance by +1 and let the // promotion guard above resolve the real successor on the next iteration. - this.#recordPresented(this.#groups[0]); + this.#recordPresented(head); this.#active += 1; } const group = this.#groups.shift(); if (group) { const seq = group.consumer.sequence; - if (group.truncated) this.#gap = true; + if (group.truncated && affectsPlayhead) this.#gap = true; // A group that carried wire frames but no media is the publisher's declared // break. Raised as the group closes rather than on the marker frame, so the // endpoint still belongs to the run it ends and the reader only re-anchors // once every terminal packet behind the marker has been delivered. - if (!group.empty && !group.media) this.#markPlayhead(); + if (marker) this.#markPlayhead(); this.#updateBuffered(); this.#pulled = Moq.Time.Milli.now(); return { diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 4820e3deba..8c8f44f35e 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -354,10 +354,54 @@ for (const container of ["legacy", "cmaf"] as const) { }); } +test("historical age loss does not erase the live GOP", async () => { + const warn = spyOn(console, "warn").mockImplementation(() => {}); + const fx = fixture(); + const live = new Moq.Group.Producer(10); + const old = new Moq.Group.Producer(5); + const expired = new Moq.Group.Producer(4); + try { + expect(await fx.subscriptions(1)).toBe(1); + fx.sync.track("audio").clock.set({ + timestamp: Time.Micro(10_000_000), + reference: Time.Milli.now(), + rate: 0, + }); + + fx.track.writeGroup(live); + writeTestFrame("legacy", live, Time.Micro(10_000_000), true, 0); + writeTestFrame("legacy", live, Time.Micro(10_033_000), false, 1); + await settle(); + + fx.track.writeGroup(old); + writeTestFrame("legacy", old, Time.Micro(5_000_000), true, 2); + await settle(); + fx.track.writeGroup(expired); + writeTestFrame("legacy", expired, Time.Micro(4_000_000), true, 3); + await settle(); + expired.close(); + old.close(); + + writeTestFrame("legacy", live, Time.Micro(10_066_000), false, 4); + writeTestFrame("legacy", live, Time.Micro(10_100_000), false, 5); + await settle(); + + expect(built[0].timestamps).toEqual([10_000_000, 10_033_000, 10_066_000, 10_100_000]); + expect(built[0].chunks).toEqual(["key", "delta", "delta", "delta"]); + } finally { + expired.close(); + old.close(); + live.close(); + fx.close(); + warn.mockRestore(); + } +}); + test("a declared marker still resets video when its group arrives behind live media", async () => { const fx = fixture(); const live = new Moq.Group.Producer(10); const marker = new Moq.Group.Producer(6); + const old = new Moq.Group.Producer(5); try { expect(await fx.subscriptions(1)).toBe(1); fx.track.writeGroup(live); @@ -368,11 +412,17 @@ test("a declared marker still resets video when its group arrives behind live me fx.track.writeGroup(marker); marker.writeFrame({ payload: Moq.Varint.encode(Time.Micro(5_033_000)), timestamp: Time.Timestamp.now() }); + await settle(); + fx.track.writeGroup(old); + writeTestFrame("legacy", old, Time.Micro(5_000_000), true, 2); + old.close(); + await settle(); marker.close(); await settle(); expect(fx.sync.out.reference.peek()).toBeUndefined(); } finally { + old.close(); live.close(); marker.close(); fx.close(); From 3db9ba704b8a173a44cd7f22b033432f84db18bf Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:05:42 -0400 Subject: [PATCH 043/127] perf(net): skip drained resume anchor work Use the live cursor's expiry once earlier cursors and their outstanding groups and frames have drained. Keep route and cap changes invalidating that identity, and read each live group's latest timestamp once. Add warm-cache fanout benchmarks and held-frame, continuation, and cap coverage. Co-Authored-By: Codex --- rs/moq-net/benches/resume.rs | 135 ++++++++++++++++++++++++- rs/moq-net/src/model/resume.rs | 173 +++++++++++++++++++++++++++++---- rs/moq-net/src/model/track.rs | 12 ++- 3 files changed, 295 insertions(+), 25 deletions(-) diff --git a/rs/moq-net/benches/resume.rs b/rs/moq-net/benches/resume.rs index ad40b788de..f65056162f 100644 --- a/rs/moq-net/benches/resume.rs +++ b/rs/moq-net/benches/resume.rs @@ -5,7 +5,7 @@ use std::task::Poll; use std::time::{Duration, Instant}; use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use moq_net::{Timestamp, broadcast, cache, group, origin, track}; +use moq_net::{Hop, Hops, Timestamp, broadcast, cache, group, origin, track}; const SETUP_TIMEOUT: Duration = Duration::from_secs(10); @@ -133,5 +133,136 @@ fn delivery(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, delivery); +async fn warm_source(groups: u64) -> (Source, origin::Dynamic, broadcast::Dynamic) { + let mut config = origin::Config::new(Hop::new(1).unwrap()); + config.pool = cache::Pool::new(cache::Config::default().with_expiry(None)); + let (origin, driver) = origin::Producer::new(config); + tokio::spawn(moq_net::time::run(driver)); + let consumer = origin.consume(); + let mut hops = Hops::new(); + hops.push(Hop::new(10).unwrap()).unwrap(); + let server = origin + .dynamic("bench/warm", origin::Route::default().with_hops(hops)) + .unwrap(); + let pending = consumer.request_broadcast("bench/warm"); + let upstream = broadcast::Info::new().produce(); + let mut dynamic = upstream.dynamic(); + server.requested_broadcast().await.unwrap().accept(&upstream); + let broadcast = pending.await.unwrap(); + let track = broadcast.track("data").unwrap(); + let subscribing = tokio::spawn({ + let track = track.clone(); + async move { track.subscribe(replay()).await.unwrap() } + }); + let old = dynamic.requested_track().await.unwrap().accept(None); + for sequence in 0..groups { + let mut group = old.create_group(sequence.into()).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence).unwrap(), b"frame".as_ref()) + .unwrap(); + group.finish().unwrap(); + } + let mut reader = subscribing.await.unwrap(); + for sequence in 0..groups { + assert_eq!(reader.recv_group().await.unwrap().unwrap().sequence, sequence); + } + drop(reader); + drop(track); + old.unused().await.unwrap(); + drop(old); + + let track = broadcast.track("data").unwrap(); + let subscribing = tokio::spawn({ + let track = track.clone(); + async move { track.subscribe(replay()).await.unwrap() } + }); + let mut live = dynamic.requested_track().await.unwrap().accept(None); + live.start_at(groups - 1).unwrap(); + for sequence in groups..=2 * groups { + let mut group = live.create_group(sequence.into()).unwrap(); + group + .write_frame(Timestamp::from_millis(sequence).unwrap(), b"frame".as_ref()) + .unwrap(); + group.finish().unwrap(); + } + let mut reader = subscribing.await.unwrap(); + for sequence in 0..=2 * groups { + assert_eq!(reader.recv_group().await.unwrap().unwrap().sequence, sequence); + } + ( + Source { + _origin: origin, + _broadcasts: vec![upstream], + _tracks: vec![live], + _broadcast: broadcast, + _reader: reader, + track, + }, + server, + dynamic, + ) +} + +fn warm_delivery(c: &mut Criterion) { + let mut group = c.benchmark_group("resume_warm"); + for groups in [1, 10, 100] { + for subscribers in [1, 10, 100] { + let id = BenchmarkId::from_parameter(format!("{groups}groups_{subscribers}subscribers")); + group.throughput(Throughput::Elements(groups * subscribers)); + group.bench_function(id, |b| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (source, _server, _dynamic) = runtime.block_on(async { + tokio::time::timeout(SETUP_TIMEOUT, warm_source(groups)) + .await + .expect("warm route setup timed out") + }); + let waiter = kio::Waiter::noop(); + b.iter_custom(|iterations| { + runtime.block_on(async { + let mut elapsed = Duration::ZERO; + for _ in 0..iterations { + let mut cursors = tokio::time::timeout(SETUP_TIMEOUT, async { + let mut cursors = Vec::with_capacity(subscribers as usize); + for _ in 0..subscribers { + let mut cursor = source.track.subscribe(replay()).await.unwrap(); + // Drain the real warm cache and its boundary before measuring live delivery. + for sequence in 0..=groups { + assert_eq!(cursor.recv_group().await.unwrap().unwrap().sequence, sequence); + } + cursors.push(cursor); + } + cursors + }) + .await + .expect("warm subscriptions timed out"); + let start = Instant::now(); + for sequence in groups + 1..=2 * groups { + for cursor in &mut cursors { + let Poll::Ready(Ok(Some(received))) = cursor.poll_recv_group(&waiter) else { + panic!("live cached group must be ready"); + }; + assert_eq!(received.sequence, sequence); + black_box(received); + assert!(cursor.poll_recv_datagram(&waiter).is_pending()); + assert!(cursor.poll_finished(&waiter).is_pending()); + } + } + for cursor in &mut cursors { + assert!(cursor.poll_recv_group(&waiter).is_pending()); + } + elapsed += start.elapsed(); + } + elapsed + }) + }); + }); + } + } + group.finish(); +} + +criterion_group!(benches, delivery, warm_delivery); criterion_main!(benches); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 0059fbf891..3fb58b59c2 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1496,9 +1496,9 @@ pub struct Subscriber { outer: Anchor, /// The logical drift anchor ([`Self::end_sequence`] and [`Self::outer`] combined, /// with the newest edge across the segments), shared with groups that outlive this - /// cursor poll. Reconciled on each poll; a sole unbounded source uses its own edge. + /// cursor poll. Reconciled on each poll while earlier segments still have readers. drift_anchor: kio::Producer, - /// The sole unbounded segment's anchors are clear; its own track supplies expiry. + /// Only the newest segment can still hand out or expire content, using its own edge. identity_anchor: bool, } @@ -1741,28 +1741,33 @@ impl Subscriber { /// The cap is the tightest any reader of this subscriber imposes: its own /// [`Self::end_at`] and whatever a wrapping splice pushed down. The edge is the /// newest across the producer's segments within that cap, or a wrapping splice's if - /// newer. A sole unbounded segment uses its own track's edge; other shapes resolve - /// on every sync, since each segment can grow independently. + /// newer. Once earlier segments and their handed-out groups and frames drain, the + /// newest segment's own edge suffices until the next route or bound change. fn refresh_anchor(&mut self) { - if let [seg] = self.segments.as_mut_slice() - && seg.start.is_none() + if self.identity_anchor { + return; + } + if let Some((seg, before)) = self.segments.split_last_mut() && seg.end.is_none() && self.end_sequence.is_none() && self.outer == Anchor::default() - { - if !self.identity_anchor { - if let Ok(mut current) = self.drift_anchor.write() { - *current = Anchor::default(); - } - seg.anchor = Anchor::default(); - if let Some(sub) = seg.stale_sub_mut() { - sub.set_anchor(Anchor::default()); - } - self.identity_anchor = true; + && ((before.is_empty() && seg.start.is_none()) + || (!self.drift_anchor.is_used() + && before.iter().all(|old| { + matches!(old.sub, SubState::Done(_)) + && old.parked.is_empty() + && old.terminal.as_ref().is_none_or(|sub| !sub.has_expiry_readers()) + }))) { + if let Ok(mut current) = self.drift_anchor.write() { + *current = Anchor::default(); + } + seg.anchor = Anchor::default(); + if let Some(sub) = seg.stale_sub_mut() { + sub.set_anchor(Anchor::default()); } + self.identity_anchor = true; return; } - self.identity_anchor = false; let outer = self.outer.clone().capped(self.end_sequence); let state = self.state.read(); let edge = state @@ -1786,6 +1791,15 @@ impl Subscriber { } } + /// Whether handed-out groups or frames still use this cursor's expiry anchors. + pub(crate) fn has_expiry_readers(&self) -> bool { + self.drift_anchor.is_used() + || self.segments.iter().any(|seg| match &seg.sub { + SubState::Active(sub) => sub.has_expiry_readers(), + _ => seg.terminal.as_ref().is_some_and(track::Subscriber::has_expiry_readers), + }) + } + /// Bound every segment's drift anchor from outside; the spliced arm of /// [`track::Subscriber::set_anchor`], for this subscriber nested as a segment of /// another splice. @@ -2504,6 +2518,10 @@ mod test { let mut producer = Producer::new(); producer.switch(&consumer, None).unwrap(); let mut sub = producer.consume().subscribe(None); + let mut open = track.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(Timestamp::ZERO, b"held".to_vec()).unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut held), b"held"); if outer { sub.set_anchor(Anchor { cap: Some(1), @@ -2512,12 +2530,8 @@ mod test { } else { sub.end_at(..1); } - let mut open = track.create_group(group::Info { sequence: 0 }).unwrap(); - open.write_frame(Timestamp::ZERO, b"held".to_vec()).unwrap(); - let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); write_group_at(&mut track, 1, "next", Duration::from_secs(1)); write_group_at(&mut track, 2, "new", Duration::from_secs(2)); - assert_eq!(read(&mut held), b"held"); assert!(held.read_frame().now_or_never().is_none()); if outer { sub.set_anchor(Anchor::default()); @@ -2528,6 +2542,123 @@ mod test { } } + #[tokio::test] + async fn drained_predecessors_keep_detached_frames_on_the_logical_edge() { + for nested in [false, true] { + let (old, old_consumer) = track_pair("old"); + let mut inner = Producer::new(); + let old_consumer = if nested { + inner.switch(&old_consumer, None).unwrap(); + track::Consumer::spliced("inner".into(), Arc::new(broadcast::Info::default()), inner.consume()) + } else { + old_consumer + }; + let (mut live, live_consumer) = track_pair("live"); + let mut producer = Producer::new(); + producer.switch(&old_consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_millis(100))); + let mut open = old.create_group(group::Info { sequence: 0 }).unwrap(); + let mut writing = open + .create_frame(frame::Info { + size: 6, + timestamp: Timestamp::ZERO, + }) + .unwrap(); + writing.write(b"old".as_ref()).unwrap(); + let mut group = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + let mut frame = group.next_frame().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!( + frame.read_chunk().now_or_never().unwrap().unwrap().unwrap(), + b"old".as_ref() + ); + drop(group); + old.finish().unwrap(); + if nested { + inner.finish().unwrap(); + } + producer.switch(&live_consumer, Position::group(1)).unwrap(); + write_group_at(&mut live, 1, "next", Duration::from_secs(1)); + assert_eq!(recv(&mut sub), 1); + assert!(frame.read_chunk().now_or_never().is_none()); + write_group_at(&mut live, 2, "new", Duration::from_secs(2)); + assert_eq!(recv(&mut sub), 2); + assert!(matches!(frame.read_chunk().now_or_never(), Some(Err(Error::Old)))); + writing.abort(Error::Cancel).unwrap(); + } + } + + #[tokio::test] + async fn drained_predecessors_keep_a_held_continuation_on_the_logical_edge() { + let (first, first_consumer) = track_pair("first"); + let (mut middle, middle_consumer) = track_pair("middle"); + let (mut live, live_consumer) = track_pair("live"); + let mut producer = Producer::new(); + producer.switch(&first_consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_secs(2))); + let mut head = first.create_group(group::Info { sequence: 0 }).unwrap(); + head.write_frame(Timestamp::ZERO, b"head".as_ref()).unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut held), b"head"); + first.finish().unwrap(); + producer + .switch(&middle_consumer, Position { group: 0, frame: 1 }) + .unwrap(); + let mut tail = middle.create_group(group::Info { sequence: 0 }).unwrap(); + tail.write_frame(Timestamp::ZERO, b"duplicate".as_ref()).unwrap(); + tail.write_frame(Timestamp::from_millis(10).unwrap(), b"tail".as_ref()) + .unwrap(); + assert_eq!(read(&mut held), b"tail"); + write_group_at(&mut middle, 1, "middle next", Duration::from_secs(1)); + middle.finish().unwrap(); + producer.switch(&live_consumer, Position::group(2)).unwrap(); + write_group_at(&mut live, 2, "next", Duration::from_secs(2)); + assert_eq!(recv(&mut sub), 1); + assert_eq!(recv(&mut sub), 2); + assert!(held.read_frame().now_or_never().is_none()); + write_group_at(&mut live, 3, "new", Duration::from_secs(3)); + assert_eq!(recv(&mut sub), 3); + assert!(matches!(held.read_frame().now_or_never(), Some(Ok(None)))); + } + + #[tokio::test] + async fn a_new_takeover_restores_expiry_after_predecessors_drain() { + let (mut first, first_consumer) = track_pair("first"); + let (mut middle, middle_consumer) = track_pair("middle"); + let (mut live, live_consumer) = track_pair("live"); + let mut producer = Producer::new(); + producer.switch(&first_consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_millis(100))); + write_group(&mut first, 0, "first"); + assert_eq!(recv(&mut sub), 0); + first.finish().unwrap(); + producer.switch(&middle_consumer, Position::group(1)).unwrap(); + write_group_at(&mut middle, 1, "middle", Duration::from_secs(1)); + assert_eq!(recv(&mut sub), 1); + recv_pending(&mut sub); + sub.start_at(0); + recv_pending(&mut sub); + sub.update(Subscription::default().with_max_age(Duration::from_millis(100))); + recv_pending(&mut sub); + + let mut open = middle.create_group(group::Info { sequence: 2 }).unwrap(); + open.write_frame(Timestamp::from_secs(2).unwrap(), b"held".as_ref()) + .unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut held), b"held"); + write_group_at(&mut middle, 3, "middle next", Duration::from_secs(3)); + producer.switch(&live_consumer, Position::group(4)).unwrap(); + write_group_at(&mut live, 4, "next", Duration::from_secs(4)); + write_group_at(&mut live, 5, "new", Duration::from_secs(5)); + assert_eq!(recv(&mut sub), 4); + assert!(matches!(held.read_frame().now_or_never(), Some(Ok(None)))); + } + /// A waker that counts its wakes, for asserting a pending poll left a live /// registration behind. struct CountWaker(std::sync::atomic::AtomicUsize); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index ba83c78728..2deedb7380 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -488,11 +488,11 @@ impl TrackState { } // The map is ordered by sequence, so the first stamped group from the // back is the newest content that exists. - let timestamp = slot.group.timestamp()?; + let timestamp = slot.group.latest()?; Some(PresentationEdge { sequence: slot.group.sequence, stamp: slot.stamp, - timestamp: slot.group.latest().unwrap_or(timestamp), + timestamp, }) }) } @@ -3659,6 +3659,14 @@ impl Control { } impl Subscriber { + /// Whether handed-out groups or frames still use this cursor's expiry anchors. + pub(crate) fn has_expiry_readers(&self) -> bool { + match &self.inner { + SubscriberKind::Plain(plain) => plain.drift_anchor.is_used(), + SubscriberKind::Spliced(spliced) => spliced.has_expiry_readers(), + } + } + /// The track's [`Info`], resolved when the subscription was established. /// /// Free, unlike [`Consumer::query`]: subscribing already waited for the info From 84e98b3a8f4d8725a4f61c024db70348e22a0a79 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:07:09 -0400 Subject: [PATCH 044/127] test(net): track live fanout cost in nightly Run the existing subscriber sweep alongside the resumed-track benchmark and document both commands. Co-Authored-By: GPT-6 --- .github/workflows/nightly.yml | 6 ++++++ bench/README.md | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 244c4449d3..365104630c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -102,6 +102,12 @@ jobs: nix develop --command cargo bench --locked -p moq-net --bench resume -- --warm-up-time 1 --measurement-time 2 --sample-size 10 --noplot + - name: Track fanout benchmark + if: ${{ !cancelled() }} + run: >- + nix develop --command cargo bench --locked -p moq-net --bench track -- + track_fanout_group --warm-up-time 1 --measurement-time 2 --sample-size 10 --noplot + - name: Audio playout benchmark if: ${{ !cancelled() }} run: >- diff --git a/bench/README.md b/bench/README.md index 7916285cf7..ab1275cdfe 100644 --- a/bench/README.md +++ b/bench/README.md @@ -38,6 +38,16 @@ nix develop --command cargo bench --locked -p moq-net --bench resume The cases sweep 1, 10, and 100 cached groups per route and concurrent subscribers. Route setup and subscription creation are excluded from the delivery measurement. +The warm-route cases drain a finished cached segment before measuring its live +successor. + +Measure live group delivery to 1, 8, 64, and 512 subscribers: + +```bash +nix develop --command cargo bench --locked -p moq-net --bench track -- track_fanout_group +``` + +Both resumed-track and live group delivery benchmarks run in Nightly. Compare one multi-threaded Tokio runtime with the same number of independent Tokio/epoll and io\_uring workers: From 33374f1e9fcf63f413a354539c350a3cfcce7653 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:05:28 -0400 Subject: [PATCH 045/127] fix(publish): disconnect each capture edge once Let the scoped worklet connection own its cleanup. Check publisher errors when the media test stops or republishes a fixture. Co-Authored-By: GPT-6 --- js/publish/src/audio/capture.test.ts | 30 +++++++++++++++++++++++++++- js/publish/src/audio/capture.ts | 1 - test/interop/clients/js/media.ts | 3 +++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/js/publish/src/audio/capture.test.ts b/js/publish/src/audio/capture.test.ts index cd7d9a79c0..1f8880fa08 100644 --- a/js/publish/src/audio/capture.test.ts +++ b/js/publish/src/audio/capture.test.ts @@ -529,7 +529,9 @@ function installGatedWebAudio() { } disconnect(node?: unknown): void { if (node === undefined) this.outputs.clear(); - else this.outputs.delete(node); + else if (!this.outputs.delete(node)) { + throw new DOMException("The destination is not connected", "InvalidAccessError"); + } } } @@ -603,6 +605,32 @@ test("captures once a gesture resumes a context built before one", async () => { } }); +test("closes an active capture without disconnecting an edge twice", async () => { + using webaudio = installGatedWebAudio(); + const errors = spyOn(console, "error").mockImplementation(() => {}); + const capture = new Capture({ enabled: true, source: new Signal(fakeSource()) as never }); + + try { + await settle(); + webaudio.gesture(); + await settle(); + webaudio.worklets[0].render(); + expect(capture.out.format.peek()).toEqual({ sampleRate: 48_000, channelCount: 1 }); + expect(webaudio.roots[0].outputs.size).toBe(1); + + capture.close(); + await settle(); + + expect(webaudio.roots[0].outputs.size).toBe(0); + expect(capture.out.frames.peek()).toBeUndefined(); + expect(capture.out.format.peek()).toBeUndefined(); + expect(errors).not.toHaveBeenCalled(); + } finally { + capture.close(); + errors.mockRestore(); + } +}); + // A suspended graph carries nothing, so an interrupted context (Safari, on a phone call) must not // leave the format behind for the encoder to keep advertising. test("drops the format while the context is interrupted", async () => { diff --git a/js/publish/src/audio/capture.ts b/js/publish/src/audio/capture.ts index 3a0c33cc0a..c4885467ea 100644 --- a/js/publish/src/audio/capture.ts +++ b/js/publish/src/audio/capture.ts @@ -160,7 +160,6 @@ export class Capture { const root = new MediaStreamAudioSourceNode(context, { mediaStream: new MediaStream([source.track]), }); - effect.cleanup(() => root.disconnect()); const loaded = new Signal(false); diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index 7f172510bf..a2478de489 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -672,6 +672,7 @@ try { console.error("=== stop and republish ==="); const before = await readPlayerState(player); await command(publisher, "stop"); + throwPageErrors(publisherErrors); await waitFrozen( player, playerErrors, @@ -722,6 +723,8 @@ try { } if (wants("capture-denial")) await captureDenial(`${broadcast}-capture.hang`); + await command(publisher, "stop"); + throwPageErrors(publisherErrors); } catch (err) { failure = err instanceof Error ? err : new Error(String(err)); } From d23787ae91d86e151b24e3e2805b5cabaa077795 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:24:34 -0400 Subject: [PATCH 046/127] fix(publish): stop retired capture worklets Co-Authored-By: GPT-5.6 Sol --- .../src/audio/capture-worklet.port.test.ts | 64 +++++++++++++++++++ js/publish/src/audio/capture-worklet.ts | 18 ++++++ js/publish/src/audio/capture.test.ts | 8 ++- js/publish/src/audio/capture.ts | 9 ++- 4 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 js/publish/src/audio/capture-worklet.port.test.ts diff --git a/js/publish/src/audio/capture-worklet.port.test.ts b/js/publish/src/audio/capture-worklet.port.test.ts new file mode 100644 index 0000000000..7732690645 --- /dev/null +++ b/js/publish/src/audio/capture-worklet.port.test.ts @@ -0,0 +1,64 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import type { Quantum } from "./capture-worklet"; + +const QUANTUM = 128; + +interface Processor { + process(inputs: Float32Array[][]): boolean; +} + +type ProcessorClass = new () => Processor; + +const scope = globalThis as unknown as Record; +const saved = new Map(); +let Capture: ProcessorClass | undefined; +let nextPort: MessagePort | undefined; + +beforeAll(async () => { + for (const name of ["AudioWorkletProcessor", "registerProcessor", "currentFrame"]) { + saved.set(name, scope[name]); + } + scope.AudioWorkletProcessor = class { + readonly port = nextPort; + }; + scope.registerProcessor = (name: string, processor: ProcessorClass) => { + if (name === "capture") Capture = processor; + }; + scope.currentFrame = 0; + const worklet = "./capture-worklet.ts?ports"; + await import(worklet); +}); + +afterAll(() => { + for (const [name, value] of saved) { + if (value === undefined) delete scope[name]; + else scope[name] = value; + } +}); + +function settle(): Promise { + return new Promise((resolve) => setTimeout(resolve, 20)); +} + +test("lets its processor end once the capture node is closed", async () => { + if (!Capture) throw new Error("capture-worklet.ts registered no 'capture' processor"); + + const node = new MessageChannel(); + nextPort = node.port1; + const capture = new Capture(); + const messages: Quantum[] = []; + node.port2.onmessage = (event: MessageEvent) => messages.push(event.data); + + expect(capture.process([[new Float32Array(QUANTUM)]])).toBe(true); + await settle(); + expect(messages).toHaveLength(1); + + node.port2.postMessage({ type: "close" }); + await settle(); + scope.currentFrame = QUANTUM; + expect(capture.process([[new Float32Array(QUANTUM)]])).toBe(false); + await settle(); + expect(messages).toHaveLength(1); + + node.port2.close(); +}); diff --git a/js/publish/src/audio/capture-worklet.ts b/js/publish/src/audio/capture-worklet.ts index 508f827624..e9000a78d2 100644 --- a/js/publish/src/audio/capture-worklet.ts +++ b/js/publish/src/audio/capture-worklet.ts @@ -12,8 +12,26 @@ export interface Quantum { channels: Float32Array[]; } +/** Stops a retired capture processor. */ +export interface Close { + type: "close"; +} + class Capture extends AudioWorkletProcessor { + #closed = false; + + constructor() { + super(); + this.port.onmessage = (event: MessageEvent) => { + if (event.data.type !== "close") return; + this.#closed = true; + this.port.onmessage = null; + this.port.close(); + }; + } + process(input: Float32Array[][]) { + if (this.#closed) return false; if (input.length > 1) throw new Error("only one input is supported."); const channels = input[0]; diff --git a/js/publish/src/audio/capture.test.ts b/js/publish/src/audio/capture.test.ts index 1f8880fa08..38959c96f7 100644 --- a/js/publish/src/audio/capture.test.ts +++ b/js/publish/src/audio/capture.test.ts @@ -504,7 +504,11 @@ function installGatedWebAudio() { } class GatedWorklet { - port = Object.assign(new EventTarget(), { start: () => {} }); + messages: unknown[] = []; + port = Object.assign(new EventTarget(), { + start: () => {}, + postMessage: (message: unknown) => this.messages.push(message), + }); constructor(_context: unknown, _name: string) { worklets.push(this); } @@ -622,6 +626,7 @@ test("closes an active capture without disconnecting an edge twice", async () => await settle(); expect(webaudio.roots[0].outputs.size).toBe(0); + expect(webaudio.worklets[0].messages).toEqual([{ type: "close" }]); expect(capture.out.frames.peek()).toBeUndefined(); expect(capture.out.format.peek()).toBeUndefined(); expect(errors).not.toHaveBeenCalled(); @@ -649,6 +654,7 @@ test("drops the format while the context is interrupted", async () => { expect(capture.out.frames.peek()).toBeUndefined(); // The retired worklet is cut from the source, or it keeps posting alongside its replacement. expect(webaudio.roots[0].outputs.size).toBe(0); + expect(webaudio.worklets[0].messages).toEqual([{ type: "close" }]); // Back to running rebuilds the worklet on a fresh anchor. webaudio.contexts[0].transition("running"); diff --git a/js/publish/src/audio/capture.ts b/js/publish/src/audio/capture.ts index c4885467ea..a92336aff1 100644 --- a/js/publish/src/audio/capture.ts +++ b/js/publish/src/audio/capture.ts @@ -3,7 +3,7 @@ import { Time } from "@moq/net"; import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Signal } from "@moq/signals"; // Compiled and inlined as a blob URL via vite-plugin-worklet. import { Fanout } from "../fanout"; -import type { Quantum } from "./capture-worklet"; +import type { Close, Quantum } from "./capture-worklet"; import CaptureWorklet from "./capture-worklet.ts?worklet"; import { isSampleSource, normalizeSource, type SampleSource, type Source, type SourceConfig } from "./types"; @@ -190,9 +190,12 @@ export class Capture { // path on macOS. Only force it when we actually have a requested count to honor. channelCountMode: requestedChannels !== undefined ? "explicit" : "max", }); - // The edge originates at root, so only root can remove it; the worklet has no outputs. root.connect(worklet); - inner.cleanup(() => root.disconnect(worklet)); + inner.cleanup(() => { + const close: Close = { type: "close" }; + worklet.port.postMessage(close); + root.disconnect(worklet); + }); const fanout = new Fanout(this.#drain(worklet, context, inner), { queue: QUEUE }); inner.cleanup(() => fanout.close()); From ac22867ea29188bf0649e6830fcb54cd2c11c189 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:12:46 -0400 Subject: [PATCH 047/127] test(tokio): cover open warm edge rejoin Co-Authored-By: GPT-5.6 Sol --- rs/moq-tokio/tests/broadcast.rs | 48 ++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 69140e2d87..554b87d8b9 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -1015,11 +1015,13 @@ async fn broadcast_rejoin_skips_a_stale_warm_cache() { "moq-lite-05", ]; for version in moq_net::Version::names().filter(|version| !pre06.contains(version)) { - rejoin_skips_a_stale_warm_cache(version).await; + for open in [false, true] { + rejoin_skips_a_stale_warm_cache(version, open).await; + } } } -async fn rejoin_skips_a_stale_warm_cache(version: &str) { +async fn rejoin_skips_a_stale_warm_cache(version: &str, open: bool) { use moq_net::Timestamp; let ms = |ms: u64| Timestamp::from_millis(ms).unwrap(); @@ -1036,9 +1038,19 @@ async fn rejoin_skips_a_stale_warm_cache(version: &str) { broadcast.announce(Default::default()).expect("announce"); let track = broadcast.create_track("audio", None).expect("create track"); let live = track.clone(); - for sequence in 0..4u64 { + for sequence in 0..3u64 { write(&track, sequence, sequence * 20); } + // The edge can be complete or still open when the front parks it. The latter is + // the stale owner that used to survive beside the replacement feed and hold an + // ordered consumer on a group no source would ever finish. + let mut edge = track + .create_group(moq_net::group::Info { sequence: 3 }) + .expect("create edge group"); + edge.write_frame(ms(60), b"edge".as_ref()).expect("write edge frame"); + if !open { + edge.finish().expect("finish edge group"); + } let mut config = moq_tokio::listen::Config::default(); config.bind = Some("[::]:0".parse().unwrap()); @@ -1079,13 +1091,12 @@ async fn rejoin_skips_a_stale_warm_cache(version: &str) { .expect("request timeout") .expect("broadcast resolves"); let budget = moq_net::track::Subscription::default().with_max_age(Duration::from_millis(100)); - async fn recv(sub: &mut moq_net::track::Subscriber) -> u64 { + async fn recv(sub: &mut moq_net::track::Subscriber) -> moq_net::group::Consumer { tokio::time::timeout(TIMEOUT, sub.recv_group()) .await .expect("recv timeout") .expect("recv failed") .expect("track ended") - .sequence } let mut sub = remote @@ -1094,7 +1105,19 @@ async fn rejoin_skips_a_stale_warm_cache(version: &str) { .subscribe(budget.clone()) .await .expect("subscribe"); - recv(&mut sub).await; + loop { + let mut cached = recv(&mut sub).await; + if cached.sequence != 3 { + continue; + } + let frame = tokio::time::timeout(TIMEOUT, cached.read_frame()) + .await + .expect("edge frame timeout") + .expect("edge frame failed") + .expect("edge group ended before its frame"); + assert_eq!(&frame.payload[..], b"edge"); + break; + } drop(sub); // The front parks the track and cancels upstream, while the publisher moves on. @@ -1112,21 +1135,26 @@ async fn rejoin_skips_a_stale_warm_cache(version: &str) { .subscribe(budget) .await .expect("resubscribe"); - let first = recv(&mut sub).await; + let first = recv(&mut sub).await.sequence; assert!( first > 4, - "{version}: a rejoining reader was served the stale cache first: group {first}" + "{version} open={open}: a rejoining reader was served the stale cache first: group {first}" ); let mut sequence = first; while sequence < 20 { - sequence = recv(&mut sub).await; + sequence = recv(&mut sub).await.sequence; assert!( sequence >= 4, - "{version}: a rejoining reader was served stale group {sequence}" + "{version} open={open}: a rejoining reader was served stale group {sequence}" ); } drop(sub); + if open { + // The publisher retained this handle only to keep the original edge open while + // the relay exercised its own warm-copy ownership. It may already be stale. + let _ = edge.finish(); + } drop(session); server.await.expect("server panicked").expect("server failed"); } From 57bf82416f4ad737ad59f5a3bdbbbc777aaf7394 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:40:19 -0400 Subject: [PATCH 048/127] test(publish): await capture messages without sleeps Co-Authored-By: GPT-6 --- .../src/audio/capture-worklet.port.test.ts | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/js/publish/src/audio/capture-worklet.port.test.ts b/js/publish/src/audio/capture-worklet.port.test.ts index 7732690645..c4ff59ff5f 100644 --- a/js/publish/src/audio/capture-worklet.port.test.ts +++ b/js/publish/src/audio/capture-worklet.port.test.ts @@ -36,29 +36,33 @@ afterAll(() => { } }); -function settle(): Promise { - return new Promise((resolve) => setTimeout(resolve, 20)); -} - test("lets its processor end once the capture node is closed", async () => { if (!Capture) throw new Error("capture-worklet.ts registered no 'capture' processor"); const node = new MessageChannel(); nextPort = node.port1; const capture = new Capture(); - const messages: Quantum[] = []; - node.port2.onmessage = (event: MessageEvent) => messages.push(event.data); - - expect(capture.process([[new Float32Array(QUANTUM)]])).toBe(true); - await settle(); - expect(messages).toHaveLength(1); + try { + const quantum = new Promise((resolve) => { + node.port2.onmessage = (event: MessageEvent) => resolve(event.data); + }); + expect(capture.process([[new Float32Array(QUANTUM)]])).toBe(true); + expect(await quantum).toEqual({ frame: 0, channels: [new Float32Array(QUANTUM)] }); - node.port2.postMessage({ type: "close" }); - await settle(); - scope.currentFrame = QUANTUM; - expect(capture.process([[new Float32Array(QUANTUM)]])).toBe(false); - await settle(); - expect(messages).toHaveLength(1); - - node.port2.close(); + const receive = node.port1.onmessage; + if (!receive) throw new Error("capture registered no message handler"); + const closed = new Promise((resolve) => { + node.port1.onmessage = (event) => { + receive.call(node.port1, event); + resolve(); + }; + }); + node.port2.postMessage({ type: "close" }); + await closed; + scope.currentFrame = QUANTUM; + expect(capture.process([[new Float32Array(QUANTUM)]])).toBe(false); + } finally { + node.port1.close(); + node.port2.close(); + } }); From fbec46ae745960856ca3de402e04171e2e9e5814 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:40:19 -0400 Subject: [PATCH 049/127] test(tokio): release stale edge ownership directly Co-Authored-By: GPT-6 --- rs/moq-tokio/tests/broadcast.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/rs/moq-tokio/tests/broadcast.rs b/rs/moq-tokio/tests/broadcast.rs index 554b87d8b9..6dd04688a9 100644 --- a/rs/moq-tokio/tests/broadcast.rs +++ b/rs/moq-tokio/tests/broadcast.rs @@ -1041,9 +1041,6 @@ async fn rejoin_skips_a_stale_warm_cache(version: &str, open: bool) { for sequence in 0..3u64 { write(&track, sequence, sequence * 20); } - // The edge can be complete or still open when the front parks it. The latter is - // the stale owner that used to survive beside the replacement feed and hold an - // ordered consumer on a group no source would ever finish. let mut edge = track .create_group(moq_net::group::Info { sequence: 3 }) .expect("create edge group"); @@ -1150,11 +1147,7 @@ async fn rejoin_skips_a_stale_warm_cache(version: &str, open: bool) { } drop(sub); - if open { - // The publisher retained this handle only to keep the original edge open while - // the relay exercised its own warm-copy ownership. It may already be stale. - let _ = edge.finish(); - } + drop(edge); drop(session); server.await.expect("server panicked").expect("server failed"); } From 5545dfd21a26ddbf71b730092ee825b4ced36184 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:38:10 -0400 Subject: [PATCH 050/127] fix(bench): send QUIC closes before exiting Stop connection tasks and await Client::close before the runtime exits. Preserve the stop result through cleanup so report failures still fail the run. Prove the actual CLI sends clean closes to all eight peers before it exits. Co-Authored-By: Codex --- rs/moq-bench/src/main.rs | 22 +++++++--- rs/moq-bench/tests/shutdown.rs | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 rs/moq-bench/tests/shutdown.rs diff --git a/rs/moq-bench/src/main.rs b/rs/moq-bench/src/main.rs index 02111a2073..433427934a 100644 --- a/rs/moq-bench/src/main.rs +++ b/rs/moq-bench/src/main.rs @@ -108,14 +108,24 @@ async fn main() -> anyhow::Result<()> { let drained = async { while tasks.join_next().await.is_some() {} }; - tokio::select! { - _ = stop => tracing::info!("duration elapsed, stopping"), - _ = tokio::signal::ctrl_c() => tracing::info!("interrupted, stopping"), - _ = drained => anyhow::bail!("all benchmark connections ended"), + let stopped = tokio::select! { + _ = stop => { + tracing::info!("duration elapsed, stopping"); + Ok(()) + }, + _ = tokio::signal::ctrl_c() => { + tracing::info!("interrupted, stopping"); + Ok(()) + }, + _ = drained => Err(anyhow::anyhow!("all benchmark connections ended")), // The reporter only returns on a stats-output failure; die loudly rather // than exit green with a partial JSONL file. - res = &mut reporter => res??, - } + res = &mut reporter => res.map_err(anyhow::Error::from).flatten(), + }; + + tasks.shutdown().await; + client.close().await; + stopped?; stats.ensure_delivery(config.expects_delivery()) } diff --git a/rs/moq-bench/tests/shutdown.rs b/rs/moq-bench/tests/shutdown.rs new file mode 100644 index 0000000000..37c2bb1acb --- /dev/null +++ b/rs/moq-bench/tests/shutdown.rs @@ -0,0 +1,74 @@ +#![cfg(feature = "noq")] + +use std::process::Stdio; +use std::time::Duration; + +use moq_net::Error; + +#[tokio::test] +async fn duration_sends_close_before_process_exit() { + moq_tokio::crypto::install_default().unwrap(); + let mut listen = moq_tokio::listen::Config::default(); + listen.bind = Some("127.0.0.1:0".parse().unwrap()); + listen.tls.generate = vec!["localhost".into()]; + let mut server = listen.init(Default::default()).unwrap().listen().await.unwrap(); + let url = format!("moqt://127.0.0.1:{}", server.local_addr().unwrap().port()); + + // The child owns a separate runtime, so exiting before queued closes are sent is visible. + let child = tokio::process::Command::new(env!("CARGO_BIN_EXE_moq-bench")) + .args([ + "--connect", + &url, + "--connect-tls-insecure", + "--connect-tls-host-name", + "localhost", + "--connections", + "8", + "--broadcasts", + "0", + "--subscribe", + "0", + "--startup", + "0s", + "--duration", + "1s", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let sessions = tokio::time::timeout_at(deadline, async { + let mut handshakes = tokio::task::JoinSet::new(); + for _ in 0..8 { + let request = server.accept().await.expect("benchmark connection"); + handshakes.spawn(async move { request.ok().await.unwrap() }); + } + let mut sessions = Vec::new(); + while let Some(session) = handshakes.join_next().await { + sessions.push(session.unwrap()); + } + sessions + }) + .await + .expect("benchmark did not connect all eight peers"); + let output = tokio::time::timeout_at(deadline, child.wait_with_output()) + .await + .expect("benchmark did not exit") + .unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + for session in sessions { + let closed = tokio::time::timeout_at(deadline, session.closed()) + .await + .expect("benchmark peer never received close"); + assert!( + matches!(&closed, Error::Transport(reason) if matches!(reason.as_str(), + "connection error: closed by peer: dropped (code 0)" + | "connection error: closed by peer: client shutdown (code 0)" + )), + "{closed:?}" + ); + } +} From b0578b09dace81bd6345beb53ee50b908e40610f Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:24:11 -0400 Subject: [PATCH 051/127] fix(audio): count playback skips at the ring Read explicit ring counters in browser quality tests instead of inferring skips from asynchronous timestamp reports. Keep startup trims and late input discards separate, and reject measured windows that change graph or timeline. Co-Authored-By: GPT-6 --- js/watch/src/audio/counters.test.ts | 93 ++++++++++++++ js/watch/src/audio/playout/index.ts | 6 +- js/watch/src/audio/ring-buffer.ts | 17 ++- js/watch/src/audio/shared-ring-buffer.ts | 41 +++++-- test/audio-quality/README.md | 38 +++--- test/audio-quality/clients/js/analyze.ts | 116 ++++++++---------- .../clients/js/src/analyze.test.ts | 110 +++++++++++++++++ .../clients/js/src/probe.test.ts | 28 +++++ test/audio-quality/clients/js/src/probe.ts | 7 +- test/audio-quality/clients/js/src/schema.ts | 18 +-- 10 files changed, 365 insertions(+), 109 deletions(-) create mode 100644 js/watch/src/audio/counters.test.ts create mode 100644 test/audio-quality/clients/js/src/analyze.test.ts diff --git a/js/watch/src/audio/counters.test.ts b/js/watch/src/audio/counters.test.ts new file mode 100644 index 0000000000..e662391542 --- /dev/null +++ b/js/watch/src/audio/counters.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, test } from "bun:test"; +import { Time } from "@moq/net"; +import { AudioRingBuffer } from "./ring-buffer"; +import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; + +for (const shared of [false, true]) { + describe(`${shared ? "shared" : "message"} ring accounting`, () => { + const create = (buffered = false) => { + if (!shared) return new AudioRingBuffer({ rate: 1000, channels: 1, latency: Time.Milli(100), buffered }); + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, 275, 1000, buffered)); + ring.setLatency(100); + return ring; + }; + const write = (ring: AudioRingBuffer | SharedRingBuffer, at: number) => { + const pcm = [new Float32Array(50).fill(0.5)]; + if (ring instanceof AudioRingBuffer) ring.write(Time.Micro(at * 1000), pcm); + else ring.insert(Time.Micro(at * 1000), pcm); + }; + + test("overflow advances the playing timeline and counts the exact skipped media", () => { + const ring = create(); + for (let at = 0; at < 150; at += 50) write(ring, at); + ring.read([new Float32Array(20)]); + for (let at = 150; at < 600; at += 50) write(ring, at); + const debug = ring.debug(); + expect(debug.skips).toBeGreaterThan(0); + expect(debug.skipped).toBe(shared ? 68 : 380); + expect(debug.discarded).toBe(0); + expect(ring.timestamp).toBe(Time.Micro((shared ? 88 : 400) * 1000)); + const pcm = [new Float32Array(20)]; + expect(ring.read(pcm)).toBe(20); + expect(Array.from(pcm[0])).toEqual(Array(20).fill(0.5)); + }); + + test("repeated overflow before any read remains startup trimming", () => { + const ring = create(true); + for (let at = 0; at < 1200; at += 50) write(ring, at); + expect(ring.debug().skips).toBe(0); + expect(ring.debug().skipped).toBe(0); + expect(ring.debug().trimmed).toBe(shared ? 688 : 1000); + }); + + test("late duplicate writes discard input without skipping playback", () => { + const ring = create(); + for (let at = 0; at < 150; at += 50) write(ring, at); + ring.read([new Float32Array(100)]); + write(ring, 0); + expect(ring.debug().discarded).toBe(50); + expect(ring.debug().skips).toBe(0); + expect(ring.debug().skipped).toBe(0); + expect(ring.timestamp).toBe(Time.Micro(100_000)); + }); + }); +} + +test("shrinking a playing message ring counts the dropped media", () => { + const ring = new AudioRingBuffer({ rate: 1000, channels: 1, latency: Time.Milli(400) }); + for (let at = 0; at < 800; at += 100) ring.write(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); + ring.read([new Float32Array(20)]); + ring.resize(Time.Milli(100)); + expect(ring.debug().skips).toBe(1); + expect(ring.debug().skipped).toBe(205); + expect(ring.timestamp).toBe(Time.Micro(525_000)); +}); + +test("shrinking a playing shared ring counts the dropped media", () => { + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, 1024, 1000)); + ring.setLatency(400); + for (let at = 0; at < 500; at += 100) ring.insert(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); + ring.read([new Float32Array(20)]); + for (let at = 500; at < 1000; at += 100) ring.insert(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); + const resized = ring.resize(275); + expect(resized.debug().skips).toBe(1); + expect(resized.debug().skipped).toBe(468); + expect(resized.timestamp).toBe(Time.Micro(488_000)); +}); + +test("shrinking before playback trims instead of skipping either timeline", () => { + const post = new AudioRingBuffer({ rate: 1000, channels: 1, latency: Time.Milli(400) }); + for (let at = 0; at < 500; at += 100) post.write(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); + post.resize(Time.Milli(100)); + expect(post.debug().trimmed).toBe(225); + expect(post.debug().skipped).toBe(0); + const shared = new SharedRingBuffer(allocSharedRingBuffer(1, 1024, 1000, true)); + shared.setLatency(400); + for (let at = 0; at < 800; at += 100) shared.insert(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); + const resized = shared.resize(128); + expect(resized.debug().trimmed).toBe(672); + expect(resized.debug().skipped).toBe(0); + resized.insert(Time.Micro(800_000), [new Float32Array(100).fill(0.5)]); + expect(resized.debug().skipped).toBe(0); + expect(resized.debug().trimmed).toBe(772); +}); diff --git a/js/watch/src/audio/playout/index.ts b/js/watch/src/audio/playout/index.ts index 41e54fd99a..1a66843e2c 100644 --- a/js/watch/src/audio/playout/index.ts +++ b/js/watch/src/audio/playout/index.ts @@ -136,11 +136,11 @@ export interface Snapshot extends Counters { stalled: boolean; /** Times the reader ran dry mid-playback. */ underruns: number; - /** Times the reader skipped ahead because the ring sat past the band. */ + /** Times playback jumped over media after the timeline started. */ skips: number; - /** Samples those skips threw away. */ + /** Media samples those jumps passed over. */ skipped: number; - /** Samples the writer dropped: too old for the playhead, or past the ring's capacity. */ + /** Incoming samples the writer rejected because they were behind the playhead. */ discarded: number; /** * Samples the writer dropped off the first fill on a timeline, before anything had been played. diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index b80f54a16c..06ec75b98f 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -292,7 +292,20 @@ export class AudioRingBuffer implements RingReader { // Samples left behind are media the reader skips, which is a new timeline. A copy that kept // them all, an empty ring included, is the one it was playing: a new generation there would // make it forget what it learned about the stream on every step of the target. - if (dropped > 0) this.#generation++; + if (dropped > 0) { + this.#generation++; + this.#drop(dropped); + } + } + + #drop(samples: number): void { + if (samples === 0) return; + if (this.#fresh) { + this.#trimmed += samples; + } else { + this.#skips++; + this.#skipped += samples; + } } write(timestamp: Time.Micro, data: Float32Array[]): void { @@ -398,7 +411,7 @@ export class AudioRingBuffer implements RingReader { if (surplus || depth > this.capacity) { const to = end - (surplus ? Math.min(hold, this.capacity) : this.capacity); const dropped = Math.max(0, to - this.#readIndex); - this.#discarded += dropped; + this.#drop(dropped); this.#jumped += dropped; this.#readIndex = to; } diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index 8f917740b8..6c77b99166 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -46,10 +46,10 @@ const OUTPUT = 9; const ACCELERATES = 10; const EXPANDS = 11; const SHORT = 12; -// Skip-aheads, and the samples they threw away. Reader only. +// Playback jumps and the media samples they passed over. Both ends increment atomically. const SKIPS = 13; const SKIPPED = 14; -// Samples the writer dropped: too old for the playhead, or past the ring's capacity. Writer only. +// Incoming samples already behind the playhead. Writer only. const DISCARDED = 15; // Frames of OUTPUT the reader synthesized to cover a gap, so they carried no media. Reader only. const CONCEALED = 16; @@ -194,6 +194,9 @@ export class SharedRingBuffer implements RingReader { // from it. While it stands, nothing on this timeline has been heard. Writer only: the reader's own // advances are visible as READ moving past it. See {@link #trim}. #resumed: number | undefined; + // Writer-only diagnostic cursor. Unlike #resumed, follows overflow too, so repeated drops + // before the first reader advance remain startup trims without changing playout's trim policy. + #unplayed: number | undefined; // What the last `view` sampled: the packed word its exchange has to match, and the cursor the // skip-ahead left it on. Reader thread only, since only the worklet reads. @@ -302,13 +305,18 @@ export class SharedRingBuffer implements RingReader { * overflow path; the reader publishes with its own exchange so it can tell a rebase apart * from losing a race. */ - #advance(candidate: number): void { + #advance(candidate: number): number { for (;;) { const state = Atomics.load(this.#state, 0); - if (((candidate - readOf(state)) | 0) <= 0) return; + if (this.#unplayed !== readOf(state)) this.#unplayed = undefined; + const advanced = (candidate - readOf(state)) | 0; + if (advanced <= 0) return 0; const next = pack(epochOf(state), candidate); - if (Atomics.compareExchange(this.#state, 0, state, next) === state) return; + if (Atomics.compareExchange(this.#state, 0, state, next) === state) { + if (this.#unplayed !== undefined) this.#unplayed = candidate; + return advanced; + } } } @@ -359,6 +367,7 @@ export class SharedRingBuffer implements RingReader { this.#anchored = true; // Nothing on this timeline has been played, so the fill is still free to trim. this.#resumed = 0; + this.#unplayed = 0; this.#position = 0; this.#lastRead = 0; this.#lastMedia = Number.NEGATIVE_INFINITY; @@ -429,8 +438,15 @@ export class SharedRingBuffer implements RingReader { const bounded = readOf(Atomics.load(this.#state, 0)); if (((end - bounded) | 0) > this.capacity) { const to = (end - this.capacity) | 0; - Atomics.add(this.#control, DISCARDED, (to - bounded) | 0); - this.#advance(to); + const dropped = this.#advance(to); + if (dropped > 0) { + if (this.#unplayed !== undefined) { + Atomics.add(this.#control, TRIMMED, dropped); + } else { + Atomics.add(this.#control, SKIPS, 1); + Atomics.add(this.#control, SKIPPED, dropped); + } + } } // Write sample data @@ -811,6 +827,7 @@ export class SharedRingBuffer implements RingReader { } } + dst.#unplayed = this.#unplayed === read ? copyStart : undefined; Atomics.store(dst.#control, TIMELINE, Atomics.load(this.#control, TIMELINE)); Atomics.store(dst.#state, 0, pack(epochOf(state), copyStart)); Atomics.store(dst.#control, WRITE, write); @@ -837,6 +854,16 @@ export class SharedRingBuffer implements RingReader { Atomics.store(dst.#control, control, Atomics.load(this.#control, control)); } + const dropped = available - copyCount; + if (dropped > 0) { + if (this.#unplayed === read) { + Atomics.add(dst.#control, TRIMMED, dropped); + } else { + Atomics.add(dst.#control, SKIPS, 1); + Atomics.add(dst.#control, SKIPPED, dropped); + } + } + // Carry the unwrapped playhead over, rebased onto dst's READ. Fold the same `read` // snapshot the copy used so both sides agree on one observation; `copyStart` is at or // ahead of it whenever the copy dropped the oldest samples. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index ea2ebea989..e81c7761ce 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -219,9 +219,8 @@ that hears the pause as a gap grades it as thrown-away audio. What it cannot say is anything about the transport, the container consumer, the device, or the wall clock, because there is no session and no audio hardware. Those metrics report null. What it can say -exactly, and the browser lanes cannot, is what the ring and the engine did: `short_quanta`, -`discarded_samples`, `accelerates`, `expands` and `stretched_samples` are read straight off the -counters rather than inferred from a 250 ms sampling grid. +exactly is what the ring and the engine did. The browser probe reads the same counters through +`audio.out.debug`; replay does not depend on asynchronous reports or device scheduling. ## The metric schema @@ -277,8 +276,9 @@ definition is a judgement call are: - **`stalled_quanta`** is the share of the run the ring spent re-stalled, refilling rather than playing. Graded separately from underruns: it is silence the player chose. -- **`skip_aheads`** is a step in the lag between wall time and the playhead, not a single large - advance. See "What the sampling grid can and cannot see" below. +- **`skip_aheads`** counts explicit ring jumps over media after playback starts. The accompanying + `skipped_samples` duration converts the ring's skipped frames at the graph's actual sample rate. + Late incoming duplicates are `discarded_samples`; initial fill trimming is not a playback skip. - **`silence_share`** is the share of sampled windows whose RMS at the graph output was below about -60 dBFS. It is the only metric read from the audio itself rather than from a counter: a counter @@ -306,27 +306,20 @@ definition is a judgement call are: playback, from `sync.out.clock`. A sample from a build without that signal is left out of the denominator rather than counted as a zero, so an older build reports `null`. -`short_quanta`, `silent_quanta`, `discarded_samples`, `accelerates`, `expands`, -`stretched_samples`, and `budget_aborts` are in the schema and report `null` in the browser lanes: -the signals they need are not on the element's public surface yet. They are null rather than zero, -and the summary says which change would fill each one in. The replay lane reads five of them -straight off the ring, because there is no page between the counter and the summary. +`short_quanta`, `discarded_samples`, `accelerates`, `expands`, and `stretched_samples` come from +`audio.out.debug`. A build without the counters reports null. `silent_quanta` and `budget_aborts` +remain unmeasured because neither lane has a counter that distinguishes those events. ### What the sampling grid can and cannot see -The page samples every 250 ms, and the playhead it reads is quantized: it moves when the ring -reports a new position, not continuously. Measured on a clean local path, consecutive samples show -the playhead advancing anywhere from 240 to 296 ms with no net drift. +The page samples every 250 ms. Worklet and worker reports arrive asynchronously, so a change +in report age can move the sampled playhead by tens of milliseconds without skipping any audio. +A continuous stream with a 60 ms report delay produces a false skip under timestamp inference. +The analyzer therefore grades the ring's cumulative skip counters and reports media drift separately. -Two derivations have to survive that, and both were wrong before they were measured: - -- A rule that calls a single sample's excess advance a skip reports about forty skips a minute on a - run with none. A skip-ahead is therefore a step in the de-trended lag, judged on the median of the - four samples either side of it, and has to exceed 40 ms: more than the quantization band, and more - than one estimator bucket plus a render quantum. -- The lag also drifts when the media timeline does not advance at wall rate, which is a property of - the source or the publisher rather than of the player. It is fitted by least squares and removed - before skips are counted, and reported on its own as `media_drift`. +A graph replacement, timeline re-anchor, or counter reset during the measured window voids the +row. Its discontinuity cannot disappear into a counter delta of zero. A build that lacks these +counters reports null instead of an estimated skip count. ### Void rules @@ -345,6 +338,7 @@ main thread with no reason: a reason is a fallback, meaning the page tried the w | `transport` | The page's session, or its audio worker's own, negotiated something other than WebTransport. A WebSocket fallback is TCP and never touches the UDP shaper. The page denies the fallback outright (below), for the worker too, so this is a backstop rather than the usual outcome. The Safari lane expects a WebSocket instead. | | `thread` | The audio did not come from the page's audio worker, the player's default: the page played it on its main thread (the detail says why), or the worker never started. Under `--offload false`, the audio did not stay on the main thread by choice: it played on the worker, or fell back with a reason. Checked once the audio plays and again at the end, since the page takes the audio back for good. A build that predates the worker cannot say, and is not voided for it. | | `ring` | The document's `crossOriginIsolated` does not match the ring the row asked for, so the other ring ran. | +| `playout` | The graph, timeline anchor, sample rate, or monotonic counters changed during the measured window. | | `clock` | `AudioContext.currentTime` drifted more than 1% from wall time over the first ten seconds, or was never readable. | | `window` | No samples survived the warmup. | | `driver` | The driver threw. A Playwright trace is saved into the run directory. | diff --git a/test/audio-quality/clients/js/analyze.ts b/test/audio-quality/clients/js/analyze.ts index 4361e93d62..6f3fa5cf7b 100644 --- a/test/audio-quality/clients/js/analyze.ts +++ b/test/audio-quality/clients/js/analyze.ts @@ -9,8 +9,8 @@ * conflating the two makes a player that re-buffers cleanly look like one that glitches. The * `underruns` counter is reported next to the episodes rather than instead of them, because one * long gap and forty scattered ones grade the same by quanta and sound nothing alike. - * - A **skip-ahead** is a sample in which the playhead advanced by more than the wall time since the - * last sample, plus a tolerance. That is audio that was buffered and then discarded. + * - A **skip-ahead** is a jump the ring counted. Message delivery delay changes a sampled + * playhead's apparent speed, so wall time cannot identify playback skips. * - **converge_s** is the first moment after which the resolved target stayed within one bucket of * its final value for the rest of the run. A target that settles and then moves again has not * converged, so it is measured backwards from the end rather than forwards from the start. @@ -169,30 +169,11 @@ if (window.length === 0) { // ── playhead ──────────────────────────────────────────────────────────────── // -// The playhead a page can read is quantized: it moves when the ring reports a new position, not -// continuously, and on the postMessage ring those reports arrive on their own cadence. Measured on a -// clean local path, consecutive 250 ms samples show the playhead advancing anywhere from 240 to 296 -// ms with no net drift. Every derivation below has to survive that, because a rule that treats a -// single sample's excess as a skip reports forty of them a minute on a run with none. +// Playhead reports are asynchronous. Only explicit ring counters identify discrete skips; +// delayed reports and time stretching can both change the apparent wall/media lag. /** A sample's playhead advanced by less than this counts as not having advanced at all. */ const PLATEAU_MS = 1; -/** - * A net playhead advance beyond wall time larger than this is a skip. - * - * Larger than the observed quantization band, and larger than one estimator bucket plus a render - * quantum, which is the least a re-anchor can discard and still have discarded anything. - */ -const SKIP_MS = 40; -/** Samples either side of a candidate that are reduced to a median before it is judged. */ -const SKIP_WINDOW = 4; - -const median = (values: number[]): number | undefined => { - if (values.length === 0) return undefined; - const sorted = [...values].sort((a, b) => a - b); - return sorted[Math.floor(sorted.length / 2)]; -}; - let stalledSamples = 0; for (const s of window) if (s.stalled) stalledSamples++; @@ -212,9 +193,7 @@ function slope(xs: number[], ys: number[]): number | null { } // The lag between wall time and the playhead drifts when the media timeline does not advance at wall -// rate, which is a property of the source or the publisher rather than of the player: a steady drift -// is not the receiver discarding anything. Left in, it reads as a skip every few seconds, so it is -// fitted and removed first, and reported on its own as `media_drift_ms_per_s`. +// rate. Report the fitted drift separately from the ring's explicit playback counters. const rawLags = window.flatMap((s) => typeof s.timestamp === "number" && !s.stalled ? [{ at: s.at, lag: s.at - s.timestamp }] : [], ); @@ -223,32 +202,6 @@ const mediaDrift = slope( rawLags.map((l) => l.lag), ); -// A skip-ahead is a step in that de-trended lag, not a single large advance. Quantization makes the -// lag oscillate inside a band; discarding buffered audio moves the band. So each candidate is judged -// on the median lag either side of it, which neither the oscillation nor the drift can fake. -const detrend = (at: number, lag: number) => lag - ((mediaDrift ?? 0) / 1000) * (at - (rawLags[0]?.at ?? 0)); -const lags = window.map((s) => - typeof s.timestamp === "number" - ? { at: s.at, lag: detrend(s.at, s.at - s.timestamp), stalled: s.stalled } - : undefined, -); -let skipAheads = 0; -let skippedMs = 0; -for (let i = SKIP_WINDOW; i < lags.length - SKIP_WINDOW; i++) { - const here = lags[i]; - if (!here || here.stalled) continue; - const before = median(lags.slice(i - SKIP_WINDOW, i).flatMap((l) => (l && !l.stalled ? [l.lag] : []))); - const after = median(lags.slice(i + 1, i + 1 + SKIP_WINDOW).flatMap((l) => (l && !l.stalled ? [l.lag] : []))); - if (before === undefined || after === undefined) continue; - const jumped = before - after; - if (jumped > SKIP_MS) { - skipAheads++; - skippedMs += jumped; - // One step is one skip, and the trailing window still straddles it for several samples. - i += SKIP_WINDOW; - } -} - // The ring's own underrun counter is cumulative, and it is the authoritative count: it sees every // partly-filled quantum, including the ones that begin and end between two 250 ms samples. // @@ -273,6 +226,41 @@ function rise(counts: (number | undefined)[]): number | null { return total; } +const playout = window.flatMap((sample) => (sample.playout ? [sample.playout] : [])); +const firstPlayout = playout[0]; +if ( + playout.some((state, i) => { + const previous = playout[i - 1]; + return ( + state.generation !== firstPlayout?.generation || + state.anchor !== firstPlayout?.anchor || + state.rate !== firstPlayout?.rate || + (previous !== undefined && + (state.skips < previous.skips || state.skipped < previous.skipped || state.output < previous.output)) + ); + }) || + (playout.length > 0 && playout.length !== window.length) +) { + voids.push({ + assertion: "playout", + detail: "the audio graph, timeline, or counters changed during the measured window", + }); +} + +const counter = (key: "skips" | "skipped" | "discarded" | "short" | "accelerates" | "expands" | "stretched") => + rise(window.map((sample) => sample.playout?.[key])); +const duration = (key: "skipped" | "discarded" | "stretched") => { + const count = counter(key); + return count === null || firstPlayout === undefined ? null : (count * 1000) / firstPlayout.rate; +}; +const skipAheads = counter("skips"); +const skippedMs = duration("skipped"); +const discardedMs = duration("discarded"); +const shortQuanta = counter("short"); +const accelerates = counter("accelerates"); +const expands = counter("expands"); +const stretchedMs = duration("stretched"); + const underrunCounts = window.map((s) => s.underruns); const hasCounter = underrunCounts.some((c) => typeof c === "number"); const underruns = rise(underrunCounts); @@ -476,23 +464,23 @@ const metrics: Record = { underrun_episodes_max: round1(episodeStats.max), underrun_samples_total: round1(underrunMs), underrun_samples_per_min: round1(underrunMs / minutes), - short_quanta_total: null, - short_quanta_per_min: null, + short_quanta_total: shortQuanta, + short_quanta_per_min: shortQuanta === null ? null : round1(shortQuanta / minutes), silent_quanta_total: null, silent_quanta_per_min: null, stalled_quanta_share: stalledShare === null ? null : Math.round(stalledShare * 1000) / 1000, - discarded_samples_total: null, - discarded_samples_per_min: null, + discarded_samples_total: round1(discardedMs), + discarded_samples_per_min: discardedMs === null ? null : round1(discardedMs / minutes), skip_aheads_total: skipAheads, - skip_aheads_per_min: round1(skipAheads / minutes), + skip_aheads_per_min: skipAheads === null ? null : round1(skipAheads / minutes), skipped_samples_total: round1(skippedMs), - skipped_samples_per_min: round1(skippedMs / minutes), - accelerates_total: null, - accelerates_per_min: null, - expands_total: null, - expands_per_min: null, - stretched_samples_total: null, - stretched_samples_per_min: null, + skipped_samples_per_min: skippedMs === null ? null : round1(skippedMs / minutes), + accelerates_total: accelerates, + accelerates_per_min: accelerates === null ? null : round1(accelerates / minutes), + expands_total: expands, + expands_per_min: expands === null ? null : round1(expands / minutes), + stretched_samples_total: round1(stretchedMs), + stretched_samples_per_min: stretchedMs === null ? null : round1(stretchedMs / minutes), skipped_groups_total: skippedGroups, skipped_groups_per_min: skippedGroups === null ? null : round1(skippedGroups / minutes), budget_aborts_total: null, diff --git a/test/audio-quality/clients/js/src/analyze.test.ts b/test/audio-quality/clients/js/src/analyze.test.ts new file mode 100644 index 0000000000..8d4a3fac68 --- /dev/null +++ b/test/audio-quality/clients/js/src/analyze.test.ts @@ -0,0 +1,110 @@ +import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Sample, Summary } from "./schema.ts"; + +const tag = "chromium-aac-44100-fixed-250-plain"; +const diagnostic = { + generation: 1, + rate: 44100, + anchor: 44100, + skips: 0, + skipped: 0, + discarded: 0, + trimmed: 0, + buffered: 11025, + target: 11025, + chunk: 1024, + skip: 3308, + stalled: false, + fresh: false, + underruns: 0, + queued: 0, + stretched: 0, + output: 0, + concealed: 0, + accelerates: 0, + expands: 0, + merges: 0, + short: 0, + budget: 250, +}; + +function analyze(change: (sample: Sample) => Sample): Summary { + const out = mkdtempSync(join(tmpdir(), "moq-audio-analysis-")); + try { + const samples = Array.from({ length: 241 }, (_, i) => { + const at = i * 250; + return change({ + at, + timestamp: 1000 + at, + stalled: false, + underruns: 0, + skipped: 0, + buffered: 250, + delay: 250, + contextTime: at, + contextRate: 44100, + rms: 0.1, + clock: "audio", + playout: { ...diagnostic }, + }); + }); + writeFileSync(join(out, `${tag}.ndjson`), JSON.stringify({ tag, samples })); + const result = spawnSync( + process.execPath, + [new URL("../analyze.ts", import.meta.url).pathname, "--run", out, "--row", tag], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + return JSON.parse(readFileSync(join(out, `${tag}.summary.json`), "utf8")); + } finally { + rmSync(out, { recursive: true, force: true }); + } +} + +test("report age does not invent a playback skip", () => { + const summary = analyze((sample) => ({ + ...sample, + timestamp: 1000 + sample.at - (sample.at >= 15000 && sample.at < 20000 ? 60 : 0), + })); + expect(summary.metrics.skip_aheads_total).toBe(0); + expect(summary.metrics.skipped_samples_total).toBe(0); +}); + +test("a real ring skip remains visible despite delayed reports", () => { + const summary = analyze((sample) => ({ + ...sample, + playout: { ...diagnostic, skips: sample.at >= 20000 ? 1 : 0, skipped: sample.at >= 20000 ? 2646 : 0 }, + })); + expect(summary.metrics.skip_aheads_total).toBe(1); + expect(summary.metrics.skipped_samples_total).toBe(60); +}); + +test("late writes and startup trim are separate from skipped playback", () => { + const summary = analyze((sample) => ({ + ...sample, + playout: { ...diagnostic, discarded: sample.at >= 20000 ? 441 : 0, trimmed: 4410 }, + })); + expect(summary.metrics.skip_aheads_total).toBe(0); + expect(summary.metrics.skipped_samples_total).toBe(0); + expect(summary.metrics.discarded_samples_total).toBe(10); +}); + +test("a missing diagnostic counter remains unmeasured", () => { + const summary = analyze((sample) => ({ ...sample, playout: undefined })); + expect(summary.metrics.skip_aheads_total).toBeNull(); + expect(summary.metrics.skipped_samples_total).toBeNull(); +}); + +for (const changed of [{ generation: 2 }, { anchor: 88200 }, { output: -1 }]) { + test(`a changed playback timeline cannot pass as zero skips (${JSON.stringify(changed)})`, () => { + const summary = analyze((sample) => ({ + ...sample, + playout: { ...diagnostic, ...(sample.at >= 20000 ? changed : {}) }, + })); + expect(summary.voids.some((entry) => entry.assertion === "playout")).toBe(true); + }); +} diff --git a/test/audio-quality/clients/js/src/probe.test.ts b/test/audio-quality/clients/js/src/probe.test.ts index efa3aa6e6e..7b602a0aad 100644 --- a/test/audio-quality/clients/js/src/probe.test.ts +++ b/test/audio-quality/clients/js/src/probe.test.ts @@ -52,3 +52,31 @@ test("the probe sums buffered ranges and tolerates builds without a range list", jest.useRealTimers(); } }); + +test("the probe copies ring counters and identifies replacement graphs", () => { + jest.useFakeTimers(); + let root = {}; + const debug = { anchor: 0, skips: 1, skipped: 1024 }; + const running = probe( + watch({ + root: { peek: () => root }, + context: signal({ sampleRate: 44100 }), + debug: signal(debug), + }), + ); + try { + jest.advanceTimersByTime(250); + const first = running.drain()[0]; + debug.skips = 2; + root = {}; + jest.advanceTimersByTime(250); + const next = running.drain()[0]; + expect(first.playout?.skips).toBe(1); + expect(first.playout?.rate).toBe(44100); + expect(next.playout?.skips).toBe(2); + expect(next.playout?.generation).toBe((first.playout?.generation ?? 0) + 1); + } finally { + running.stop(); + jest.useRealTimers(); + } +}); diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index dd52650ee8..4c81ddd68d 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -104,6 +104,7 @@ export function probe(watch: MoqWatch): Probe { // and would report a perfect run as a fully silent one. let analyser: AnalyserNode | undefined; let attachedTo: AudioNode | undefined; + let generation = 0; let pcm: Float32Array | undefined; // Render capacity arrives on its own event rather than on demand, so the latest reading is held @@ -114,6 +115,7 @@ export function probe(watch: MoqWatch): Probe { const attach = () => { const root = maybe(() => watch.audio.out.root.peek()); if (!root || root === attachedTo) return; + generation++; try { const node = new AnalyserNode(root.context, { fftSize: 2048 }); root.connect(node); @@ -163,6 +165,7 @@ export function probe(watch: MoqWatch): Probe { const audio = watch.audio.out; const sync = watch.sync.out; const context = maybe(() => audio.context.peek()); + const debug = maybe(() => audio.debug.peek()); // `buffered` is a list of ranges, not a depth. What the grader wants is how much audio is // ready to play, so the ranges are summed; a gap in the middle is not playable time. @@ -181,9 +184,7 @@ export function probe(watch: MoqWatch): Probe { spread: num(audio.spread), buffered: ranges.length > 0 ? buffered : undefined, skipped: num(audio.skipped), - // `audio.out.debug` carries the ring's own short, discarded and stretch counters. This - // probe does not read it, so those metrics are null on the browser lanes and filled only - // by the replay lane, which drives the engine directly. + playout: debug && context ? { ...debug, generation, rate: context.sampleRate } : undefined, stats: maybe(() => audio.stats.peek()) as Record | undefined, thread: threadOf(watch), diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index 746042cc85..8e00c92ff4 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -21,6 +21,8 @@ * @module */ +import type MoqWatch from "@moq/watch/element"; + /** Milliseconds, as a float. The unit of every duration in this schema. */ export type Ms = number; @@ -100,7 +102,6 @@ export const METRICS: Record = { clock: "viewer", aggregations: ["total", "per_min"], description: "Quanta delivered with fewer samples than the render quantum asked for.", - pending: "the browser probe does not read `audio.out.debug`; the replay lane reads it directly", }, silent_quanta: { unit: "count", @@ -120,14 +121,13 @@ export const METRICS: Record = { unit: "samples", clock: "viewer", aggregations: ["total", "per_min"], - description: "Buffered samples thrown away without being played, reported in ms.", - pending: "the browser probe does not read `audio.out.debug`; the replay lane reads it directly", + description: "Incoming samples rejected because they were behind the playhead, reported in ms.", }, skip_aheads: { unit: "count", clock: "viewer", aggregations: ["total", "per_min"], - description: "Re-anchors that jumped the playhead forward, discarding buffered audio.", + description: "Playback jumps over media after the timeline started, counted by the ring.", }, skipped_samples: { unit: "samples", @@ -140,21 +140,18 @@ export const METRICS: Record = { clock: "viewer", aggregations: ["total", "per_min"], description: "Time-stretch decisions that played the buffer down faster than real time.", - pending: "the browser probe does not read `audio.out.debug`; the replay lane reads it directly", }, expands: { unit: "count", clock: "viewer", aggregations: ["total", "per_min"], description: "Concealment decisions that generated audio to cover a gap.", - pending: "the browser probe does not read `audio.out.debug`; the replay lane reads it directly", }, stretched_samples: { unit: "samples", clock: "viewer", aggregations: ["total", "per_min"], description: "Samples whose duration was altered by stretching or concealment, reported in ms.", - pending: "the browser probe does not read `audio.out.debug`; the replay lane reads it directly", }, skipped_groups: { unit: "count", @@ -216,7 +213,7 @@ export const METRICS: Record = { clock: "publisher", aggregations: ["last"], description: - "Rate at which the media timeline runs away from wall time, in ms per second. A property of the source or the publisher, not of the player, and removed before skip-aheads are counted.", + "Rate at which the media timeline runs away from wall time, in ms per second. Fitted independently of the ring counters, which count skip-aheads directly.", }, }; @@ -347,6 +344,11 @@ export type Sample = { buffered?: Ms; /** `audio.out.skipped`: cumulative groups the container consumer abandoned. */ skipped?: number; + /** Ring counters, with their graph identity and sample rate. Missing on older builds. */ + playout?: Omit>, "budget"> & { + generation: number; + rate: number; + }; /** `audio.out.stats`, passed through as-is: whatever the build publishes. */ stats?: Record; /** From 66716464e4c3a7581e8b95a37a1baabd474adba4 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:35:07 -0400 Subject: [PATCH 052/127] fix(audio-quality): clear failed analyser replacements Co-Authored-By: GPT-6 --- .../clients/js/src/probe.test.ts | 42 +++++++++++++++++++ test/audio-quality/clients/js/src/probe.ts | 10 ++--- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/test/audio-quality/clients/js/src/probe.test.ts b/test/audio-quality/clients/js/src/probe.test.ts index 7b602a0aad..e4de2132ba 100644 --- a/test/audio-quality/clients/js/src/probe.test.ts +++ b/test/audio-quality/clients/js/src/probe.test.ts @@ -80,3 +80,45 @@ test("the probe copies ring counters and identifies replacement graphs", () => { jest.useRealTimers(); } }); + +for (const failure of ["constructor", "connect"] as const) { + test(`a replacement analyser ${failure} failure cannot sample the previous graph`, () => { + jest.useFakeTimers(); + const original = Object.getOwnPropertyDescriptor(globalThis, "AnalyserNode"); + let replacing = false; + Object.defineProperty(globalThis, "AnalyserNode", { + configurable: true, + value: class { + fftSize = 2048; + constructor() { + if (replacing && failure === "constructor") throw new Error("constructor failed"); + } + getFloatTimeDomainData(pcm: Float32Array) { + pcm.fill(0.5); + } + }, + }); + const node = () => ({ + context: {}, + connect() { + if (replacing && failure === "connect") throw new Error("connect failed"); + }, + }); + let root = node(); + const running = probe(watch({ root: { peek: () => root } })); + try { + jest.advanceTimersByTime(250); + expect(running.drain()[0].rms).toBe(0.5); + replacing = true; + root = node(); + jest.advanceTimersByTime(500); + expect(running.drain().map((sample) => sample.rms)).toEqual([undefined, undefined]); + expect(running.notes()).toEqual([`analyser: ${failure} failed`]); + } finally { + running.stop(); + if (original) Object.defineProperty(globalThis, "AnalyserNode", original); + else Reflect.deleteProperty(globalThis, "AnalyserNode"); + jest.useRealTimers(); + } + }); +} diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index 4c81ddd68d..d5d476ed6f 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -114,19 +114,19 @@ export function probe(watch: MoqWatch): Probe { const attach = () => { const root = maybe(() => watch.audio.out.root.peek()); - if (!root || root === attachedTo) return; + if (root === attachedTo) return; + attachedTo = root; generation++; + analyser = undefined; + pcm = undefined; + if (!root) return; try { const node = new AnalyserNode(root.context, { fftSize: 2048 }); root.connect(node); analyser = node; pcm = new Float32Array(node.fftSize); - attachedTo = root; } catch (err) { notes.push(`analyser: ${err instanceof Error ? err.message : String(err)}`); - // Claim the root even though the analyser never attached, or every later sample retries - // the same failing construction and pushes the same note again. - attachedTo = root; } const context = maybe(() => watch.audio.out.context.peek()); From 0251594baeaaf2f8677180b2ec23314fb4a149c0 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:36:00 -0400 Subject: [PATCH 053/127] test(audio): attest the concrete ring in browser quality runs Co-Authored-By: GPT-6 --- .github/workflows/audio-quality.yml | 5 +++++ .github/workflows/nightly.yml | 5 +++++ js/watch/src/audio/buffer.test.ts | 1 + js/watch/src/audio/counters.test.ts | 1 + js/watch/src/audio/playout/index.ts | 2 ++ js/watch/src/audio/ring-buffer.ts | 1 + js/watch/src/audio/shared-ring-buffer.ts | 1 + test/audio-quality/README.md | 13 ++++++++++--- test/audio-quality/clients/js/driver.ts | 18 ++++++++---------- .../clients/js/src/analyze.test.ts | 4 ++-- test/audio-quality/clients/js/src/page.ts | 1 + .../clients/js/src/schema.test.ts | 9 ++++++++- test/audio-quality/clients/js/src/schema.ts | 12 +++++++++++- test/audio-quality/run.sh | 4 ++-- 14 files changed, 58 insertions(+), 19 deletions(-) diff --git a/.github/workflows/audio-quality.yml b/.github/workflows/audio-quality.yml index f4889ad04c..f9661deda1 100644 --- a/.github/workflows/audio-quality.yml +++ b/.github/workflows/audio-quality.yml @@ -143,6 +143,11 @@ jobs: run: nix develop --command just test audio-quality --enforce shell: bash -leo pipefail {0} + - name: Shared audio ring quality + if: ${{ !cancelled() }} + run: nix develop --command just test audio-quality --profiles fixed-250 --rings isolated --offload false --enforce + shell: bash -leo pipefail {0} + # The run directory is the whole diagnosis: each process's log, the # shaper's counters, the raw ndjson, the per-row summaries, and a # Playwright trace of any page that failed. The harness keeps it on a diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 365104630c..00337274d5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -233,6 +233,11 @@ jobs: run: nix develop --command just test audio-quality --enforce shell: bash -leo pipefail {0} + - name: Shared audio ring quality + if: ${{ !cancelled() }} + run: nix develop --command just test audio-quality --profiles fixed-250 --rings isolated --offload false --enforce + shell: bash -leo pipefail {0} + # The run directory is the whole diagnosis: each process's log, the # shaper's counters, the raw ndjson, the per-row summaries, and a # Playwright trace of any page that failed. The harness keeps it on a diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index c7aa4b2f59..0aaefcabe2 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -152,6 +152,7 @@ function state(worklet: FakeWorklet, reader: Playhead | undefined, stalled: bool timeline: worklet.timeline, playhead: reader, debug: { + backend: "message", buffered: 0, target: 0, chunk: 0, diff --git a/js/watch/src/audio/counters.test.ts b/js/watch/src/audio/counters.test.ts index e662391542..de44bc3b53 100644 --- a/js/watch/src/audio/counters.test.ts +++ b/js/watch/src/audio/counters.test.ts @@ -23,6 +23,7 @@ for (const shared of [false, true]) { ring.read([new Float32Array(20)]); for (let at = 150; at < 600; at += 50) write(ring, at); const debug = ring.debug(); + expect(debug.backend).toBe(shared ? "shared" : "message"); expect(debug.skips).toBeGreaterThan(0); expect(debug.skipped).toBe(shared ? 68 : 380); expect(debug.discarded).toBe(0); diff --git a/js/watch/src/audio/playout/index.ts b/js/watch/src/audio/playout/index.ts index 1a66843e2c..f24d51533c 100644 --- a/js/watch/src/audio/playout/index.ts +++ b/js/watch/src/audio/playout/index.ts @@ -124,6 +124,8 @@ export interface Counters { * the worklet's state message on the postMessage one. */ export interface Snapshot extends Counters { + /** The ring implementation that produced this snapshot. */ + backend: "shared" | "message"; /** Media samples the ring holds. */ buffered: number; /** The playout target, in samples. */ diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 06ec75b98f..2b807ad1bc 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -96,6 +96,7 @@ export class AudioRingBuffer implements RingReader { }; readonly #playhead: Playhead = { timestamp: Time.Micro.zero, rate: 0 }; readonly #snapshot: Snapshot = { + backend: "message", queued: 0, stretched: 0, output: 0, diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index 6c77b99166..89970e6655 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -956,6 +956,7 @@ export class SharedRingBuffer implements RingReader { debug(): Snapshot { const load = (index: number) => Atomics.load(this.#control, index); return { + backend: "shared", buffered: this.length, target: load(LATENCY), chunk: load(CHUNK), diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index e81c7761ce..347d4250eb 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -16,7 +16,7 @@ just test audio-quality --runtime replay # the recorded traces just test audio-quality --offload false # the audio on the page's main thread ``` -The matrix is codec x jitter profile x ring path: 24 rows, about 30 minutes at the default 60 +The matrix is codec x jitter profile x document isolation: 24 rows, about 30 minutes at the default 60 seconds a row. `--profiles`, `--rings`, and `--codecs` take comma-separated lists; `--seed` replays a given impairment; `--duration` shortens a row. @@ -38,7 +38,7 @@ Three rows are enforced today: | Codec | Enforced | | --- | --- | | opus | `fixed-250` isolated | -| aac | `fixed-250` both rings | +| aac | `fixed-250` both isolation contexts | Twenty-one keep the marker, and the reason is the target rather than a fault in the measurement: an `auto` row's target follows the path for the whole window, so where inside sixty seconds the path's @@ -78,6 +78,12 @@ native twin, or this harness: that lane takes seconds and cannot be unlucky. | `safari` | Real Safari over a WebSocket, the two control profiles | no | locally, on macOS | | `replay` | The recorded traces through the same player, on a simulated clock | no | pull requests, nightly, and anywhere else, in a second | +The default audio worker uses the message ring in both document isolation contexts. These rows +prove isolation handling, not SharedArrayBuffer playback. The nightly also runs +`--profiles fixed-250 --rings isolated --offload false --enforce` for both codecs. Those rows keep +audio on the page and require the concrete ring's debug snapshot to report `shared`. The default +rows require `message`. Unknown or unexpected implementations void the row. + ## How the browser reaches the relay ```text @@ -337,7 +343,8 @@ main thread with no reason: a reason is a fallback, meaning the page tried the w | `shaper` | An active profile's `delayed` counter is zero, so the impairment never applied and an impaired run became an unimpaired pass. `near-zero` and `fixed-250` are exempt: zero is the right answer for the control. | | `transport` | The page's session, or its audio worker's own, negotiated something other than WebTransport. A WebSocket fallback is TCP and never touches the UDP shaper. The page denies the fallback outright (below), for the worker too, so this is a backstop rather than the usual outcome. The Safari lane expects a WebSocket instead. | | `thread` | The audio did not come from the page's audio worker, the player's default: the page played it on its main thread (the detail says why), or the worker never started. Under `--offload false`, the audio did not stay on the main thread by choice: it played on the worker, or fell back with a reason. Checked once the audio plays and again at the end, since the page takes the audio back for good. A build that predates the worker cannot say, and is not voided for it. | -| `ring` | The document's `crossOriginIsolated` does not match the ring the row asked for, so the other ring ran. | +| `ring` | The document's `crossOriginIsolated` does not match the requested context. | +| `backend` | The concrete ring's debug snapshot is absent or names a different implementation from the requested execution path. | | `playout` | The graph, timeline anchor, sample rate, or monotonic counters changed during the measured window. | | `clock` | `AudioContext.currentTime` drifted more than 1% from wall time over the first ten seconds, or was never readable. | | `window` | No samples survived the warmup. | diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts index 76ac20f625..44f75d02f3 100644 --- a/test/audio-quality/clients/js/driver.ts +++ b/test/audio-quality/clients/js/driver.ts @@ -5,7 +5,7 @@ * The driver's own job is small: stand the page up, wait for it to actually be playing, let it run, * and then decide whether what it measured is allowed to count. That last part is the point. A row * that ran on the WebSocket fallback never went through the UDP shaper, and a row whose document was - * not isolated the way the matrix asked for ran the other ring; both would otherwise pass quietly + * not isolated the way the matrix asked for ran a different context; both would otherwise pass quietly * against a budget written for something else, which is worse than failing. * * bun driver.ts --url http://127.0.0.1:4499 --broadcast bbb.hang --page dist \ @@ -19,7 +19,7 @@ import { join, resolve } from "node:path"; import { parseArgs } from "node:util"; import type { Page } from "playwright"; import { Failure, launch, open, saveTrace, serve } from "../../../interop/clients/js/harness.ts"; -import { type Ring, type Thread, threadVoid, type Void } from "./src/schema.ts"; +import { type Backend, backendVoid, type Ring, type Thread, threadVoid, type Void } from "./src/schema.ts"; const { values } = parseArgs({ options: { @@ -63,13 +63,8 @@ const expectThread = values.offload === "false" ? "main" : "worker"; const out = resolve(values.out); mkdirSync(out, { recursive: true }); -/** - * The two prefixes the same build is served under. - * - * Cross-origin isolation is a property of the document, not of the bundle, so one build served twice - * is all it takes to run both rings. `/plain` is the production path: most viewers are not isolated, - * and the postMessage ring is what they get. - */ +// Isolation permits shared memory. The default worker still uses the message ring; +// an isolated page runs the shared ring only when --offload false keeps audio on the page. const server = serve({ root: resolve(values.page), port: Number.parseInt(values.port, 10), @@ -101,6 +96,7 @@ type Status = { crossOriginIsolated: boolean; transport?: string; thread?: Thread; + backend?: Backend; timestamp?: number; stalled?: boolean; underruns?: number; @@ -129,7 +125,9 @@ const note = (assertion: string, detail: string) => { // session is a second one, and takes it back for good when the worker cannot play it. So it is checked // once the audio plays and again at the end. A row run with `--offload false` expects the main thread. const checkThread = (status: Status | undefined) => { - const found = threadVoid(status?.thread, "webtransport", expectThread); + const found = + threadVoid(status?.thread, "webtransport", expectThread) ?? + backendVoid(status?.backend, expectThread === "main" && ring === "isolated" ? "shared" : "message"); if (found && !voids.some((v) => v.assertion === found.assertion && v.detail === found.detail)) { note(found.assertion, found.detail); } diff --git a/test/audio-quality/clients/js/src/analyze.test.ts b/test/audio-quality/clients/js/src/analyze.test.ts index 8d4a3fac68..b78197a2c7 100644 --- a/test/audio-quality/clients/js/src/analyze.test.ts +++ b/test/audio-quality/clients/js/src/analyze.test.ts @@ -6,7 +6,8 @@ import { join } from "node:path"; import type { Sample, Summary } from "./schema.ts"; const tag = "chromium-aac-44100-fixed-250-plain"; -const diagnostic = { +const diagnostic: NonNullable = { + backend: "message", generation: 1, rate: 44100, anchor: 44100, @@ -29,7 +30,6 @@ const diagnostic = { expands: 0, merges: 0, short: 0, - budget: 250, }; function analyze(change: (sample: Sample) => Sample): Summary { diff --git a/test/audio-quality/clients/js/src/page.ts b/test/audio-quality/clients/js/src/page.ts index 097f2743bb..30adcd4050 100644 --- a/test/audio-quality/clients/js/src/page.ts +++ b/test/audio-quality/clients/js/src/page.ts @@ -110,6 +110,7 @@ setInterval(() => { // The audio's own path: the page's audio worker has a session of its own, which `transport` // does not see. thread: threadOf(watch), + backend: peek(() => watch.audio.out.debug.peek()?.backend), timestamp: peek(() => watch.audio.out.timestamp.peek()), stalled: peek(() => watch.audio.out.stalled.peek()), underruns: peek(() => watch.audio.out.underruns.peek()), diff --git a/test/audio-quality/clients/js/src/schema.test.ts b/test/audio-quality/clients/js/src/schema.test.ts index 0f887998b3..f95d03b89a 100644 --- a/test/audio-quality/clients/js/src/schema.test.ts +++ b/test/audio-quality/clients/js/src/schema.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { frameFloor, threadVoid } from "./schema.ts"; +import { backendVoid, frameFloor, threadVoid } from "./schema.ts"; test("the replay floor comes from distinct sorted media timestamps", () => { expect(frameFloor([60.5, 0, 20.2, 20.2, 40.4])).toBe(21); @@ -56,3 +56,10 @@ test("a row that keeps its audio on the page counts only the main thread it chos detail: "the page fell back to the main thread rather than keeping the audio there: the worker failed to load", }); }); + +test("backend coverage requires an observed concrete ring", () => { + expect(backendVoid("shared", "shared")).toBeUndefined(); + expect(backendVoid("message", "message")).toBeUndefined(); + expect(backendVoid("message", "shared")?.assertion).toBe("backend"); + expect(backendVoid(undefined, "message")?.detail).toContain("observed unknown"); +}); diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index 8e00c92ff4..c1a325bfad 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -305,7 +305,7 @@ export type Row = { rate: number; /** The shaper profile the path ran under. */ profile: string; - /** Which ring ran. */ + /** Document isolation in browser rows; concrete ring selection in replay rows. */ ring: Ring; }; @@ -321,6 +321,16 @@ export const rowKey = (row: Row): string => `${row.runtime}-${row.codec}-${row.r */ export type Thread = { kind: "worker"; transport?: string } | { kind: "main"; reason?: string } | { kind: "pending" }; +/** The concrete ring reported by the running player. */ +export type Backend = NonNullable>["backend"]; + +/** Reject a row whose concrete ring differs from the requested execution path. */ +export function backendVoid(actual: Backend | undefined, expected: Backend): Void | undefined { + return actual === expected + ? undefined + : { assertion: "backend", detail: `expected ${expected} ring, observed ${actual ?? "unknown"}` }; +} + /** One 250 ms probe sample: everything public the page could read at that instant. */ export type Sample = { /** Milliseconds since the page started sampling, on the viewer clock. */ diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index b7e27c7850..cdef880e0e 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -13,8 +13,8 @@ # WebSocket fallback is TCP and never touches the UDP shaper. # - The audio came from the page's audio worker, the player's default, not its main thread; or, # under `--offload false`, from the main thread the page was told to keep it on. -# - The ring that ran is the one the row asked for, which is decided by whether the document is -# cross-origin isolated, not by anything the page can assert about itself. +# - The document has the requested isolation and the ring identifies its implementation. +# The worker uses messages in both contexts; isolated --offload false uses shared memory. # # `--runtime` picks which of three lanes runs. `chromium` is the matrix above. `safari` is real # Safari through safaridriver, whose session is a WebSocket and therefore never traverses the UDP From 74aaab832718f27c730923e739abd4173d88f533 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:39:32 -0400 Subject: [PATCH 054/127] refactor(audio): limit shared ring resizing to growth Co-Authored-By: GPT-6 --- js/watch/src/audio/counters.test.ts | 23 +--------------- js/watch/src/audio/shared-ring-buffer.test.ts | 27 +++++++++---------- js/watch/src/audio/shared-ring-buffer.ts | 22 ++++----------- 3 files changed, 18 insertions(+), 54 deletions(-) diff --git a/js/watch/src/audio/counters.test.ts b/js/watch/src/audio/counters.test.ts index de44bc3b53..2f8a82c6c3 100644 --- a/js/watch/src/audio/counters.test.ts +++ b/js/watch/src/audio/counters.test.ts @@ -64,31 +64,10 @@ test("shrinking a playing message ring counts the dropped media", () => { expect(ring.timestamp).toBe(Time.Micro(525_000)); }); -test("shrinking a playing shared ring counts the dropped media", () => { - const ring = new SharedRingBuffer(allocSharedRingBuffer(1, 1024, 1000)); - ring.setLatency(400); - for (let at = 0; at < 500; at += 100) ring.insert(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); - ring.read([new Float32Array(20)]); - for (let at = 500; at < 1000; at += 100) ring.insert(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); - const resized = ring.resize(275); - expect(resized.debug().skips).toBe(1); - expect(resized.debug().skipped).toBe(468); - expect(resized.timestamp).toBe(Time.Micro(488_000)); -}); - -test("shrinking before playback trims instead of skipping either timeline", () => { +test("shrinking a message ring before playback trims instead of skipping", () => { const post = new AudioRingBuffer({ rate: 1000, channels: 1, latency: Time.Milli(400) }); for (let at = 0; at < 500; at += 100) post.write(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); post.resize(Time.Milli(100)); expect(post.debug().trimmed).toBe(225); expect(post.debug().skipped).toBe(0); - const shared = new SharedRingBuffer(allocSharedRingBuffer(1, 1024, 1000, true)); - shared.setLatency(400); - for (let at = 0; at < 800; at += 100) shared.insert(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); - const resized = shared.resize(128); - expect(resized.debug().trimmed).toBe(672); - expect(resized.debug().skipped).toBe(0); - resized.insert(Time.Micro(800_000), [new Float32Array(100).fill(0.5)]); - expect(resized.debug().skipped).toBe(0); - expect(resized.debug().trimmed).toBe(772); }); diff --git a/js/watch/src/audio/shared-ring-buffer.test.ts b/js/watch/src/audio/shared-ring-buffer.test.ts index e0fde8e3d0..5ee1ada715 100644 --- a/js/watch/src/audio/shared-ring-buffer.test.ts +++ b/js/watch/src/audio/shared-ring-buffer.test.ts @@ -851,21 +851,18 @@ describe("SharedRingBuffer.resize", () => { expect(dst.stalled).toBe(false); }); - it("truncates to the newest samples when shrinking below the unread span", () => { - const src = create({ rate: 1000, channels: 1, capacity: 64, latency: 64 }); - // Fill [0, 48) with value 1, then [48, 64) with value 2. - insert(src, 0, 48, { channels: 1, value: 1.0 }); - insert(src, 48, 16, { channels: 1, value: 2.0 }); - - const dst = src.resize(16); - expect(dst.capacity).toBe(16); - - // Only the most recent 16 samples fit. - const out = read(dst, 16, 1); - for (let i = 0; i < 16; i++) { - expect(out[0][i]).toBe(2.0); - } - }); + for (const consumed of [0, 16]) { + it(`refuses shrinking without changing buffered media after ${consumed} samples`, () => { + const src = create({ rate: 1000, channels: 1, capacity: 64, latency: 32 }); + fill(src, 0, 64, { value: 0.5 }); + if (consumed) read(src, consumed, 1); + const before = src.debug(); + + expect(() => src.resize(16)).toThrow("cannot shrink a shared audio ring"); + expect(src.debug()).toEqual(before); + expect(read(src, 64 - consumed, 1)[0]).toEqual(new Float32Array(64 - consumed).fill(0.5)); + }); + } }); describe("buffered mode", () => { diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index 89970e6655..0378d23b67 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -791,10 +791,8 @@ export class SharedRingBuffer implements RingReader { } /** - * Allocate a new ring with `newCapacity` samples and copy the unread window - * [READ, WRITE) plus control state into it. Used when growing capacity so - * we don't drop buffered audio. If `newCapacity` is smaller than the unread - * span, the oldest samples are truncated. + * Grow the ring to `newCapacity`, preserving unread samples and control state. + * Shrinking is unsupported: the old reader keeps playing until the replacement arrives. * * Main thread only. `resize()` reads from the source `SharedRingBuffer` and * writes into a freshly allocated buffer from `allocSharedRingBuffer`, so it @@ -803,6 +801,7 @@ export class SharedRingBuffer implements RingReader { * by READ/WRITE elsewhere. */ resize(newCapacity: number): SharedRingBuffer { + if (newCapacity < this.capacity) throw new Error("cannot shrink a shared audio ring"); const init = allocSharedRingBuffer(this.channels, newCapacity, this.rate, this.buffered); const dst = new SharedRingBuffer(init); dst.#anchored = this.#anchored; @@ -815,7 +814,7 @@ export class SharedRingBuffer implements RingReader { const stalled = Atomics.load(this.#control, STALLED); const available = (write - read) | 0; - const copyCount = Math.max(0, Math.min(available, dst.capacity)); + const copyCount = Math.max(0, available); const copyStart = (write - copyCount) | 0; for (let channel = 0; channel < this.channels; channel++) { @@ -854,19 +853,8 @@ export class SharedRingBuffer implements RingReader { Atomics.store(dst.#control, control, Atomics.load(this.#control, control)); } - const dropped = available - copyCount; - if (dropped > 0) { - if (this.#unplayed === read) { - Atomics.add(dst.#control, TRIMMED, dropped); - } else { - Atomics.add(dst.#control, SKIPS, 1); - Atomics.add(dst.#control, SKIPPED, dropped); - } - } - // Carry the unwrapped playhead over, rebased onto dst's READ. Fold the same `read` - // snapshot the copy used so both sides agree on one observation; `copyStart` is at or - // ahead of it whenever the copy dropped the oldest samples. + // snapshot the copy used so both sides agree on one observation. dst.#position = this.#foldRead(read) + ((copyStart - read) | 0); dst.#lastRead = copyStart; dst.#lastMedia = this.#lastMedia; From 10a1a5bff53a6c2aa75a87ace056006537b31a6a Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:42:50 -0400 Subject: [PATCH 055/127] docs(audio-quality): distinguish isolation from ring transport Co-Authored-By: GPT-6 --- test/audio-quality/clients/js/src/probe.ts | 7 ++----- test/audio-quality/clients/js/src/schema.ts | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index d5d476ed6f..a4fc28fd1b 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -1,11 +1,8 @@ /** * Samples what a `` will tell anyone who asks, every 250 ms. * - * Public signals only. Nothing here reaches past the element's `out` surface, patches a module, or - * knows which ring is running, so the same probe reads a build from the npm package and a build from - * this checkout, and a number it reports is one a consumer could have read too. Where a counter does - * not exist yet the sample carries `undefined` and the analyzer reports null, rather than this file - * growing a private hook to fill the gap. + * Reads the element's output signals, including internal ring diagnostics when available. + * Missing counters remain undefined and the analyzer reports null. * * Two things are measured rather than read, because no signal carries them: * diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index c1a325bfad..8b08576b09 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -287,14 +287,14 @@ export type Drift = { // ── the row identity ──────────────────────────────────────────────────────── -/** Which ring the page actually ran, which is decided by whether the document is isolated. */ +/** Document isolation for browser rows, or shared/message ring selection for replay. */ export type Ring = "isolated" | "plain"; /** * One matrix cell. * * A budget is keyed by the whole thing. Keying by profile alone would grade one codec's floor - * against another's, and the sample rate and the ring path each move it as much as the profile does. + * against another's, and preserves the isolation context in browser results. */ export type Row = { /** The runtime that played it. */ @@ -409,7 +409,7 @@ export type Sample = { /** What the page reports once, at the start, rather than every sample. */ export type Environment = { - /** Whether the document is cross-origin isolated, and therefore which ring can run. */ + /** Whether the document is cross-origin isolated, permitting shared memory. */ crossOriginIsolated: boolean; /** * The transport the page's session negotiated. Anything but WebTransport bypasses the UDP shaper. The From 8b2fcea7fda3fe814efe2a55db7e568783fa0229 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:45:52 -0400 Subject: [PATCH 056/127] fix(audio-quality): measure signed stretch counters consistently Co-Authored-By: GPT-6 --- test/audio-quality/clients/js/analyze.ts | 10 +++++++--- .../audio-quality/clients/js/src/analyze.test.ts | 16 ++++++++++++++++ test/audio-quality/clients/js/src/schema.ts | 4 ++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/test/audio-quality/clients/js/analyze.ts b/test/audio-quality/clients/js/analyze.ts index 6f3fa5cf7b..be2f8f841b 100644 --- a/test/audio-quality/clients/js/analyze.ts +++ b/test/audio-quality/clients/js/analyze.ts @@ -247,9 +247,9 @@ if ( }); } -const counter = (key: "skips" | "skipped" | "discarded" | "short" | "accelerates" | "expands" | "stretched") => +const counter = (key: "skips" | "skipped" | "discarded" | "short" | "accelerates" | "expands") => rise(window.map((sample) => sample.playout?.[key])); -const duration = (key: "skipped" | "discarded" | "stretched") => { +const duration = (key: "skipped" | "discarded") => { const count = counter(key); return count === null || firstPlayout === undefined ? null : (count * 1000) / firstPlayout.rate; }; @@ -259,7 +259,11 @@ const discardedMs = duration("discarded"); const shortQuanta = counter("short"); const accelerates = counter("accelerates"); const expands = counter("expands"); -const stretchedMs = duration("stretched"); +const lastPlayout = playout.at(-1); +const stretchedMs = + firstPlayout && lastPlayout + ? (Math.abs(lastPlayout.stretched - firstPlayout.stretched) * 1000) / firstPlayout.rate + : null; const underrunCounts = window.map((s) => s.underruns); const hasCounter = underrunCounts.some((c) => typeof c === "number"); diff --git a/test/audio-quality/clients/js/src/analyze.test.ts b/test/audio-quality/clients/js/src/analyze.test.ts index b78197a2c7..712d3f8ffa 100644 --- a/test/audio-quality/clients/js/src/analyze.test.ts +++ b/test/audio-quality/clients/js/src/analyze.test.ts @@ -93,6 +93,22 @@ test("late writes and startup trim are separate from skipped playback", () => { expect(summary.metrics.discarded_samples_total).toBe(10); }); +test("expansion contributes to the magnitude of net time stretching", () => { + const summary = analyze((sample) => ({ + ...sample, + playout: { ...diagnostic, stretched: sample.at >= 20000 ? -2646 : 0 }, + })); + expect(summary.metrics.stretched_samples_total).toBe(60); +}); + +test("opposite time stretches report their net, as the replay lane does", () => { + const summary = analyze((sample) => ({ + ...sample, + playout: { ...diagnostic, stretched: sample.at >= 20000 && sample.at < 30000 ? 2646 : 0 }, + })); + expect(summary.metrics.stretched_samples_total).toBe(0); +}); + test("a missing diagnostic counter remains unmeasured", () => { const summary = analyze((sample) => ({ ...sample, playout: undefined })); expect(summary.metrics.skip_aheads_total).toBeNull(); diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index 8b08576b09..ed077b2989 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -145,13 +145,13 @@ export const METRICS: Record = { unit: "count", clock: "viewer", aggregations: ["total", "per_min"], - description: "Concealment decisions that generated audio to cover a gap.", + description: "Time-stretch decisions that lengthened buffered audio.", }, stretched_samples: { unit: "samples", clock: "viewer", aggregations: ["total", "per_min"], - description: "Samples whose duration was altered by stretching or concealment, reported in ms.", + description: "Magnitude of net compression minus expansion, in ms; a lower bound on altered duration.", }, skipped_groups: { unit: "count", From 2f010a06cdfc33c96cd5df86c326ed1519eecab2 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:43:19 -0400 Subject: [PATCH 057/127] test(browser): release console argument handles Co-Authored-By: Codex --- test/interop/clients/js/harness.ts | 7 +++++-- test/wasm/driver.ts | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/interop/clients/js/harness.ts b/test/interop/clients/js/harness.ts index aeff6187e8..3b78c0eac2 100644 --- a/test/interop/clients/js/harness.ts +++ b/test/interop/clients/js/harness.ts @@ -182,8 +182,11 @@ export async function open( const page = await browser.newPage(options); const errors: BrowserErrors = { page: [], console: [] }; page.on("console", (message) => { - console.error(`[${label}] ${message.text()}`); - if (message.type() === "error") errors.console.push(message.text()); + const text = message.text(); + const type = message.type(); + void Promise.allSettled(message.args().map((argument) => argument.dispose())); + console.error(`[${label}] ${text}`); + if (type === "error") errors.console.push(text); }); page.on("pageerror", (error) => { console.error(`[${label} error] ${error.message}`); diff --git a/test/wasm/driver.ts b/test/wasm/driver.ts index d3d6b7f604..982df9c89d 100644 --- a/test/wasm/driver.ts +++ b/test/wasm/driver.ts @@ -82,6 +82,7 @@ try { const fatal: string[] = []; page.on("console", (message) => { const text = message.text(); + void Promise.allSettled(message.args().map((argument) => argument.dispose())); console.error(`[page] ${text}`); if (text.includes("panicked at")) fatal.push(`panic: ${text}`); }); From 818c9aebb24b446661c24bda191e613b6e306330 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:14:24 -0400 Subject: [PATCH 058/127] fix(audio): observe playback jumps at reader commit Keep operational replay counters and budgets unchanged. Record actual forward reader discontinuities across capacity changes and worklet handoff without counting startup or reset as playback loss. Co-Authored-By: GPT-6 --- js/watch/src/audio/buffer.test.ts | 2 + js/watch/src/audio/counters.test.ts | 93 ++++++++++++++++--- js/watch/src/audio/playout/index.ts | 10 +- js/watch/src/audio/ring-buffer.ts | 33 ++++--- js/watch/src/audio/shared-ring-buffer.test.ts | 2 +- js/watch/src/audio/shared-ring-buffer.ts | 76 +++++++++------ test/audio-quality/README.md | 14 ++- test/audio-quality/clients/js/analyze.ts | 14 ++- test/audio-quality/clients/js/replay.ts | 6 ++ .../clients/js/src/analyze.test.ts | 6 +- .../clients/js/src/grade.test.ts | 55 +++++++++++ test/audio-quality/clients/js/src/schema.ts | 18 +++- 12 files changed, 259 insertions(+), 70 deletions(-) create mode 100644 test/audio-quality/clients/js/src/grade.test.ts diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 0aaefcabe2..27f27ac683 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -153,6 +153,8 @@ function state(worklet: FakeWorklet, reader: Playhead | undefined, stalled: bool playhead: reader, debug: { backend: "message", + jumps: 0, + jumped: 0, buffered: 0, target: 0, chunk: 0, diff --git a/js/watch/src/audio/counters.test.ts b/js/watch/src/audio/counters.test.ts index 2f8a82c6c3..e74a610f14 100644 --- a/js/watch/src/audio/counters.test.ts +++ b/js/watch/src/audio/counters.test.ts @@ -22,23 +22,27 @@ for (const shared of [false, true]) { for (let at = 0; at < 150; at += 50) write(ring, at); ring.read([new Float32Array(20)]); for (let at = 150; at < 600; at += 50) write(ring, at); - const debug = ring.debug(); - expect(debug.backend).toBe(shared ? "shared" : "message"); - expect(debug.skips).toBeGreaterThan(0); - expect(debug.skipped).toBe(shared ? 68 : 380); - expect(debug.discarded).toBe(0); + expect(ring.debug().jumps).toBe(0); expect(ring.timestamp).toBe(Time.Micro((shared ? 88 : 400) * 1000)); const pcm = [new Float32Array(20)]; expect(ring.read(pcm)).toBe(20); expect(Array.from(pcm[0])).toEqual(Array(20).fill(0.5)); + const debug = ring.debug(); + expect(debug.backend).toBe(shared ? "shared" : "message"); + expect(debug.jumps).toBe(1); + expect(debug.jumped).toBe(shared ? 68 : 380); + expect(debug.discarded).toBe(shared ? 68 : 380); }); - test("repeated overflow before any read remains startup trimming", () => { + test("repeated startup overflow is not an observed playback jump", () => { const ring = create(true); for (let at = 0; at < 1200; at += 50) write(ring, at); expect(ring.debug().skips).toBe(0); expect(ring.debug().skipped).toBe(0); - expect(ring.debug().trimmed).toBe(shared ? 688 : 1000); + expect(ring.debug().jumps).toBe(0); + expect(ring.read([new Float32Array(20)])).toBe(20); + expect(ring.debug().jumps).toBe(0); + expect(ring.debug().jumped).toBe(0); }); test("late duplicate writes discard input without skipping playback", () => { @@ -49,6 +53,7 @@ for (const shared of [false, true]) { expect(ring.debug().discarded).toBe(50); expect(ring.debug().skips).toBe(0); expect(ring.debug().skipped).toBe(0); + expect(ring.debug().jumps).toBe(0); expect(ring.timestamp).toBe(Time.Micro(100_000)); }); }); @@ -59,15 +64,81 @@ test("shrinking a playing message ring counts the dropped media", () => { for (let at = 0; at < 800; at += 100) ring.write(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); ring.read([new Float32Array(20)]); ring.resize(Time.Milli(100)); - expect(ring.debug().skips).toBe(1); - expect(ring.debug().skipped).toBe(205); + expect(ring.debug().jumps).toBe(0); expect(ring.timestamp).toBe(Time.Micro(525_000)); + expect(ring.read([new Float32Array(20)])).toBe(20); + expect(ring.debug().jumps).toBe(1); + expect(ring.debug().jumped).toBe(205); }); test("shrinking a message ring before playback trims instead of skipping", () => { const post = new AudioRingBuffer({ rate: 1000, channels: 1, latency: Time.Milli(400) }); for (let at = 0; at < 500; at += 100) post.write(Time.Micro(at * 1000), [new Float32Array(100).fill(0.5)]); post.resize(Time.Milli(100)); - expect(post.debug().trimmed).toBe(225); - expect(post.debug().skipped).toBe(0); + expect(post.read([new Float32Array(20)])).toBe(20); + expect(post.debug().jumps).toBe(0); + expect(post.debug().jumped).toBe(0); +}); + +for (const consumed of [0, 20]) { + test(`handoff counts only unheard overflow after ${consumed} initial samples`, () => { + const source = new SharedRingBuffer(allocSharedRingBuffer(1, 128, 1000, true)); + source.setLatency(50); + for (const at of [0, 50]) source.insert(Time.Micro(at * 1000), [new Float32Array(50).fill(0.5)]); + if (consumed) expect(source.read([new Float32Array(consumed)])).toBe(consumed); + const replacement = source.resize(256); + replacement.insert(Time.Micro(100_000), [new Float32Array(300).fill(0.5)]); + expect(source.read([new Float32Array(40)])).toBe(40); + const reader = new SharedRingBuffer(replacement.init, source); + expect(replacement.debug().jumps).toBe(0); + expect(reader.read([new Float32Array(20)])).toBe(20); + expect(replacement.debug().jumps).toBe(1); + expect(replacement.debug().jumped).toBe(104 - consumed); + }); +} + +test("an overflow that invalidates a reader commit is counted only after media resumes", () => { + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, 128, 1000, true)); + ring.setLatency(50); + for (const at of [0, 50]) ring.insert(Time.Micro(at * 1000), [new Float32Array(50).fill(0.5)]); + ring.read([new Float32Array(20)]); + ring.view(); + ring.peek([new Float32Array(20)], 20); + ring.insert(Time.Micro(100_000), [new Float32Array(200).fill(0.5)]); + expect(ring.commit(20)).toBe(false); + expect(ring.debug().jumps).toBe(0); + expect(ring.read([new Float32Array(20)])).toBe(20); + expect(ring.debug().jumps).toBe(1); + expect(ring.debug().jumped).toBe(152); +}); + +test("a handoff to a reset timeline does not observe the previous cursor as a jump", () => { + const source = new SharedRingBuffer(allocSharedRingBuffer(1, 128, 1000, true)); + source.setLatency(50); + for (const at of [0, 50]) source.insert(Time.Micro(at * 1000), [new Float32Array(50).fill(0.5)]); + source.read([new Float32Array(20)]); + const replacement = source.resize(256); + replacement.reset(); + for (const at of [1000, 1050]) replacement.insert(Time.Micro(at * 1000), [new Float32Array(50).fill(0.5)]); + const reader = new SharedRingBuffer(replacement.init, source); + expect(reader.read([new Float32Array(20)])).toBe(20); + expect(replacement.debug().jumps).toBe(0); + expect(replacement.debug().jumped).toBe(0); + expect(replacement.timestamp).toBe(Time.Micro(1_020_000)); +}); + +test("handoff preserves a late observation even when the replacement resets", () => { + const source = new SharedRingBuffer(allocSharedRingBuffer(1, 128, 1000, true)); + source.setLatency(50); + for (const at of [0, 50]) source.insert(Time.Micro(at * 1000), [new Float32Array(50).fill(0.5)]); + source.read([new Float32Array(20)]); + source.insert(Time.Micro(100_000), [new Float32Array(200).fill(0.5)]); + const replacement = source.resize(256); + expect(source.read([new Float32Array(20)])).toBe(20); + replacement.reset(); + for (const at of [1000, 1050]) replacement.insert(Time.Micro(at * 1000), [new Float32Array(50).fill(0.5)]); + const reader = new SharedRingBuffer(replacement.init, source); + expect(reader.read([new Float32Array(20)])).toBe(20); + expect(replacement.debug().jumps).toBe(1); + expect(replacement.debug().jumped).toBe(152); }); diff --git a/js/watch/src/audio/playout/index.ts b/js/watch/src/audio/playout/index.ts index f24d51533c..6489990023 100644 --- a/js/watch/src/audio/playout/index.ts +++ b/js/watch/src/audio/playout/index.ts @@ -138,12 +138,16 @@ export interface Snapshot extends Counters { stalled: boolean; /** Times the reader ran dry mid-playback. */ underruns: number; - /** Times playback jumped over media after the timeline started. */ + /** Explicit skip operations, excluding writer capacity bounds. */ skips: number; - /** Media samples those jumps passed over. */ + /** Media samples those skip operations requested. */ skipped: number; - /** Incoming samples the writer rejected because they were behind the playhead. */ + /** Writer discards from late input or capacity bounds. */ discarded: number; + /** Forward discontinuities observed when the reader commits media after playback began. */ + jumps: number; + /** Media samples those observed discontinuities passed over. */ + jumped: number; /** * Samples the writer dropped off the first fill on a timeline, before anything had been played. * diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 2b807ad1bc..0c056b4232 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -57,6 +57,9 @@ export class AudioRingBuffer implements RingReader { #skips = 0; #skipped = 0; #discarded = 0; + #jumps = 0; + #jumpedSamples = 0; + #observedRead: number | undefined; // Samples dropped off the front of the first fill on a timeline, which no listener waited on. #trimmed = 0; // Whether nothing on this timeline has been played yet, so what is buffered is still free to @@ -97,6 +100,8 @@ export class AudioRingBuffer implements RingReader { readonly #playhead: Playhead = { timestamp: Time.Micro.zero, rate: 0 }; readonly #snapshot: Snapshot = { backend: "message", + jumps: 0, + jumped: 0, queued: 0, stretched: 0, output: 0, @@ -234,6 +239,8 @@ export class AudioRingBuffer implements RingReader { snapshot.skips = this.#skips; snapshot.skipped = this.#skipped; snapshot.discarded = this.#discarded; + snapshot.jumps = this.#jumps; + snapshot.jumped = this.#jumpedSamples; snapshot.trimmed = this.#trimmed; // `#fresh` stands until the reader commits a sample from this timeline, which is exactly // what "nothing has been played yet" means; an unanchored ring has no timeline at all. @@ -293,20 +300,7 @@ export class AudioRingBuffer implements RingReader { // Samples left behind are media the reader skips, which is a new timeline. A copy that kept // them all, an empty ring included, is the one it was playing: a new generation there would // make it forget what it learned about the stream on every step of the target. - if (dropped > 0) { - this.#generation++; - this.#drop(dropped); - } - } - - #drop(samples: number): void { - if (samples === 0) return; - if (this.#fresh) { - this.#trimmed += samples; - } else { - this.#skips++; - this.#skipped += samples; - } + if (dropped > 0) this.#generation++; } write(timestamp: Time.Micro, data: Float32Array[]): void { @@ -412,7 +406,7 @@ export class AudioRingBuffer implements RingReader { if (surplus || depth > this.capacity) { const to = end - (surplus ? Math.min(hold, this.capacity) : this.capacity); const dropped = Math.max(0, to - this.#readIndex); - this.#drop(dropped); + this.#discarded += dropped; this.#jumped += dropped; this.#readIndex = to; } @@ -530,6 +524,7 @@ export class AudioRingBuffer implements RingReader { this.#ended = false; this.#anchored = false; this.#fresh = true; + this.#observedRead = undefined; this.#generation++; } @@ -577,7 +572,15 @@ export class AudioRingBuffer implements RingReader { * timeline part way through a read the way the shared transport's main thread can. */ commit(count: number): boolean { + if (count > 0 && this.#observedRead !== undefined) { + const jumped = this.#readIndex - this.#observedRead; + if (jumped > 0) { + this.#jumps++; + this.#jumpedSamples += jumped; + } + } this.#readIndex += count; + if (count > 0) this.#observedRead = this.#readIndex; this.#jumped = 0; // Something of this timeline has now been heard, so the rest stops being free to drop. if (count > 0) this.#fresh = false; diff --git a/js/watch/src/audio/shared-ring-buffer.test.ts b/js/watch/src/audio/shared-ring-buffer.test.ts index 5ee1ada715..86c7580b52 100644 --- a/js/watch/src/audio/shared-ring-buffer.test.ts +++ b/js/watch/src/audio/shared-ring-buffer.test.ts @@ -86,7 +86,7 @@ describe("initialization", () => { expect(init.capacity).toBe(128); expect(init.rate).toBe(1000); expect(init.samples.byteLength).toBe(2 * 128 * 4); // 2 channels * 128 samples * Float32 - expect(init.control.byteLength).toBe(20 * 4); // 20 control slots * Int32 + expect(init.control.byteLength).toBe(22 * 4); // 22 control slots * Int32 expect(init.state.byteLength).toBe(8); // packed epoch + read cursor }); diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index 0378d23b67..15affc84d1 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -46,10 +46,10 @@ const OUTPUT = 9; const ACCELERATES = 10; const EXPANDS = 11; const SHORT = 12; -// Playback jumps and the media samples they passed over. Both ends increment atomically. +// Explicit skip operations and their requested samples, excluding capacity bounds. const SKIPS = 13; const SKIPPED = 14; -// Incoming samples already behind the playhead. Writer only. +// Samples the writer dropped: too old for the playhead, or past the ring's capacity. Writer only. const DISCARDED = 15; // Frames of OUTPUT the reader synthesized to cover a gap, so they carried no media. Reader only. const CONCEALED = 16; @@ -70,7 +70,10 @@ const ENDED = 18; * to close: nothing had been played, so the playhead simply started further in. See `#trim`. */ const TRIMMED = 19; -const CONTROL_SLOTS = 20; +// Discontinuities actually observed when the reader commits media. Reader only. +const JUMPS = 20; +const JUMPED = 21; +const CONTROL_SLOTS = 22; /** * The playhead and its mutation epoch, packed into one 64-bit word: epoch in the high @@ -194,9 +197,6 @@ export class SharedRingBuffer implements RingReader { // from it. While it stands, nothing on this timeline has been heard. Writer only: the reader's own // advances are visible as READ moving past it. See {@link #trim}. #resumed: number | undefined; - // Writer-only diagnostic cursor. Unlike #resumed, follows overflow too, so repeated drops - // before the first reader advance remain startup trims without changing playout's trim policy. - #unplayed: number | undefined; // What the last `view` sampled: the packed word its exchange has to match, and the cursor the // skip-ahead left it on. Reader thread only, since only the worklet reads. @@ -225,6 +225,10 @@ export class SharedRingBuffer implements RingReader { // is the writer having dropped the oldest samples out from under the reader, which the playout // engine has to treat as a step rather than as the buffer draining. #expected: number | undefined; + // The last positive reader commit, preserved across resize but not a new media timeline. + #observedRead: number | undefined; + #observedTimeline = 0; + #observedEpoch = 0; // Absolute sample index of that first sample. READ/WRITE are stored relative to it, so // `timestamp` adds it back to recover media time. Main-thread only: the worklet reads by @@ -287,14 +291,21 @@ export class SharedRingBuffer implements RingReader { const timeline = Atomics.load(source.#control, TIMELINE); const from = Atomics.load(source.#state, 0); + for (const counter of [JUMPS, JUMPED]) { + Atomics.store(this.#control, counter, Atomics.load(source.#control, counter)); + } for (;;) { const state = Atomics.load(this.#state, 0); if (Atomics.load(this.#control, TIMELINE) !== timeline) return; - if (((readOf(from) - readOf(state)) | 0) <= 0) return; + if (((readOf(from) - readOf(state)) | 0) <= 0) break; const next = pack(epochOf(state), readOf(from)); - if (Atomics.compareExchange(this.#state, 0, state, next) === state) return; + if (Atomics.compareExchange(this.#state, 0, state, next) === state) break; + } + if (source.#observedTimeline === timeline) { + this.#observedRead = source.#observedRead; + this.#observedTimeline = timeline; } } @@ -305,18 +316,13 @@ export class SharedRingBuffer implements RingReader { * overflow path; the reader publishes with its own exchange so it can tell a rebase apart * from losing a race. */ - #advance(candidate: number): number { + #advance(candidate: number): void { for (;;) { const state = Atomics.load(this.#state, 0); - if (this.#unplayed !== readOf(state)) this.#unplayed = undefined; - const advanced = (candidate - readOf(state)) | 0; - if (advanced <= 0) return 0; + if (((candidate - readOf(state)) | 0) <= 0) return; const next = pack(epochOf(state), candidate); - if (Atomics.compareExchange(this.#state, 0, state, next) === state) { - if (this.#unplayed !== undefined) this.#unplayed = candidate; - return advanced; - } + if (Atomics.compareExchange(this.#state, 0, state, next) === state) return; } } @@ -367,7 +373,6 @@ export class SharedRingBuffer implements RingReader { this.#anchored = true; // Nothing on this timeline has been played, so the fill is still free to trim. this.#resumed = 0; - this.#unplayed = 0; this.#position = 0; this.#lastRead = 0; this.#lastMedia = Number.NEGATIVE_INFINITY; @@ -438,15 +443,8 @@ export class SharedRingBuffer implements RingReader { const bounded = readOf(Atomics.load(this.#state, 0)); if (((end - bounded) | 0) > this.capacity) { const to = (end - this.capacity) | 0; - const dropped = this.#advance(to); - if (dropped > 0) { - if (this.#unplayed !== undefined) { - Atomics.add(this.#control, TRIMMED, dropped); - } else { - Atomics.add(this.#control, SKIPS, 1); - Atomics.add(this.#control, SKIPPED, dropped); - } - } + Atomics.add(this.#control, DISCARDED, (to - bounded) | 0); + this.#advance(to); } // Write sample data @@ -539,6 +537,13 @@ export class SharedRingBuffer implements RingReader { */ view(): RingView { const state = Atomics.load(this.#state, 0); + const epoch = epochOf(state); + if (epoch !== this.#observedEpoch) { + const timeline = Atomics.load(this.#control, TIMELINE); + if (timeline !== this.#observedTimeline) this.#observedRead = undefined; + this.#observedTimeline = timeline; + this.#observedEpoch = epoch; + } this.#snapshot = state; const stalled = Atomics.load(this.#control, STALLED) === 1; @@ -587,7 +592,7 @@ export class SharedRingBuffer implements RingReader { view.unstable = unstable; view.converge = !this.buffered; view.skipped = jumped + this.#pending; - view.generation = epochOf(state); + view.generation = epoch; return view; } @@ -620,6 +625,16 @@ export class SharedRingBuffer implements RingReader { if (((next - readOf(state)) | 0) !== 0) { if (Atomics.compareExchange(this.#state, 0, state, pack(epochOf(state), next)) !== state) return false; } + if (count > 0) { + if (this.#observedRead !== undefined) { + const jumped = (this.#cursor - this.#observedRead) | 0; + if (jumped > 0) { + Atomics.add(this.#control, JUMPS, 1); + Atomics.add(this.#control, JUMPED, jumped); + } + } + this.#observedRead = next; + } if (this.#pending > 0) { Atomics.add(this.#control, SKIPS, 1); Atomics.add(this.#control, SKIPPED, this.#pending); @@ -826,7 +841,8 @@ export class SharedRingBuffer implements RingReader { } } - dst.#unplayed = this.#unplayed === read ? copyStart : undefined; + dst.#observedRead = this.#observedRead; + dst.#observedTimeline = this.#observedTimeline; Atomics.store(dst.#control, TIMELINE, Atomics.load(this.#control, TIMELINE)); Atomics.store(dst.#state, 0, pack(epochOf(state), copyStart)); Atomics.store(dst.#control, WRITE, write); @@ -848,6 +864,8 @@ export class SharedRingBuffer implements RingReader { SKIPPED, DISCARDED, TRIMMED, + JUMPS, + JUMPED, ENDED, ]) { Atomics.store(dst.#control, control, Atomics.load(this.#control, control)); @@ -962,6 +980,8 @@ export class SharedRingBuffer implements RingReader { skips: load(SKIPS), skipped: load(SKIPPED), discarded: load(DISCARDED), + jumps: load(JUMPS), + jumped: load(JUMPED), trimmed: load(TRIMMED), // Where the writer left READ, against where READ is now: the same question `#trim` asks, // and asked here rather than read off `#resumed` because only an insert clears that, so diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 347d4250eb..5c7b04ac5a 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -282,9 +282,17 @@ definition is a judgement call are: - **`stalled_quanta`** is the share of the run the ring spent re-stalled, refilling rather than playing. Graded separately from underruns: it is silence the player chose. -- **`skip_aheads`** counts explicit ring jumps over media after playback starts. The accompanying - `skipped_samples` duration converts the ring's skipped frames at the graph's actual sample rate. - Late incoming duplicates are `discarded_samples`; initial fill trimming is not a playback skip. +- **`observed_jumps`** counts forward discontinuities when the reader commits media after playback + starts. `observed_skipped_samples` measures the media passed over in milliseconds. Both lanes + publish these comparable observations; startup trimming and timeline resets are excluded. + They remain informational under the existing budgets. + +- **`skip_aheads`** and **`skipped_samples`** use those observations in browser rows. Replay retains + its original operational definitions: explicit latency or empty-ring skip requests, excluding + capacity bounds. Its `discarded_samples` counts writer discards from capacity bounds or late + input. Keeping these operational categories preserves the existing independent replay limits; + an allowance for writer discards does not permit additional explicit skip requests. No budget + values or keys changed. These replay fields must not be read as aggregate playback loss. - **`silence_share`** is the share of sampled windows whose RMS at the graph output was below about -60 dBFS. It is the only metric read from the audio itself rather than from a counter: a counter diff --git a/test/audio-quality/clients/js/analyze.ts b/test/audio-quality/clients/js/analyze.ts index be2f8f841b..938fea4c9b 100644 --- a/test/audio-quality/clients/js/analyze.ts +++ b/test/audio-quality/clients/js/analyze.ts @@ -236,7 +236,7 @@ if ( state.anchor !== firstPlayout?.anchor || state.rate !== firstPlayout?.rate || (previous !== undefined && - (state.skips < previous.skips || state.skipped < previous.skipped || state.output < previous.output)) + (state.jumps < previous.jumps || state.jumped < previous.jumped || state.output < previous.output)) ); }) || (playout.length > 0 && playout.length !== window.length) @@ -247,14 +247,14 @@ if ( }); } -const counter = (key: "skips" | "skipped" | "discarded" | "short" | "accelerates" | "expands") => +const counter = (key: "jumps" | "jumped" | "discarded" | "short" | "accelerates" | "expands") => rise(window.map((sample) => sample.playout?.[key])); -const duration = (key: "skipped" | "discarded") => { +const duration = (key: "jumped" | "discarded") => { const count = counter(key); return count === null || firstPlayout === undefined ? null : (count * 1000) / firstPlayout.rate; }; -const skipAheads = counter("skips"); -const skippedMs = duration("skipped"); +const skipAheads = counter("jumps"); +const skippedMs = duration("jumped"); const discardedMs = duration("discarded"); const shortQuanta = counter("short"); const accelerates = counter("accelerates"); @@ -479,6 +479,10 @@ const metrics: Record = { skip_aheads_per_min: skipAheads === null ? null : round1(skipAheads / minutes), skipped_samples_total: round1(skippedMs), skipped_samples_per_min: skippedMs === null ? null : round1(skippedMs / minutes), + observed_jumps_total: skipAheads, + observed_jumps_per_min: skipAheads === null ? null : round1(skipAheads / minutes), + observed_skipped_samples_total: round1(skippedMs), + observed_skipped_samples_per_min: skippedMs === null ? null : round1(skippedMs / minutes), accelerates_total: accelerates, accelerates_per_min: accelerates === null ? null : round1(accelerates / minutes), expands_total: expands, diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts index 6ee84af598..edf3b584d3 100644 --- a/test/audio-quality/clients/js/replay.ts +++ b/test/audio-quality/clients/js/replay.ts @@ -177,6 +177,8 @@ for (const { recording, row } of matrix) { const underruns = rise((s) => s.debug.underruns); const skips = rise((s) => s.debug.skips); const skippedSamples = rise((s) => s.debug.skipped); + const observedJumps = rise((s) => s.debug.jumps); + const observedSkipped = rise((s) => s.debug.jumped); const discarded = rise((s) => s.debug.discarded); const accelerates = rise((s) => s.debug.accelerates); const expands = rise((s) => s.debug.expands); @@ -228,6 +230,10 @@ for (const { recording, row } of matrix) { skip_aheads_per_min: round1(skips / minutes), skipped_samples_total: round1(asMs(skippedSamples, recording.rate)), skipped_samples_per_min: round1(asMs(skippedSamples, recording.rate) / minutes), + observed_jumps_total: observedJumps, + observed_jumps_per_min: round1(observedJumps / minutes), + observed_skipped_samples_total: round1(asMs(observedSkipped, recording.rate)), + observed_skipped_samples_per_min: round1(asMs(observedSkipped, recording.rate) / minutes), accelerates_total: accelerates, accelerates_per_min: round1(accelerates / minutes), expands_total: expands, diff --git a/test/audio-quality/clients/js/src/analyze.test.ts b/test/audio-quality/clients/js/src/analyze.test.ts index 712d3f8ffa..ea9ce23441 100644 --- a/test/audio-quality/clients/js/src/analyze.test.ts +++ b/test/audio-quality/clients/js/src/analyze.test.ts @@ -11,6 +11,8 @@ const diagnostic: NonNullable = { generation: 1, rate: 44100, anchor: 44100, + jumps: 0, + jumped: 0, skips: 0, skipped: 0, discarded: 0, @@ -77,10 +79,12 @@ test("report age does not invent a playback skip", () => { test("a real ring skip remains visible despite delayed reports", () => { const summary = analyze((sample) => ({ ...sample, - playout: { ...diagnostic, skips: sample.at >= 20000 ? 1 : 0, skipped: sample.at >= 20000 ? 2646 : 0 }, + playout: { ...diagnostic, jumps: sample.at >= 20000 ? 1 : 0, jumped: sample.at >= 20000 ? 2646 : 0 }, })); expect(summary.metrics.skip_aheads_total).toBe(1); expect(summary.metrics.skipped_samples_total).toBe(60); + expect(summary.metrics.observed_jumps_total).toBe(1); + expect(summary.metrics.observed_skipped_samples_total).toBe(60); }); test("late writes and startup trim are separate from skipped playback", () => { diff --git a/test/audio-quality/clients/js/src/grade.test.ts b/test/audio-quality/clients/js/src/grade.test.ts new file mode 100644 index 0000000000..3c7d389b56 --- /dev/null +++ b/test/audio-quality/clients/js/src/grade.test.ts @@ -0,0 +1,55 @@ +import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Summary } from "./schema.ts"; + +test("replay keeps independent skip and writer-discard limits while reporting observed jumps", () => { + const out = mkdtempSync(join(tmpdir(), "moq-audio-grade-")); + try { + const replay = spawnSync( + process.execPath, + [ + new URL("../replay.ts", import.meta.url).pathname, + "--out", + out, + "--fixtures", + "mic-firefox", + "--rings", + "plain", + ], + { encoding: "utf8" }, + ); + expect(replay.status, replay.stderr).toBe(0); + const path = join(out, "replay-opus-48000-mic-firefox-plain.summary.json"); + const summary: Summary = JSON.parse(readFileSync(path, "utf8")); + expect(summary.metrics.skip_aheads_per_min).toBe(0); + expect(summary.metrics.skipped_samples_per_min).toBe(0); + expect(summary.metrics.discarded_samples_per_min).toBe(13127.3); + expect(summary.metrics.observed_skipped_samples_per_min).toBeGreaterThan(0); + + const grade = (changed: Record = {}) => { + writeFileSync(path, JSON.stringify({ ...summary, metrics: { ...summary.metrics, ...changed } })); + return spawnSync( + process.execPath, + [ + new URL("../grade.ts", import.meta.url).pathname, + "--run", + out, + "--budgets", + new URL("../../../budgets.json", import.meta.url).pathname, + "--enforce", + ], + { encoding: "utf8" }, + ); + }; + expect(grade().status).toBe(0); + expect(grade({ observed_jumps_per_min: 1e6, observed_skipped_samples_per_min: 1e6 }).status).toBe(0); + expect(grade({ skip_aheads_per_min: 1, discarded_samples_per_min: 0 }).status).toBe(1); + expect(grade({ skipped_samples_per_min: 1, discarded_samples_per_min: 0 }).status).toBe(1); + expect(grade({ discarded_samples_per_min: 13128.3 }).status).toBe(1); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}); diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index ed077b2989..1583762696 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -121,19 +121,31 @@ export const METRICS: Record = { unit: "samples", clock: "viewer", aggregations: ["total", "per_min"], - description: "Incoming samples rejected because they were behind the playhead, reported in ms.", + description: "Writer discard operations from late input or capacity bounds, reported in ms.", }, skip_aheads: { unit: "count", clock: "viewer", aggregations: ["total", "per_min"], - description: "Playback jumps over media after the timeline started, counted by the ring.", + description: "Browser: observed playback jumps. Replay: explicit skip operations, excluding capacity bounds.", }, skipped_samples: { unit: "samples", clock: "viewer", aggregations: ["total", "per_min"], - description: "Media time the playhead jumped over, reported in ms.", + description: "Media time attributed to skip_aheads, reported in ms.", + }, + observed_jumps: { + unit: "count", + clock: "viewer", + aggregations: ["total", "per_min"], + description: "Forward discontinuities observed when the reader commits media, excluding startup and resets.", + }, + observed_skipped_samples: { + unit: "samples", + clock: "viewer", + aggregations: ["total", "per_min"], + description: "Media passed over by observed playback discontinuities, reported in ms.", }, accelerates: { unit: "count", From 75280d49ea22acacd570a445e45ccb2de214720a Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:20:15 -0400 Subject: [PATCH 059/127] fix(watch): keep pause clickable while buffering Let the buffering indicator inherit its container pointer handling. Exercise the real Pause control with the indicator visible in the browser media suite. Co-Authored-By: GPT-6 --- js/watch/src/ui/styles/buffering-indicator.css | 1 - test/interop/clients/js/media.ts | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/js/watch/src/ui/styles/buffering-indicator.css b/js/watch/src/ui/styles/buffering-indicator.css index 6cb99e082a..4d5efec8d7 100644 --- a/js/watch/src/ui/styles/buffering-indicator.css +++ b/js/watch/src/ui/styles/buffering-indicator.css @@ -10,7 +10,6 @@ z-index: 1; background-color: var(--color-black-alpha-40); backdrop-filter: blur(var(--spacing-2)); - pointer-events: auto; } .buffering-spinner { diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index a2478de489..3a9dc33d60 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -562,9 +562,21 @@ try { // ── pause and resume ───────────────────────────────────────────────────── if (wants("pause")) { console.error("=== pause and resume ==="); + // Keep the buffering indicator visible so hit testing covers an interrupted stream. + await player.locator(SELECTORS.ui).evaluate((element) => { + if (!element.shadowRoot) throw new Error("player UI has no shadow root"); + const style = document.createElement("style"); + style.dataset.interop = "buffering"; + style.textContent = ".buffering { display: flex !important; }"; + element.shadowRoot.append(style); + }); // The chrome auto-hides while playing; pointer activity reveals the real control. await player.dispatchEvent(SELECTORS.ui, "pointermove"); await player.locator(SELECTORS.ui).locator(SELECTORS.pauseControl).click(); + await player + .locator(SELECTORS.ui) + .locator('style[data-interop="buffering"]') + .evaluate((element) => element.remove()); await waitForState(player, playerErrors, { deadline: Date.now() + SETTLE_MS, assertion: "pause takes effect", From 59c6a23dddbd6c79163382f09ddd3436e34aa906 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:18:58 -0400 Subject: [PATCH 060/127] fix(net): expire takeover tails from the serving route Co-Authored-By: GPT-5.6 Sol --- rs/moq-net/src/model/resume.rs | 201 ++++++++++++++++++++++++++++++++- rs/moq-net/src/model/track.rs | 119 ++++++++++++------- 2 files changed, 278 insertions(+), 42 deletions(-) diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 3fb58b59c2..2861247912 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -156,6 +156,20 @@ fn served_start(segments: &[Segment], from: u64, cap: Option) -> Option, waiter: &kio::Waiter) -> Option { + let mut successor = None; + for segment in segments { + let start = segment.start.map_or(0, |start| start.group).max(from); + let candidate = segment + .track + .poll_served_start(start, min_some(cap, last_group(segment.end)), waiter); + if successor.is_none() { + successor = candidate; + } + } + successor +} + /// How many segments a logical track keeps before pruning terminal ones from the /// front: the live segment plus a couple of predecessors still draining to slow /// readers. Without a bound, every failover leaves one dead segment (pinning a @@ -249,8 +263,7 @@ impl ResumeState { } /// Where the logical track continues past the exclusive group `boundary` of segment - /// `id`, below the reader's `cap`: the start of the first group the later segments - /// serve there. `None` while none is cached, or it has no frame yet. + /// `id`, below the reader's `cap`: the first group the later segments serve there. fn successor(&self, id: u64, boundary: u64, cap: Option) -> Option { let index = self.segments.iter().position(|segment| segment.id == id)?; served_start(&self.segments[index + 1..], boundary, cap) @@ -328,6 +341,38 @@ impl ResumeState { } } +pub(super) struct ExpiryBound { + state: kio::ConsumerWeak, + boundary: u64, +} + +impl ExpiryBound { + fn new(state: kio::ConsumerWeak, boundary: u64) -> Self { + Self { state, boundary } + } + + pub(super) fn poll_anchor(&self, outer: Anchor, waiter: &kio::Waiter) -> Anchor { + let mut anchor = outer.clone().capped(Some(self.boundary)); + let _ = self.state.poll(waiter, |state| { + let edge = state + .live_edge(outer.cap) + .into_iter() + .chain(outer.edge.clone()) + .max_by_key(|edge| edge.sequence); + anchor = Anchor { edge, ..outer.clone() }; + let cap = anchor.cap; + anchor = anchor.clone().capped(Some(self.boundary)); + if anchor.cap != cap { + // Scan from the boundary rather than locating its original segment: a + // handed-out group may outlive that segment's producer-side prune. + anchor.successor = poll_served_start(&state.segments, self.boundary, cap, waiter); + } + Poll::<()>::Pending + }); + anchor + } +} + /// Splices tracks into one logical track by switching at group boundaries. /// /// Created with [`Self::new`]; hand out read access via [`Self::consume`]. Call @@ -708,6 +753,15 @@ impl Consumer { served_start(&self.state.read().segments, from, cap) } + pub(crate) fn poll_served_start(&self, from: u64, cap: Option, waiter: &kio::Waiter) -> Option { + let mut successor = None; + let _ = self.state.poll(waiter, |state| { + successor = poll_served_start(&state.segments, from, cap, waiter); + Poll::<()>::Pending + }); + successor + } + /// The newest cached group across every spliced segment; see /// [`track::Consumer::peek_latest`]. pub(crate) fn peek_latest(&self) -> Option { @@ -1185,7 +1239,8 @@ impl Group { // `start_at` clamps up to the first frame the copy still holds, so landing // higher than asked means this route can't cover the seam after all. Treat it // like a dead copy and wait for one that can. - let mut group = track.guard_group(group, self.subscription.clone(), self.anchor.clone(), bound); + let expiry = bound.map(|boundary| ExpiryBound::new(self.state.weak(), boundary)); + let mut group = track.guard_group(group, self.subscription.clone(), self.anchor.clone(), expiry); group.set_stale_meter(self.stale_stats.clone()); group.start_at(self.index); if group.index() != self.index { @@ -1343,8 +1398,9 @@ impl Group { // The continuation's copy declares the count: its own count // already includes the frames it skipped. Some(continuation) => { + let expiry = bound.map(|boundary| ExpiryBound::new(self.state.weak(), boundary)); let mut continuation = - track.guard_group(continuation, self.subscription.clone(), self.anchor.clone(), bound); + track.guard_group(continuation, self.subscription.clone(), self.anchor.clone(), expiry); continuation.set_stale_meter(self.stale_stats.clone()); return continuation.poll_finished(waiter); } @@ -2659,6 +2715,143 @@ mod test { assert!(matches!(held.read_frame().now_or_never(), Some(Ok(None)))); } + #[tokio::test] + async fn takeover_tail_uses_the_next_routes_successor() { + let (mut first, first_consumer) = track_pair("first"); + let (mut middle, middle_consumer) = track_pair("middle"); + let (mut live, live_consumer) = track_pair("live"); + let mut producer = Producer::new(); + producer.switch(&first_consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_millis(100))); + write_group(&mut first, 0, "first"); + assert_eq!(recv(&mut sub), 0); + first.finish().unwrap(); + producer.switch(&middle_consumer, Position::group(1)).unwrap(); + write_group_at(&mut middle, 1, "middle", Duration::from_secs(1)); + assert_eq!(recv(&mut sub), 1); + + let mut open = middle.create_group(group::Info { sequence: 2 }).unwrap(); + open.write_frame(Timestamp::from_secs(2).unwrap(), b"held".as_ref()) + .unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut held), b"held"); + producer.switch(&live_consumer, Position::group(3)).unwrap(); + write_group_at(&mut live, 3, "next", Duration::from_secs(3)); + write_group_at(&mut live, 4, "new", Duration::from_secs(4)); + assert_eq!(recv(&mut sub), 3); + // The handed-out group reads the final ResumeState after its producer closes. + drop(producer); + assert!(matches!(held.read_frame().now_or_never(), Some(Ok(None)))); + } + + #[tokio::test] + async fn takeover_tail_after_a_continuation_uses_the_next_routes_successor() { + let (first, first_consumer) = track_pair("first"); + let (middle, middle_consumer) = track_pair("middle"); + let (mut live, live_consumer) = track_pair("live"); + let mut producer = Producer::new(); + producer.switch(&first_consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_millis(100))); + + let mut head = first.create_group(group::Info { sequence: 0 }).unwrap(); + head.write_frame(Timestamp::ZERO, b"head".as_ref()).unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut held), b"head"); + + producer + .switch(&middle_consumer, Position { group: 0, frame: 1 }) + .unwrap(); + let mut continuation = middle.create_group(group::Info { sequence: 0 }).unwrap(); + continuation.start_at(1).unwrap(); + continuation + .write_frame(Timestamp::from_millis(10).unwrap(), b"continuation".as_ref()) + .unwrap(); + assert_eq!(read(&mut held), b"continuation"); + + producer.switch(&live_consumer, Position::group(1)).unwrap(); + write_group_at(&mut live, 1, "next", Duration::from_secs(1)); + write_group_at(&mut live, 2, "new", Duration::from_secs(2)); + assert_eq!(recv(&mut sub), 1); + assert!(matches!(held.read_frame().now_or_never(), Some(Ok(None)))); + } + + #[tokio::test] + async fn takeover_tail_wakes_when_an_unpolled_successor_gets_its_first_timestamp() { + use std::task::Context; + + let (mut first, first_consumer) = track_pair("first"); + let (mut middle, middle_consumer) = track_pair("middle"); + let (mut live, live_consumer) = track_pair("live"); + let mut producer = Producer::new(); + producer.switch(&first_consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_millis(100))); + write_group(&mut first, 0, "first"); + assert_eq!(recv(&mut sub), 0); + first.finish().unwrap(); + producer.switch(&middle_consumer, Position::group(1)).unwrap(); + write_group_at(&mut middle, 1, "middle", Duration::from_secs(1)); + assert_eq!(recv(&mut sub), 1); + + let mut open = middle.create_group(group::Info { sequence: 2 }).unwrap(); + open.write_frame(Timestamp::from_secs(2).unwrap(), b"held".as_ref()) + .unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut held), b"held"); + + producer.switch(&live_consumer, Position::group(3)).unwrap(); + let mut successor = live.create_group(group::Info { sequence: 3 }).unwrap(); + write_group_at(&mut live, 4, "edge", Duration::from_secs(4)); + let (counter, waker) = CountWaker::new(); + let mut cx = Context::from_waker(&waker); + let mut pending = std::pin::pin!(held.read_frame()); + assert!(pending.as_mut().poll(&mut cx).is_pending()); + let before = counter.count(); + successor + .write_frame(Timestamp::from_secs(3).unwrap(), b"successor".as_ref()) + .unwrap(); + assert!(counter.count() > before, "the successor timestamp wakes the held tail"); + assert!(matches!(pending.as_mut().poll(&mut cx), Poll::Ready(Ok(None)))); + } + + #[tokio::test] + async fn takeover_tail_keeps_a_successor_inside_the_budget() { + let (mut first, first_consumer) = track_pair("first"); + let (mut middle, middle_consumer) = track_pair("middle"); + let (mut live, live_consumer) = track_pair("live"); + let mut producer = Producer::new(); + producer.switch(&first_consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_millis(100))); + write_group(&mut first, 0, "first"); + assert_eq!(recv(&mut sub), 0); + first.finish().unwrap(); + producer.switch(&middle_consumer, Position::group(1)).unwrap(); + write_group_at(&mut middle, 1, "middle", Duration::from_secs(1)); + assert_eq!(recv(&mut sub), 1); + + let mut open = middle.create_group(group::Info { sequence: 2 }).unwrap(); + open.write_frame(Timestamp::from_secs(2).unwrap(), b"held".as_ref()) + .unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut held), b"held"); + + producer.switch(&live_consumer, Position::group(3)).unwrap(); + write_group_at(&mut live, 3, "late tail", Duration::from_millis(9_990)); + write_group_at(&mut live, 4, "edge", Duration::from_secs(10)); + assert_eq!(recv(&mut sub), 3); + assert!(held.read_frame().now_or_never().is_none()); + + write_group_at(&mut live, 5, "new edge", Duration::from_millis(10_200)); + assert!(matches!(held.read_frame().now_or_never(), Some(Ok(None)))); + } + /// A waker that counts its wakes, for asserting a pending poll left a live /// registration behind. struct CountWaker(std::sync::atomic::AtomicUsize); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 2deedb7380..41e101a3c4 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -549,10 +549,9 @@ impl TrackState { Some(slot.group.timestamp()) } - /// The first servable group's start in `from..cap`, with the slot identity a later - /// judgment needs to tell that group from whatever replaces it. `None` when no such - /// group is cached or it has no frame yet: an unstamped successor leaves reach - /// unbounded, and this does not skip past it to a later group. + /// The first servable group in `from..cap`, with enough identity to watch its + /// first timestamp. `None` means no such group is cached; an unstamped group is + /// still the immediate successor and prevents a later group from replacing it. fn served_start(&self, from: u64, cap: Option) -> Option { let slot = self .lookup @@ -563,7 +562,6 @@ impl TrackState { Some(ServedStart { sequence: slot.group.sequence, stamp: slot.stamp, - timestamp: slot.group.timestamp()?, }) } @@ -606,7 +604,7 @@ impl TrackState { .filter(|live| self.holds(live.sequence, live.stamp)) .map(|live| (live.sequence, live.timestamp)); // `outer` and `successor` were revalidated on their own tracks before this lock - // was taken ([`LiveEdge::is_live`], [`Successor::start`]). There is no slot for + // was taken ([`LiveEdge::is_live`], [`Successor::poll_start`]). There is no slot for // them here, and taking their locks here would nest. let Some((_, timestamp)) = local .into_iter() @@ -2439,15 +2437,14 @@ impl Consumer { } } - /// Where the first servable group in `from..cap` starts, with enough identity to - /// revalidate it later. A splice answers from the first segment holding one. + /// The first servable group in `from..cap`, with enough identity to revalidate it + /// and watch its first timestamp. A splice answers from the first segment holding one. pub(crate) fn served_start(&self, from: u64, cap: Option) -> Option { match &self.inner { ConsumerKind::Plain(state) => { let served = state.read().served_start(from, cap)?; Some(Successor { sequence: served.sequence, - timestamp: served.timestamp, stamp: served.stamp, track: state.weak(), }) @@ -2456,6 +2453,28 @@ impl Consumer { } } + /// Resolve the first servable group in `from..cap` while registering for cache + /// changes that can move it. An unstamped group is still returned: its + /// [`Successor`] registers directly for the first timestamp when judged. + pub(crate) fn poll_served_start(&self, from: u64, cap: Option, waiter: &kio::Waiter) -> Option { + match &self.inner { + ConsumerKind::Plain(state) => { + let mut successor = None; + let track = state.weak(); + let _ = state.poll(waiter, |state| { + successor = state.served_start(from, cap).map(|served| Successor { + sequence: served.sequence, + stamp: served.stamp, + track: track.clone(), + }); + Poll::<()>::Pending + }); + successor + } + ConsumerKind::Spliced(resume) => resume.poll_served_start(from, cap, waiter), + } + } + /// The nearest cached group below `sequence`, under the same terms as /// [`Self::peek_group`]. Walks the cache's own order, so gaps in the group numbering /// are crossed and aborted (evicted) entries are skipped. @@ -2494,12 +2513,12 @@ impl Consumer { } /// Attach a subscription's drift policy to a cached group resolved outside its cursor. - pub(crate) fn guard_group( + pub(super) fn guard_group( &self, group: group::Consumer, subscription: kio::Consumer, anchor: kio::Consumer, - bound: Option, + bound: Option, ) -> group::Consumer { let ConsumerKind::Plain(state) = &self.inner else { return group; @@ -2509,7 +2528,10 @@ impl Consumer { state: state.weak(), subscription, anchor, - bound, + bound: match bound { + Some(bound) => ExpiryBound::Resume(Box::new(bound)), + None => ExpiryBound::Plain, + }, sequence, })) } @@ -3085,10 +3107,24 @@ struct GroupExpiry { state: kio::ConsumerWeak, subscription: kio::Consumer, anchor: kio::Consumer, - bound: Option, + bound: ExpiryBound, sequence: u64, } +enum ExpiryBound { + Plain, + Resume(Box), +} + +impl ExpiryBound { + fn anchor(&self, anchor: Anchor, waiter: &kio::Waiter) -> Anchor { + match self { + Self::Plain => anchor, + Self::Resume(bound) => bound.poll_anchor(anchor, waiter), + } + } +} + impl group::Expiry for GroupExpiry { fn is_expired(&self, waiter: &kio::Waiter) -> bool { let mut max_age = Duration::default(); @@ -3102,14 +3138,17 @@ impl group::Expiry for GroupExpiry { anchor = (**current).clone(); Poll::<()>::Pending }); - let anchor = anchor.capped(self.bound); + let anchor = self.bound.anchor(anchor, waiter); let cap = anchor.cap; // Before this track's lock: both may name another track, and nesting deadlocks. let outer = anchor .edge .filter(LiveEdge::is_live) .map(|live| (live.sequence, live.timestamp)); - let successor = anchor.successor.as_ref().and_then(Successor::start); + let successor = anchor + .successor + .as_ref() + .and_then(|successor| successor.poll_start(waiter)); let mut expired = false; let _ = self.state.poll(waiter, |state| { @@ -3198,8 +3237,9 @@ pub(crate) struct Anchor { /// Where the reader's next group past `cap` starts presenting, when another track /// serves it (a splice's next segment). The last group below the cap has no /// successor in its own track, so without this nothing bounds its reach and it is - /// never judged stale. `None` while unknown or unstamped. Revalidated like `edge`: - /// a cached start must not convict once that group is gone. + /// never judged stale. `None` while no immediate successor is cached; an unstamped + /// successor resolves no start until its first frame. Revalidated like `edge`: a + /// cached start must not convict once that group is gone. pub successor: Option, } @@ -3250,40 +3290,40 @@ impl PartialEq for LiveEdge { struct ServedStart { sequence: u64, stamp: u32, - timestamp: Timestamp, } -/// A successor pushed onto another track's cursor. The timestamp alone is not enough: -/// once the group is evicted, a later group can keep the outer edge valid while this -/// start is no longer where the track continues. +/// A successor pushed onto another track's cursor. Its slot identity keeps an evicted +/// group from bounding the previous segment after the track has moved on. #[derive(Clone)] pub(crate) struct Successor { sequence: u64, - timestamp: Timestamp, stamp: u32, track: kio::ConsumerWeak, } impl Successor { - /// The start this still names, re-read from its own track, or `None` once that - /// group is gone or no longer stamped. Takes that track's lock, so never call it - /// under another's. - fn start(&self) -> Option { - let state = self.track.read(); - let slot = state.lookup.get(&self.sequence)?; - if slot.stamp != self.stamp || slot.group.is_aborted() { - return None; - } - slot.group.timestamp() + /// The start this still names, registering for its first timestamp and cache + /// changes. Takes that track's lock, so never call it under another's. + fn poll_start(&self, waiter: &kio::Waiter) -> Option { + let mut start = None; + let _ = self.track.poll(waiter, |state| { + let Some(slot) = state.lookup.get(&self.sequence) else { + return Poll::<()>::Pending; + }; + if slot.stamp != self.stamp || slot.group.is_aborted() { + return Poll::<()>::Pending; + } + let _ = slot.group.poll_timestamp(waiter); + start = slot.group.timestamp(); + Poll::<()>::Pending + }); + start } } impl PartialEq for Successor { fn eq(&self, other: &Self) -> bool { - self.sequence == other.sequence - && self.timestamp == other.timestamp - && self.stamp == other.stamp - && self.track.same_channel(&other.track) + self.sequence == other.sequence && self.stamp == other.stamp && self.track.same_channel(&other.track) } } @@ -3426,7 +3466,10 @@ impl PlainSubscriber { .as_ref() .filter(|live| live.is_live()) .map(|live| (live.sequence, live.timestamp)); - let successor = drift.successor.as_ref().and_then(Successor::start); + let successor = drift + .successor + .as_ref() + .and_then(|successor| successor.poll_start(waiter)); let presentation = drift.edge.presentation; let cap = drift.edge.cap; let budget = drift.budget; @@ -3447,7 +3490,7 @@ impl PlainSubscriber { state: self.state.weak(), subscription: self.subscription.consume(), anchor: self.drift_anchor.consume(), - bound: None, + bound: ExpiryBound::Plain, sequence, })) } From c7faf59406165560a5632c794f9ac1b0b6ae4931 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:24:21 -0400 Subject: [PATCH 061/127] fix(audio): count only concurrent overflow discards Charge the cursor advance that wins the CAS, excluding samples the reader consumed while the writer was taking its snapshot. Keep operational replay categories and media behavior unchanged. Co-Authored-By: GPT-6 --- js/watch/src/audio/counters.test.ts | 31 ++++++++++++++++++++++++ js/watch/src/audio/shared-ring-buffer.ts | 11 +++++---- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/js/watch/src/audio/counters.test.ts b/js/watch/src/audio/counters.test.ts index e74a610f14..d322122abc 100644 --- a/js/watch/src/audio/counters.test.ts +++ b/js/watch/src/audio/counters.test.ts @@ -112,6 +112,37 @@ test("an overflow that invalidates a reader commit is counted only after media r expect(ring.debug().jumped).toBe(152); }); +for (const incoming of [60, 200]) { + test(`overflow accounts for a concurrent reader before inserting ${incoming} samples`, () => { + const writer = new SharedRingBuffer(allocSharedRingBuffer(1, 128, 1000, true)); + writer.setLatency(50); + for (const at of [0, 50]) writer.insert(Time.Micro(at * 1000), [new Float32Array(50).fill(0.5)]); + const reader = new SharedRingBuffer(writer.init); + expect(reader.read([new Float32Array(20)])).toBe(20); + const exchange = Atomics.compareExchange; + const atomics = Atomics as { compareExchange: unknown }; + let raced = false; + atomics.compareExchange = (array: BigInt64Array, index: number, expected: bigint, next: bigint) => { + if (!raced && array.buffer === writer.init.state) { + raced = true; + const output = [new Float32Array(40)]; + expect(reader.read(output)).toBe(40); + expect(output[0]).toEqual(new Float32Array(40).fill(0.5)); + } + return exchange(array, index, expected, next); + }; + try { + writer.insert(Time.Micro(100_000), [new Float32Array(incoming).fill(0.5)]); + } finally { + atomics.compareExchange = exchange; + } + expect(raced).toBe(true); + expect(writer.debug().discarded).toBe(incoming === 60 ? 0 : 112); + expect(reader.read([new Float32Array(20)])).toBe(20); + expect(writer.debug().jumped).toBe(incoming === 60 ? 0 : 112); + }); +} + test("a handoff to a reset timeline does not observe the previous cursor as a jump", () => { const source = new SharedRingBuffer(allocSharedRingBuffer(1, 128, 1000, true)); source.setLatency(50); diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index 15affc84d1..e96604ae16 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -316,13 +316,14 @@ export class SharedRingBuffer implements RingReader { * overflow path; the reader publishes with its own exchange so it can tell a rebase apart * from losing a race. */ - #advance(candidate: number): void { + #advance(candidate: number): number { for (;;) { const state = Atomics.load(this.#state, 0); - if (((candidate - readOf(state)) | 0) <= 0) return; + const advanced = (candidate - readOf(state)) | 0; + if (advanced <= 0) return 0; const next = pack(epochOf(state), candidate); - if (Atomics.compareExchange(this.#state, 0, state, next) === state) return; + if (Atomics.compareExchange(this.#state, 0, state, next) === state) return advanced; } } @@ -443,8 +444,8 @@ export class SharedRingBuffer implements RingReader { const bounded = readOf(Atomics.load(this.#state, 0)); if (((end - bounded) | 0) > this.capacity) { const to = (end - this.capacity) | 0; - Atomics.add(this.#control, DISCARDED, (to - bounded) | 0); - this.#advance(to); + const discarded = this.#advance(to); + if (discarded > 0) Atomics.add(this.#control, DISCARDED, discarded); } // Write sample data From f21282589556a3bbe4ce51d7c5ea76a42af55da7 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:28:42 -0400 Subject: [PATCH 062/127] fix(net): wake takeover tails as the live edge advances Co-Authored-By: GPT-5.6 Sol --- rs/moq-net/src/model/group.rs | 10 +++++ rs/moq-net/src/model/resume.rs | 77 +++++++++++++++++++++++++++++++++- rs/moq-net/src/model/track.rs | 41 ++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 0205ae10b4..082be1e7d7 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -939,6 +939,16 @@ impl Producer { } } + /// Read the newest frame timestamp and register for any change to it. + pub(crate) fn poll_latest(&self, waiter: &kio::Waiter) -> Option { + let mut latest = None; + let _ = self.state.poll(waiter, |state| { + latest = state.latest; + Poll::<()>::Pending + }); + latest + } + /// Block until the group is closed or aborted. pub async fn closed(&self) -> Error { kio::wait(|waiter| self.poll_closed(waiter)).await diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 2861247912..e2d8869cbb 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -170,6 +170,19 @@ fn poll_served_start(segments: &[Segment], from: u64, cap: Option, waiter: successor } +fn poll_live_edge(segments: &[Segment], cap: Option, waiter: &kio::Waiter) -> Option { + segments + .iter() + .filter_map(|segment| { + let edge = segment + .track + .poll_live_edge(min_some(cap, last_group(segment.end)), waiter)?; + let start = segment.start.map_or(0, |start| start.group); + (edge.sequence >= start).then_some(edge) + }) + .max_by_key(|edge| edge.sequence) +} + /// How many segments a logical track keeps before pruning terminal ones from the /// front: the live segment plus a couple of predecessors still draining to slow /// readers. Without a bound, every failover leaves one dead segment (pinning a @@ -354,8 +367,7 @@ impl ExpiryBound { pub(super) fn poll_anchor(&self, outer: Anchor, waiter: &kio::Waiter) -> Anchor { let mut anchor = outer.clone().capped(Some(self.boundary)); let _ = self.state.poll(waiter, |state| { - let edge = state - .live_edge(outer.cap) + let edge = poll_live_edge(&state.segments, outer.cap, waiter) .into_iter() .chain(outer.edge.clone()) .max_by_key(|edge| edge.sequence); @@ -747,6 +759,15 @@ impl Consumer { self.state.read().live_edge(cap) } + pub(crate) fn poll_live_edge(&self, cap: Option, waiter: &kio::Waiter) -> Option { + let mut edge = None; + let _ = self.state.poll(waiter, |state| { + edge = poll_live_edge(&state.segments, cap, waiter); + Poll::<()>::Pending + }); + edge + } + /// Where the first servable group in `from..cap` starts, with the identity to /// revalidate it; see [`track::Consumer::served_start`]. pub(crate) fn served_start(&self, from: u64, cap: Option) -> Option { @@ -2819,6 +2840,58 @@ mod test { assert!(matches!(pending.as_mut().poll(&mut cx), Poll::Ready(Ok(None)))); } + #[tokio::test] + async fn takeover_tail_wakes_as_the_unpolled_edge_advances() { + use std::task::Context; + + let (mut first, first_consumer) = track_pair("first"); + let (mut middle, middle_consumer) = track_pair("middle"); + let (mut live, live_consumer) = track_pair("live"); + let mut producer = Producer::new(); + producer.switch(&first_consumer, None).unwrap(); + let mut sub = producer + .consume() + .subscribe(Subscription::default().with_max_age(Duration::from_millis(100))); + write_group(&mut first, 0, "first"); + assert_eq!(recv(&mut sub), 0); + first.finish().unwrap(); + producer.switch(&middle_consumer, Position::group(1)).unwrap(); + write_group_at(&mut middle, 1, "middle", Duration::from_secs(1)); + assert_eq!(recv(&mut sub), 1); + + let mut open = middle.create_group(group::Info { sequence: 2 }).unwrap(); + open.write_frame(Timestamp::from_secs(2).unwrap(), b"held".as_ref()) + .unwrap(); + let mut held = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut held), b"held"); + + producer.switch(&live_consumer, Position::group(3)).unwrap(); + write_group_at(&mut live, 3, "successor", Duration::from_secs(3)); + let mut edge = live.create_group(group::Info { sequence: 4 }).unwrap(); + let (counter, waker) = CountWaker::new(); + let mut cx = Context::from_waker(&waker); + let mut pending = std::pin::pin!(held.read_frame()); + assert!(pending.as_mut().poll(&mut cx).is_pending()); + + let before = counter.count(); + edge.write_frame(Timestamp::from_millis(3_050).unwrap(), b"edge".as_ref()) + .unwrap(); + assert!( + counter.count() > before, + "the edge's first timestamp wakes the held tail" + ); + assert!(pending.as_mut().poll(&mut cx).is_pending()); + + let before = counter.count(); + edge.write_frame(Timestamp::from_millis(3_200).unwrap(), b"new edge".as_ref()) + .unwrap(); + assert!( + counter.count() > before, + "the edge's newest timestamp wakes the held tail" + ); + assert!(matches!(pending.as_mut().poll(&mut cx), Poll::Ready(Ok(None)))); + } + #[tokio::test] async fn takeover_tail_keeps_a_successor_inside_the_budget() { let (mut first, first_consumer) = track_pair("first"); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 41e101a3c4..cdb1e650ce 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -497,6 +497,26 @@ impl TrackState { }) } + /// The live edge plus the waits that can move it: every newer unstamped group and + /// the selected group's newest timestamp. + fn poll_live_edge(&self, cap: Option, waiter: &kio::Waiter) -> Option { + self.lookup + .range(..) + .rev() + .filter(|(seq, _)| super::subscription::before_end(**seq, cap)) + .find_map(|(_, slot)| { + if !slot.visible || slot.group.is_aborted() { + return None; + } + let timestamp = slot.group.poll_latest(waiter)?; + Some(PresentationEdge { + sequence: slot.group.sequence, + stamp: slot.stamp, + timestamp, + }) + }) + } + /// This track's own edge under the exclusive `cap`, for measuring drift. An outer /// edge and a successor live on other tracks; the caller revalidates those before /// taking this lock and passes them in, so the locks never nest. @@ -2437,6 +2457,27 @@ impl Consumer { } } + /// Resolve the live edge while registering for timestamp changes that can move it. + pub(crate) fn poll_live_edge(&self, cap: Option, waiter: &kio::Waiter) -> Option { + match &self.inner { + ConsumerKind::Plain(state) => { + let mut edge = None; + let track = state.weak(); + let _ = state.poll(waiter, |state| { + edge = state.poll_live_edge(cap, waiter).map(|edge| LiveEdge { + sequence: edge.sequence, + timestamp: edge.timestamp, + stamp: edge.stamp, + track: track.clone(), + }); + Poll::<()>::Pending + }); + edge + } + ConsumerKind::Spliced(resume) => resume.poll_live_edge(cap, waiter), + } + } + /// The first servable group in `from..cap`, with enough identity to revalidate it /// and watch its first timestamp. A splice answers from the first segment holding one. pub(crate) fn served_start(&self, from: u64, cap: Option) -> Option { From ca4817459aab8fd71998d90043bb87e309a78dbd Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 07:56:07 -0400 Subject: [PATCH 063/127] fix(cli): drain stdin in pipe-sized reads `moq import` read stdin into an empty `BytesMut`, which offers 64 bytes per read, and every tokio stdin read is a blocking-pool round trip. On a loaded Linux runner that capped the importer near 1 MB/s, so each video keyframe burst held the audio muxed behind it for hundreds of milliseconds, past the player's fixed 250 ms delay. Reads now take up to 64 KiB, a default Linux pipe. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/publish.rs | 134 ++++++++++++++++++++++++++------------ 1 file changed, 93 insertions(+), 41 deletions(-) diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 5fa8036fc1..e90587ca03 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -342,47 +342,7 @@ impl Publish { /// Drive the source until stdin EOF (or the capture devices stop). pub async fn run(self) -> anyhow::Result<()> { match self.source { - Source::Stream(mut decoder) => { - let mut stdin = tokio::io::stdin(); - let mut buffer = bytes::BytesMut::new(); - - // Damage reported so far, so only the change is logged. A live feed is - // diagnosed by the rate at which these climb, and stdin may never end, so - // they have to surface as they accumulate rather than at exit. - let mut reported = decoder.stats(); - - // Run the read/decode loop so an error surfaces here rather than - // dropping the decoder (and its tracks) with a bare Error::Dropped. - let result: anyhow::Result<()> = async { - loop { - buffer.clear(); - let n = tokio::io::AsyncReadExt::read_buf(&mut stdin, &mut buffer).await?; - if n == 0 { - return Ok(()); // EOF - } - decoder.decode_chunk(&buffer)?; - - let latest = decoder.stats(); - if latest != reported { - log_stats(latest.as_ref(), reported.as_ref()); - reported = latest; - } - } - } - .await; - - // Flush on a clean EOF; on any error (read, decode, or the flush - // itself) abort with the real cause so subscribers see it instead of - // a bare Error::Dropped. - let outcome = result.and_then(|()| decoder.finish()); - // The drain at end of input can publish a frame nothing vouched for, so the - // final snapshot is only complete after `finish`. - log_stats(decoder.stats().as_ref(), reported.as_ref()); - if let Err(err) = &outcome { - decoder.abort(moq_net::Error::Transport(err.to_string())); - } - outcome - } + Source::Stream(decoder) => decode(decoder, tokio::io::stdin()).await, #[cfg(feature = "capture")] Source::Capture { catalog, video, audio } => { // Each enabled medium publishes its own track onto the shared @@ -432,6 +392,55 @@ impl Publish { } } +/// The most one read of the input takes: the default Linux pipe capacity. +/// +/// Each stdin read is a round trip through tokio's blocking pool, so a read has to take whatever the +/// pipe holds. At the 64 bytes an empty buffer offers, a loaded host drains a keyframe slower than +/// it arrives, and the audio muxed behind it reaches subscribers late. +const READ_SIZE: usize = 64 * 1024; + +/// Decode `input` into the broadcast until EOF. +async fn decode(mut decoder: PublishDecoder, mut input: impl tokio::io::AsyncRead + Unpin) -> anyhow::Result<()> { + let mut buffer = bytes::BytesMut::with_capacity(READ_SIZE); + + // Damage reported so far, so only the change is logged. A live feed is + // diagnosed by the rate at which these climb, and stdin may never end, so + // they have to surface as they accumulate rather than at exit. + let mut reported = decoder.stats(); + + // Run the read/decode loop so an error surfaces here rather than + // dropping the decoder (and its tracks) with a bare Error::Dropped. + let result: anyhow::Result<()> = async { + loop { + buffer.clear(); + let n = tokio::io::AsyncReadExt::read_buf(&mut input, &mut buffer).await?; + if n == 0 { + return Ok(()); // EOF + } + decoder.decode_chunk(&buffer)?; + + let latest = decoder.stats(); + if latest != reported { + log_stats(latest.as_ref(), reported.as_ref()); + reported = latest; + } + } + } + .await; + + // Flush on a clean EOF; on any error (read, decode, or the flush + // itself) abort with the real cause so subscribers see it instead of + // a bare Error::Dropped. + let outcome = result.and_then(|()| decoder.finish()); + // The drain at end of input can publish a frame nothing vouched for, so the + // final snapshot is only complete after `finish`. + log_stats(decoder.stats().as_ref(), reported.as_ref()); + if let Err(err) = &outcome { + decoder.abort(moq_net::Error::Transport(err.to_string())); + } + outcome +} + #[cfg(feature = "capture")] impl CaptureArgs { /// The video source named by the flags, defaulting to the default camera. @@ -757,6 +766,49 @@ mod tests { ); } + /// A pipe already holding all of `data`, counting the reads that drain it. + struct Pipe { + data: bytes::Bytes, + reads: usize, + } + + impl tokio::io::AsyncRead for Pipe { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let n = self.data.len().min(buf.remaining()); + let chunk = self.data.split_to(n); + buf.put_slice(&chunk); + self.reads += 1; + std::task::Poll::Ready(Ok(())) + } + } + + /// Every stdin read is a round trip through tokio's blocking pool, so a burst the pipe already + /// holds has to drain in pipe-sized reads, not a few bytes at a time. + #[tokio::test] + async fn stdin_drains_in_pipe_sized_reads() { + const INPUT: &[u8] = include_bytes!("../../moq-mux/src/container/ts/test_data/bbb_cbr.ts"); + + let broadcast = moq_net::broadcast::Info::new().produce(); + let publish = Publish::new(broadcast, &PublishFormat::Ts, Default::default()).unwrap(); + #[allow(irrefutable_let_patterns)] + let Source::Stream(decoder) = publish.source else { + panic!("expected a stream source"); + }; + + let mut pipe = Pipe { + data: bytes::Bytes::from_static(INPUT), + reads: 0, + }; + decode(decoder, &mut pipe).await.unwrap(); + + // Full reads of a default Linux pipe, then the one that finds EOF. + assert_eq!(pipe.reads, INPUT.len().div_ceil(64 * 1024) + 1); + } + /// Read the first frame of a verbatim track back as raw bytes. async fn read_frame(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); From 6b25efc3c9f3c16f531ea4d845c576b1f847cd5d Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:20:10 -0400 Subject: [PATCH 064/127] fix(watch): close the audio context once its render processor has stopped Chromium keeps a closed AudioContext and its AudioWorkletNode alive for as long as the node's processor has not stopped, and a processor only stops in a quantum its context renders. Closing the context right after posting the close message left one context and node pair in the heap per detach. The render processor now reports `stopped` on the node's own port in the quantum that ends it, and the decoder closes a running context once every processor built in it has said so. A context that is not running renders no such quantum, so it is still closed at once, as is one that stops running while its processor is stopping. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.test.ts | 162 +++++++++++++++++- js/watch/src/audio/decoder.ts | 77 ++++++++- .../src/audio/render-worklet.port.test.ts | 51 ++++++ js/watch/src/audio/render-worklet.ts | 13 +- js/watch/src/audio/render.ts | 18 +- 5 files changed, 304 insertions(+), 17 deletions(-) diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index d5858cd307..2af3a7f35a 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -143,13 +143,52 @@ class MockAudioDecoder { async flush(): Promise {} } -/** Enough of an AudioWorkletNode for the ring to be built against. */ -class MockWorkletNode { +/** + * Enough of an AudioWorkletNode for the ring to be built against, behind a processor that stops the + * way the render worklet's does: in the next quantum its context renders after it is told to close, + * saying so on the node's port. A running context renders on its own; a suspended one never does. + */ +class MockWorkletNode extends EventTarget { static built: MockContext[] = []; + static nodes: MockWorkletNode[] = []; + // Whether quanta are rendered only by `render`, so a case can hold a processor between its close + // and its stop. + static manual = false; + + readonly context: MockContext; + /** Everything the page sent the processor, in order. */ + readonly messages: unknown[] = []; + readonly port = Object.assign(new EventTarget(), { + postMessage: (message: unknown) => { + this.messages.push(message); + if ((message as { type?: string }).type !== "close") return; + this.#closed = true; + this.#tick(); + }, + start: () => {}, + }); + #closed = false; + #stopped = false; + constructor(context: MockContext) { + super(); + this.context = context; MockWorkletNode.built.push(context); + MockWorkletNode.nodes.push(this); + context.addEventListener("statechange", () => this.#tick()); } - readonly port = { postMessage: () => {}, onmessage: null, addEventListener: () => {}, start: () => {} }; + + #tick(): void { + if (this.#closed && !MockWorkletNode.manual) setTimeout(() => this.render(), 0); + } + + /** Render one quantum, which a context only does while running. */ + render(): void { + if (this.context.state !== "running" || !this.#closed || this.#stopped) return; + this.#stopped = true; + this.port.dispatchEvent(new MessageEvent("message", { data: { type: "stopped" } })); + } + connect(): void {} disconnect(): void {} } @@ -204,6 +243,8 @@ beforeEach(() => { (globalThis as Record).EncodedAudioChunk = MockEncodedChunk; MockContext.built = []; MockWorkletNode.built = []; + MockWorkletNode.nodes = []; + MockWorkletNode.manual = false; MockContext.activation = false; MockContext.grace = false; MockContext.autoplay = false; @@ -550,6 +591,121 @@ test("a player put back on the page builds a context again", async () => { close(); }); +test("a player taken off the page closes its context once the processor in it has stopped", async () => { + // Chromium keeps a closed context, and the worklet node in it, for as long as the node's processor + // has not stopped, and a processor only stops in a quantum its context renders. Closed straight + // after the close message, every detach leaves one of each in the page's heap for good. + MockWorkletNode.manual = true; + const { decoder: built, attached, close } = decoder(true); + await flush(); + click(); + await flush(); + const context = built.out.context.peek() as unknown as MockContext; + const [node] = MockWorkletNode.nodes; + expect(context.state).toBe("running"); + + attached.set(false); + await flush(); + + // Told to stop, and gone from the outputs, but the context renders on until the processor says so. + expect(node.messages).toContainEqual({ type: "close" }); + expect(built.out.context.peek()).toBeUndefined(); + expect(context.state).toBe("running"); + + node.render(); + await flush(); + expect(context.state).toBe("closed"); + + close(); +}); + +test("a player taken off the page closes a context that is not running at once", async () => { + // A suspended context renders no quantum, so its processor never stops and waiting would only hold + // the context open. Chromium keeps this pair: nothing short of rendering releases it. + MockWorkletNode.manual = true; + const { decoder: built, attached, close } = decoder(true); + await flush(); + const context = built.out.context.peek() as unknown as MockContext; + expect(context.state).toBe("suspended"); + expect(MockWorkletNode.nodes.length).toBe(1); + + attached.set(false); + await flush(); + expect(MockWorkletNode.nodes[0].messages).toContainEqual({ type: "close" }); + expect(context.state).toBe("closed"); + + close(); +}); + +test("a context that stops running while its processor stops is closed then", async () => { + // The browser suspends a context whose device fails, and interrupts one for a call: either way the + // quantum the processor would stop in never comes. + MockWorkletNode.manual = true; + const { decoder: built, attached, close } = decoder(true); + await flush(); + click(); + await flush(); + const context = built.out.context.peek() as unknown as MockContext; + + attached.set(false); + await flush(); + expect(context.state).toBe("running"); + + context.state = "suspended"; + context.dispatchEvent(new Event("statechange")); + await flush(); + expect(context.state).toBe("closed"); + + close(); +}); + +test("a processor that failed does not hold its context open", async () => { + // A processor that throws is stopped by the browser, which says so on the node rather than the port. + MockWorkletNode.manual = true; + const { decoder: built, attached, close } = decoder(true); + await flush(); + click(); + await flush(); + const context = built.out.context.peek() as unknown as MockContext; + const [node] = MockWorkletNode.nodes; + + attached.set(false); + await flush(); + expect(context.state).toBe("running"); + + node.dispatchEvent(new Event("processorerror")); + await flush(); + expect(context.state).toBe("closed"); + + close(); +}); + +test("a rate change builds the new context at once and closes the old one once its processor stopped", async () => { + MockWorkletNode.manual = true; + const { decoder: built, catalog: root, close } = decoder(true); + await flush(); + click(); + await flush(); + const first = built.out.context.peek() as unknown as MockContext; + const [node] = MockWorkletNode.nodes; + + MockContext.grace = true; + root.set(catalog({ rate: 44100 })); + await flush(); + + const second = built.out.context.peek() as unknown as MockContext; + expect(second.sampleRate).toBe(44100); + expect(second.state).toBe("running"); + expect(first.state).toBe("running"); + + node.render(); + await flush(); + expect(first.state).toBe("closed"); + expect(MockContext.live()).toEqual([second]); + + close(); +}); + test("a viewer muting a tile keeps the context it is playing", async () => { // Muting stops the download, and the graph stays up behind it: rebuilding would spend a gesture // on the unmute, which is the one thing a viewer cannot be asked for twice. diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 5aefb5cc0c..ed4e903b99 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -19,7 +19,7 @@ import type { Delay, Sync } from "../sync"; import { reportTransport, supportsSharedArrayBuffer } from "./buffer"; import { audioMaxAge, type DecoderConfig, decoderConfig } from "./config"; import type * as Playout from "./playout"; -import type { Close } from "./render"; +import type { Close, ToMain } from "./render"; // Compiled and inlined as a blob URL via vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; @@ -176,7 +176,7 @@ export class Decoder { // The AudioContext the graph runs in, owned here rather than by an effect: it is built by the // gesture that starts it (see #buildContext) and outlives every other change to the graph short // of the player leaving the page (see #runContext). - #context: { audio: AudioContext; effects: Effect } | undefined; + #context: { audio: AudioContext; effects: Effect; processors: Processors } | undefined; // Everything that feeds the ring (the subscription, the estimator, the decoder and the writes) and the // graph it writes into: on the page, or in the page's worker. See #runSupply. @@ -448,12 +448,13 @@ export class Decoder { ...(rate !== undefined && { sampleRate: rate }), }); const effects = new Effect(); - this.#context = { audio: context, effects }; + const processors = new Processors(context); + this.#context = { audio: context, effects, processors }; // Expose the rate the graph actually runs at. this.#out.sampleRate.set(context.sampleRate); this.#out.context.set(context); - effects.run((effect) => this.#runWorklet(effect, context)); + effects.run((effect) => this.#runWorklet(effect, context, processors)); return context; } @@ -464,16 +465,16 @@ export class Decoder { if (!context) return; this.#context = undefined; - // Cancel module loads before close rejects them; signal propagation happens later. + // Cancel module loads before close rejects them; signal propagation happens later. This also + // tells the processor to stop, which the context is closed after. context.effects.close(); this.#out.context.set(undefined); this.#out.sampleRate.set(undefined); - // A context closed twice rejects, and there is nothing to do about a close that fails anyway. - context.audio.close().catch(() => {}); + context.processors.close(); } - #runWorklet(effect: Effect, context: AudioContext): void { + #runWorklet(effect: Effect, context: AudioContext, processors: Processors): void { // It takes a second or so to initialize the AudioWorklet, so do it even if disabled. This is // less efficient for video-only playback but makes muting/unmuting instant, since the first // gesture on the page builds a context for every tile whether or not it is the one clicked. @@ -506,6 +507,7 @@ export class Decoder { channelCountMode: "explicit", outputChannelCount: [channelCount], }); + processors.add(worklet); effect.cleanup(() => { // The context outlives this node, so the processor has to be told to end. See `Close`. const close: Close = { type: "close" }; @@ -604,3 +606,62 @@ export class Decoder { // Whether the WebCodecs audio decoder can play this config. static supported = supported; } + +/** + * The processors built in one AudioContext, which closes it once every one of them has stopped. + * + * Chromium keeps a closed context, and every node in it, for as long as one of its processors has not + * stopped, and a processor only stops in a quantum its context renders. So a running context is closed + * once each processor has said it stopped (see `Stopped`), and one that renders nothing (suspended for + * want of a gesture, interrupted, failed) is closed at once: its processors can never stop, and waiting + * would only hold it open. + */ +class Processors { + readonly #context: AudioContext; + // Every processor that has not said it stopped. + readonly #active = new Set(); + // Owns every listener, all released once the context is closed. + readonly #signals = new Effect(); + #closing = false; + + constructor(context: AudioContext) { + this.#context = context; + } + + /** Follow the processor behind `node` until it says it stopped, or fails, which stops it too. */ + add(node: AudioWorkletNode): void { + this.#active.add(node); + const dispose = this.#signals.run((effect) => { + const stop = () => { + dispose(); + this.#active.delete(node); + if (this.#closing && this.#active.size === 0) this.#close(); + }; + effect.event(node.port, "message", (event) => { + if ((event as MessageEvent).data?.type === "stopped") stop(); + }); + effect.event(node, "processorerror", stop); + // A port only delivers to listeners added with addEventListener once it is started. + node.port.start(); + }); + } + + /** Close the context once every processor in it has stopped, or now if it renders nothing. */ + close(): void { + this.#closing = true; + if (this.#active.size === 0 || this.#context.state !== "running") { + this.#close(); + return; + } + // A context that stops rendering never runs the quantum a processor would stop in. + this.#signals.event(this.#context, "statechange", () => { + if (this.#context.state !== "running") this.#close(); + }); + } + + #close(): void { + this.#signals.close(); + // A context closed twice rejects, and there is nothing to do about a close that fails anyway. + this.#context.close().catch(() => {}); + } +} diff --git a/js/watch/src/audio/render-worklet.port.test.ts b/js/watch/src/audio/render-worklet.port.test.ts index 70715149da..1cd0db72d4 100644 --- a/js/watch/src/audio/render-worklet.port.test.ts +++ b/js/watch/src/audio/render-worklet.port.test.ts @@ -186,4 +186,55 @@ describe("render worklet ports", () => { node.port2.close(); }); + + it("says on the node's own port that its processor stopped, in the quantum that stops it", async () => { + // Chromium keeps a closed context, and every node in it, for as long as one of its processors + // has not stopped, and a processor only stops in a quantum it renders. The page closes the + // context on this, not on the close it sent. + if (!Render) throw new Error("render-worklet.ts registered no 'render' processor"); + const node = new MessageChannel(); + nextPort = node.port1; + const render = new Render(); + const extra = new MessageChannel(); + const handoff: Port = { type: "port", port: extra.port1 }; + const page: ToMain[] = []; + const writer: ToMain[] = []; + try { + const receive = node.port1.onmessage; + if (!receive) throw new Error("render registered no message handler"); + // Resolves once the processor has handled the close the page sent. + const handled = Promise.withResolvers(); + node.port1.onmessage = (event) => { + receive.call(node.port1, event); + if ((event.data as Message).type === "close") handled.resolve(); + }; + node.port2.postMessage(handoff, [extra.port1]); + extra.port2.onmessage = (event: MessageEvent) => writer.push(event.data); + node.port2.postMessage({ type: "close" }); + await handled.promise; + + // Nothing yet: a close is not a stop. A marker from the processor's end lands after anything + // the processor already said on it. + const marked = new Promise((resolve) => { + node.port2.onmessage = (event: MessageEvent) => { + if (event.data === "marker") resolve(); + else page.push(event.data); + }; + }); + node.port1.postMessage("marker"); + await marked; + expect(page).toEqual([]); + + const stopped = new Promise((resolve) => { + node.port2.onmessage = (event: MessageEvent) => resolve(event.data); + }); + expect(render.process([], [[new Float32Array(QUANTUM)]], {})).toBe(false); + expect(await stopped).toEqual({ type: "stopped" }); + // Only the page closes a context, so a writer's port hears nothing of it. + expect(writer).toEqual([]); + } finally { + node.port2.close(); + extra.port2.close(); + } + }); }); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index e1e0c88bd8..0ccbaa518e 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -1,6 +1,6 @@ import { Time } from "@moq/net"; import { Stretcher } from "./playout"; -import type { Message, State, Unreadable } from "./render"; +import type { Message, State, Stopped, Unreadable } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; import { SharedRingBuffer } from "./shared-ring-buffer"; @@ -76,7 +76,8 @@ class Render extends AudioWorkletProcessor { for (const port of this.#ports) { port.onmessage = null; port.onmessageerror = null; - port.close(); + // The node's own port stays open to say when `process` has stopped. + if (port !== this.port) port.close(); } this.#ports.length = 0; } else if (msg.type === "end") { @@ -117,7 +118,13 @@ class Render extends AudioWorkletProcessor { } process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record) { - if (this.#closed) return false; + if (this.#closed) { + // Nothing feeds the node, so returning false ends the processor in this quantum. + const stopped: Stopped = { type: "stopped" }; + this.port.postMessage(stopped); + this.port.close(); + return false; + } const output = outputs[0]; const backend = this.#backend; const engine = this.#engine; diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 8e48f51b4d..f700dba961 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -5,8 +5,8 @@ import type { SharedRingBufferInit } from "./shared-ring-buffer"; /** Everything a writer sends the render worklet: over the node's own port, or over one handed to it as a {@link Port}. */ export type Message = InitShared | InitPost | Data | End | Latency | Reset | Stall | Truncate | Port | Close; -/** Playback reports and errors sent by the render worklet. */ -export type ToMain = State | Unreadable; +/** Playback reports, errors and the end of the processor, sent by the render worklet. */ +export type ToMain = State | Unreadable | Stopped; /** * A message reached the worklet and could not be deserialized, so whatever it carried never arrived: @@ -77,12 +77,24 @@ export interface End { /** * The node is done with: the processor stops rendering and lets the browser collect it. A processor * whose `process` keeps returning true keeps running after its node is disconnected, for as long as the - * context is open, and the decoder rebuilds nodes in a context that stays open. + * context is open, and the decoder rebuilds nodes in a context that stays open. It answers with + * {@link Stopped} once it has. */ export interface Close { type: "close"; } +/** + * The processor has stopped, said on the node's own port in the quantum a {@link Close} ends it. + * + * Chromium keeps a closed context, and every node in it, for as long as a processor in it has not + * stopped, and a processor only stops in a quantum it renders. So the page closes the context once + * every processor in it has said this. + */ +export interface Stopped { + type: "stopped"; +} + /** * Drop buffered samples at or after `timestamp`, keeping what is already due (fallback path only; * the shared path truncates via Atomics). From 78923c1ea4565e4e75e20b04127345abcb78d8dc Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:20:10 -0400 Subject: [PATCH 065/127] fix(publish): close the capture context once its processor has stopped The capture closed its AudioContext before telling its processor to stop, which in Chromium keeps the closed context and its worklet node alive for the page's lifetime. The capture processor now runs on until its input is cut, since Chromium ends a processor that returned false at the first quantum after that without calling it again, and then reports `stopped`. The capture closes a running context once every processor built in it has said so, and one that is not running at once. Co-Authored-By: Claude Opus 5.5 --- .../src/audio/capture-worklet.port.test.ts | 19 ++++- js/publish/src/audio/capture-worklet.ts | 24 ++++++- js/publish/src/audio/capture.test.ts | 72 ++++++++++++++++++- js/publish/src/audio/capture.ts | 67 ++++++++++++++++- 4 files changed, 173 insertions(+), 9 deletions(-) diff --git a/js/publish/src/audio/capture-worklet.port.test.ts b/js/publish/src/audio/capture-worklet.port.test.ts index c4ff59ff5f..b6f512917f 100644 --- a/js/publish/src/audio/capture-worklet.port.test.ts +++ b/js/publish/src/audio/capture-worklet.port.test.ts @@ -36,7 +36,12 @@ afterAll(() => { } }); -test("lets its processor end once the capture node is closed", async () => { +// Chromium keeps a closed context, and every node in it, for as long as one of its processors has not +// stopped, so the page closes the context only once the processor says it has. A processor with a +// connected input is not stopped by returning false: Chromium ends it at the next quantum after the +// input is cut, without calling it again, so a processor that stopped while still fed would never +// get to say so. +test("ends its processor once the capture node is closed and cut from its input, and says so", async () => { if (!Capture) throw new Error("capture-worklet.ts registered no 'capture' processor"); const node = new MessageChannel(); @@ -59,8 +64,18 @@ test("lets its processor end once the capture node is closed", async () => { }); node.port2.postMessage({ type: "close" }); await closed; + + // Closed but still fed: it runs on, capturing nothing. + const next = new Promise((resolve) => { + node.port2.onmessage = (event: MessageEvent) => resolve(event.data); + }); scope.currentFrame = QUANTUM; - expect(capture.process([[new Float32Array(QUANTUM)]])).toBe(false); + expect(capture.process([[new Float32Array(QUANTUM)]])).toBe(true); + + // Cut from its input: it stops, and says so in the same quantum. + scope.currentFrame = 2 * QUANTUM; + expect(capture.process([[]])).toBe(false); + expect(await next).toEqual({ type: "stopped" }); } finally { node.port1.close(); node.port2.close(); diff --git a/js/publish/src/audio/capture-worklet.ts b/js/publish/src/audio/capture-worklet.ts index e9000a78d2..ed779700f4 100644 --- a/js/publish/src/audio/capture-worklet.ts +++ b/js/publish/src/audio/capture-worklet.ts @@ -12,11 +12,22 @@ export interface Quantum { channels: Float32Array[]; } -/** Stops a retired capture processor. */ +/** Stops a retired capture processor, which answers with {@link Stopped} once it has. */ export interface Close { type: "close"; } +/** + * The processor has stopped, said in the quantum that ends it. + * + * Chromium keeps a closed context, and every node in it, for as long as a processor in it has not + * stopped, and a processor only stops in a quantum it renders. So the page closes the context once + * every processor in it has said this. + */ +export interface Stopped { + type: "stopped"; +} + class Capture extends AudioWorkletProcessor { #closed = false; @@ -26,12 +37,19 @@ class Capture extends AudioWorkletProcessor { if (event.data.type !== "close") return; this.#closed = true; this.port.onmessage = null; - this.port.close(); }; } process(input: Float32Array[][]) { - if (this.#closed) return false; + if (this.#closed) { + // Chromium ends a processor that returned false at the first quantum its input is cut, without + // calling it again, so it runs on until then to be the one that says it stopped. + if (input[0]?.length) return true; + const stopped: Stopped = { type: "stopped" }; + this.port.postMessage(stopped); + this.port.close(); + return false; + } if (input.length > 1) throw new Error("only one input is supported."); const channels = input[0]; diff --git a/js/publish/src/audio/capture.test.ts b/js/publish/src/audio/capture.test.ts index 38959c96f7..daf9c14857 100644 --- a/js/publish/src/audio/capture.test.ts +++ b/js/publish/src/audio/capture.test.ts @@ -252,9 +252,10 @@ function installRenderingWebAudio() { postMessage(): void {} } - class FakeAudioWorkletNode { + class FakeAudioWorkletNode extends EventTarget { port = new FakePort(); constructor(_context: unknown, _name: string) { + super(); node = this; } connect(): void {} @@ -494,6 +495,7 @@ function installGatedWebAudio() { return Promise.resolve(); } close(): Promise { + this.state = "closed"; return Promise.resolve(); } transition(state: string): void { @@ -503,13 +505,14 @@ function installGatedWebAudio() { } } - class GatedWorklet { + class GatedWorklet extends EventTarget { messages: unknown[] = []; port = Object.assign(new EventTarget(), { start: () => {}, postMessage: (message: unknown) => this.messages.push(message), }); constructor(_context: unknown, _name: string) { + super(); worklets.push(this); } connect(): void {} @@ -520,6 +523,10 @@ function installGatedWebAudio() { new MessageEvent("message", { data: { frame, channels: [new Float32Array(128)] } }), ); } + // What a closed processor posts in the quantum it stops in. + stop(): void { + this.port.dispatchEvent(new MessageEvent("message", { data: { type: "stopped" } })); + } } class FakeGraphNode { @@ -667,3 +674,64 @@ test("drops the format while the context is interrupted", async () => { capture.close(); await settle(); }); + +// Chromium keeps a closed context, and the worklet node in it, for as long as the node's processor has +// not stopped, and a processor only stops in a quantum its context renders. Closed first, every +// publish that ends leaves one of each in the page's heap for good. +test("closes the context once the capture processor has stopped", async () => { + using webaudio = installGatedWebAudio(); + const capture = new Capture({ enabled: true, source: new Signal(fakeSource()) as never }); + await settle(); + webaudio.gesture(); + await settle(); + const [context] = webaudio.contexts; + const [worklet] = webaudio.worklets; + expect(context.state).toBe("running"); + + capture.close(); + await settle(); + + // Told to stop and cut from the microphone, but the context renders on until the processor says so. + expect(worklet.messages).toEqual([{ type: "close" }]); + expect(webaudio.roots[0].outputs.size).toBe(0); + expect(context.state).toBe("running"); + + worklet.stop(); + await settle(); + expect(context.state).toBe("closed"); +}); + +// Waiting on a context that renders nothing would only hold it open: its processor can never stop. +test("closes a context that is not running at once", async () => { + using webaudio = installGatedWebAudio(); + const capture = new Capture({ enabled: true, source: new Signal(fakeSource()) as never }); + await settle(); + webaudio.gesture(); + await settle(); + const [context] = webaudio.contexts; + + context.transition("interrupted"); + await settle(); + expect(webaudio.worklets[0].messages).toEqual([{ type: "close" }]); + + capture.close(); + await settle(); + expect(context.state).toBe("closed"); +}); + +test("closes the context when it stops running while the processor stops", async () => { + using webaudio = installGatedWebAudio(); + const capture = new Capture({ enabled: true, source: new Signal(fakeSource()) as never }); + await settle(); + webaudio.gesture(); + await settle(); + const [context] = webaudio.contexts; + + capture.close(); + await settle(); + expect(context.state).toBe("running"); + + context.transition("suspended"); + await settle(); + expect(context.state).toBe("closed"); +}); diff --git a/js/publish/src/audio/capture.ts b/js/publish/src/audio/capture.ts index a92336aff1..3df3762945 100644 --- a/js/publish/src/audio/capture.ts +++ b/js/publish/src/audio/capture.ts @@ -3,7 +3,7 @@ import { Time } from "@moq/net"; import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Signal } from "@moq/signals"; // Compiled and inlined as a blob URL via vite-plugin-worklet. import { Fanout } from "../fanout"; -import type { Close, Quantum } from "./capture-worklet"; +import type { Close, Quantum, Stopped } from "./capture-worklet"; import CaptureWorklet from "./capture-worklet.ts?worklet"; import { isSampleSource, normalizeSource, type SampleSource, type Source, type SourceConfig } from "./types"; @@ -151,7 +151,10 @@ export class Capture { latencyHint: "interactive", sampleRate, }); - effect.cleanup(() => context.close()); + // Closes the context once every processor built below has stopped, which the graph tells them to + // later in this same teardown. + const processors = new Processors(context); + effect.cleanup(() => processors.close()); // Nothing guarantees a gesture has happened yet: a pre-granted microphone reaches here on page // load. A context built then starts suspended and renders nothing until one arrives. @@ -190,6 +193,7 @@ export class Capture { // path on macOS. Only force it when we actually have a requested count to honor. channelCountMode: requestedChannels !== undefined ? "explicit" : "max", }); + processors.add(worklet); root.connect(worklet); inner.cleanup(() => { const close: Close = { type: "close" }; @@ -265,6 +269,65 @@ export class Capture { } } +/** + * The processors built in one AudioContext, which closes it once every one of them has stopped. + * + * Chromium keeps a closed context, and every node in it, for as long as one of its processors has not + * stopped, and a processor only stops in a quantum its context renders. So a running context is closed + * once each processor has said it stopped (see `Stopped`), and one that renders nothing (suspended for + * want of a gesture, interrupted, failed) is closed at once: its processors can never stop, and waiting + * would only hold it open. + */ +class Processors { + readonly #context: AudioContext; + // Every processor that has not said it stopped. + readonly #active = new Set(); + // Owns every listener, all released once the context is closed. + readonly #signals = new Effect(); + #closing = false; + + constructor(context: AudioContext) { + this.#context = context; + } + + /** Follow the processor behind `node` until it says it stopped, or fails, which stops it too. */ + add(node: AudioWorkletNode): void { + this.#active.add(node); + const dispose = this.#signals.run((effect) => { + const stop = () => { + dispose(); + this.#active.delete(node); + if (this.#closing && this.#active.size === 0) this.#close(); + }; + effect.event(node.port, "message", (event) => { + if ((event as MessageEvent>).data?.type === "stopped") stop(); + }); + effect.event(node, "processorerror", stop); + // A port only delivers to listeners added with addEventListener once it is started. + node.port.start(); + }); + } + + /** Close the context once every processor in it has stopped, or now if it renders nothing. */ + close(): void { + this.#closing = true; + if (this.#active.size === 0 || this.#context.state !== "running") { + this.#close(); + return; + } + // A context that stops rendering never runs the quantum a processor would stop in. + this.#signals.event(this.#context, "statechange", () => { + if (this.#context.state !== "running") this.#close(); + }); + } + + #close(): void { + this.#signals.close(); + // There is nothing to do about a close that fails. + this.#context.close().catch(() => {}); + } +} + // Split each AudioData into one planar float buffer per channel, converting the sample format when // the decoder hands us something else. function planar(): TransformStream { From 7f02213e7e9d37a394fe675f4a1c2bdddf32985f Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:01:37 -0400 Subject: [PATCH 066/127] test(watch): a reopened track must not step the picture back A track opened after a gap starts at the keyframe of the group it joins, and a relay can hand a returning viewer the groups it kept from before the gap, so its first pictures can be older than the one held on screen. The viewer sees the picture step back before it jumps to live. Covers a hidden tab shown again (with and without its first group skipped), a reattached element (a new consumer of the same publisher), and an audio clock that comes back behind the held picture. Three guards stay green today and must stay so: a rewind from a publisher that names no clock, and a republish on its own clock or on none. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.backwards.test.ts | 387 +++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 js/watch/src/video/decoder.backwards.test.ts diff --git a/js/watch/src/video/decoder.backwards.test.ts b/js/watch/src/video/decoder.backwards.test.ts new file mode 100644 index 0000000000..4b40d56db4 --- /dev/null +++ b/js/watch/src/video/decoder.backwards.test.ts @@ -0,0 +1,387 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import * as Catalog from "@moq/hang/catalog"; +import * as Moq from "@moq/net"; +import { Time } from "@moq/net"; +import { type Effect, Signal } from "@moq/signals"; +import type { Broadcast } from "../broadcast"; +import { Sync } from "../sync"; +import { Decoder } from "./decoder"; +import type { Source } from "./source"; + +// What a track opened after a gap (a tab shown again, a tile scrolled back, a resume, a reattached +// element) may put on screen. Its first group can start before the picture the tile is holding, and +// nothing in that group older than the held picture is due. WebCodecs is not in bun, so a fake codec +// records chunks and emits pictures on demand. Real timers throughout, each wait a poll bounded at 3 s. + +type Codec = { + chunks: string[]; + emit(timestamp: number): void; +}; +let codecs: Codec[] = []; + +const real = { VideoDecoder: globalThis.VideoDecoder, EncodedVideoChunk: globalThis.EncodedVideoChunk }; + +beforeEach(() => { + codecs = []; + class FakeVideoFrame { + readonly displayWidth = 16; + readonly displayHeight = 16; + readonly timestamp: number; + constructor(timestamp: number) { + this.timestamp = timestamp; + } + clone(): FakeVideoFrame { + return new FakeVideoFrame(this.timestamp); + } + close(): void {} + } + class FakeVideoDecoder { + state = "unconfigured"; + readonly #codec: Codec; + constructor(init: { output: (frame: unknown) => void; error: (error: Error) => void }) { + this.#codec = { chunks: [], emit: (timestamp) => init.output(new FakeVideoFrame(timestamp)) }; + codecs.push(this.#codec); + } + configure(): void { + this.state = "configured"; + } + decode(chunk: { type: string }): void { + this.#codec.chunks.push(chunk.type); + } + close(): void { + this.state = "closed"; + } + static isConfigSupported(): Promise<{ supported: boolean }> { + return Promise.resolve({ supported: true }); + } + } + class FakeEncodedVideoChunk { + readonly type: string; + readonly timestamp: number; + readonly byteLength: number; + constructor(init: { type: string; timestamp: number; data: Uint8Array }) { + this.type = init.type; + this.timestamp = init.timestamp; + this.byteLength = init.data.byteLength; + } + } + globalThis.VideoDecoder = FakeVideoDecoder as unknown as typeof VideoDecoder; + globalThis.EncodedVideoChunk = FakeEncodedVideoChunk as unknown as typeof EncodedVideoChunk; +}); + +afterEach(() => { + globalThis.VideoDecoder = real.VideoDecoder; + globalThis.EncodedVideoChunk = real.EncodedVideoChunk; +}); + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); +async function settle(rounds = 20): Promise { + for (let i = 0; i < rounds; i++) await flush(); +} +async function until(done: () => boolean, ms = 3_000): Promise { + const start = Date.now(); + while (!done() && Date.now() - start < ms) await flush(); +} + +function write(group: Moq.Group.Producer, timestamp: number): void { + const header = Moq.Varint.encode(timestamp); + const body = new Uint8Array(header.length + 1); + body.set(header); + body[header.length] = 1; + group.writeFrame({ payload: body, timestamp: Time.Timestamp.now() }); +} + +// The broadcast clock every catalog here advertises unless a case says otherwise: the one mapping a +// publisher fixes for its whole run. +const CLOCK: Catalog.Clock = { wall: Catalog.u53(1_000_000), timescale: 1_000_000 }; + +function fixture(options: { clock?: Catalog.Clock } = { clock: CLOCK }) { + const clock = options.clock; + let broadcast = new Moq.Broadcast.Producer(); + const handle = new Signal(broadcast.consume()); + const config = Catalog.VideoConfigSchema.parse({ codec: "avc1.640028", container: { kind: "legacy" } }); + const catalog = new Signal({ + video: { renditions: { video: config } }, + clock, + } as Catalog.Root); + const source = { + in: { + broadcast: new Signal({ + relativeBroadcast: (effect: Effect) => effect.get(handle), + out: { catalog }, + } as unknown as Broadcast), + }, + out: { + track: new Signal("video"), + config: new Signal(config), + catalog: new Signal(undefined), + }, + } as unknown as Source; + const sync = new Sync({ delay: Time.Milli(100) }); + const enabled = new Signal(true); + const decoder = new Decoder({ source, sync, enabled }); + let track = new Moq.Track.Producer("video").accept({}); + broadcast.insertTrack(track); + + // Every picture the decoder hands the renderer, in order. + const shown: number[] = []; + const dispose = decoder.out.frame.subscribe((frame) => { + if (frame) shown.push(frame.timestamp); + }); + + // The audio playhead, parked so nothing here depends on how fast the machine is. + const park = (micro: number) => + sync.track("audio").clock.set({ timestamp: Time.Micro(micro), reference: Time.Milli.now(), rate: 0 }); + + const close = () => { + dispose(); + decoder.close(); + sync.close(); + broadcast.close(); + }; + return { + decoder, + sync, + enabled, + get track() { + return track; + }, + shown, + park, + close, + // A reattached element or a replaced session: the same publisher's broadcast comes back as a + // new consumer, with the catalog it always had. + reconnect() { + handle.set(broadcast.consume()); + }, + // The session dropped while the tile stayed on screen: the broadcast goes away. + offline() { + handle.set(undefined); + }, + // A publisher restarted under the same name: a new broadcast whose catalog names its own clock. + republish(next: Catalog.Clock | undefined) { + broadcast.close(); + broadcast = new Moq.Broadcast.Producer(); + track = new Moq.Track.Producer("video").accept({}); + broadcast.insertTrack(track); + handle.set(broadcast.consume()); + catalog.set({ video: { renditions: { video: config } }, clock: next } as Catalog.Root); + }, + }; +} + +// One group, keyframe at 10.000 s and a delta at 12.000 s, painted up to 12.000 s. +async function hold(fx: ReturnType): Promise { + await until(() => codecs.length >= 1); + fx.park(12_000_000); + const group = fx.track.appendGroup(); + write(group, 10_000_000); + write(group, 12_000_000); + await until(() => (codecs[0]?.chunks.length ?? 0) > 1); + codecs[0].emit(10_000_000); + codecs[0].emit(12_000_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 12_000_000); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(12_000_000); +} + +// `hold`, then the video is taken away (a hidden tab, a scroll out) and, after `during`, brought back. +async function holdThenReopen(fx: ReturnType, during?: () => void): Promise { + await hold(fx); + fx.enabled.set(false); + await settle(); + during?.(); + fx.shown.length = 0; + fx.enabled.set(true); + await until(() => codecs.length >= 2); +} + +test("a track reopened after a gap never steps the picture back", async () => { + const fx = fixture(); + try { + await holdThenReopen(fx); + + // Back within the same group: the new subscription starts at its keyframe, 10.000 s, which is + // older than the held picture, while the playhead is still at 12.000 s. Nothing in it is due. + await until(() => (codecs[1]?.chunks.length ?? 0) > 1); + codecs[1].emit(10_000_000); + await settle(); + + // The live group follows and plays as usual. + fx.park(12_100_000); + write(fx.track.appendGroup(), 12_040_000); + await until(() => (codecs[1]?.chunks.length ?? 0) > 2); + codecs[1].emit(12_040_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 12_040_000); + + expect(fx.shown.filter((ts) => ts < 12_000_000)).toEqual([]); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(12_040_000); + } finally { + fx.close(); + } +}); + +test("a track reopened after a gap never steps back, even once its first group was skipped", async () => { + const fx = fixture(); + try { + await holdThenReopen(fx); + + // The live group lands before the joined group's keyframe comes out of the codec, so the + // consumer skips the joined group (a discontinuity) while its 10.000 s picture is in flight. + await until(() => (codecs[1]?.chunks.length ?? 0) > 1); + fx.park(12_100_000); + write(fx.track.appendGroup(), 12_040_000); + await until(() => (codecs[1]?.chunks.length ?? 0) > 2); + codecs[1].emit(10_000_000); + await settle(); + codecs[1].emit(12_040_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 12_040_000); + + expect(fx.shown.filter((ts) => ts < 12_000_000)).toEqual([]); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(12_040_000); + } finally { + fx.close(); + } +}); + +test("a track reopened on a rewound timeline still shows its pictures", async () => { + // Only a publisher that names no clock can rewind: one that does fixes its mapping for good. + const fx = fixture({}); + try { + // The publisher rewound while the video was away: its latest group starts at 0.5 s, and the + // shared clock was re-anchored there, so a picture older than the held one is what is due. + await holdThenReopen(fx, () => { + fx.sync.reset(); + fx.park(500_000); + write(fx.track.appendGroup(), 500_000); + }); + await until(() => (codecs[1]?.chunks.length ?? 0) > 0); + codecs[1].emit(500_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 500_000); + + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(500_000); + } finally { + fx.close(); + } +}); + +test("a reattached track never steps the picture back", async () => { + const fx = fixture(); + try { + // The element was detached and reattached: a new consumer of the same publisher, whose + // catalog still names the clock the held picture was painted on. + await holdThenReopen(fx, () => fx.reconnect()); + + // The first group served is the one the viewer was watching when it left. + await until(() => (codecs[1]?.chunks.length ?? 0) > 1); + codecs[1].emit(10_000_000); + await settle(); + + fx.park(12_100_000); + write(fx.track.appendGroup(), 12_040_000); + await until(() => (codecs[1]?.chunks.length ?? 0) > 2); + codecs[1].emit(12_040_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 12_040_000); + + expect(fx.shown.filter((ts) => ts < 12_000_000)).toEqual([]); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(12_040_000); + } finally { + fx.close(); + } +}); + +test("a track whose broadcast went away and came back never steps the picture back", async () => { + const fx = fixture(); + try { + await hold(fx); + + // The session dropped while the tile stayed on screen, which clears the picture, and the same + // publisher's broadcast came back on a new one. + fx.offline(); + await until(() => fx.decoder.out.frame.peek() === undefined); + expect(fx.decoder.out.frame.peek()).toBeUndefined(); + fx.shown.length = 0; + fx.reconnect(); + await until(() => codecs.length >= 2); + + await until(() => (codecs[1]?.chunks.length ?? 0) > 1); + codecs[1].emit(10_000_000); + await settle(); + + fx.park(12_100_000); + write(fx.track.appendGroup(), 12_040_000); + await until(() => (codecs[1]?.chunks.length ?? 0) > 2); + codecs[1].emit(12_040_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 12_040_000); + + expect(fx.shown.filter((ts) => ts < 12_000_000)).toEqual([]); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(12_040_000); + } finally { + fx.close(); + } +}); + +test("a reopened track never steps back, even once the audio clock parks behind the held picture", async () => { + const fx = fixture(); + try { + await holdThenReopen(fx); + + // Audio came back first, with media from before the gap: its playhead sits behind the + // picture on screen, so by the clock alone the older picture would be due. + fx.park(11_000_000); + await until(() => fx.sync.now() === 11_000); + expect(fx.sync.now()).toBe(Time.Milli(11_000)); + await until(() => (codecs[1]?.chunks.length ?? 0) > 1); + codecs[1].emit(10_000_000); + await settle(); + + fx.park(12_100_000); + write(fx.track.appendGroup(), 12_040_000); + await until(() => (codecs[1]?.chunks.length ?? 0) > 2); + codecs[1].emit(12_040_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 12_040_000); + + expect(fx.shown.filter((ts) => ts < 12_000_000)).toEqual([]); + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(12_040_000); + } finally { + fx.close(); + } +}); + +test("a republished broadcast on its own clock still shows its pictures", async () => { + const fx = fixture(); + try { + // The publisher restarted while the video was away: a new broadcast on a new clock, whose + // timeline starts near zero, far below the held picture. + await holdThenReopen(fx, () => { + fx.republish({ wall: Catalog.u53(9_000_000), timescale: 1_000_000 }); + fx.park(500_000); + write(fx.track.appendGroup(), 500_000); + }); + await until(() => (codecs[1]?.chunks.length ?? 0) > 0); + codecs[1].emit(500_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 500_000); + + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(500_000); + } finally { + fx.close(); + } +}); + +test("a republished broadcast without a clock still shows its pictures", async () => { + const fx = fixture(); + try { + // Nothing names the timeline, so a new broadcast is taken for a new one. + await holdThenReopen(fx, () => { + fx.republish(undefined); + fx.park(500_000); + write(fx.track.appendGroup(), 500_000); + }); + await until(() => (codecs[1]?.chunks.length ?? 0) > 0); + codecs[1].emit(500_000); + await until(() => fx.decoder.out.frame.peek()?.timestamp === 500_000); + + expect(fx.decoder.out.frame.peek()?.timestamp).toBe(500_000); + } finally { + fx.close(); + } +}); From 4844bc2dd4bf474ae281a313d26883a03f9b0596 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:21:52 -0400 Subject: [PATCH 067/127] fix(watch): never step a reopened track's picture back A track opened after a gap (a tab shown again, a resume, a rebuild, a reattached element, a replaced session) is promoted at once, and its first pictures can be older than the one the viewer last saw: its subscription starts at the keyframe of the group it joins, and a relay can hand a returning viewer the groups it kept from before the gap. The viewer saw the picture step back, then jump to live. The new track now carries the last picture shown and the broadcast clock it was painted on. A publisher fixes the catalog clock for its whole run, so on the same clock an older timestamp is older content however the playhead was re-anchored since: a new consumer resets it, and audio that comes back first with media from before the gap parks it behind. Such a picture is dropped. With no shared clock (a republish, or a publisher that names none) it is dropped only while the playhead is still past the last one shown, so a rewind and a republish keep playing at once. The last picture is remembered apart from the frame on screen, which going offline clears. Public API: none. No wire impact. Co-Authored-By: Claude Opus 5.5 --- doc/concept/playout.md | 9 +++++ js/watch/src/video/decoder.test.ts | 9 ++++- js/watch/src/video/decoder.ts | 58 ++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/doc/concept/playout.md b/doc/concept/playout.md index 96254ed4cb..3671dd2051 100644 --- a/doc/concept/playout.md +++ b/doc/concept/playout.md @@ -435,6 +435,15 @@ republish in place, rather than being rebuilt around it, treats the replacement broadcast as a tune-in and resets the ring and the clock there. A rendition swap is not one: it reopens a subscription on the timeline already playing. +A reattached element reaches the same reset, since its new session hands it a +new broadcast, but its catalog names the same broadcast clock. A track reopened +after any gap can start on pictures older than the last one shown (the keyframe +of the group it joins, or groups a relay kept from before the gap), so the video +decoder never shows a picture older than that one on the same clock, wherever +the reset or a returning audio clock put the playhead. Without a shared clock it +holds such a picture back only while the playhead is past the last one shown, +which a rewind or a republish moves below it. + ## The "plus one frame" question The first attempt added a learned frame duration on top of the quantile. It diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 8c8f44f35e..0777d04587 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -230,7 +230,12 @@ function fixture(config = testConfig("legacy")) { const broadcast = new Moq.Broadcast.Producer(); const consumer = broadcast.consume(); const source = { - in: { broadcast: new Signal({ relativeBroadcast: () => consumer } as unknown as Broadcast) }, + in: { + broadcast: new Signal({ + relativeBroadcast: () => consumer, + out: { catalog: new Signal(undefined) }, + } as unknown as Broadcast), + }, out: { track: new Signal(TRACK), config: new Signal(config), @@ -631,6 +636,7 @@ test("a replaced session re-subscribes to video", async () => { in: { broadcast: new Signal({ relativeBroadcast: (effect: Effect) => effect.get(handle), + out: { catalog: new Signal(undefined) }, } as unknown as Broadcast), }, out: { @@ -688,6 +694,7 @@ test("a republished broadcast re-anchors the clock", async () => { in: { broadcast: new Signal({ relativeBroadcast: (effect: Effect) => effect.get(handle), + out: { catalog: new Signal(undefined) }, } as unknown as Broadcast), }, out: { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index e13157a7fd..1243e52afc 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -139,6 +139,11 @@ export class Decoder { // publisher rather than a continuation. See `#runPending`. #broadcast?: Moq.Broadcast.Consumer; + // The last picture handed to the renderer and the broadcast clock it was painted on. It outlives + // the picture itself (going offline clears that) so a track opened after a gap knows what the + // viewer last saw. See `#runPending`. + #shown?: { timestamp: Time.Milli; clock?: Catalog.Clock }; + // Bumped to rebuild the track without anything else about the rendition changing: a codec that // errored, or a picture that stayed frozen past RECOVER. `#runPending` reads it, so a bump tears // the old subscription down and opens a new one at the live edge. @@ -267,14 +272,32 @@ export class Decoder { const spread = effect.get(this.#spread); if (!spread) return; + // Peeked: a catalog update must not rebuild the track. + const clock = broadcast.out.catalog.peek()?.clock; + + // A track opened after a gap (a hidden tab shown, a resume, a rebuild, a reattached element, + // a replaced session) is promoted at once, so it must not step back from what the viewer last + // saw. + const shown = this.#active.peek() ? undefined : this.#shown; + const held = shown && { + timestamp: shown.timestamp, + sameClock: + clock !== undefined && + shown.clock !== undefined && + clock.wall === shown.clock.wall && + clock.timescale === shown.clock.timescale, + }; + // Start a new pending effect. let pending: DecoderTrack | undefined = new DecoderTrack({ sync: this.sync, broadcast: active, track, config: identity.decoder, + clock, stats: this.#out.stats, spread, + held, }); effect.set(this.#pendingJitter, pending.jitter); @@ -338,6 +361,9 @@ export class Decoder { // proxy() would share the same reference, allowing the source to close our frame. effect.run((inner) => { const frame = inner.get(active.frame); + if (frame) { + this.#shown = { timestamp: Time.Milli.fromMicro(frame.timestamp as Time.Micro), clock: active.clock }; + } this.#out.frame.update((prev) => { prev?.close(); return frame?.clone(); @@ -448,6 +474,15 @@ interface DecoderTrackProps { /** The rendition's arrival estimator, which outlives this subscription. */ spread: Container.Jitter; + + /** The broadcast clock the catalog named when this track opened, if it named one. */ + clock?: Catalog.Clock; + + /** + * The picture the viewer last saw when this track opened after a gap, and whether it was painted + * on this track's broadcast clock, which puts both on one timeline. + */ + held?: { timestamp: Time.Milli; sameClock: boolean }; } class DecoderTrack { @@ -458,6 +493,7 @@ class DecoderTrack { stats: Signal; spread: Container.Jitter; jitter: Time.Milli | undefined; + clock: Catalog.Clock | undefined; timestamp = new Signal(undefined); frame = new Signal(undefined); @@ -479,6 +515,11 @@ class DecoderTrack { // Decoded frames waiting to be rendered. #buffered = new Signal([]); + // See `DecoderTrackProps.held`. Kept across a discontinuity: a skipped group is not a new + // timeline. Only a publisher that names no clock can rewind, and a rewind re-anchors the + // playhead below the held picture, which the playhead check sees. + #held: { timestamp: Time.Milli; sameClock: boolean } | undefined; + // The last discontinuity count seen from the container consumer; doubles as a generation // so in-flight decodes from before a rewind can be dropped on output. #discontinuity = 0; @@ -498,6 +539,8 @@ class DecoderTrack { this.stats = props.stats; this.spread = props.spread; this.jitter = renditionJitter(props.config); + this.clock = props.clock; + this.#held = props.held; this.#signals.run(this.#run.bind(this)); } @@ -530,6 +573,21 @@ class DecoderTrack { // set `timestamp` from the old timeline and late-reject the whole rewind. if (this.sync.out.reference.peek() === undefined) return; + // The subscription starts at the keyframe of the group it joins, and a relay can + // hand a returning viewer the groups it kept from before the gap, so the first + // pictures can be older than the one last shown. On that picture's own broadcast + // clock such a picture is older content outright, however the playhead was + // re-anchored since (a new consumer resets it, and audio back first with media + // from before the gap parks it behind). Without a shared clock it is not due + // while the playhead is still past the held picture: only a rewind puts it below. + // Either way it would only step the picture back until the live one replaces it. + const held = this.#held; + if (held !== undefined && timestamp < held.timestamp) { + if (held.sameClock) return; + const playhead = this.sync.now(); + if (playhead !== undefined && playhead >= held.timestamp) return; + } + if (this.frame.peek() === undefined) { // This preview is already visible. Older backlog must not replace it // while its timestamp is still waiting for the shared clock. From 2e6f92cacae152fd2ba3c327e4ed1e18ec7edb92 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:33:05 -0400 Subject: [PATCH 068/127] test(relay): a returning viewer is never served media from before it left When the last subscriber leaves, the relay cancels upstream and keeps the track's newest groups warm; a returning subscriber may be served them only if the publisher's feed still continues from there. The upstream warm-cache tests cover that without a relay, so this runs it through a real one, with the edge group open and finished, on every version that resolves a rejoin against the publisher (lite-06 onward and IETF). Two publisher shapes: one that keeps producing while nobody watches, and one that stops on demand loss and marks the break the way @moq/publish does (the group finished at its end and a marker group of one empty frame after it), resuming only once a subscriber returns. The second runs over IETF, where SUBSCRIBE_OK's Largest shows the relay the marker past its cache. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/tests/rejoin.rs | 302 +++++++++++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 rs/moq-relay/tests/rejoin.rs diff --git a/rs/moq-relay/tests/rejoin.rs b/rs/moq-relay/tests/rejoin.rs new file mode 100644 index 0000000000..004fa55530 --- /dev/null +++ b/rs/moq-relay/tests/rejoin.rs @@ -0,0 +1,302 @@ +//! A viewer that leaves and comes back through a real relay is never handed media from +//! before it left. +//! +//! When the last subscriber leaves, the relay cancels its upstream subscription and keeps +//! the track's newest groups warm. A returning subscriber may be served that cache only if +//! the publisher's feed still continues from it. The relay meets two publisher shapes: +//! +//! - one that keeps producing while nobody watches, the shape the warm-cache tests in +//! `moq-tokio` cover without a relay; +//! - one that stops on demand loss and marks the break, the way `@moq/publish` does: its +//! encoder only runs while a subscriber is attached, and when the last one leaves it +//! finishes the current group at its end and appends a marker group of one empty frame +//! there (`Container.Legacy.Producer.cut`). It resumes with a new keyframe group once a +//! subscriber returns, some time after the subscription arrives. +//! +//! The assertion is the same for both: the returning subscriber's first group, and every +//! group after it, comes from after the gap. +#![cfg(feature = "noq")] + +use std::time::Duration; + +use moq_relay::{Config, Relay}; +use moq_tokio::moq_net; + +/// Ceiling for every wait, so a broken rejoin fails with a message instead of hanging. +const TIMEOUT: Duration = Duration::from_secs(10); + +/// The track every case publishes. +const TRACK: &str = "video"; + +/// The age budget both viewers subscribe with, as a live player's is: far below the gap. +const BUDGET: Duration = Duration::from_millis(100); + +/// Where the first group after the gap starts, well past the budget from anything before it. +const RESUMED_MS: u64 = 10_000; + +/// What the publisher does while nobody is watching. +#[derive(Clone, Copy, Debug)] +enum Idle { + /// Keeps producing, like an encoder that never stops. + Produces, + /// Stops producing and marks the break, like `@moq/publish` (see the module docs). + Pauses, +} + +fn ms(ms: u64) -> moq_net::Timestamp { + moq_net::Timestamp::from_millis(ms).expect("timestamp") +} + +/// Bind a public relay on an ephemeral loopback port and serve it on this runtime. +async fn relay() -> (url::Url, tokio::task::JoinHandle<()>) { + // Process-global; every case in this binary races to be first. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let mut config = Config::default(); + config.listen.bind = Some("127.0.0.1:0".parse().unwrap()); + config.listen.tls.generate = vec!["localhost".into()]; + config.auth.public = vec![moq_auth::Pattern::all()]; + + let relay = Relay::load(config).await.expect("load relay"); + let addr = relay.quic_addr().expect("relay bound no QUIC address"); + let url = format!("https://127.0.0.1:{}/rejoin", addr.port()).parse().unwrap(); + let handle = tokio::spawn(async move { + let _ = relay.run().await; + }); + (url, handle) +} + +/// A one-shot QUIC client pinned to `version`, trusting the relay's generated certificate. +fn client(version: moq_net::Version) -> moq_tokio::Client { + let mut config = moq_tokio::connect::Config::default(); + config.bind = Some("127.0.0.1:0".parse().unwrap()); + config.tls.insecure = Some(true); + config.websocket.enabled = Some(false); + config.version = vec![version]; + config.init(Default::default()).expect("client init") +} + +/// Write a finished group of two frames, at `at` and 50ms later. +fn write_group(track: &moq_net::track::Producer, sequence: u64, at: u64) { + let mut group = track + .create_group(moq_net::group::Info { sequence }) + .expect("create group"); + group.write_frame(ms(at), b"frame".as_ref()).expect("write frame"); + group.write_frame(ms(at + 50), b"frame".as_ref()).expect("write frame"); + group.finish().expect("finish group"); +} + +/// Open a subscriber session and subscribe to [`TRACK`] once `live` is announced. +async fn subscribe(url: &url::Url, version: moq_net::Version) -> (moq_tokio::Connection, moq_net::track::Subscriber) { + let origin = moq_tokio::origin::spawn(); + let consumer = origin.consume(); + let session = tokio::time::timeout( + TIMEOUT, + client(version) + .with_subscriber(origin) + .with_reconnect(false) + .connect(url.clone()) + .established(), + ) + .await + .expect("subscriber connect timed out") + .expect("subscriber connect failed"); + + let broadcast = tokio::time::timeout(TIMEOUT, consumer.routed_broadcast("live")) + .await + .expect("announcement timed out") + .expect("origin closed before announcing"); + let subscriber = tokio::time::timeout( + TIMEOUT, + broadcast + .track(TRACK) + .expect("track") + .subscribe(moq_net::track::Subscription::default().with_max_age(BUDGET)), + ) + .await + .expect("subscribe timed out") + .expect("subscribe rejected"); + (session, subscriber) +} + +async fn recv(subscriber: &mut moq_net::track::Subscriber, what: &str) -> moq_net::group::Consumer { + tokio::time::timeout(TIMEOUT, subscriber.recv_group()) + .await + .unwrap_or_else(|_| panic!("{what}: no group within {TIMEOUT:?}")) + .unwrap_or_else(|err| panic!("{what}: track aborted: {err}")) + .unwrap_or_else(|| panic!("{what}: track finished")) +} + +async fn rejoin(version: moq_net::Version, idle: Idle, open: bool) { + let case = format!("{version} {idle:?} open={open}"); + let (url, relay) = relay().await; + + let publisher = moq_tokio::origin::spawn(); + let broadcast = publisher.create_broadcast("live").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + let track = broadcast.create_track(TRACK, None).expect("create track"); + for sequence in 0..3u64 { + write_group(&track, sequence, sequence * 100); + } + // The group live when the viewer leaves: still open, or finished just before. + let mut edge = track + .create_group(moq_net::group::Info { sequence: 3 }) + .expect("create edge group"); + edge.write_frame(ms(300), b"edge".as_ref()).expect("write edge frame"); + if !open { + edge.write_frame(ms(350), b"edge".as_ref()).expect("write edge frame"); + edge.finish().expect("finish edge group"); + } + + let publish_session = tokio::time::timeout( + TIMEOUT, + client(version) + .with_publisher(&publisher) + .with_reconnect(false) + .connect(url.clone()) + .established(), + ) + .await + .expect("publisher connect timed out") + .expect("publisher connect failed"); + + // The viewer watches up to the edge, then leaves with its whole session, as a detached + // player does. + let (session, mut subscriber) = subscribe(&url, version).await; + loop { + let mut group = recv(&mut subscriber, &format!("{case}: first viewer")).await; + if group.sequence != 3 { + continue; + } + let frame = tokio::time::timeout(TIMEOUT, group.read_frame()) + .await + .expect("edge frame timed out") + .expect("edge frame failed") + .expect("edge group ended before its frame"); + assert_eq!(&frame.payload[..], b"edge"); + break; + } + drop(subscriber); + drop(session); + + // The relay cancels upstream once nobody reads. + tokio::time::timeout(TIMEOUT, track.unused()) + .await + .unwrap_or_else(|_| panic!("{case}: the relay never cancelled upstream")) + .expect("track open"); + + // The first group after the gap. + let resumed = 4; + match idle { + Idle::Produces => { + for sequence in resumed..=20u64 { + write_group(&track, sequence, RESUMED_MS + (sequence - resumed) * 100); + } + } + Idle::Pauses => { + // The break, as `cut` marks it: a group still open ends with an empty frame one + // interval after its last one, then a marker group holds one empty frame at the + // last frame written. + let last = if open { 300 } else { 350 }; + if open { + edge.write_frame(ms(last + 50), b"".as_ref()).expect("write end frame"); + edge.finish().expect("finish edge group"); + } + let mut marker = track + .create_group(moq_net::group::Info { sequence: resumed }) + .expect("create marker group"); + marker.write_frame(ms(last), b"".as_ref()).expect("write marker"); + marker.finish().expect("finish marker group"); + } + } + + let (session, mut subscriber) = subscribe(&url, version).await; + let first = recv(&mut subscriber, &format!("{case}: returning viewer")) + .await + .sequence; + assert!( + first >= resumed, + "{case}: the returning viewer was first served group {first}, from before it left" + ); + + // A paused publisher resumes only once the encoder has produced its first frame, after + // the subscription is already being served. + let last = match idle { + Idle::Produces => 20, + Idle::Pauses => { + write_group(&track, resumed + 1, RESUMED_MS); + resumed + 1 + } + }; + let mut sequence = first; + while sequence < last { + sequence = recv(&mut subscriber, &format!("{case}: returning viewer")) + .await + .sequence; + assert!( + sequence >= resumed, + "{case}: the returning viewer was served group {sequence}, from before it left" + ); + } + + drop(subscriber); + drop(session); + drop(edge); + drop(publish_session); + relay.abort(); +} + +/// Every version whose relay resolves a rejoin against the publisher (lite-06 onward and IETF) +/// that `wanted` keeps. +fn versions(wanted: fn(&moq_net::Version) -> bool) -> impl Iterator { + let pre06 = [ + "moq-lite-01", + "moq-lite-02", + "moq-lite-03", + "moq-lite-04", + "moq-lite-05", + ]; + moq_net::Version::names() + .filter(move |name| !pre06.contains(name)) + .map(|name| name.parse().expect("version")) + .filter(wanted) +} + +/// Run each version with the edge group open and finished, and report every case that fails. +async fn every_case(idle: Idle, wanted: fn(&moq_net::Version) -> bool) { + let mut failed = Vec::new(); + let mut cases = 0; + for version in versions(wanted) { + for open in [false, true] { + cases += 1; + if let Err(err) = tokio::spawn(rejoin(version, idle, open)).await { + let panic = err.into_panic(); + let message = panic + .downcast_ref::() + .cloned() + .or_else(|| panic.downcast_ref::<&str>().map(|s| s.to_string())) + .unwrap_or_default(); + failed.push(message); + } + } + } + assert!(cases > 0, "no version to run"); + assert!( + failed.is_empty(), + "{} of {cases} cases failed:\n{}", + failed.len(), + failed.join("\n") + ); +} + +#[tokio::test] +async fn rejoin_skips_a_stale_cache_of_a_publisher_that_kept_producing() { + every_case(Idle::Produces, |_| true).await; +} + +/// IETF learns the publisher's newest group from SUBSCRIBE_OK's Largest, and the break's marker +/// group is past the cache. +#[tokio::test] +async fn rejoin_over_ietf_skips_a_stale_cache_of_a_publisher_that_paused() { + every_case(Idle::Pauses, |version| matches!(version, moq_net::Version::Ietf(_))).await; +} From bdec360cfe1d813ec07b46ef5f087f5a9925abb7 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:56:50 -0400 Subject: [PATCH 069/127] test(audio-quality): pass a silence breach only when every quiet window is authored The raw silence share keeps its ceilings. A share over its ceiling now passes only when the analyzer places every quiet window it counted over quiet audio at the same media time; one quiet window over audible audio, one it cannot place, or a failed alignment keeps the failure. Rows within the ceiling are unchanged. The reference is the audio the page decoded: the publisher's own encode of the looped file, replayed offline (byte for byte on the same host), decoded with the page's decoder and mixed as the AnalyserNode mixes. Windows are placed with the context clock and the ring's playhead less its output counter, bracketed by the neighbouring reports; one constant per row is fitted by log RMS correlation. Co-Authored-By: Claude Opus 5.5 --- test/audio-quality/README.md | 39 ++- test/audio-quality/clients/js/analyze.ts | 106 +++++- test/audio-quality/clients/js/grade.ts | 70 +++- .../clients/js/src/grade.test.ts | 103 +++++- test/audio-quality/clients/js/src/probe.ts | 4 +- test/audio-quality/clients/js/src/schema.ts | 77 +++++ .../clients/js/src/silence.fixture.ts | 140 ++++++++ .../clients/js/src/silence.test.ts | 133 +++++++ test/audio-quality/clients/js/src/silence.ts | 325 ++++++++++++++++++ test/audio-quality/run.sh | 23 +- 10 files changed, 1000 insertions(+), 20 deletions(-) create mode 100644 test/audio-quality/clients/js/src/silence.fixture.ts create mode 100644 test/audio-quality/clients/js/src/silence.test.ts create mode 100644 test/audio-quality/clients/js/src/silence.ts diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 5c7b04ac5a..9d8e8cdfa4 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -296,7 +296,9 @@ definition is a judgement call are: - **`silence_share`** is the share of sampled windows whose RMS at the graph output was below about -60 dBFS. It is the only metric read from the audio itself rather than from a counter: a counter - says the ring was fed, and only the PCM says the listener heard anything. + says the ring was fed, and only the PCM says the listener heard anything. The film has quiet + scenes of its own, so the same row reads 0.11 or 0.26 depending on which minute it plays; see + [the quiet proof](#the-quiet-proof) for how a share over its ceiling can still pass. - **`converge_s`** is measured backwards from the end of the run, to the last moment the resolved target was more than one bucket from its final value. A target that settles and then moves again @@ -324,6 +326,36 @@ definition is a judgement call are: `audio.out.debug`. A build without the counters reports null. `silent_quanta` and `budget_aborts` remain unmeasured because neither lane has a counter that distinguishes those events. +### The quiet proof + +The raw `silence_share` and its ceilings are graded exactly as before. A share over its ceiling +passes only when every quiet window it counted lines up with a quiet window of the audio the page +decoded, at the same media time. One quiet window over audible audio, one that cannot be placed, or +a proof counting other windows than the share did, keeps the failure. A row within its ceiling keeps +the verdict it had. The grade prints both: the raw share, and how many quiet windows were placed over +quiet audio. + +The reference is rebuilt after the row, in `analyze.ts`, never in the page. It replays the +publisher's own audio encode of the file it loops (`audio_of` in `run.sh`, with `-stream_loop -1` +and without `-re`), which reproduces the published packets byte for byte on the same host, decodes +it with the page's decoder (FFmpeg's AAC, libopus), and mixes it to mono as the AnalyserNode mixes +its input. Against the film itself the codec's level change alone moves a window within a percent of +the floor across it. + +A window is placed with what the probe already records ([`src/silence.ts`](clients/js/src/silence.ts)): + +- `AudioContext.currentTime`, read with the RMS, ends the window. +- The ring's playhead less its `output` counter, from one report, is the media frame paired with + each output frame. Only a time stretch moves it while the ring plays, and the reports either side + bracket it across the window, widened by one maximal stretch for each further stretch between + them and for the final window, which has no later report. +- A concealment, underrun, short quantum, skip, jump, discard, trim, stall, new graph, or new + timeline between those reports refuses the window rather than placing it. + +That leaves one constant per row, fitted by correlating log RMS over the audible windows with the +reference. The fit needs 40 audible windows, a correlation of 0.999, and a level within 5% before +any quiet window is placed with it; otherwise the row is unproven. + ### What the sampling grid can and cannot see The page samples every 250 ms. Worklet and worker reports arrive asynchronously, so a change @@ -395,13 +427,14 @@ clients/js/ src/probe.ts samples the element's public signals every 250ms src/beacon.ts batches to the sink, sendBeacon on pagehide src/schema.ts the metric contract - src/*.test.ts the thread void rule and how the probe reads the thread, under `just test` + src/silence.ts places each quiet window in the audio the page decoded + src/*.test.ts the void rules, the probe, the analyzer, the grader and the quiet proof, under `just test` driver.ts one row in headless Chromium, and the void checks replay.ts the recorded traces, with no relay, shaper, or browser safari.ts one row in real Safari, and the void checks it needs instead webdriver.ts a dependency-free W3C WebDriver client over safaridriver sink.ts one ndjson file per row - analyze.ts ndjson to summary.json and summary.md + analyze.ts ndjson to summary.json and summary.md, with the quiet proof grade.ts summaries against budgets.json compare.ts a before/after table across two run directories ``` diff --git a/test/audio-quality/clients/js/analyze.ts b/test/audio-quality/clients/js/analyze.ts index 938fea4c9b..7c4edcb00c 100644 --- a/test/audio-quality/clients/js/analyze.ts +++ b/test/audio-quality/clients/js/analyze.ts @@ -14,8 +14,12 @@ * - **converge_s** is the first moment after which the resolved target stayed within one bucket of * its final value for the rest of the run. A target that settles and then moves again has not * converged, so it is measured backwards from the end rather than forwards from the start. + * - **silence** is graded raw, and each quiet window is also placed in the audio the page decoded, + * rebuilt here from the file the publisher loops (`--media`) and its audio options (`--encode=`) + * rather than in the page, so the player does no extra work. The grader excuses a share over its + * ceiling only when every quiet window lands on quiet source; see `src/silence.ts`. * - * bun analyze.ts --run --row [--warmup 5] + * bun analyze.ts --run --row [--warmup 5] [--media --encode=