From 15b54d8f4c3d6e3bd94a4ac8ba1044ab2780b04d Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:51 +0000 Subject: [PATCH 01/13] Add offline TaskData dictionary evaluator Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- turbopack/crates/turbo-persistence/Cargo.toml | 4 + turbopack/crates/turbo-persistence/README.md | 22 + .../turbo-persistence/src/bin/sst_inspect.rs | 318 +------ .../src/bin/taskdata_dictionary.rs | 850 ++++++++++++++++++ turbopack/crates/turbo-persistence/src/lib.rs | 2 + .../crates/turbo-persistence/src/offline.rs | 448 +++++++++ 6 files changed, 1347 insertions(+), 297 deletions(-) create mode 100644 turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs create mode 100644 turbopack/crates/turbo-persistence/src/offline.rs diff --git a/turbopack/crates/turbo-persistence/Cargo.toml b/turbopack/crates/turbo-persistence/Cargo.toml index 8698edaf4c82..3686c3c7276d 100644 --- a/turbopack/crates/turbo-persistence/Cargo.toml +++ b/turbopack/crates/turbo-persistence/Cargo.toml @@ -52,6 +52,10 @@ turbo-tasks-malloc = { workspace = true, features = ["custom_allocator"] } name = "sst_inspect" path = "src/bin/sst_inspect.rs" +[[bin]] +name = "taskdata_dictionary" +path = "src/bin/taskdata_dictionary.rs" + [lints] workspace = true diff --git a/turbopack/crates/turbo-persistence/README.md b/turbopack/crates/turbo-persistence/README.md index 52c0f31305cc..64643a49192c 100644 --- a/turbopack/crates/turbo-persistence/README.md +++ b/turbopack/crates/turbo-persistence/README.md @@ -362,6 +362,28 @@ Configuration options for compactions are: - max number of SST files that are merged at once - coverage when compaction is triggered (otherwise calling compact is a noop) +## Evaluating zstd dictionaries offline + +`taskdata_dictionary` compares zstd dictionaries against active blocks from existing database +copies without modifying them or running the application that created them: + +```sh +cargo run -p turbo-persistence --bin taskdata_dictionary -- \ + --dictionary candidate-a.zdict \ + --dictionary candidate-b.zdict \ + --json report.json \ + path/to/database-a path/to/database-b +``` + +The no-dictionary zstd level 3 baseline is always included. Family 2 (TaskData) is selected by +default; `--family ` overrides it. The evaluator follows `CURRENT`, deletion files, and meta-file +supersession, verifies checksums, and models the same 12.5% minimum-savings threshold used by the +writer. Index blocks and ineligible key blocks remain unchanged in modeled SST sizes. + +Only SST blocks are evaluated. External blob references are counted and reported, but `.blob` +payloads are not read. Timing fields are single-pass diagnostics; use the byte/count fields for +repeatable comparisons of a copied cache snapshot. + ## Opening - Read the `CURRENT` file diff --git a/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs b/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs index 740f6b4757bc..fcc30fdb9158 100644 --- a/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs +++ b/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs @@ -10,33 +10,27 @@ //! `type - KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN`. use std::{ - collections::{BTreeMap, HashSet}, + collections::BTreeMap, path::{Path, PathBuf}, }; use anyhow::{Context, Result, bail}; -use byteorder::{BE, ReadBytesExt}; -use fs_err::{self as fs, File}; -use lzzzz::lz4::decompress; +use fs_err::File; use memmap2::Mmap; use turbo_persistence::{ - BLOCK_HEADER_SIZE, Compression, MAX_INLINE_VALUE_SIZE, checksum_block, - meta_file::MetaFile, + MAX_INLINE_VALUE_SIZE, mmap_helper::advise_mmap_for_persistence, - read_current_version, - sst_filter::SstFilter, + offline::{ + KeyBlockHeader, SstInfo, collect_sst_info, key_block_entry_types, parse_key_block_header, + parse_key_block_indices, read_block, + }, static_sorted_file::{ - BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, BLOCK_TYPE_KEY_NO_HASH, - BLOCK_TYPE_KEY_WITH_HASH, FIXED_KEY_BLOCK_MIXED_VALUE_TYPE, KEY_BLOCK_ENTRY_TYPE_BLOB, - KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, KEY_BLOCK_ENTRY_TYPE_KEY_DELETED, - KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN, KEY_BLOCK_ENTRY_TYPE_MEDIUM, - KEY_BLOCK_ENTRY_TYPE_SMALL, + KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, + KEY_BLOCK_ENTRY_TYPE_KEY_DELETED, KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN, + KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL, }, }; -/// Size of the key block header (1B type + 3B entry count). -const KEY_BLOCK_HEADER_SIZE: usize = 4; - /// Block size information #[derive(Default, Debug, Clone)] struct BlockSizeInfo { @@ -129,13 +123,6 @@ impl SstStats { } } -/// Information about an SST file from the meta file -struct SstInfo { - sequence_number: u32, - block_count: u16, - compression: Compression, -} - /// Accumulates statistics for a single entry of the given type. fn track_entry_type(stats: &mut SstStats, entry_type: u8) { *stats.entry_type_counts.entry(entry_type).or_insert(0) += 1; @@ -220,265 +207,6 @@ fn format_bytes(bytes: u64) -> String { } } -/// Collect SST info from all active meta files in the database directory, -/// mirroring the DB's own open logic: read CURRENT, filter by .del files, -/// and apply SstFilter to skip superseded entries. -fn collect_sst_info(db_path: &Path) -> Result>> { - // Read the CURRENT sequence number — only files with seq <= current are valid. - let current = read_current_version(db_path)? - .context("CURRENT file is missing")? - .max_sequence_number; - - // Read .del files to find sequences that were deleted but not yet cleaned up. - let mut deleted_seqs: HashSet = HashSet::new(); - for entry in fs::read_dir(db_path)? { - let path = entry?.path(); - if path.extension().and_then(|s| s.to_str()) == Some("del") { - let content = fs::read(&path)?; - let mut cursor: &[u8] = &content; - while !cursor.is_empty() { - deleted_seqs.insert(cursor.read_u32::()?); - } - } - } - - // Collect valid meta sequence numbers. - let mut meta_seqs: Vec = fs::read_dir(db_path)? - .filter_map(|e| e.ok()) - .filter_map(|e| { - let path = e.path(); - if path.extension().and_then(|s| s.to_str()) != Some("meta") { - return None; - } - let seq: u32 = path.file_stem()?.to_str()?.parse().ok()?; - if seq > current || deleted_seqs.contains(&seq) { - return None; - } - Some(seq) - }) - .collect(); - - if meta_seqs.is_empty() { - bail!("No active .meta files found in {}", db_path.display()); - } - - meta_seqs.sort_unstable(); - - let mut meta_files: Vec = meta_seqs - .iter() - .map(|&seq| { - MetaFile::open(db_path, seq, None, turbo_persistence::AccessMode::Mmap) - .with_context(|| format!("Failed to open {seq:08}.meta")) - }) - .collect::>()?; - - // Apply SstFilter (newest first) to drop entries superseded by a newer meta file. - let mut sst_filter = SstFilter::new(); - for meta in meta_files.iter_mut().rev() { - sst_filter.apply_filter(meta); - } - - let mut family_sst_info: BTreeMap> = BTreeMap::new(); - for meta in &meta_files { - let family = meta.family(); - for entry in meta.entries() { - family_sst_info.entry(family).or_default().push(SstInfo { - sequence_number: entry.sequence_number(), - block_count: entry.block_count(), - compression: meta.compression(), - }); - } - } - - Ok(family_sst_info) -} - -/// Information about a raw block read from disk. -struct RawBlock { - data: Box<[u8]>, - compressed_size: u64, - actual_size: u64, - was_compressed: bool, -} - -/// Reads, checksums, and decompresses a single block from the mmap. -fn read_block( - mmap: &Mmap, - block_offsets_start: usize, - block_index: u16, - sequence_number: u32, - compression: Compression, -) -> Result { - let offset = block_offsets_start + block_index as usize * size_of::(); - - let block_start = if block_index == 0 { - 0 - } else { - (&mmap[offset - size_of::()..offset]).read_u32::()? as usize - }; - let block_end = (&mmap[offset..offset + size_of::()]).read_u32::()? as usize; - - let uncompressed_length = - (&mmap[block_start..block_start + size_of::()]).read_u32::()?; - let expected_checksum = (&mmap - [block_start + size_of::()..block_start + BLOCK_HEADER_SIZE]) - .read_u32::()?; - let compressed_data = &mmap[block_start + BLOCK_HEADER_SIZE..block_end]; - let compressed_size = compressed_data.len() as u64; - - let was_compressed = uncompressed_length > 0; - let actual_size = if was_compressed { - uncompressed_length as u64 - } else { - compressed_size - }; - - let actual_checksum = checksum_block(compressed_data); - if actual_checksum != expected_checksum { - bail!( - "Cache corruption detected: checksum mismatch in block {} of {:08}.sst (expected \ - {:08x}, got {:08x})", - block_index, - sequence_number, - expected_checksum, - actual_checksum - ); - } - - let data = if was_compressed { - let mut buffer = vec![0u8; uncompressed_length as usize]; - let bytes_written = match compression { - Compression::Lz4 => { - decompress(compressed_data, &mut buffer).context("LZ4 decompression failed")? - } - Compression::Zstd3 => zstd::bulk::decompress_to_buffer(compressed_data, &mut buffer) - .context("zstd decompression failed")?, - }; - assert_eq!( - bytes_written, uncompressed_length as usize, - "Decompressed length does not match expected" - ); - buffer.into_boxed_slice() - } else { - Box::from(compressed_data) - }; - - Ok(RawBlock { - data, - compressed_size, - actual_size, - was_compressed, - }) -} - -/// Parses an index block to extract all referenced key block indices. -/// -/// Index block format: `[1B type][2B first_block][N * (8B hash + 2B block_index)]`. -fn parse_key_block_indices(index_block: &[u8]) -> HashSet { - assert!(index_block.len() >= 3, "Index block too small"); - let mut data = &index_block[1..]; // skip block type byte - let first_block = data.read_u16::().unwrap(); - let mut indices = HashSet::new(); - indices.insert(first_block); - const ENTRY_SIZE: usize = size_of::() + size_of::(); - let entry_count = data.len() / ENTRY_SIZE; - for i in 0..entry_count { - let block_index = (&data[i * ENTRY_SIZE + 8..]).read_u16::().unwrap(); - indices.insert(block_index); - } - indices -} - -/// Parsed header of a key block. -#[derive(Clone, Copy)] -enum KeyBlockHeader { - Variable { - entry_count: u32, - }, - Fixed { - entry_count: u32, - value_type: u8, - }, - /// Fixed-size layout whose entries share a value size but not a value type, so each carries - /// its own type byte between its key and its value. - FixedMixedType { - entry_count: u32, - hash_len: usize, - key_size: usize, - stride: usize, - }, -} - -/// Parses the header of a key block from the full decompressed block data. -fn parse_key_block_header(block: &[u8]) -> Result { - assert!(block.len() >= 4, "Key block too small"); - let block_type = block[0]; - let entry_count = ((block[1] as u32) << 16) | ((block[2] as u32) << 8) | (block[3] as u32); - match block_type { - BLOCK_TYPE_KEY_WITH_HASH | BLOCK_TYPE_KEY_NO_HASH => { - Ok(KeyBlockHeader::Variable { entry_count }) - } - BLOCK_TYPE_FIXED_KEY_WITH_HASH | BLOCK_TYPE_FIXED_KEY_NO_HASH => { - assert!(block.len() >= 6, "Fixed key block header too small"); - if block[5] == FIXED_KEY_BLOCK_MIXED_VALUE_TYPE { - assert!(block.len() >= 7, "Mixed-type key block header too small"); - let hash_len = if block_type == BLOCK_TYPE_FIXED_KEY_WITH_HASH { - 8 - } else { - 0 - }; - let key_size = block[4] as usize; - let val_size = block[6] as usize; - Ok(KeyBlockHeader::FixedMixedType { - entry_count, - hash_len, - key_size, - // +1 for the per-entry type byte. - stride: hash_len + key_size + val_size + 1, - }) - } else { - Ok(KeyBlockHeader::Fixed { - entry_count, - value_type: block[5], - }) - } - } - _ => bail!("Invalid key block type: {block_type}"), - } -} - -/// Iterates over entry type bytes in a key block. -/// -/// For variable-size key blocks, reads byte 0 of each 4-byte offset table entry. For fixed-size -/// key blocks, yields the single `value_type` repeated `entry_count` times, or reads the per-entry -/// type byte when the block has mixed types. -fn iter_key_block_entry_types( - header: KeyBlockHeader, - block: &[u8], -) -> impl Iterator + '_ { - let entry_count = match header { - KeyBlockHeader::Variable { entry_count } - | KeyBlockHeader::Fixed { entry_count, .. } - | KeyBlockHeader::FixedMixedType { entry_count, .. } => entry_count, - }; - (0..entry_count).map(move |i| match header { - // Variable block: offset table starts at byte 4 (after 1B type + 3B count), - // each entry is 4 bytes, first byte is the entry type. - KeyBlockHeader::Variable { .. } => block[KEY_BLOCK_HEADER_SIZE + i as usize * 4], - KeyBlockHeader::Fixed { value_type, .. } => value_type, - KeyBlockHeader::FixedMixedType { - hash_len, - key_size, - stride, - .. - } => { - // Entry data starts after the 7-byte mixed-type header; the type byte sits between - // the entry's key and its value. - block[7 + i as usize * stride + hash_len + key_size] - } - }) -} - /// Analyze an SST file and return entry type statistics fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { let compression = info.compression; @@ -509,10 +237,10 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { info.sequence_number, compression, )?; - let key_block_indices = parse_key_block_indices(&index_raw.data); + let key_block_indices = parse_key_block_indices(&index_raw.data)?; stats.index_blocks.add( - index_raw.compressed_size, + index_raw.stored_size, index_raw.actual_size, index_raw.was_compressed, ); @@ -540,7 +268,7 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { // Value block — no type header, just raw data. stats .value_blocks - .add(raw.compressed_size, raw.actual_size, raw.was_compressed); + .add(raw.stored_size, raw.actual_size, raw.was_compressed); continue; } @@ -548,7 +276,7 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { stats .key_blocks - .add(raw.compressed_size, raw.actual_size, raw.was_compressed); + .add(raw.stored_size, raw.actual_size, raw.was_compressed); let key_block_header = parse_key_block_header(block).with_context(|| { format!( @@ -558,22 +286,18 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { })?; match key_block_header { KeyBlockHeader::Variable { .. } => { - stats.variable_key_blocks.add( - raw.compressed_size, - raw.actual_size, - raw.was_compressed, - ); + stats + .variable_key_blocks + .add(raw.stored_size, raw.actual_size, raw.was_compressed); } KeyBlockHeader::Fixed { .. } | KeyBlockHeader::FixedMixedType { .. } => { - stats.fixed_key_blocks.add( - raw.compressed_size, - raw.actual_size, - raw.was_compressed, - ); + stats + .fixed_key_blocks + .add(raw.stored_size, raw.actual_size, raw.was_compressed); } }; - for entry_type in iter_key_block_entry_types(key_block_header, block) { + for entry_type in key_block_entry_types(key_block_header, block)? { track_entry_type(&mut stats, entry_type); } } diff --git a/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs new file mode 100644 index 000000000000..4743ef792764 --- /dev/null +++ b/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs @@ -0,0 +1,850 @@ +//! Evaluate zstd dictionaries against TaskData blocks from existing persistence caches. + +use std::{ + collections::{BTreeMap, BTreeSet}, + env, + ffi::OsString, + fs::File, + io::BufWriter, + mem::size_of, + path::{Path, PathBuf}, + time::Instant, +}; + +use anyhow::{Context, Result, bail, ensure}; +use serde::Serialize; +use turbo_persistence::{ + BLOCK_HEADER_SIZE, + offline::{ + KeyBlockHeader, MIN_KEY_SIZE_FOR_COMPRESSION, SstInfo, collect_sst_info, + key_block_entry_types, max_key_length, open_sst, parse_key_block_header, + parse_key_block_indices, read_block, + }, + static_sorted_file::KEY_BLOCK_ENTRY_TYPE_BLOB, +}; +use xxhash_rust::xxh3::xxh3_64; + +const SCHEMA_VERSION: u32 = 1; + +#[derive(Default)] +struct Options { + family: u32, + dictionaries: Vec, + json: Option, + caches: Vec, +} + +#[derive(Clone, Serialize)] +struct DictionaryInfo { + name: String, + path: Option, + bytes: usize, + dictionary_id: Option, + xxh3_64: Option, +} + +struct Candidate { + info: DictionaryInfo, + compressor: zstd::bulk::Compressor<'static>, + decompressor: zstd::bulk::Decompressor<'static>, + setup_ns: u64, +} + +#[derive(Default, Clone, Serialize)] +struct BlockGroup { + count: u64, + bytes: u64, +} + +impl BlockGroup { + fn add(&mut self, bytes: usize) { + self.count += 1; + self.bytes += bytes as u64; + } + + fn merge(&mut self, other: &Self) { + self.count += other.count; + self.bytes += other.bytes; + } +} + +#[derive(Default, Clone, Serialize)] +struct BlockSummary { + eligible_key: BlockGroup, + eligible_value: BlockGroup, + excluded_key: BlockGroup, + excluded_index: BlockGroup, + size_buckets: BTreeMap<&'static str, BlockGroup>, + blob_references: u64, +} + +impl BlockSummary { + fn add_eligible_size(&mut self, bytes: usize) { + self.size_buckets + .entry(size_bucket(bytes)) + .or_default() + .add(bytes); + } + + fn merge(&mut self, other: &Self) { + self.eligible_key.merge(&other.eligible_key); + self.eligible_value.merge(&other.eligible_value); + self.excluded_key.merge(&other.excluded_key); + self.excluded_index.merge(&other.excluded_index); + self.blob_references += other.blob_references; + for (bucket, group) in &other.size_buckets { + self.size_buckets.entry(bucket).or_default().merge(group); + } + } +} + +#[derive(Default, Clone, Serialize)] +struct CandidateResult { + dictionary: Option, + raw_compressed_bytes: u64, + modeled_payload_bytes: u64, + modeled_complete_sst_bytes: u64, + raw_compression_ratio: Option, + modeled_delta_vs_baseline_pct: Option, + modeled_delta_vs_current_pct: Option, + compressed_blocks: u64, + fallback_blocks: u64, + became_compressed: u64, + became_uncompressed: u64, + setup_ns: u64, + encode_ns: u64, + decode_ns: u64, +} + +impl CandidateResult { + fn merge(&mut self, other: &Self) { + self.raw_compressed_bytes += other.raw_compressed_bytes; + self.modeled_payload_bytes += other.modeled_payload_bytes; + self.modeled_complete_sst_bytes += other.modeled_complete_sst_bytes; + self.compressed_blocks += other.compressed_blocks; + self.fallback_blocks += other.fallback_blocks; + self.became_compressed += other.became_compressed; + self.became_uncompressed += other.became_uncompressed; + self.encode_ns += other.encode_ns; + self.decode_ns += other.decode_ns; + } +} + +#[derive(Serialize)] +struct CacheReport { + path: PathBuf, + family: u32, + recorded_codecs: BTreeSet, + active_ssts: u64, + original_complete_sst_bytes: u64, + original_eligible_payload_bytes: u64, + blocks: BlockSummary, + candidates: Vec, +} + +#[derive(Serialize)] +struct Report { + schema_version: u32, + family: u32, + timing_note: &'static str, + caches: Vec, + combined: CombinedReport, +} + +#[derive(Serialize)] +struct CombinedReport { + cache_count: usize, + active_ssts: u64, + original_complete_sst_bytes: u64, + original_eligible_payload_bytes: u64, + blocks: BlockSummary, + candidates: Vec, +} + +fn size_bucket(bytes: usize) -> &'static str { + match bytes { + 0..=4095 => "<4KiB", + 4096..=16383 => "4-16KiB", + 16384..=65535 => "16-64KiB", + 65536..=1048575 => "64KiB-1MiB", + _ => ">=1MiB", + } +} + +fn parse_args_from(args: impl IntoIterator) -> Result { + let mut options = Options { + family: 2, + ..Default::default() + }; + let mut args = args.into_iter(); + while let Some(arg) = args.next() { + match arg.to_str() { + Some("--dictionary" | "-d") => options.dictionaries.push(PathBuf::from( + args.next().context("--dictionary requires a path")?, + )), + Some("--json") => { + options.json = Some(PathBuf::from( + args.next().context("--json requires a path")?, + )); + } + Some("--family") => { + let value = args.next().context("--family requires an integer")?; + options.family = value + .to_str() + .context("--family must be UTF-8")? + .parse() + .context("--family must be an unsigned integer")?; + } + Some("--help" | "-h") => { + print_help(); + std::process::exit(0); + } + Some(value) if value.starts_with('-') => bail!("Unknown option {value}"), + _ => options.caches.push(PathBuf::from(arg)), + } + } + ensure!( + !options.caches.is_empty(), + "At least one cache directory is required" + ); + Ok(options) +} + +fn parse_args() -> Result { + parse_args_from(env::args_os().skip(1)) +} + +fn print_help() { + println!( + "Usage: taskdata_dictionary [OPTIONS] ...\n\nEvaluate candidate zstd \ + dictionaries against active TaskData SST blocks.\n\nOptions:\n-d, --dictionary \ + Candidate dictionary (repeatable)\n--family Family ID to evaluate (default: 2 \ + / TaskData)\n--json Write a JSON report\n-h, --help Show this \ + help" + ); +} + +fn make_candidates(paths: &[PathBuf]) -> Result> { + let mut candidates = Vec::with_capacity(paths.len() + 1); + let started = Instant::now(); + let compressor = zstd::bulk::Compressor::new(3)?; + let decompressor = zstd::bulk::Decompressor::new()?; + candidates.push(Candidate { + info: DictionaryInfo { + name: "zstd3 (no dictionary)".into(), + path: None, + bytes: 0, + dictionary_id: None, + xxh3_64: None, + }, + compressor, + decompressor, + setup_ns: started.elapsed().as_nanos() as u64, + }); + + let mut names = BTreeSet::new(); + for path in paths { + let dictionary = std::fs::read(path) + .with_context(|| format!("Failed to read dictionary {}", path.display()))?; + ensure!( + !dictionary.is_empty(), + "Dictionary {} is empty", + path.display() + ); + let name = path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or("dictionary") + .to_owned(); + ensure!( + names.insert(name.clone()), + "Duplicate dictionary name {name}" + ); + let started = Instant::now(); + let compressor = zstd::bulk::Compressor::with_dictionary(3, &dictionary) + .with_context(|| format!("Failed to load dictionary {}", path.display()))?; + let decompressor = zstd::bulk::Decompressor::with_dictionary(&dictionary) + .with_context(|| format!("Failed to load dictionary {}", path.display()))?; + let setup_ns = started.elapsed().as_nanos() as u64; + candidates.push(Candidate { + info: DictionaryInfo { + name, + path: Some(path.clone()), + bytes: dictionary.len(), + dictionary_id: zstd::zstd_safe::get_dict_id_from_dict(&dictionary) + .map(|id| id.get()), + xxh3_64: Some(format!("{:016x}", xxh3_64(&dictionary))), + }, + compressor, + decompressor, + setup_ns, + }); + } + Ok(candidates) +} + +fn production_stored_size(original_len: usize, compressed_len: usize) -> (usize, bool) { + if compressed_len < original_len - original_len / 8 { + (compressed_len, true) + } else { + (original_len, false) + } +} + +fn evaluate_block( + candidates: &mut [Candidate], + results: &mut [CandidateResult], + data: &[u8], + was_compressed: bool, +) -> Result<()> { + for (candidate, result) in candidates.iter_mut().zip(results) { + let started = Instant::now(); + let compressed = candidate + .compressor + .compress(data) + .with_context(|| format!("Failed to compress with {}", candidate.info.name))?; + result.encode_ns += started.elapsed().as_nanos() as u64; + + let started = Instant::now(); + let decoded = candidate + .decompressor + .decompress(&compressed, data.len()) + .with_context(|| format!("Failed to decompress with {}", candidate.info.name))?; + result.decode_ns += started.elapsed().as_nanos() as u64; + ensure!( + decoded == data, + "Round trip mismatch with {}", + candidate.info.name + ); + + result.raw_compressed_bytes += compressed.len() as u64; + let (stored, compressed_decision) = production_stored_size(data.len(), compressed.len()); + result.modeled_payload_bytes += stored as u64; + if compressed_decision { + result.compressed_blocks += 1; + } else { + result.fallback_blocks += 1; + } + if compressed_decision && !was_compressed { + result.became_compressed += 1; + } else if !compressed_decision && was_compressed { + result.became_uncompressed += 1; + } + } + Ok(()) +} + +fn count_blob_references(header: KeyBlockHeader, data: &[u8]) -> Result { + Ok(key_block_entry_types(header, data)? + .into_iter() + .filter(|&entry_type| entry_type == KEY_BLOCK_ENTRY_TYPE_BLOB) + .count() as u64) +} + +fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Result { + ensure!(path.is_dir(), "Not a cache directory: {}", path.display()); + let families = collect_sst_info(path) + .with_context(|| format!("Failed to inspect cache {}", path.display()))?; + let ssts = families.get(&family).with_context(|| { + format!( + "Cache {} has no active SSTs for family {family}", + path.display() + ) + })?; + + let mut report = CacheReport { + path: path.to_path_buf(), + family, + recorded_codecs: BTreeSet::new(), + active_ssts: ssts.len() as u64, + original_complete_sst_bytes: 0, + original_eligible_payload_bytes: 0, + blocks: BlockSummary::default(), + candidates: candidates + .iter() + .map(|candidate| CandidateResult { + dictionary: Some(candidate.info.clone()), + setup_ns: candidate.setup_ns, + ..Default::default() + }) + .collect(), + }; + + for info in ssts { + evaluate_sst(path, info, candidates, &mut report) + .with_context(|| format!("Failed to evaluate {:08}.sst", info.sequence_number))?; + } + + for result in &mut report.candidates { + result.modeled_complete_sst_bytes = report.original_complete_sst_bytes + - report.original_eligible_payload_bytes + + result.modeled_payload_bytes; + } + finalize_results( + &mut report.candidates, + report.blocks.eligible_key.bytes + report.blocks.eligible_value.bytes, + report.original_complete_sst_bytes, + ); + Ok(report) +} + +fn evaluate_sst( + db_path: &Path, + info: &SstInfo, + candidates: &mut [Candidate], + report: &mut CacheReport, +) -> Result<()> { + ensure!(info.block_count > 0, "SST contains no blocks"); + let (mmap, file_size, offsets_start) = open_sst(db_path, info)?; + report.original_complete_sst_bytes += file_size; + report + .recorded_codecs + .insert(format!("{:?}", info.compression)); + + let index_index = info.block_count - 1; + let index = read_block( + &mmap, + offsets_start, + index_index, + info.sequence_number, + info.compression, + )?; + let key_indices = parse_key_block_indices(&index.data)?; + report.blocks.excluded_index.add(index.data.len()); + + for block_index in 0..index_index { + let block = read_block( + &mmap, + offsets_start, + block_index, + info.sequence_number, + info.compression, + )?; + let eligible = if key_indices.contains(&block_index) { + let header = parse_key_block_header(&block.data).with_context(|| { + format!( + "Invalid key block {block_index} in {:08}.sst", + info.sequence_number + ) + })?; + report.blocks.blob_references += count_blob_references(header, &block.data)?; + if max_key_length(header, &block.data)? >= MIN_KEY_SIZE_FOR_COMPRESSION { + report.blocks.eligible_key.add(block.data.len()); + true + } else { + report.blocks.excluded_key.add(block.data.len()); + false + } + } else { + report.blocks.eligible_value.add(block.data.len()); + true + }; + if eligible { + report.blocks.add_eligible_size(block.data.len()); + report.original_eligible_payload_bytes += block.stored_size; + evaluate_block( + candidates, + &mut report.candidates, + &block.data, + block.was_compressed, + )?; + } + } + + let modeled_fixed_bytes = + info.block_count as u64 * (BLOCK_HEADER_SIZE as u64 + size_of::() as u64); + ensure!( + file_size >= modeled_fixed_bytes, + "SST is smaller than its headers and block directory" + ); + Ok(()) +} + +fn percentage_delta(value: u64, baseline: u64) -> Option { + (baseline > 0).then(|| (value as f64 / baseline as f64 - 1.0) * 100.0) +} + +fn finalize_results(results: &mut [CandidateResult], uncompressed_bytes: u64, current_bytes: u64) { + let baseline = results + .first() + .map_or(0, |result| result.modeled_complete_sst_bytes); + for result in results { + result.raw_compression_ratio = (uncompressed_bytes > 0) + .then(|| result.raw_compressed_bytes as f64 / uncompressed_bytes as f64); + result.modeled_delta_vs_baseline_pct = + percentage_delta(result.modeled_complete_sst_bytes, baseline); + result.modeled_delta_vs_current_pct = + percentage_delta(result.modeled_complete_sst_bytes, current_bytes); + } +} + +fn combine(caches: &[CacheReport], candidates: &[Candidate]) -> CombinedReport { + let mut combined = CombinedReport { + cache_count: caches.len(), + active_ssts: 0, + original_complete_sst_bytes: 0, + original_eligible_payload_bytes: 0, + blocks: BlockSummary::default(), + candidates: candidates + .iter() + .map(|candidate| CandidateResult { + dictionary: Some(candidate.info.clone()), + setup_ns: candidate.setup_ns, + ..Default::default() + }) + .collect(), + }; + for cache in caches { + combined.active_ssts += cache.active_ssts; + combined.original_complete_sst_bytes += cache.original_complete_sst_bytes; + combined.original_eligible_payload_bytes += cache.original_eligible_payload_bytes; + combined.blocks.merge(&cache.blocks); + for (total, result) in combined.candidates.iter_mut().zip(&cache.candidates) { + total.merge(result); + } + } + finalize_results( + &mut combined.candidates, + combined.blocks.eligible_key.bytes + combined.blocks.eligible_value.bytes, + combined.original_complete_sst_bytes, + ); + combined +} + +fn print_report(report: &Report) { + for cache in &report.caches { + println!( + "Cache {}: {} SSTs, {} bytes; eligible {} key + {} value blocks; blobs omitted {}", + cache.path.display(), + cache.active_ssts, + cache.original_complete_sst_bytes, + cache.blocks.eligible_key.count, + cache.blocks.eligible_value.count, + cache.blocks.blob_references, + ); + let baseline = cache.candidates[0].modeled_complete_sst_bytes; + for result in &cache.candidates { + let name = &result.dictionary.as_ref().unwrap().name; + let delta = if baseline == 0 { + 0.0 + } else { + (result.modeled_complete_sst_bytes as f64 / baseline as f64 - 1.0) * 100.0 + }; + println!( + " {name}: modeled SST {} bytes ({delta:+.2}% vs baseline)", + result.modeled_complete_sst_bytes + ); + } + } + println!(); + println!( + "Combined family {}: {} cache directories, {} active SSTs", + report.family, + report.caches.len(), + report.combined.active_ssts + ); + println!( + "Eligible: {} key + {} value blocks, {} bytes uncompressed; blobs omitted: {}", + report.combined.blocks.eligible_key.count, + report.combined.blocks.eligible_value.count, + report.combined.blocks.eligible_key.bytes + report.combined.blocks.eligible_value.bytes, + report.combined.blocks.blob_references + ); + println!( + "Original active SST bytes: {}", + report.combined.original_complete_sst_bytes + ); + println!(); + println!( + "{:<28} {:>15} {:>9} {:>15} {:>10} {:>12} {:>12}", + "Candidate", + "Raw compressed", + "ratio", + "Modeled SST", + "vs baseline", + "encode ms", + "decode ms" + ); + let baseline = report.combined.candidates[0].modeled_complete_sst_bytes; + for result in &report.combined.candidates { + let name = &result.dictionary.as_ref().unwrap().name; + let delta = if baseline == 0 { + 0.0 + } else { + (result.modeled_complete_sst_bytes as f64 / baseline as f64 - 1.0) * 100.0 + }; + println!( + "{name:<28} {:>15} {:>8.2}% {:>15} {:>+9.2}% {:>12.3} {:>12.3}", + result.raw_compressed_bytes, + result.raw_compression_ratio.unwrap_or_default() * 100.0, + result.modeled_complete_sst_bytes, + delta, + result.encode_ns as f64 / 1_000_000.0, + result.decode_ns as f64 / 1_000_000.0, + ); + } + println!(); + println!("Note: timings are single-pass wall-clock diagnostics, not benchmarks."); +} + +fn run() -> Result<()> { + let options = parse_args()?; + let mut candidates = make_candidates(&options.dictionaries)?; + let mut caches = Vec::with_capacity(options.caches.len()); + for path in &options.caches { + caches.push(evaluate_cache(path, options.family, &mut candidates)?); + } + let combined = combine(&caches, &candidates); + let report = Report { + schema_version: SCHEMA_VERSION, + family: options.family, + timing_note: "Single-pass wall-clock diagnostics; size/count fields are the comparison \ + contract.", + caches, + combined, + }; + print_report(&report); + if let Some(path) = options.json { + let file = File::create(&path) + .with_context(|| format!("Failed to create JSON report {}", path.display()))?; + serde_json::to_writer_pretty(BufWriter::new(file), &report)?; + } + Ok(()) +} + +fn main() { + if let Err(error) = run() { + eprintln!("Error: {error:#}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use std::{fs, path::Path}; + + use anyhow::Result; + use byteorder::{BE, WriteBytesExt}; + use tempfile::TempDir; + use turbo_persistence::{Compression, DbConfig, SerialScheduler, TurboPersistence}; + + use super::{evaluate_cache, make_candidates, parse_args_from, production_stored_size}; + + fn make_cache(family: usize, compression: Compression) -> Result { + let tempdir = tempfile::tempdir()?; + let mut config = DbConfig::<8>::default(); + config.family_configs[family].compression = compression; + let db = TurboPersistence::::open_with_config( + tempdir.path().to_path_buf(), + config, + )?; + let batch = db.write_batch()?; + for index in 0..50u8 { + let key = format!("long-task-data-key-{index:04}").into_bytes(); + let value = format!("function component{index}() {{ return null; }}") + .repeat(32) + .into_bytes(); + batch.put(family as u32, key, value.into())?; + } + for index in 0..4u8 { + let key = format!("long-medium-task-data-key-{index:04}").into_bytes(); + batch.put(family as u32, key, vec![b'a' + index; 5000].into())?; + } + let mut state = 0x1234_5678_u32; + let incompressible = (0..5000) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect::>(); + batch.put( + family as u32, + b"long-incompressible-task-data-key".to_vec(), + incompressible.into(), + )?; + db.commit_write_batch(batch)?; + db.shutdown()?; + Ok(tempdir) + } + + fn write_dictionary(path: &Path, byte: u8) -> Result<()> { + let mut dictionary = b"function component return const import export className".repeat(32); + dictionary.push(byte); + fs::write(path, dictionary)?; + Ok(()) + } + + #[test] + fn cli_requires_cache_and_rejects_unknown_options() { + assert!(parse_args_from([]).is_err()); + assert!(parse_args_from(["--unknown".into()]).is_err()); + let options = parse_args_from([ + "--family".into(), + "7".into(), + "--dictionary".into(), + "candidate.dict".into(), + "cache".into(), + ]) + .unwrap(); + assert_eq!(options.family, 7); + assert_eq!( + options.dictionaries, + [std::path::PathBuf::from("candidate.dict")] + ); + assert_eq!(options.caches, [std::path::PathBuf::from("cache")]); + } + + #[test] + fn production_threshold_is_strict() { + assert_eq!(production_stored_size(800, 699), (699, true)); + assert_eq!(production_stored_size(800, 700), (800, false)); + } + + #[test] + fn evaluates_multiple_dictionaries_without_mutating_cache() -> Result<()> { + let cache = make_cache(2, Compression::Zstd3)?; + let dictionary_dir = tempfile::tempdir()?; + let first = dictionary_dir.path().join("first.dict"); + let second = dictionary_dir.path().join("second.dict"); + write_dictionary(&first, 1)?; + write_dictionary(&second, 2)?; + let before = fs::read(cache.path().join("00000001.sst"))?; + + let mut candidates = make_candidates(&[first, second])?; + let report = evaluate_cache(cache.path(), 2, &mut candidates)?; + + assert_eq!(report.active_ssts, 1); + assert_eq!(report.candidates.len(), 3); + assert!(report.blocks.eligible_value.count > 0); + assert!(report.blocks.eligible_key.count > 0); + assert_eq!(report.blocks.excluded_index.count, 1); + assert_eq!(report.blocks.blob_references, 0); + assert_eq!( + report.candidates[0].modeled_complete_sst_bytes, + report.original_complete_sst_bytes + ); + assert!(report.candidates[0].fallback_blocks > 0); + assert!( + report + .candidates + .iter() + .all(|candidate| candidate.modeled_complete_sst_bytes > 0) + ); + + let mut repeated_candidates = make_candidates(&[ + dictionary_dir.path().join("first.dict"), + dictionary_dir.path().join("second.dict"), + ])?; + let repeated = evaluate_cache(cache.path(), 2, &mut repeated_candidates)?; + assert_eq!( + report.original_complete_sst_bytes, + repeated.original_complete_sst_bytes + ); + assert_eq!( + report.blocks.eligible_key.count, + repeated.blocks.eligible_key.count + ); + assert_eq!( + report.blocks.eligible_value.count, + repeated.blocks.eligible_value.count + ); + assert_eq!( + report + .candidates + .iter() + .map(|candidate| candidate.modeled_complete_sst_bytes) + .collect::>(), + repeated + .candidates + .iter() + .map(|candidate| candidate.modeled_complete_sst_bytes) + .collect::>() + ); + assert_eq!(before, fs::read(cache.path().join("00000001.sst"))?); + Ok(()) + } + + #[test] + fn counts_blob_references_in_key_blocks() -> Result<()> { + use turbo_persistence::{ + offline::parse_key_block_header, + static_sorted_file::{BLOCK_TYPE_FIXED_KEY_NO_HASH, KEY_BLOCK_ENTRY_TYPE_BLOB}, + }; + + let block = [ + BLOCK_TYPE_FIXED_KEY_NO_HASH, + 0, + 0, + 1, + 1, + KEY_BLOCK_ENTRY_TYPE_BLOB, + b'k', + 0, + 0, + 0, + 42, + ]; + let header = parse_key_block_header(&block)?; + assert_eq!(super::count_blob_references(header, &block)?, 1); + Ok(()) + } + + #[test] + fn corrupt_blocks_are_rejected_with_context() -> Result<()> { + let cache = make_cache(2, Compression::Zstd3)?; + let sst_path = cache.path().join("00000001.sst"); + let mut bytes = fs::read(&sst_path)?; + bytes[8] ^= 1; + fs::write(&sst_path, bytes)?; + let mut candidates = make_candidates(&[])?; + let error = evaluate_cache(cache.path(), 2, &mut candidates) + .err() + .expect("corrupt block should fail"); + let message = format!("{error:#}"); + assert!(message.contains("00000001.sst")); + assert!(message.contains("Checksum mismatch")); + Ok(()) + } + + #[test] + fn active_ssts_follow_current_deletions_and_supersession() -> Result<()> { + let cache = make_cache(2, Compression::Zstd3)?; + fs::copy( + cache.path().join("00000002.meta"), + cache.path().join("00000003.meta"), + )?; + let mut current: serde_json::Value = + serde_json::from_slice(&fs::read(cache.path().join("CURRENT"))?)?; + current["max_sequence_number"] = 3.into(); + fs::write(cache.path().join("CURRENT"), serde_json::to_vec(¤t)?)?; + + let mut candidates = make_candidates(&[])?; + let superseded = evaluate_cache(cache.path(), 2, &mut candidates)?; + assert_eq!(superseded.active_ssts, 1); + + let mut deletion = Vec::new(); + deletion.write_u32::(3)?; + fs::write(cache.path().join("00000004.del"), deletion)?; + let mut candidates = make_candidates(&[])?; + let deleted = evaluate_cache(cache.path(), 2, &mut candidates)?; + assert_eq!(deleted.active_ssts, 1); + assert_eq!( + superseded.original_complete_sst_bytes, + deleted.original_complete_sst_bytes + ); + Ok(()) + } + + #[test] + fn family_override_selects_non_taskdata_family() -> Result<()> { + let cache = make_cache(7, Compression::Lz4)?; + let mut candidates = make_candidates(&[])?; + let report = evaluate_cache(cache.path(), 7, &mut candidates)?; + assert_eq!(report.family, 7); + assert_eq!(report.recorded_codecs, ["Lz4".to_string()].into()); + assert!(report.blocks.eligible_value.count > 0); + Ok(()) + } +} diff --git a/turbopack/crates/turbo-persistence/src/lib.rs b/turbopack/crates/turbo-persistence/src/lib.rs index 67fe6ad4ad09..1d693f64a5fe 100644 --- a/turbopack/crates/turbo-persistence/src/lib.rs +++ b/turbopack/crates/turbo-persistence/src/lib.rs @@ -16,6 +16,8 @@ mod merge_iter; pub mod meta_file; mod meta_file_builder; pub mod mmap_helper; +#[doc(hidden)] +pub mod offline; mod parallel_scheduler; mod rc_bytes; mod shared_bytes; diff --git a/turbopack/crates/turbo-persistence/src/offline.rs b/turbopack/crates/turbo-persistence/src/offline.rs new file mode 100644 index 000000000000..25fd880bf86c --- /dev/null +++ b/turbopack/crates/turbo-persistence/src/offline.rs @@ -0,0 +1,448 @@ +//! Shared helpers for offline inspection of persistence SST files. + +use std::{ + collections::{BTreeMap, HashSet}, + mem::size_of, + path::Path, +}; + +use anyhow::{Context, Result, bail, ensure}; +use byteorder::{BE, ReadBytesExt}; +use fs_err::{self as fs, File}; +use lzzzz::lz4::decompress; +use memmap2::Mmap; + +use crate::{ + BLOCK_HEADER_SIZE, Compression, checksum_block, + meta_file::MetaFile, + mmap_helper::advise_mmap_for_persistence, + read_current_version, + sst_filter::SstFilter, + static_sorted_file::{ + BLOB_VALUE_REF_SIZE, BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, + BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY_NO_HASH, BLOCK_TYPE_KEY_WITH_HASH, + FIXED_KEY_BLOCK_MIXED_VALUE_TYPE, KEY_BLOCK_ENTRY_TYPE_BLOB, + KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, KEY_BLOCK_ENTRY_TYPE_KEY_DELETED, + KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN, KEY_BLOCK_ENTRY_TYPE_MEDIUM, + KEY_BLOCK_ENTRY_TYPE_SMALL, KEY_DELETED_REF_SIZE, MEDIUM_VALUE_REF_SIZE, + SMALL_VALUE_REF_SIZE, + }, +}; + +const KEY_BLOCK_HEADER_SIZE: usize = 4; +pub const MIN_KEY_SIZE_FOR_COMPRESSION: usize = 16; + +/// Information about an active SST file recorded by a meta file. +#[derive(Clone, Copy, Debug)] +pub struct SstInfo { + pub sequence_number: u32, + pub block_count: u16, + pub compression: Compression, +} + +/// Collects active SSTs by family, mirroring database open logic. +pub fn collect_sst_info(db_path: &Path) -> Result>> { + let current = read_current_version(db_path)? + .context("CURRENT file is missing")? + .max_sequence_number; + + let mut deleted_seqs = HashSet::new(); + for entry in fs::read_dir(db_path) + .with_context(|| format!("Failed to read database directory {}", db_path.display()))? + { + let path = entry?.path(); + if path.extension().and_then(|s| s.to_str()) == Some("del") { + let content = fs::read(&path) + .with_context(|| format!("Failed to read deletion file {}", path.display()))?; + let mut cursor: &[u8] = &content; + while !cursor.is_empty() { + deleted_seqs.insert( + cursor.read_u32::().with_context(|| { + format!("Truncated sequence number in {}", path.display()) + })?, + ); + } + } + } + + let mut meta_seqs: Vec = fs::read_dir(db_path)? + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) != Some("meta") { + return None; + } + let sequence: u32 = path.file_stem()?.to_str()?.parse().ok()?; + (sequence <= current && !deleted_seqs.contains(&sequence)).then_some(sequence) + }) + .collect(); + + if meta_seqs.is_empty() { + bail!("No active .meta files found in {}", db_path.display()); + } + meta_seqs.sort_unstable(); + + let mut meta_files: Vec = meta_seqs + .iter() + .map(|&sequence| { + MetaFile::open(db_path, sequence, None) + .with_context(|| format!("Failed to open {sequence:08}.meta")) + }) + .collect::>()?; + + let mut sst_filter = SstFilter::new(); + for meta in meta_files.iter_mut().rev() { + sst_filter.apply_filter(meta); + } + + let mut families: BTreeMap> = BTreeMap::new(); + for meta in &meta_files { + for entry in meta.entries() { + families.entry(meta.family()).or_default().push(SstInfo { + sequence_number: entry.sequence_number(), + block_count: entry.block_count(), + compression: meta.compression(), + }); + } + } + Ok(families) +} + +/// A checksummed block reconstructed to its original bytes. +pub struct RawBlock { + pub data: Box<[u8]>, + pub stored_size: u64, + pub actual_size: u64, + pub was_compressed: bool, +} + +/// Reads, checksums, and decompresses a single SST block. +pub fn read_block( + mmap: &Mmap, + block_offsets_start: usize, + block_index: u16, + sequence_number: u32, + compression: Compression, +) -> Result { + let offset = block_offsets_start + .checked_add(block_index as usize * size_of::()) + .context("Block offset overflow")?; + let end_bytes = mmap + .get(offset..offset + size_of::()) + .with_context(|| { + format!( + "Block {block_index} directory entry is out of bounds in {sequence_number:08}.sst" + ) + })?; + let block_end = (&end_bytes[..]).read_u32::()? as usize; + let block_start = if block_index == 0 { + 0 + } else { + let start_bytes = mmap + .get(offset - size_of::()..offset) + .with_context(|| format!("Block {block_index} start offset is out of bounds"))?; + (&start_bytes[..]).read_u32::()? as usize + }; + ensure!( + block_end >= block_start + BLOCK_HEADER_SIZE && block_end <= block_offsets_start, + "Invalid bounds {block_start}..{block_end} for block {block_index} in \ + {sequence_number:08}.sst" + ); + + let header = mmap + .get(block_start..block_start + BLOCK_HEADER_SIZE) + .with_context(|| format!("Truncated header for block {block_index}"))?; + let uncompressed_length = (&header[..4]).read_u32::()?; + let expected_checksum = (&header[4..]).read_u32::()?; + let stored_data = mmap + .get(block_start + BLOCK_HEADER_SIZE..block_end) + .with_context(|| format!("Truncated payload for block {block_index}"))?; + let actual_checksum = checksum_block(stored_data); + ensure!( + actual_checksum == expected_checksum, + "Checksum mismatch in block {block_index} of {sequence_number:08}.sst (expected \ + {expected_checksum:08x}, got {actual_checksum:08x})" + ); + + let was_compressed = uncompressed_length > 0; + let data = if was_compressed { + let mut output = vec![0; uncompressed_length as usize]; + let written = match compression { + Compression::Lz4 => decompress(stored_data, &mut output) + .map_err(anyhow::Error::from) + .context("LZ4 decompression failed"), + Compression::Zstd3 => zstd::bulk::decompress_to_buffer(stored_data, &mut output) + .map_err(anyhow::Error::from) + .context("zstd decompression failed"), + } + .with_context(|| { + format!("Failed to decompress block {block_index} of {sequence_number:08}.sst") + })?; + ensure!( + written == uncompressed_length as usize, + "Decompressed block {block_index} of {sequence_number:08}.sst to {written} bytes, \ + expected {uncompressed_length}" + ); + output.into_boxed_slice() + } else { + Box::from(stored_data) + }; + + Ok(RawBlock { + actual_size: data.len() as u64, + stored_size: stored_data.len() as u64, + data, + was_compressed, + }) +} + +/// Parses an index block and returns all key-block indices. +pub fn parse_key_block_indices(index_block: &[u8]) -> Result> { + ensure!(index_block.len() >= 3, "Index block is too small"); + ensure!( + index_block[0] == BLOCK_TYPE_INDEX, + "Invalid index block type" + ); + let mut data = &index_block[1..]; + let first_block = data.read_u16::()?; + let mut indices = HashSet::from([first_block]); + const ENTRY_SIZE: usize = size_of::() + size_of::(); + let (entries, remainder) = data.as_chunks::(); + ensure!(remainder.is_empty(), "Index block has a truncated entry"); + for entry in entries { + indices.insert((&entry[size_of::()..]).read_u16::()?); + } + Ok(indices) +} + +/// Parsed key-block layout used by both offline tools. +#[derive(Clone, Copy)] +pub enum KeyBlockHeader { + Variable { + entry_count: u32, + hash_len: usize, + }, + Fixed { + entry_count: u32, + hash_len: usize, + key_size: usize, + value_type: u8, + }, + FixedMixedType { + entry_count: u32, + hash_len: usize, + key_size: usize, + stride: usize, + }, +} + +impl KeyBlockHeader { + pub fn entry_count(self) -> u32 { + match self { + Self::Variable { entry_count, .. } + | Self::Fixed { entry_count, .. } + | Self::FixedMixedType { entry_count, .. } => entry_count, + } + } +} + +/// Parses a key-block header. +pub fn parse_key_block_header(block: &[u8]) -> Result { + ensure!( + block.len() >= KEY_BLOCK_HEADER_SIZE, + "Key block is too small" + ); + let block_type = block[0]; + let entry_count = ((block[1] as u32) << 16) | ((block[2] as u32) << 8) | block[3] as u32; + let hash_len = match block_type { + BLOCK_TYPE_KEY_WITH_HASH | BLOCK_TYPE_FIXED_KEY_WITH_HASH => size_of::(), + BLOCK_TYPE_KEY_NO_HASH | BLOCK_TYPE_FIXED_KEY_NO_HASH => 0, + _ => bail!("Invalid key block type {block_type}"), + }; + match block_type { + BLOCK_TYPE_KEY_WITH_HASH | BLOCK_TYPE_KEY_NO_HASH => Ok(KeyBlockHeader::Variable { + entry_count, + hash_len, + }), + BLOCK_TYPE_FIXED_KEY_WITH_HASH | BLOCK_TYPE_FIXED_KEY_NO_HASH => { + ensure!(block.len() >= 6, "Fixed key block header is too small"); + let key_size = block[4] as usize; + if block[5] == FIXED_KEY_BLOCK_MIXED_VALUE_TYPE { + ensure!(block.len() >= 7, "Mixed key block header is too small"); + Ok(KeyBlockHeader::FixedMixedType { + entry_count, + hash_len, + key_size, + stride: hash_len + key_size + block[6] as usize + 1, + }) + } else { + Ok(KeyBlockHeader::Fixed { + entry_count, + hash_len, + key_size, + value_type: block[5], + }) + } + } + _ => unreachable!(), + } +} + +/// Returns the entry type bytes from a key block after validating its layout. +pub fn key_block_entry_types(header: KeyBlockHeader, block: &[u8]) -> Result> { + let count = header.entry_count() as usize; + match header { + KeyBlockHeader::Variable { .. } => { + let end = KEY_BLOCK_HEADER_SIZE + count * size_of::(); + let offsets = block + .get(KEY_BLOCK_HEADER_SIZE..end) + .context("Variable key block offset table is truncated")?; + Ok(offsets + .as_chunks::<4>() + .0 + .iter() + .map(|entry| entry[0]) + .collect()) + } + KeyBlockHeader::Fixed { + hash_len, + key_size, + value_type, + .. + } => { + let stride = hash_len + key_size + entry_value_size(value_type)?; + ensure!( + block.len() == 6 + count * stride, + "Fixed key block has an invalid length" + ); + Ok(vec![value_type; count]) + } + KeyBlockHeader::FixedMixedType { + hash_len, + key_size, + stride, + .. + } => { + ensure!( + block.len() == 7 + count * stride, + "Mixed key block has an invalid length" + ); + Ok((0..count) + .map(|index| block[7 + index * stride + hash_len + key_size]) + .collect()) + } + } +} + +fn entry_value_size(entry_type: u8) -> Result { + match entry_type { + KEY_BLOCK_ENTRY_TYPE_SMALL => Ok(SMALL_VALUE_REF_SIZE), + KEY_BLOCK_ENTRY_TYPE_MEDIUM => Ok(MEDIUM_VALUE_REF_SIZE), + KEY_BLOCK_ENTRY_TYPE_BLOB => Ok(BLOB_VALUE_REF_SIZE), + KEY_BLOCK_ENTRY_TYPE_KEY_DELETED => Ok(KEY_DELETED_REF_SIZE), + value if value >= KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN => { + Ok((value - KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN) as usize) + } + value if value >= KEY_BLOCK_ENTRY_TYPE_INLINE_MIN => { + Ok((value - KEY_BLOCK_ENTRY_TYPE_INLINE_MIN) as usize) + } + value => bail!("Invalid key block entry type {value}"), + } +} + +/// Returns the maximum stored key length in a parsed key block. +pub fn max_key_length(header: KeyBlockHeader, block: &[u8]) -> Result { + match header { + KeyBlockHeader::Fixed { key_size, .. } + | KeyBlockHeader::FixedMixedType { key_size, .. } => Ok(key_size), + KeyBlockHeader::Variable { + entry_count, + hash_len, + } => { + let entry_count = entry_count as usize; + let header_size = KEY_BLOCK_HEADER_SIZE + entry_count * size_of::(); + ensure!( + header_size <= block.len(), + "Variable key block header is truncated" + ); + let offsets = &block[KEY_BLOCK_HEADER_SIZE..header_size]; + let mut max_key = 0; + for index in 0..entry_count { + let word = (&offsets[index * 4..]).read_u32::()?; + let entry_type = (word >> 24) as u8; + let start = header_size + (word & 0x00ff_ffff) as usize; + let end = if index + 1 < entry_count { + let next = (&offsets[(index + 1) * 4..]).read_u32::()?; + header_size + (next & 0x00ff_ffff) as usize + } else { + block.len() + }; + let overhead = hash_len + entry_value_size(entry_type)?; + ensure!( + end >= start + overhead && end <= block.len(), + "Invalid entry bounds in variable key block" + ); + max_key = max_key.max(end - start - overhead); + } + Ok(max_key) + } + } +} + +/// Opens and mmaps an SST for offline analysis. +pub fn open_sst(db_path: &Path, info: &SstInfo) -> Result<(Mmap, u64, usize)> { + let path = db_path.join(format!("{:08}.sst", info.sequence_number)); + let file = File::open(&path).with_context(|| format!("Failed to open {}", path.display()))?; + let file_size = file.metadata()?.len(); + let mmap = unsafe { Mmap::map(file.file()) } + .with_context(|| format!("Failed to mmap {}", path.display()))?; + advise_mmap_for_persistence(&mmap)?; + let directory_size = info.block_count as usize * size_of::(); + ensure!( + mmap.len() >= directory_size, + "SST block directory is truncated" + ); + let block_offsets_start = mmap.len() - directory_size; + Ok((mmap, file_size, block_offsets_start)) +} + +#[cfg(test)] +mod tests { + use byteorder::{BE, WriteBytesExt}; + + use super::{max_key_length, parse_key_block_header, parse_key_block_indices}; + use crate::static_sorted_file::{ + BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY_NO_HASH, KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, + }; + + #[test] + fn parses_index_block_indices() { + let mut block = vec![BLOCK_TYPE_INDEX]; + block.write_u16::(3).unwrap(); + block.write_u64::(42).unwrap(); + block.write_u16::(7).unwrap(); + assert_eq!(parse_key_block_indices(&block).unwrap(), [3, 7].into()); + } + + #[test] + fn finds_maximum_variable_key_length() { + let mut block = vec![BLOCK_TYPE_KEY_NO_HASH, 0, 0, 2]; + block + .write_u32::((KEY_BLOCK_ENTRY_TYPE_INLINE_MIN as u32) << 24) + .unwrap(); + block + .write_u32::(((KEY_BLOCK_ENTRY_TYPE_INLINE_MIN as u32) << 24) | 3) + .unwrap(); + block.extend_from_slice(b"abc"); + block.extend_from_slice(b"a-much-longer-key"); + let header = parse_key_block_header(&block).unwrap(); + assert_eq!(max_key_length(header, &block).unwrap(), 17); + } + + #[test] + fn rejects_truncated_variable_key_table() { + let block = [BLOCK_TYPE_KEY_NO_HASH, 0, 0, 2, 0, 0, 0, 0]; + let header = parse_key_block_header(&block).unwrap(); + assert!(max_key_length(header, &block).is_err()); + } +} From ed286de6483679e992edc4b2578038b028b60af7 Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:51 +0000 Subject: [PATCH 02/13] Train TaskData dictionaries offline Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- turbopack/crates/turbo-persistence/README.md | 34 +- .../src/bin/taskdata_dictionary.rs | 453 +++++++++++++++--- 2 files changed, 414 insertions(+), 73 deletions(-) diff --git a/turbopack/crates/turbo-persistence/README.md b/turbopack/crates/turbo-persistence/README.md index 64643a49192c..78e367cd1cbb 100644 --- a/turbopack/crates/turbo-persistence/README.md +++ b/turbopack/crates/turbo-persistence/README.md @@ -362,27 +362,35 @@ Configuration options for compactions are: - max number of SST files that are merged at once - coverage when compaction is triggered (otherwise calling compact is a noop) -## Evaluating zstd dictionaries offline +## Training and evaluating zstd dictionaries offline -`taskdata_dictionary` compares zstd dictionaries against active blocks from existing database -copies without modifying them or running the application that created them: +`taskdata_dictionary` trains and compares zstd dictionaries using active blocks from existing +database copies without modifying them or running the application that created them: ```sh -cargo run -p turbo-persistence --bin taskdata_dictionary -- \ - --dictionary candidate-a.zdict \ - --dictionary candidate-b.zdict \ +cargo run -p turbo-persistence --bin taskdata_dictionary -- train \ + --output candidate.zdict \ + path/to/database-a path/to/database-b + +cargo run -p turbo-persistence --bin taskdata_dictionary -- evaluate \ + --dictionary candidate.zdict \ --json report.json \ path/to/database-a path/to/database-b ``` -The no-dictionary zstd level 3 baseline is always included. Family 2 (TaskData) is selected by -default; `--family ` overrides it. The evaluator follows `CURRENT`, deletion files, and meta-file -supersession, verifies checksums, and models the same 12.5% minimum-savings threshold used by the -writer. Index blocks and ineligible key blocks remain unchanged in modeled SST sizes. +Training defaults to a 64 KiB dictionary and at most 10,000 samples. It first counts eligible blocks, +then scans again and selects every `ceil(block_count / max_samples)`th block, so only selected blocks +are retained for zstd's trainer. `--max-dictionary-size`, `--max-samples`, and `--force` override the +defaults and output behavior. + +The no-dictionary zstd level 3 baseline is always included during evaluation. Family 2 (TaskData) is +selected by default; `--family ` overrides it. The tool follows `CURRENT`, deletion files, and +meta-file supersession, verifies checksums, and models the same 12.5% minimum-savings threshold used +by the writer. Index blocks and ineligible key blocks remain unchanged in modeled SST sizes. -Only SST blocks are evaluated. External blob references are counted and reported, but `.blob` -payloads are not read. Timing fields are single-pass diagnostics; use the byte/count fields for -repeatable comparisons of a copied cache snapshot. +Only SST blocks are used. External blob references are counted and reported, but `.blob` payloads +are not read. Timing fields are single-pass diagnostics; use the byte/count fields for repeatable +comparisons of a copied cache snapshot. ## Opening diff --git a/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs index 4743ef792764..0bf26d6476f7 100644 --- a/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs @@ -1,11 +1,11 @@ -//! Evaluate zstd dictionaries against TaskData blocks from existing persistence caches. +//! Train and evaluate zstd dictionaries from TaskData blocks in existing persistence caches. use std::{ collections::{BTreeMap, BTreeSet}, env, ffi::OsString, - fs::File, - io::BufWriter, + fs::{self, File, OpenOptions}, + io::{BufWriter, Write}, mem::size_of, path::{Path, PathBuf}, time::Instant, @@ -26,11 +26,21 @@ use xxhash_rust::xxh3::xxh3_64; const SCHEMA_VERSION: u32 = 1; -#[derive(Default)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Command { + Train, + Evaluate, +} + struct Options { + command: Command, family: u32, dictionaries: Vec, json: Option, + output: Option, + max_dictionary_size: usize, + max_samples: usize, + force: bool, caches: Vec, } @@ -161,6 +171,29 @@ struct CombinedReport { candidates: Vec, } +#[derive(Serialize)] +struct TrainingCacheReport { + path: PathBuf, + scanned_samples: u64, + selected_samples: u64, + selected_bytes: u64, +} + +#[derive(Serialize)] +struct TrainingReport { + schema_version: u32, + family: u32, + zstd_version: &'static str, + max_dictionary_size: usize, + max_samples: usize, + scanned_samples: u64, + stride: u64, + selected_samples: u64, + selected_bytes: u64, + caches: Vec, + dictionary: DictionaryInfo, +} + fn size_bucket(bytes: usize) -> &'static str { match bytes { 0..=4095 => "<4KiB", @@ -171,12 +204,40 @@ fn size_bucket(bytes: usize) -> &'static str { } } +fn parse_number(value: OsString, option: &str) -> Result +where + T::Err: std::error::Error + Send + Sync + 'static, +{ + value + .to_str() + .with_context(|| format!("{option} must be UTF-8"))? + .parse() + .with_context(|| format!("{option} must be an unsigned integer")) +} + fn parse_args_from(args: impl IntoIterator) -> Result { + let mut args = args.into_iter(); + let command = match args.next().as_deref().and_then(|value| value.to_str()) { + Some("train") => Command::Train, + Some("evaluate") => Command::Evaluate, + Some("--help" | "-h") => { + print_help(); + std::process::exit(0); + } + Some(value) => bail!("Expected `train` or `evaluate`, got {value}"), + None => bail!("A `train` or `evaluate` subcommand is required"), + }; let mut options = Options { + command, family: 2, - ..Default::default() + dictionaries: Vec::new(), + json: None, + output: None, + max_dictionary_size: 64 * 1024, + max_samples: 10_000, + force: false, + caches: Vec::new(), }; - let mut args = args.into_iter(); while let Some(arg) = args.next() { match arg.to_str() { Some("--dictionary" | "-d") => options.dictionaries.push(PathBuf::from( @@ -187,14 +248,31 @@ fn parse_args_from(args: impl IntoIterator) -> Result args.next().context("--json requires a path")?, )); } + Some("--output" | "-o") => { + options.output = Some(PathBuf::from( + args.next().context("--output requires a path")?, + )); + } Some("--family") => { - let value = args.next().context("--family requires an integer")?; - options.family = value - .to_str() - .context("--family must be UTF-8")? - .parse() - .context("--family must be an unsigned integer")?; + options.family = parse_number( + args.next().context("--family requires an integer")?, + "--family", + )?; + } + Some("--max-dictionary-size") => { + options.max_dictionary_size = parse_number( + args.next() + .context("--max-dictionary-size requires an integer")?, + "--max-dictionary-size", + )?; + } + Some("--max-samples") => { + options.max_samples = parse_number( + args.next().context("--max-samples requires an integer")?, + "--max-samples", + )?; } + Some("--force") => options.force = true, Some("--help" | "-h") => { print_help(); std::process::exit(0); @@ -207,6 +285,27 @@ fn parse_args_from(args: impl IntoIterator) -> Result !options.caches.is_empty(), "At least one cache directory is required" ); + ensure!( + options.max_dictionary_size > 0, + "--max-dictionary-size must be positive" + ); + ensure!(options.max_samples > 0, "--max-samples must be positive"); + match options.command { + Command::Train => { + ensure!(options.output.is_some(), "train requires --output"); + ensure!( + options.dictionaries.is_empty(), + "train does not accept --dictionary" + ); + } + Command::Evaluate => { + ensure!( + options.output.is_none(), + "evaluate does not accept --output" + ); + ensure!(!options.force, "evaluate does not accept --force"); + } + } Ok(options) } @@ -216,11 +315,15 @@ fn parse_args() -> Result { fn print_help() { println!( - "Usage: taskdata_dictionary [OPTIONS] ...\n\nEvaluate candidate zstd \ - dictionaries against active TaskData SST blocks.\n\nOptions:\n-d, --dictionary \ - Candidate dictionary (repeatable)\n--family Family ID to evaluate (default: 2 \ - / TaskData)\n--json Write a JSON report\n-h, --help Show this \ - help" + "Usage:\n taskdata_dictionary train --output [OPTIONS] ...\n \ + taskdata_dictionary evaluate [OPTIONS] ...\n\nTrain or evaluate zstd \ + dictionaries using active TaskData SST blocks.\n\nShared options:\n --family \ + Family ID (default: 2 / TaskData)\n --json Write a JSON report\n\nTrain \ + options:\n -o, --output Dictionary output path\n --max-dictionary-size \ + Maximum size (default: 65536)\n --max-samples Sample \ + cap (default: 10000)\n --force Replace an existing output\n\nEvaluate \ + options:\n -d, --dictionary Candidate dictionary (repeatable)\n\n -h, --help \ + Show this help" ); } @@ -341,7 +444,11 @@ fn count_blob_references(header: KeyBlockHeader, data: &[u8]) -> Result { .count() as u64) } -fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Result { +fn scan_cache( + path: &Path, + family: u32, + mut on_eligible: impl FnMut(&[u8], bool) -> Result<()>, +) -> Result { ensure!(path.is_dir(), "Not a cache directory: {}", path.display()); let families = collect_sst_info(path) .with_context(|| format!("Failed to inspect cache {}", path.display()))?; @@ -351,7 +458,6 @@ fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Res path.display() ) })?; - let mut report = CacheReport { path: path.to_path_buf(), family, @@ -360,21 +466,28 @@ fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Res original_complete_sst_bytes: 0, original_eligible_payload_bytes: 0, blocks: BlockSummary::default(), - candidates: candidates - .iter() - .map(|candidate| CandidateResult { - dictionary: Some(candidate.info.clone()), - setup_ns: candidate.setup_ns, - ..Default::default() - }) - .collect(), + candidates: Vec::new(), }; - for info in ssts { - evaluate_sst(path, info, candidates, &mut report) - .with_context(|| format!("Failed to evaluate {:08}.sst", info.sequence_number))?; + scan_sst(path, info, &mut report, &mut on_eligible) + .with_context(|| format!("Failed to scan {:08}.sst", info.sequence_number))?; } + Ok(report) +} +fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Result { + let mut results: Vec = candidates + .iter() + .map(|candidate| CandidateResult { + dictionary: Some(candidate.info.clone()), + setup_ns: candidate.setup_ns, + ..Default::default() + }) + .collect(); + let mut report = scan_cache(path, family, |data, was_compressed| { + evaluate_block(candidates, &mut results, data, was_compressed) + })?; + report.candidates = results; for result in &mut report.candidates { result.modeled_complete_sst_bytes = report.original_complete_sst_bytes - report.original_eligible_payload_bytes @@ -388,11 +501,11 @@ fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Res Ok(report) } -fn evaluate_sst( +fn scan_sst( db_path: &Path, info: &SstInfo, - candidates: &mut [Candidate], report: &mut CacheReport, + on_eligible: &mut impl FnMut(&[u8], bool) -> Result<()>, ) -> Result<()> { ensure!(info.block_count > 0, "SST contains no blocks"); let (mmap, file_size, offsets_start) = open_sst(db_path, info)?; @@ -442,12 +555,7 @@ fn evaluate_sst( if eligible { report.blocks.add_eligible_size(block.data.len()); report.original_eligible_payload_bytes += block.stored_size; - evaluate_block( - candidates, - &mut report.candidates, - &block.data, - block.was_compressed, - )?; + on_eligible(&block.data, block.was_compressed)?; } } @@ -460,6 +568,162 @@ fn evaluate_sst( Ok(()) } +fn write_atomic(path: &Path, bytes: &[u8], force: bool) -> Result<()> { + if path.exists() && !force { + bail!( + "Output {} already exists; pass --force to replace it", + path.display() + ); + } + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent).with_context(|| format!("Failed to create {}", parent.display()))?; + let filename = path + .file_name() + .and_then(|name| name.to_str()) + .context("Output filename must be UTF-8")?; + let temporary = parent.join(format!(".{filename}.{}.tmp", std::process::id())); + let result = (|| -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .with_context(|| format!("Failed to create {}", temporary.display()))?; + file.write_all(bytes)?; + file.sync_all()?; + fs::rename(&temporary, path).with_context(|| { + format!( + "Failed to atomically rename {} to {}", + temporary.display(), + path.display() + ) + })?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result +} + +fn training_dictionary_info(path: &Path, dictionary: &[u8]) -> DictionaryInfo { + DictionaryInfo { + name: path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("dictionary") + .to_owned(), + path: Some(path.to_path_buf()), + bytes: dictionary.len(), + dictionary_id: zstd::zstd_safe::get_dict_id_from_dict(dictionary).map(|id| id.get()), + xxh3_64: Some(format!("{:016x}", xxh3_64(dictionary))), + } +} + +fn train(options: &Options) -> Result { + let output = options.output.as_deref().expect("validated by parse_args"); + if output.exists() && !options.force { + bail!( + "Output {} already exists; pass --force to replace it", + output.display() + ); + } + let mut cache_paths = options.caches.clone(); + cache_paths.sort(); + + let mut scanned_by_cache = Vec::with_capacity(cache_paths.len()); + let mut scanned_samples = 0_u64; + for path in &cache_paths { + let mut count = 0_u64; + scan_cache(path, options.family, |_, _| { + count += 1; + Ok(()) + })?; + scanned_samples += count; + scanned_by_cache.push(count); + } + ensure!(scanned_samples > 0, "No eligible blocks found for training"); + let stride = scanned_samples.div_ceil(options.max_samples as u64).max(1); + + let mut samples: Vec> = Vec::with_capacity( + usize::try_from(scanned_samples.div_ceil(stride)) + .unwrap_or(options.max_samples) + .min(options.max_samples), + ); + let mut cache_reports = Vec::with_capacity(cache_paths.len()); + let mut global_index = 0_u64; + for (path, scanned) in cache_paths.iter().zip(scanned_by_cache) { + let mut selected_samples = 0_u64; + let mut selected_bytes = 0_u64; + scan_cache(path, options.family, |data, _| { + let select = global_index.is_multiple_of(stride) && samples.len() < options.max_samples; + global_index += 1; + if select { + selected_samples += 1; + selected_bytes += data.len() as u64; + samples.push(Box::from(data)); + } + Ok(()) + })?; + cache_reports.push(TrainingCacheReport { + path: path.clone(), + scanned_samples: scanned, + selected_samples, + selected_bytes, + }); + } + ensure!(!samples.is_empty(), "No blocks selected for training"); + let selected_bytes = samples.iter().map(|sample| sample.len() as u64).sum(); + let dictionary = + zstd::dict::from_samples(&samples, options.max_dictionary_size).with_context(|| { + format!( + "Failed to train a {}-byte dictionary from {} selected samples ({} bytes); add \ + caches, raise --max-samples, or request a smaller dictionary", + options.max_dictionary_size, + samples.len(), + selected_bytes + ) + })?; + write_atomic(output, &dictionary, options.force)?; + let info = training_dictionary_info(output, &dictionary); + Ok(TrainingReport { + schema_version: SCHEMA_VERSION, + family: options.family, + zstd_version: zstd::zstd_safe::version_string(), + max_dictionary_size: options.max_dictionary_size, + max_samples: options.max_samples, + scanned_samples, + stride, + selected_samples: samples.len() as u64, + selected_bytes, + caches: cache_reports, + dictionary: info, + }) +} + +fn print_training_report(report: &TrainingReport) { + println!( + "Trained {} ({} bytes, id {:?}, xxh3 {}) from {} of {} eligible blocks (stride {}, {} \ + selected bytes)", + report.dictionary.path.as_ref().unwrap().display(), + report.dictionary.bytes, + report.dictionary.dictionary_id, + report.dictionary.xxh3_64.as_deref().unwrap_or("none"), + report.selected_samples, + report.scanned_samples, + report.stride, + report.selected_bytes, + ); + for cache in &report.caches { + println!( + " {}: selected {} / {} blocks ({} bytes)", + cache.path.display(), + cache.selected_samples, + cache.scanned_samples, + cache.selected_bytes + ); + } +} + fn percentage_delta(value: u64, baseline: u64) -> Option { (baseline > 0).then(|| (value as f64 / baseline as f64 - 1.0) * 100.0) } @@ -587,31 +851,44 @@ fn print_report(report: &Report) { println!("Note: timings are single-pass wall-clock diagnostics, not benchmarks."); } -fn run() -> Result<()> { - let options = parse_args()?; - let mut candidates = make_candidates(&options.dictionaries)?; - let mut caches = Vec::with_capacity(options.caches.len()); - for path in &options.caches { - caches.push(evaluate_cache(path, options.family, &mut candidates)?); - } - let combined = combine(&caches, &candidates); - let report = Report { - schema_version: SCHEMA_VERSION, - family: options.family, - timing_note: "Single-pass wall-clock diagnostics; size/count fields are the comparison \ - contract.", - caches, - combined, - }; - print_report(&report); - if let Some(path) = options.json { - let file = File::create(&path) +fn write_json(path: Option<&Path>, report: &impl Serialize) -> Result<()> { + if let Some(path) = path { + let file = File::create(path) .with_context(|| format!("Failed to create JSON report {}", path.display()))?; - serde_json::to_writer_pretty(BufWriter::new(file), &report)?; + serde_json::to_writer_pretty(BufWriter::new(file), report)?; } Ok(()) } +fn run() -> Result<()> { + let options = parse_args()?; + match options.command { + Command::Train => { + let report = train(&options)?; + print_training_report(&report); + write_json(options.json.as_deref(), &report) + } + Command::Evaluate => { + let mut candidates = make_candidates(&options.dictionaries)?; + let mut caches = Vec::with_capacity(options.caches.len()); + for path in &options.caches { + caches.push(evaluate_cache(path, options.family, &mut candidates)?); + } + let combined = combine(&caches, &candidates); + let report = Report { + schema_version: SCHEMA_VERSION, + family: options.family, + timing_note: "Single-pass wall-clock diagnostics; size/count fields are the \ + comparison contract.", + caches, + combined, + }; + print_report(&report); + write_json(options.json.as_deref(), &report) + } + } +} + fn main() { if let Err(error) = run() { eprintln!("Error: {error:#}"); @@ -628,7 +905,10 @@ mod tests { use tempfile::TempDir; use turbo_persistence::{Compression, DbConfig, SerialScheduler, TurboPersistence}; - use super::{evaluate_cache, make_candidates, parse_args_from, production_stored_size}; + use super::{ + Command, Options, evaluate_cache, make_candidates, parse_args_from, production_stored_size, + train, + }; fn make_cache(family: usize, compression: Compression) -> Result { let tempdir = tempfile::tempdir()?; @@ -646,7 +926,7 @@ mod tests { .into_bytes(); batch.put(family as u32, key, value.into())?; } - for index in 0..4u8 { + for index in 0..100u8 { let key = format!("long-medium-task-data-key-{index:04}").into_bytes(); batch.put(family as u32, key, vec![b'a' + index; 5000].into())?; } @@ -681,6 +961,7 @@ mod tests { assert!(parse_args_from([]).is_err()); assert!(parse_args_from(["--unknown".into()]).is_err()); let options = parse_args_from([ + "evaluate".into(), "--family".into(), "7".into(), "--dictionary".into(), @@ -694,6 +975,58 @@ mod tests { [std::path::PathBuf::from("candidate.dict")] ); assert_eq!(options.caches, [std::path::PathBuf::from("cache")]); + + let train = parse_args_from([ + "train".into(), + "--output".into(), + "output.dict".into(), + "cache".into(), + ]) + .unwrap(); + assert_eq!(train.command, Command::Train); + assert_eq!(train.max_dictionary_size, 64 * 1024); + assert_eq!(train.max_samples, 10_000); + } + + fn training_options(caches: &[&Path], output: &Path, force: bool) -> Options { + Options { + command: Command::Train, + family: 2, + dictionaries: Vec::new(), + json: None, + output: Some(output.to_path_buf()), + max_dictionary_size: 1024, + max_samples: 100, + force, + caches: caches.iter().map(|path| path.to_path_buf()).collect(), + } + } + + #[test] + fn trains_with_two_pass_sampling_and_no_clobber() -> Result<()> { + let first_cache = make_cache(2, Compression::Zstd3)?; + let second_cache = make_cache(2, Compression::Lz4)?; + let output_dir = tempfile::tempdir()?; + let first_output = output_dir.path().join("first.zdict"); + let second_output = output_dir.path().join("second.zdict"); + let caches = [first_cache.path(), second_cache.path()]; + + let first = train(&training_options(&caches, &first_output, false))?; + assert_eq!(first.max_dictionary_size, 1024); + assert_eq!(first.max_samples, 100); + assert!(first.scanned_samples > first.selected_samples); + assert!(first.selected_samples <= 100); + assert_eq!(first.caches.len(), 2); + assert!(first.caches.iter().all(|cache| cache.selected_samples > 0)); + assert_eq!(fs::read(&first_output)?.len(), first.dictionary.bytes); + + let second = train(&training_options(&caches, &second_output, false))?; + assert_eq!(fs::read(&first_output)?, fs::read(&second_output)?); + assert!(train(&training_options(&caches, &first_output, false)).is_err()); + train(&training_options(&caches, &first_output, true))?; + assert_eq!(first.selected_samples, second.selected_samples); + assert_eq!(first.selected_bytes, second.selected_bytes); + Ok(()) } #[test] From 6375dea22927b1139174a2009c3e866c36ca1799 Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:52 +0000 Subject: [PATCH 03/13] Simplify zstd dictionary tooling Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- Cargo.lock | 1 + turbopack/crates/turbo-persistence/Cargo.toml | 5 +- turbopack/crates/turbo-persistence/README.md | 34 +- .../turbo-persistence/src/bin/sst_inspect.rs | 234 +++- .../src/bin/taskdata_dictionary.rs | 1183 ----------------- .../src/bin/zstd_dictionary.rs | 992 ++++++++++++++ turbopack/crates/turbo-persistence/src/lib.rs | 3 +- .../crates/turbo-persistence/src/offline.rs | 422 ++---- .../src/static_sorted_file.rs | 2 +- .../crates/turbo-tasks-backend/README.md | 24 + 10 files changed, 1336 insertions(+), 1564 deletions(-) delete mode 100644 turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs create mode 100644 turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs create mode 100644 turbopack/crates/turbo-tasks-backend/README.md diff --git a/Cargo.lock b/Cargo.lock index ae0e36976484..6b7f53e0e816 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9964,6 +9964,7 @@ dependencies = [ "auto-hash-map", "bitfield", "byteorder", + "clap", "codspeed-criterion-compat", "crc32fast", "dashmap 7.0.0-rc2", diff --git a/turbopack/crates/turbo-persistence/Cargo.toml b/turbopack/crates/turbo-persistence/Cargo.toml index 3686c3c7276d..7fcd5dc7d616 100644 --- a/turbopack/crates/turbo-persistence/Cargo.toml +++ b/turbopack/crates/turbo-persistence/Cargo.toml @@ -17,6 +17,7 @@ anyhow = { workspace = true } auto-hash-map = { workspace = true } bitfield = { workspace = true } byteorder = { workspace = true } +clap = { workspace = true } crc32fast = { workspace = true } dashmap = { workspace = true} either = { workspace = true } @@ -53,8 +54,8 @@ name = "sst_inspect" path = "src/bin/sst_inspect.rs" [[bin]] -name = "taskdata_dictionary" -path = "src/bin/taskdata_dictionary.rs" +name = "zstd_dictionary" +path = "src/bin/zstd_dictionary.rs" [lints] workspace = true diff --git a/turbopack/crates/turbo-persistence/README.md b/turbopack/crates/turbo-persistence/README.md index 78e367cd1cbb..22a453494bad 100644 --- a/turbopack/crates/turbo-persistence/README.md +++ b/turbopack/crates/turbo-persistence/README.md @@ -364,33 +364,31 @@ Configuration options for compactions are: ## Training and evaluating zstd dictionaries offline -`taskdata_dictionary` trains and compares zstd dictionaries using active blocks from existing -database copies without modifying them or running the application that created them: +`zstd_dictionary` trains and compares zstd dictionaries from logical values in existing database +copies without modifying them or running the application that created them: ```sh -cargo run -p turbo-persistence --bin taskdata_dictionary -- train \ - --output candidate.zdict \ +cargo run -p turbo-persistence --bin zstd_dictionary -- train \ + --family --output candidate.zdict \ path/to/database-a path/to/database-b -cargo run -p turbo-persistence --bin taskdata_dictionary -- evaluate \ - --dictionary candidate.zdict \ - --json report.json \ +cargo run -p turbo-persistence --bin zstd_dictionary -- evaluate \ + --family --dictionary candidate.zdict --json report.json \ path/to/database-a path/to/database-b ``` -Training defaults to a 64 KiB dictionary and at most 10,000 samples. It first counts eligible blocks, -then scans again and selects every `ceil(block_count / max_samples)`th block, so only selected blocks -are retained for zstd's trainer. `--max-dictionary-size`, `--max-samples`, and `--force` override the -defaults and output behavior. +Training produces a 64 KiB dictionary from up to approximately 64 MiB of samples. It takes one +hash-ordered logical value from each cache in turn, so one large cache cannot monopolize the sample. +The output path is replaced atomically. -The no-dictionary zstd level 3 baseline is always included during evaluation. Family 2 (TaskData) is -selected by default; `--family ` overrides it. The tool follows `CURRENT`, deletion files, and -meta-file supersession, verifies checksums, and models the same 12.5% minimum-savings threshold used -by the writer. Index blocks and ineligible key blocks remain unchanged in modeled SST sizes. +The no-dictionary zstd level 3 baseline is always included during evaluation. The tool follows +`CURRENT`, deletion files, and meta-file supersession, and uses `StaticSortedFileIter` to read slice, +medium, and blob values. Checksums and decompressed lengths are verified. -Only SST blocks are used. External blob references are counted and reported, but `.blob` payloads -are not read. Timing fields are single-pass diagnostics; use the byte/count fields for repeatable -comparisons of a copied cache snapshot. +Small values are grouped into physical blocks in production, so the report's per-value 12.5% +minimum-savings calculation is a comparative estimate, not exact SST-size modeling. Blob estimates +include their fixed 8-byte headers. Timing fields are single-pass diagnostics; use byte/count fields +for repeatable comparisons of one copied cache snapshot. ## Opening diff --git a/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs b/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs index fcc30fdb9158..caa0b9ed555d 100644 --- a/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs +++ b/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs @@ -10,27 +10,31 @@ //! `type - KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN`. use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashSet}, path::{Path, PathBuf}, }; use anyhow::{Context, Result, bail}; +use byteorder::{BE, ReadBytesExt}; use fs_err::File; +use lzzzz::lz4::decompress; use memmap2::Mmap; use turbo_persistence::{ - MAX_INLINE_VALUE_SIZE, + BLOCK_HEADER_SIZE, Compression, MAX_INLINE_VALUE_SIZE, checksum_block, mmap_helper::advise_mmap_for_persistence, - offline::{ - KeyBlockHeader, SstInfo, collect_sst_info, key_block_entry_types, parse_key_block_header, - parse_key_block_indices, read_block, - }, + offline::{SstInfo, collect_sst_info}, static_sorted_file::{ - KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, - KEY_BLOCK_ENTRY_TYPE_KEY_DELETED, KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN, - KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL, + BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, BLOCK_TYPE_KEY_NO_HASH, + BLOCK_TYPE_KEY_WITH_HASH, FIXED_KEY_BLOCK_MIXED_VALUE_TYPE, KEY_BLOCK_ENTRY_TYPE_BLOB, + KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, KEY_BLOCK_ENTRY_TYPE_KEY_DELETED, + KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN, KEY_BLOCK_ENTRY_TYPE_MEDIUM, + KEY_BLOCK_ENTRY_TYPE_SMALL, }, }; +/// Size of the key block header (1B type + 3B entry count). +const KEY_BLOCK_HEADER_SIZE: usize = 4; + /// Block size information #[derive(Default, Debug, Clone)] struct BlockSizeInfo { @@ -207,6 +211,192 @@ fn format_bytes(bytes: u64) -> String { } } +/// Information about a raw block read from disk. +struct RawBlock { + data: Box<[u8]>, + compressed_size: u64, + actual_size: u64, + was_compressed: bool, +} + +/// Reads, checksums, and decompresses a single block from the mmap. +fn read_block( + mmap: &Mmap, + block_offsets_start: usize, + block_index: u16, + sequence_number: u32, + compression: Compression, +) -> Result { + let offset = block_offsets_start + block_index as usize * size_of::(); + + let block_start = if block_index == 0 { + 0 + } else { + (&mmap[offset - size_of::()..offset]).read_u32::()? as usize + }; + let block_end = (&mmap[offset..offset + size_of::()]).read_u32::()? as usize; + + let uncompressed_length = + (&mmap[block_start..block_start + size_of::()]).read_u32::()?; + let expected_checksum = (&mmap + [block_start + size_of::()..block_start + BLOCK_HEADER_SIZE]) + .read_u32::()?; + let compressed_data = &mmap[block_start + BLOCK_HEADER_SIZE..block_end]; + let compressed_size = compressed_data.len() as u64; + + let was_compressed = uncompressed_length > 0; + let actual_size = if was_compressed { + uncompressed_length as u64 + } else { + compressed_size + }; + + let actual_checksum = checksum_block(compressed_data); + if actual_checksum != expected_checksum { + bail!( + "Cache corruption detected: checksum mismatch in block {} of {:08}.sst (expected \ + {:08x}, got {:08x})", + block_index, + sequence_number, + expected_checksum, + actual_checksum + ); + } + + let data = if was_compressed { + let mut buffer = vec![0u8; uncompressed_length as usize]; + let bytes_written = match compression { + Compression::Lz4 => { + decompress(compressed_data, &mut buffer).context("LZ4 decompression failed")? + } + Compression::Zstd3 => zstd::bulk::decompress_to_buffer(compressed_data, &mut buffer) + .context("zstd decompression failed")?, + }; + assert_eq!( + bytes_written, uncompressed_length as usize, + "Decompressed length does not match expected" + ); + buffer.into_boxed_slice() + } else { + Box::from(compressed_data) + }; + + Ok(RawBlock { + data, + compressed_size, + actual_size, + was_compressed, + }) +} + +/// Parses an index block to extract all referenced key block indices. +/// +/// Index block format: `[1B type][2B first_block][N * (8B hash + 2B block_index)]`. +fn parse_key_block_indices(index_block: &[u8]) -> HashSet { + assert!(index_block.len() >= 3, "Index block too small"); + let mut data = &index_block[1..]; // skip block type byte + let first_block = data.read_u16::().unwrap(); + let mut indices = HashSet::new(); + indices.insert(first_block); + const ENTRY_SIZE: usize = size_of::() + size_of::(); + let entry_count = data.len() / ENTRY_SIZE; + for i in 0..entry_count { + let block_index = (&data[i * ENTRY_SIZE + 8..]).read_u16::().unwrap(); + indices.insert(block_index); + } + indices +} + +/// Parsed header of a key block. +#[derive(Clone, Copy)] +enum KeyBlockHeader { + Variable { + entry_count: u32, + }, + Fixed { + entry_count: u32, + value_type: u8, + }, + /// Fixed-size layout whose entries share a value size but not a value type, so each carries + /// its own type byte between its key and its value. + FixedMixedType { + entry_count: u32, + hash_len: usize, + key_size: usize, + stride: usize, + }, +} + +/// Parses the header of a key block from the full decompressed block data. +fn parse_key_block_header(block: &[u8]) -> Result { + assert!(block.len() >= 4, "Key block too small"); + let block_type = block[0]; + let entry_count = ((block[1] as u32) << 16) | ((block[2] as u32) << 8) | (block[3] as u32); + match block_type { + BLOCK_TYPE_KEY_WITH_HASH | BLOCK_TYPE_KEY_NO_HASH => { + Ok(KeyBlockHeader::Variable { entry_count }) + } + BLOCK_TYPE_FIXED_KEY_WITH_HASH | BLOCK_TYPE_FIXED_KEY_NO_HASH => { + assert!(block.len() >= 6, "Fixed key block header too small"); + if block[5] == FIXED_KEY_BLOCK_MIXED_VALUE_TYPE { + assert!(block.len() >= 7, "Mixed-type key block header too small"); + let hash_len = if block_type == BLOCK_TYPE_FIXED_KEY_WITH_HASH { + 8 + } else { + 0 + }; + let key_size = block[4] as usize; + let val_size = block[6] as usize; + Ok(KeyBlockHeader::FixedMixedType { + entry_count, + hash_len, + key_size, + // +1 for the per-entry type byte. + stride: hash_len + key_size + val_size + 1, + }) + } else { + Ok(KeyBlockHeader::Fixed { + entry_count, + value_type: block[5], + }) + } + } + _ => bail!("Invalid key block type: {block_type}"), + } +} + +/// Iterates over entry type bytes in a key block. +/// +/// For variable-size key blocks, reads byte 0 of each 4-byte offset table entry. For fixed-size +/// key blocks, yields the single `value_type` repeated `entry_count` times, or reads the per-entry +/// type byte when the block has mixed types. +fn iter_key_block_entry_types( + header: KeyBlockHeader, + block: &[u8], +) -> impl Iterator + '_ { + let entry_count = match header { + KeyBlockHeader::Variable { entry_count } + | KeyBlockHeader::Fixed { entry_count, .. } + | KeyBlockHeader::FixedMixedType { entry_count, .. } => entry_count, + }; + (0..entry_count).map(move |i| match header { + // Variable block: offset table starts at byte 4 (after 1B type + 3B count), + // each entry is 4 bytes, first byte is the entry type. + KeyBlockHeader::Variable { .. } => block[KEY_BLOCK_HEADER_SIZE + i as usize * 4], + KeyBlockHeader::Fixed { value_type, .. } => value_type, + KeyBlockHeader::FixedMixedType { + hash_len, + key_size, + stride, + .. + } => { + // Entry data starts after the 7-byte mixed-type header; the type byte sits between + // the entry's key and its value. + block[7 + i as usize * stride + hash_len + key_size] + } + }) +} + /// Analyze an SST file and return entry type statistics fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { let compression = info.compression; @@ -237,10 +427,10 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { info.sequence_number, compression, )?; - let key_block_indices = parse_key_block_indices(&index_raw.data)?; + let key_block_indices = parse_key_block_indices(&index_raw.data); stats.index_blocks.add( - index_raw.stored_size, + index_raw.compressed_size, index_raw.actual_size, index_raw.was_compressed, ); @@ -268,7 +458,7 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { // Value block — no type header, just raw data. stats .value_blocks - .add(raw.stored_size, raw.actual_size, raw.was_compressed); + .add(raw.compressed_size, raw.actual_size, raw.was_compressed); continue; } @@ -276,7 +466,7 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { stats .key_blocks - .add(raw.stored_size, raw.actual_size, raw.was_compressed); + .add(raw.compressed_size, raw.actual_size, raw.was_compressed); let key_block_header = parse_key_block_header(block).with_context(|| { format!( @@ -286,18 +476,22 @@ fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { })?; match key_block_header { KeyBlockHeader::Variable { .. } => { - stats - .variable_key_blocks - .add(raw.stored_size, raw.actual_size, raw.was_compressed); + stats.variable_key_blocks.add( + raw.compressed_size, + raw.actual_size, + raw.was_compressed, + ); } KeyBlockHeader::Fixed { .. } | KeyBlockHeader::FixedMixedType { .. } => { - stats - .fixed_key_blocks - .add(raw.stored_size, raw.actual_size, raw.was_compressed); + stats.fixed_key_blocks.add( + raw.compressed_size, + raw.actual_size, + raw.was_compressed, + ); } }; - for entry_type in key_block_entry_types(key_block_header, block)? { + for entry_type in iter_key_block_entry_types(key_block_header, block) { track_entry_type(&mut stats, entry_type); } } diff --git a/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs deleted file mode 100644 index 0bf26d6476f7..000000000000 --- a/turbopack/crates/turbo-persistence/src/bin/taskdata_dictionary.rs +++ /dev/null @@ -1,1183 +0,0 @@ -//! Train and evaluate zstd dictionaries from TaskData blocks in existing persistence caches. - -use std::{ - collections::{BTreeMap, BTreeSet}, - env, - ffi::OsString, - fs::{self, File, OpenOptions}, - io::{BufWriter, Write}, - mem::size_of, - path::{Path, PathBuf}, - time::Instant, -}; - -use anyhow::{Context, Result, bail, ensure}; -use serde::Serialize; -use turbo_persistence::{ - BLOCK_HEADER_SIZE, - offline::{ - KeyBlockHeader, MIN_KEY_SIZE_FOR_COMPRESSION, SstInfo, collect_sst_info, - key_block_entry_types, max_key_length, open_sst, parse_key_block_header, - parse_key_block_indices, read_block, - }, - static_sorted_file::KEY_BLOCK_ENTRY_TYPE_BLOB, -}; -use xxhash_rust::xxh3::xxh3_64; - -const SCHEMA_VERSION: u32 = 1; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Command { - Train, - Evaluate, -} - -struct Options { - command: Command, - family: u32, - dictionaries: Vec, - json: Option, - output: Option, - max_dictionary_size: usize, - max_samples: usize, - force: bool, - caches: Vec, -} - -#[derive(Clone, Serialize)] -struct DictionaryInfo { - name: String, - path: Option, - bytes: usize, - dictionary_id: Option, - xxh3_64: Option, -} - -struct Candidate { - info: DictionaryInfo, - compressor: zstd::bulk::Compressor<'static>, - decompressor: zstd::bulk::Decompressor<'static>, - setup_ns: u64, -} - -#[derive(Default, Clone, Serialize)] -struct BlockGroup { - count: u64, - bytes: u64, -} - -impl BlockGroup { - fn add(&mut self, bytes: usize) { - self.count += 1; - self.bytes += bytes as u64; - } - - fn merge(&mut self, other: &Self) { - self.count += other.count; - self.bytes += other.bytes; - } -} - -#[derive(Default, Clone, Serialize)] -struct BlockSummary { - eligible_key: BlockGroup, - eligible_value: BlockGroup, - excluded_key: BlockGroup, - excluded_index: BlockGroup, - size_buckets: BTreeMap<&'static str, BlockGroup>, - blob_references: u64, -} - -impl BlockSummary { - fn add_eligible_size(&mut self, bytes: usize) { - self.size_buckets - .entry(size_bucket(bytes)) - .or_default() - .add(bytes); - } - - fn merge(&mut self, other: &Self) { - self.eligible_key.merge(&other.eligible_key); - self.eligible_value.merge(&other.eligible_value); - self.excluded_key.merge(&other.excluded_key); - self.excluded_index.merge(&other.excluded_index); - self.blob_references += other.blob_references; - for (bucket, group) in &other.size_buckets { - self.size_buckets.entry(bucket).or_default().merge(group); - } - } -} - -#[derive(Default, Clone, Serialize)] -struct CandidateResult { - dictionary: Option, - raw_compressed_bytes: u64, - modeled_payload_bytes: u64, - modeled_complete_sst_bytes: u64, - raw_compression_ratio: Option, - modeled_delta_vs_baseline_pct: Option, - modeled_delta_vs_current_pct: Option, - compressed_blocks: u64, - fallback_blocks: u64, - became_compressed: u64, - became_uncompressed: u64, - setup_ns: u64, - encode_ns: u64, - decode_ns: u64, -} - -impl CandidateResult { - fn merge(&mut self, other: &Self) { - self.raw_compressed_bytes += other.raw_compressed_bytes; - self.modeled_payload_bytes += other.modeled_payload_bytes; - self.modeled_complete_sst_bytes += other.modeled_complete_sst_bytes; - self.compressed_blocks += other.compressed_blocks; - self.fallback_blocks += other.fallback_blocks; - self.became_compressed += other.became_compressed; - self.became_uncompressed += other.became_uncompressed; - self.encode_ns += other.encode_ns; - self.decode_ns += other.decode_ns; - } -} - -#[derive(Serialize)] -struct CacheReport { - path: PathBuf, - family: u32, - recorded_codecs: BTreeSet, - active_ssts: u64, - original_complete_sst_bytes: u64, - original_eligible_payload_bytes: u64, - blocks: BlockSummary, - candidates: Vec, -} - -#[derive(Serialize)] -struct Report { - schema_version: u32, - family: u32, - timing_note: &'static str, - caches: Vec, - combined: CombinedReport, -} - -#[derive(Serialize)] -struct CombinedReport { - cache_count: usize, - active_ssts: u64, - original_complete_sst_bytes: u64, - original_eligible_payload_bytes: u64, - blocks: BlockSummary, - candidates: Vec, -} - -#[derive(Serialize)] -struct TrainingCacheReport { - path: PathBuf, - scanned_samples: u64, - selected_samples: u64, - selected_bytes: u64, -} - -#[derive(Serialize)] -struct TrainingReport { - schema_version: u32, - family: u32, - zstd_version: &'static str, - max_dictionary_size: usize, - max_samples: usize, - scanned_samples: u64, - stride: u64, - selected_samples: u64, - selected_bytes: u64, - caches: Vec, - dictionary: DictionaryInfo, -} - -fn size_bucket(bytes: usize) -> &'static str { - match bytes { - 0..=4095 => "<4KiB", - 4096..=16383 => "4-16KiB", - 16384..=65535 => "16-64KiB", - 65536..=1048575 => "64KiB-1MiB", - _ => ">=1MiB", - } -} - -fn parse_number(value: OsString, option: &str) -> Result -where - T::Err: std::error::Error + Send + Sync + 'static, -{ - value - .to_str() - .with_context(|| format!("{option} must be UTF-8"))? - .parse() - .with_context(|| format!("{option} must be an unsigned integer")) -} - -fn parse_args_from(args: impl IntoIterator) -> Result { - let mut args = args.into_iter(); - let command = match args.next().as_deref().and_then(|value| value.to_str()) { - Some("train") => Command::Train, - Some("evaluate") => Command::Evaluate, - Some("--help" | "-h") => { - print_help(); - std::process::exit(0); - } - Some(value) => bail!("Expected `train` or `evaluate`, got {value}"), - None => bail!("A `train` or `evaluate` subcommand is required"), - }; - let mut options = Options { - command, - family: 2, - dictionaries: Vec::new(), - json: None, - output: None, - max_dictionary_size: 64 * 1024, - max_samples: 10_000, - force: false, - caches: Vec::new(), - }; - while let Some(arg) = args.next() { - match arg.to_str() { - Some("--dictionary" | "-d") => options.dictionaries.push(PathBuf::from( - args.next().context("--dictionary requires a path")?, - )), - Some("--json") => { - options.json = Some(PathBuf::from( - args.next().context("--json requires a path")?, - )); - } - Some("--output" | "-o") => { - options.output = Some(PathBuf::from( - args.next().context("--output requires a path")?, - )); - } - Some("--family") => { - options.family = parse_number( - args.next().context("--family requires an integer")?, - "--family", - )?; - } - Some("--max-dictionary-size") => { - options.max_dictionary_size = parse_number( - args.next() - .context("--max-dictionary-size requires an integer")?, - "--max-dictionary-size", - )?; - } - Some("--max-samples") => { - options.max_samples = parse_number( - args.next().context("--max-samples requires an integer")?, - "--max-samples", - )?; - } - Some("--force") => options.force = true, - Some("--help" | "-h") => { - print_help(); - std::process::exit(0); - } - Some(value) if value.starts_with('-') => bail!("Unknown option {value}"), - _ => options.caches.push(PathBuf::from(arg)), - } - } - ensure!( - !options.caches.is_empty(), - "At least one cache directory is required" - ); - ensure!( - options.max_dictionary_size > 0, - "--max-dictionary-size must be positive" - ); - ensure!(options.max_samples > 0, "--max-samples must be positive"); - match options.command { - Command::Train => { - ensure!(options.output.is_some(), "train requires --output"); - ensure!( - options.dictionaries.is_empty(), - "train does not accept --dictionary" - ); - } - Command::Evaluate => { - ensure!( - options.output.is_none(), - "evaluate does not accept --output" - ); - ensure!(!options.force, "evaluate does not accept --force"); - } - } - Ok(options) -} - -fn parse_args() -> Result { - parse_args_from(env::args_os().skip(1)) -} - -fn print_help() { - println!( - "Usage:\n taskdata_dictionary train --output [OPTIONS] ...\n \ - taskdata_dictionary evaluate [OPTIONS] ...\n\nTrain or evaluate zstd \ - dictionaries using active TaskData SST blocks.\n\nShared options:\n --family \ - Family ID (default: 2 / TaskData)\n --json Write a JSON report\n\nTrain \ - options:\n -o, --output Dictionary output path\n --max-dictionary-size \ - Maximum size (default: 65536)\n --max-samples Sample \ - cap (default: 10000)\n --force Replace an existing output\n\nEvaluate \ - options:\n -d, --dictionary Candidate dictionary (repeatable)\n\n -h, --help \ - Show this help" - ); -} - -fn make_candidates(paths: &[PathBuf]) -> Result> { - let mut candidates = Vec::with_capacity(paths.len() + 1); - let started = Instant::now(); - let compressor = zstd::bulk::Compressor::new(3)?; - let decompressor = zstd::bulk::Decompressor::new()?; - candidates.push(Candidate { - info: DictionaryInfo { - name: "zstd3 (no dictionary)".into(), - path: None, - bytes: 0, - dictionary_id: None, - xxh3_64: None, - }, - compressor, - decompressor, - setup_ns: started.elapsed().as_nanos() as u64, - }); - - let mut names = BTreeSet::new(); - for path in paths { - let dictionary = std::fs::read(path) - .with_context(|| format!("Failed to read dictionary {}", path.display()))?; - ensure!( - !dictionary.is_empty(), - "Dictionary {} is empty", - path.display() - ); - let name = path - .file_name() - .and_then(|value| value.to_str()) - .unwrap_or("dictionary") - .to_owned(); - ensure!( - names.insert(name.clone()), - "Duplicate dictionary name {name}" - ); - let started = Instant::now(); - let compressor = zstd::bulk::Compressor::with_dictionary(3, &dictionary) - .with_context(|| format!("Failed to load dictionary {}", path.display()))?; - let decompressor = zstd::bulk::Decompressor::with_dictionary(&dictionary) - .with_context(|| format!("Failed to load dictionary {}", path.display()))?; - let setup_ns = started.elapsed().as_nanos() as u64; - candidates.push(Candidate { - info: DictionaryInfo { - name, - path: Some(path.clone()), - bytes: dictionary.len(), - dictionary_id: zstd::zstd_safe::get_dict_id_from_dict(&dictionary) - .map(|id| id.get()), - xxh3_64: Some(format!("{:016x}", xxh3_64(&dictionary))), - }, - compressor, - decompressor, - setup_ns, - }); - } - Ok(candidates) -} - -fn production_stored_size(original_len: usize, compressed_len: usize) -> (usize, bool) { - if compressed_len < original_len - original_len / 8 { - (compressed_len, true) - } else { - (original_len, false) - } -} - -fn evaluate_block( - candidates: &mut [Candidate], - results: &mut [CandidateResult], - data: &[u8], - was_compressed: bool, -) -> Result<()> { - for (candidate, result) in candidates.iter_mut().zip(results) { - let started = Instant::now(); - let compressed = candidate - .compressor - .compress(data) - .with_context(|| format!("Failed to compress with {}", candidate.info.name))?; - result.encode_ns += started.elapsed().as_nanos() as u64; - - let started = Instant::now(); - let decoded = candidate - .decompressor - .decompress(&compressed, data.len()) - .with_context(|| format!("Failed to decompress with {}", candidate.info.name))?; - result.decode_ns += started.elapsed().as_nanos() as u64; - ensure!( - decoded == data, - "Round trip mismatch with {}", - candidate.info.name - ); - - result.raw_compressed_bytes += compressed.len() as u64; - let (stored, compressed_decision) = production_stored_size(data.len(), compressed.len()); - result.modeled_payload_bytes += stored as u64; - if compressed_decision { - result.compressed_blocks += 1; - } else { - result.fallback_blocks += 1; - } - if compressed_decision && !was_compressed { - result.became_compressed += 1; - } else if !compressed_decision && was_compressed { - result.became_uncompressed += 1; - } - } - Ok(()) -} - -fn count_blob_references(header: KeyBlockHeader, data: &[u8]) -> Result { - Ok(key_block_entry_types(header, data)? - .into_iter() - .filter(|&entry_type| entry_type == KEY_BLOCK_ENTRY_TYPE_BLOB) - .count() as u64) -} - -fn scan_cache( - path: &Path, - family: u32, - mut on_eligible: impl FnMut(&[u8], bool) -> Result<()>, -) -> Result { - ensure!(path.is_dir(), "Not a cache directory: {}", path.display()); - let families = collect_sst_info(path) - .with_context(|| format!("Failed to inspect cache {}", path.display()))?; - let ssts = families.get(&family).with_context(|| { - format!( - "Cache {} has no active SSTs for family {family}", - path.display() - ) - })?; - let mut report = CacheReport { - path: path.to_path_buf(), - family, - recorded_codecs: BTreeSet::new(), - active_ssts: ssts.len() as u64, - original_complete_sst_bytes: 0, - original_eligible_payload_bytes: 0, - blocks: BlockSummary::default(), - candidates: Vec::new(), - }; - for info in ssts { - scan_sst(path, info, &mut report, &mut on_eligible) - .with_context(|| format!("Failed to scan {:08}.sst", info.sequence_number))?; - } - Ok(report) -} - -fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Result { - let mut results: Vec = candidates - .iter() - .map(|candidate| CandidateResult { - dictionary: Some(candidate.info.clone()), - setup_ns: candidate.setup_ns, - ..Default::default() - }) - .collect(); - let mut report = scan_cache(path, family, |data, was_compressed| { - evaluate_block(candidates, &mut results, data, was_compressed) - })?; - report.candidates = results; - for result in &mut report.candidates { - result.modeled_complete_sst_bytes = report.original_complete_sst_bytes - - report.original_eligible_payload_bytes - + result.modeled_payload_bytes; - } - finalize_results( - &mut report.candidates, - report.blocks.eligible_key.bytes + report.blocks.eligible_value.bytes, - report.original_complete_sst_bytes, - ); - Ok(report) -} - -fn scan_sst( - db_path: &Path, - info: &SstInfo, - report: &mut CacheReport, - on_eligible: &mut impl FnMut(&[u8], bool) -> Result<()>, -) -> Result<()> { - ensure!(info.block_count > 0, "SST contains no blocks"); - let (mmap, file_size, offsets_start) = open_sst(db_path, info)?; - report.original_complete_sst_bytes += file_size; - report - .recorded_codecs - .insert(format!("{:?}", info.compression)); - - let index_index = info.block_count - 1; - let index = read_block( - &mmap, - offsets_start, - index_index, - info.sequence_number, - info.compression, - )?; - let key_indices = parse_key_block_indices(&index.data)?; - report.blocks.excluded_index.add(index.data.len()); - - for block_index in 0..index_index { - let block = read_block( - &mmap, - offsets_start, - block_index, - info.sequence_number, - info.compression, - )?; - let eligible = if key_indices.contains(&block_index) { - let header = parse_key_block_header(&block.data).with_context(|| { - format!( - "Invalid key block {block_index} in {:08}.sst", - info.sequence_number - ) - })?; - report.blocks.blob_references += count_blob_references(header, &block.data)?; - if max_key_length(header, &block.data)? >= MIN_KEY_SIZE_FOR_COMPRESSION { - report.blocks.eligible_key.add(block.data.len()); - true - } else { - report.blocks.excluded_key.add(block.data.len()); - false - } - } else { - report.blocks.eligible_value.add(block.data.len()); - true - }; - if eligible { - report.blocks.add_eligible_size(block.data.len()); - report.original_eligible_payload_bytes += block.stored_size; - on_eligible(&block.data, block.was_compressed)?; - } - } - - let modeled_fixed_bytes = - info.block_count as u64 * (BLOCK_HEADER_SIZE as u64 + size_of::() as u64); - ensure!( - file_size >= modeled_fixed_bytes, - "SST is smaller than its headers and block directory" - ); - Ok(()) -} - -fn write_atomic(path: &Path, bytes: &[u8], force: bool) -> Result<()> { - if path.exists() && !force { - bail!( - "Output {} already exists; pass --force to replace it", - path.display() - ); - } - let parent = path.parent().unwrap_or_else(|| Path::new(".")); - fs::create_dir_all(parent).with_context(|| format!("Failed to create {}", parent.display()))?; - let filename = path - .file_name() - .and_then(|name| name.to_str()) - .context("Output filename must be UTF-8")?; - let temporary = parent.join(format!(".{filename}.{}.tmp", std::process::id())); - let result = (|| -> Result<()> { - let mut file = OpenOptions::new() - .write(true) - .create_new(true) - .open(&temporary) - .with_context(|| format!("Failed to create {}", temporary.display()))?; - file.write_all(bytes)?; - file.sync_all()?; - fs::rename(&temporary, path).with_context(|| { - format!( - "Failed to atomically rename {} to {}", - temporary.display(), - path.display() - ) - })?; - Ok(()) - })(); - if result.is_err() { - let _ = fs::remove_file(&temporary); - } - result -} - -fn training_dictionary_info(path: &Path, dictionary: &[u8]) -> DictionaryInfo { - DictionaryInfo { - name: path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("dictionary") - .to_owned(), - path: Some(path.to_path_buf()), - bytes: dictionary.len(), - dictionary_id: zstd::zstd_safe::get_dict_id_from_dict(dictionary).map(|id| id.get()), - xxh3_64: Some(format!("{:016x}", xxh3_64(dictionary))), - } -} - -fn train(options: &Options) -> Result { - let output = options.output.as_deref().expect("validated by parse_args"); - if output.exists() && !options.force { - bail!( - "Output {} already exists; pass --force to replace it", - output.display() - ); - } - let mut cache_paths = options.caches.clone(); - cache_paths.sort(); - - let mut scanned_by_cache = Vec::with_capacity(cache_paths.len()); - let mut scanned_samples = 0_u64; - for path in &cache_paths { - let mut count = 0_u64; - scan_cache(path, options.family, |_, _| { - count += 1; - Ok(()) - })?; - scanned_samples += count; - scanned_by_cache.push(count); - } - ensure!(scanned_samples > 0, "No eligible blocks found for training"); - let stride = scanned_samples.div_ceil(options.max_samples as u64).max(1); - - let mut samples: Vec> = Vec::with_capacity( - usize::try_from(scanned_samples.div_ceil(stride)) - .unwrap_or(options.max_samples) - .min(options.max_samples), - ); - let mut cache_reports = Vec::with_capacity(cache_paths.len()); - let mut global_index = 0_u64; - for (path, scanned) in cache_paths.iter().zip(scanned_by_cache) { - let mut selected_samples = 0_u64; - let mut selected_bytes = 0_u64; - scan_cache(path, options.family, |data, _| { - let select = global_index.is_multiple_of(stride) && samples.len() < options.max_samples; - global_index += 1; - if select { - selected_samples += 1; - selected_bytes += data.len() as u64; - samples.push(Box::from(data)); - } - Ok(()) - })?; - cache_reports.push(TrainingCacheReport { - path: path.clone(), - scanned_samples: scanned, - selected_samples, - selected_bytes, - }); - } - ensure!(!samples.is_empty(), "No blocks selected for training"); - let selected_bytes = samples.iter().map(|sample| sample.len() as u64).sum(); - let dictionary = - zstd::dict::from_samples(&samples, options.max_dictionary_size).with_context(|| { - format!( - "Failed to train a {}-byte dictionary from {} selected samples ({} bytes); add \ - caches, raise --max-samples, or request a smaller dictionary", - options.max_dictionary_size, - samples.len(), - selected_bytes - ) - })?; - write_atomic(output, &dictionary, options.force)?; - let info = training_dictionary_info(output, &dictionary); - Ok(TrainingReport { - schema_version: SCHEMA_VERSION, - family: options.family, - zstd_version: zstd::zstd_safe::version_string(), - max_dictionary_size: options.max_dictionary_size, - max_samples: options.max_samples, - scanned_samples, - stride, - selected_samples: samples.len() as u64, - selected_bytes, - caches: cache_reports, - dictionary: info, - }) -} - -fn print_training_report(report: &TrainingReport) { - println!( - "Trained {} ({} bytes, id {:?}, xxh3 {}) from {} of {} eligible blocks (stride {}, {} \ - selected bytes)", - report.dictionary.path.as_ref().unwrap().display(), - report.dictionary.bytes, - report.dictionary.dictionary_id, - report.dictionary.xxh3_64.as_deref().unwrap_or("none"), - report.selected_samples, - report.scanned_samples, - report.stride, - report.selected_bytes, - ); - for cache in &report.caches { - println!( - " {}: selected {} / {} blocks ({} bytes)", - cache.path.display(), - cache.selected_samples, - cache.scanned_samples, - cache.selected_bytes - ); - } -} - -fn percentage_delta(value: u64, baseline: u64) -> Option { - (baseline > 0).then(|| (value as f64 / baseline as f64 - 1.0) * 100.0) -} - -fn finalize_results(results: &mut [CandidateResult], uncompressed_bytes: u64, current_bytes: u64) { - let baseline = results - .first() - .map_or(0, |result| result.modeled_complete_sst_bytes); - for result in results { - result.raw_compression_ratio = (uncompressed_bytes > 0) - .then(|| result.raw_compressed_bytes as f64 / uncompressed_bytes as f64); - result.modeled_delta_vs_baseline_pct = - percentage_delta(result.modeled_complete_sst_bytes, baseline); - result.modeled_delta_vs_current_pct = - percentage_delta(result.modeled_complete_sst_bytes, current_bytes); - } -} - -fn combine(caches: &[CacheReport], candidates: &[Candidate]) -> CombinedReport { - let mut combined = CombinedReport { - cache_count: caches.len(), - active_ssts: 0, - original_complete_sst_bytes: 0, - original_eligible_payload_bytes: 0, - blocks: BlockSummary::default(), - candidates: candidates - .iter() - .map(|candidate| CandidateResult { - dictionary: Some(candidate.info.clone()), - setup_ns: candidate.setup_ns, - ..Default::default() - }) - .collect(), - }; - for cache in caches { - combined.active_ssts += cache.active_ssts; - combined.original_complete_sst_bytes += cache.original_complete_sst_bytes; - combined.original_eligible_payload_bytes += cache.original_eligible_payload_bytes; - combined.blocks.merge(&cache.blocks); - for (total, result) in combined.candidates.iter_mut().zip(&cache.candidates) { - total.merge(result); - } - } - finalize_results( - &mut combined.candidates, - combined.blocks.eligible_key.bytes + combined.blocks.eligible_value.bytes, - combined.original_complete_sst_bytes, - ); - combined -} - -fn print_report(report: &Report) { - for cache in &report.caches { - println!( - "Cache {}: {} SSTs, {} bytes; eligible {} key + {} value blocks; blobs omitted {}", - cache.path.display(), - cache.active_ssts, - cache.original_complete_sst_bytes, - cache.blocks.eligible_key.count, - cache.blocks.eligible_value.count, - cache.blocks.blob_references, - ); - let baseline = cache.candidates[0].modeled_complete_sst_bytes; - for result in &cache.candidates { - let name = &result.dictionary.as_ref().unwrap().name; - let delta = if baseline == 0 { - 0.0 - } else { - (result.modeled_complete_sst_bytes as f64 / baseline as f64 - 1.0) * 100.0 - }; - println!( - " {name}: modeled SST {} bytes ({delta:+.2}% vs baseline)", - result.modeled_complete_sst_bytes - ); - } - } - println!(); - println!( - "Combined family {}: {} cache directories, {} active SSTs", - report.family, - report.caches.len(), - report.combined.active_ssts - ); - println!( - "Eligible: {} key + {} value blocks, {} bytes uncompressed; blobs omitted: {}", - report.combined.blocks.eligible_key.count, - report.combined.blocks.eligible_value.count, - report.combined.blocks.eligible_key.bytes + report.combined.blocks.eligible_value.bytes, - report.combined.blocks.blob_references - ); - println!( - "Original active SST bytes: {}", - report.combined.original_complete_sst_bytes - ); - println!(); - println!( - "{:<28} {:>15} {:>9} {:>15} {:>10} {:>12} {:>12}", - "Candidate", - "Raw compressed", - "ratio", - "Modeled SST", - "vs baseline", - "encode ms", - "decode ms" - ); - let baseline = report.combined.candidates[0].modeled_complete_sst_bytes; - for result in &report.combined.candidates { - let name = &result.dictionary.as_ref().unwrap().name; - let delta = if baseline == 0 { - 0.0 - } else { - (result.modeled_complete_sst_bytes as f64 / baseline as f64 - 1.0) * 100.0 - }; - println!( - "{name:<28} {:>15} {:>8.2}% {:>15} {:>+9.2}% {:>12.3} {:>12.3}", - result.raw_compressed_bytes, - result.raw_compression_ratio.unwrap_or_default() * 100.0, - result.modeled_complete_sst_bytes, - delta, - result.encode_ns as f64 / 1_000_000.0, - result.decode_ns as f64 / 1_000_000.0, - ); - } - println!(); - println!("Note: timings are single-pass wall-clock diagnostics, not benchmarks."); -} - -fn write_json(path: Option<&Path>, report: &impl Serialize) -> Result<()> { - if let Some(path) = path { - let file = File::create(path) - .with_context(|| format!("Failed to create JSON report {}", path.display()))?; - serde_json::to_writer_pretty(BufWriter::new(file), report)?; - } - Ok(()) -} - -fn run() -> Result<()> { - let options = parse_args()?; - match options.command { - Command::Train => { - let report = train(&options)?; - print_training_report(&report); - write_json(options.json.as_deref(), &report) - } - Command::Evaluate => { - let mut candidates = make_candidates(&options.dictionaries)?; - let mut caches = Vec::with_capacity(options.caches.len()); - for path in &options.caches { - caches.push(evaluate_cache(path, options.family, &mut candidates)?); - } - let combined = combine(&caches, &candidates); - let report = Report { - schema_version: SCHEMA_VERSION, - family: options.family, - timing_note: "Single-pass wall-clock diagnostics; size/count fields are the \ - comparison contract.", - caches, - combined, - }; - print_report(&report); - write_json(options.json.as_deref(), &report) - } - } -} - -fn main() { - if let Err(error) = run() { - eprintln!("Error: {error:#}"); - std::process::exit(1); - } -} - -#[cfg(test)] -mod tests { - use std::{fs, path::Path}; - - use anyhow::Result; - use byteorder::{BE, WriteBytesExt}; - use tempfile::TempDir; - use turbo_persistence::{Compression, DbConfig, SerialScheduler, TurboPersistence}; - - use super::{ - Command, Options, evaluate_cache, make_candidates, parse_args_from, production_stored_size, - train, - }; - - fn make_cache(family: usize, compression: Compression) -> Result { - let tempdir = tempfile::tempdir()?; - let mut config = DbConfig::<8>::default(); - config.family_configs[family].compression = compression; - let db = TurboPersistence::::open_with_config( - tempdir.path().to_path_buf(), - config, - )?; - let batch = db.write_batch()?; - for index in 0..50u8 { - let key = format!("long-task-data-key-{index:04}").into_bytes(); - let value = format!("function component{index}() {{ return null; }}") - .repeat(32) - .into_bytes(); - batch.put(family as u32, key, value.into())?; - } - for index in 0..100u8 { - let key = format!("long-medium-task-data-key-{index:04}").into_bytes(); - batch.put(family as u32, key, vec![b'a' + index; 5000].into())?; - } - let mut state = 0x1234_5678_u32; - let incompressible = (0..5000) - .map(|_| { - state ^= state << 13; - state ^= state >> 17; - state ^= state << 5; - state as u8 - }) - .collect::>(); - batch.put( - family as u32, - b"long-incompressible-task-data-key".to_vec(), - incompressible.into(), - )?; - db.commit_write_batch(batch)?; - db.shutdown()?; - Ok(tempdir) - } - - fn write_dictionary(path: &Path, byte: u8) -> Result<()> { - let mut dictionary = b"function component return const import export className".repeat(32); - dictionary.push(byte); - fs::write(path, dictionary)?; - Ok(()) - } - - #[test] - fn cli_requires_cache_and_rejects_unknown_options() { - assert!(parse_args_from([]).is_err()); - assert!(parse_args_from(["--unknown".into()]).is_err()); - let options = parse_args_from([ - "evaluate".into(), - "--family".into(), - "7".into(), - "--dictionary".into(), - "candidate.dict".into(), - "cache".into(), - ]) - .unwrap(); - assert_eq!(options.family, 7); - assert_eq!( - options.dictionaries, - [std::path::PathBuf::from("candidate.dict")] - ); - assert_eq!(options.caches, [std::path::PathBuf::from("cache")]); - - let train = parse_args_from([ - "train".into(), - "--output".into(), - "output.dict".into(), - "cache".into(), - ]) - .unwrap(); - assert_eq!(train.command, Command::Train); - assert_eq!(train.max_dictionary_size, 64 * 1024); - assert_eq!(train.max_samples, 10_000); - } - - fn training_options(caches: &[&Path], output: &Path, force: bool) -> Options { - Options { - command: Command::Train, - family: 2, - dictionaries: Vec::new(), - json: None, - output: Some(output.to_path_buf()), - max_dictionary_size: 1024, - max_samples: 100, - force, - caches: caches.iter().map(|path| path.to_path_buf()).collect(), - } - } - - #[test] - fn trains_with_two_pass_sampling_and_no_clobber() -> Result<()> { - let first_cache = make_cache(2, Compression::Zstd3)?; - let second_cache = make_cache(2, Compression::Lz4)?; - let output_dir = tempfile::tempdir()?; - let first_output = output_dir.path().join("first.zdict"); - let second_output = output_dir.path().join("second.zdict"); - let caches = [first_cache.path(), second_cache.path()]; - - let first = train(&training_options(&caches, &first_output, false))?; - assert_eq!(first.max_dictionary_size, 1024); - assert_eq!(first.max_samples, 100); - assert!(first.scanned_samples > first.selected_samples); - assert!(first.selected_samples <= 100); - assert_eq!(first.caches.len(), 2); - assert!(first.caches.iter().all(|cache| cache.selected_samples > 0)); - assert_eq!(fs::read(&first_output)?.len(), first.dictionary.bytes); - - let second = train(&training_options(&caches, &second_output, false))?; - assert_eq!(fs::read(&first_output)?, fs::read(&second_output)?); - assert!(train(&training_options(&caches, &first_output, false)).is_err()); - train(&training_options(&caches, &first_output, true))?; - assert_eq!(first.selected_samples, second.selected_samples); - assert_eq!(first.selected_bytes, second.selected_bytes); - Ok(()) - } - - #[test] - fn production_threshold_is_strict() { - assert_eq!(production_stored_size(800, 699), (699, true)); - assert_eq!(production_stored_size(800, 700), (800, false)); - } - - #[test] - fn evaluates_multiple_dictionaries_without_mutating_cache() -> Result<()> { - let cache = make_cache(2, Compression::Zstd3)?; - let dictionary_dir = tempfile::tempdir()?; - let first = dictionary_dir.path().join("first.dict"); - let second = dictionary_dir.path().join("second.dict"); - write_dictionary(&first, 1)?; - write_dictionary(&second, 2)?; - let before = fs::read(cache.path().join("00000001.sst"))?; - - let mut candidates = make_candidates(&[first, second])?; - let report = evaluate_cache(cache.path(), 2, &mut candidates)?; - - assert_eq!(report.active_ssts, 1); - assert_eq!(report.candidates.len(), 3); - assert!(report.blocks.eligible_value.count > 0); - assert!(report.blocks.eligible_key.count > 0); - assert_eq!(report.blocks.excluded_index.count, 1); - assert_eq!(report.blocks.blob_references, 0); - assert_eq!( - report.candidates[0].modeled_complete_sst_bytes, - report.original_complete_sst_bytes - ); - assert!(report.candidates[0].fallback_blocks > 0); - assert!( - report - .candidates - .iter() - .all(|candidate| candidate.modeled_complete_sst_bytes > 0) - ); - - let mut repeated_candidates = make_candidates(&[ - dictionary_dir.path().join("first.dict"), - dictionary_dir.path().join("second.dict"), - ])?; - let repeated = evaluate_cache(cache.path(), 2, &mut repeated_candidates)?; - assert_eq!( - report.original_complete_sst_bytes, - repeated.original_complete_sst_bytes - ); - assert_eq!( - report.blocks.eligible_key.count, - repeated.blocks.eligible_key.count - ); - assert_eq!( - report.blocks.eligible_value.count, - repeated.blocks.eligible_value.count - ); - assert_eq!( - report - .candidates - .iter() - .map(|candidate| candidate.modeled_complete_sst_bytes) - .collect::>(), - repeated - .candidates - .iter() - .map(|candidate| candidate.modeled_complete_sst_bytes) - .collect::>() - ); - assert_eq!(before, fs::read(cache.path().join("00000001.sst"))?); - Ok(()) - } - - #[test] - fn counts_blob_references_in_key_blocks() -> Result<()> { - use turbo_persistence::{ - offline::parse_key_block_header, - static_sorted_file::{BLOCK_TYPE_FIXED_KEY_NO_HASH, KEY_BLOCK_ENTRY_TYPE_BLOB}, - }; - - let block = [ - BLOCK_TYPE_FIXED_KEY_NO_HASH, - 0, - 0, - 1, - 1, - KEY_BLOCK_ENTRY_TYPE_BLOB, - b'k', - 0, - 0, - 0, - 42, - ]; - let header = parse_key_block_header(&block)?; - assert_eq!(super::count_blob_references(header, &block)?, 1); - Ok(()) - } - - #[test] - fn corrupt_blocks_are_rejected_with_context() -> Result<()> { - let cache = make_cache(2, Compression::Zstd3)?; - let sst_path = cache.path().join("00000001.sst"); - let mut bytes = fs::read(&sst_path)?; - bytes[8] ^= 1; - fs::write(&sst_path, bytes)?; - let mut candidates = make_candidates(&[])?; - let error = evaluate_cache(cache.path(), 2, &mut candidates) - .err() - .expect("corrupt block should fail"); - let message = format!("{error:#}"); - assert!(message.contains("00000001.sst")); - assert!(message.contains("Checksum mismatch")); - Ok(()) - } - - #[test] - fn active_ssts_follow_current_deletions_and_supersession() -> Result<()> { - let cache = make_cache(2, Compression::Zstd3)?; - fs::copy( - cache.path().join("00000002.meta"), - cache.path().join("00000003.meta"), - )?; - let mut current: serde_json::Value = - serde_json::from_slice(&fs::read(cache.path().join("CURRENT"))?)?; - current["max_sequence_number"] = 3.into(); - fs::write(cache.path().join("CURRENT"), serde_json::to_vec(¤t)?)?; - - let mut candidates = make_candidates(&[])?; - let superseded = evaluate_cache(cache.path(), 2, &mut candidates)?; - assert_eq!(superseded.active_ssts, 1); - - let mut deletion = Vec::new(); - deletion.write_u32::(3)?; - fs::write(cache.path().join("00000004.del"), deletion)?; - let mut candidates = make_candidates(&[])?; - let deleted = evaluate_cache(cache.path(), 2, &mut candidates)?; - assert_eq!(deleted.active_ssts, 1); - assert_eq!( - superseded.original_complete_sst_bytes, - deleted.original_complete_sst_bytes - ); - Ok(()) - } - - #[test] - fn family_override_selects_non_taskdata_family() -> Result<()> { - let cache = make_cache(7, Compression::Lz4)?; - let mut candidates = make_candidates(&[])?; - let report = evaluate_cache(cache.path(), 7, &mut candidates)?; - assert_eq!(report.family, 7); - assert_eq!(report.recorded_codecs, ["Lz4".to_string()].into()); - assert!(report.blocks.eligible_value.count > 0); - Ok(()) - } -} diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs new file mode 100644 index 000000000000..103090962bcc --- /dev/null +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -0,0 +1,992 @@ +//! Train and evaluate zstd dictionaries from logical values in persistence caches. + +use std::{ + collections::{BTreeSet, HashSet, VecDeque}, + fs::{self, File, OpenOptions}, + io::{BufWriter, Write}, + path::{Path, PathBuf}, + sync::Arc, + time::Instant, +}; + +use anyhow::{Context, Result, ensure}; +use clap::{Args, Parser, Subcommand}; +use serde::Serialize; +use turbo_persistence::{ + Compression, IterValue, MAX_INLINE_VALUE_SIZE, StaticSortedFileIter, StaticSortedFileMetaData, + offline::{SstInfo, collect_sst_info, decode_medium, read_blob}, +}; +use xxhash_rust::xxh3::xxh3_64; + +const SCHEMA_VERSION: u32 = 2; +const DICTIONARY_SIZE: usize = 64 * 1024; +const SAMPLE_BUDGET_MULTIPLIER: usize = 1000; +const SAMPLE_BYTE_BUDGET: usize = DICTIONARY_SIZE * SAMPLE_BUDGET_MULTIPLIER; +const BLOB_HEADER_SIZE: usize = 8; + +#[derive(Parser)] +#[command(about = "Train and evaluate zstd dictionaries from persistence caches")] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Train a 64 KiB zstd dictionary from logical values. + Train { + #[command(flatten)] + source: Source, + /// Dictionary output path. Existing files are replaced. + #[arg(short, long)] + output: PathBuf, + /// Optional JSON training report. + #[arg(long)] + json: Option, + }, + /// Compare dictionaries with zstd level 3 without a dictionary. + Evaluate { + #[command(flatten)] + source: Source, + /// Candidate dictionary. May be supplied more than once. + #[arg(short, long)] + dictionary: Vec, + /// Optional JSON evaluation report. + #[arg(long)] + json: Option, + }, +} + +#[derive(Args)] +struct Source { + /// Persistence family ID to inspect. + #[arg(long)] + family: u32, + /// Database directories containing CURRENT, meta, SST, and blob files. + #[arg(required = true)] + caches: Vec, +} + +#[derive(Clone, Serialize)] +struct DictionaryInfo { + name: String, + path: Option, + bytes: usize, + dictionary_id: Option, + xxh3_64: Option, +} + +struct Candidate { + info: DictionaryInfo, + compressor: zstd::bulk::Compressor<'static>, + decompressor: zstd::bulk::Decompressor<'static>, + setup_ns: u64, +} + +#[derive(Clone, Copy)] +enum SampleKind { + Slice, + Medium, + Blob, +} + +struct Sample { + kind: SampleKind, + data: Arc<[u8]>, +} + +#[derive(Default, Clone, Serialize)] +struct Metric { + count: u64, + bytes: u64, +} + +impl Metric { + fn add(&mut self, bytes: usize) { + self.count += 1; + self.bytes += bytes as u64; + } + + fn merge(&mut self, other: &Self) { + self.count += other.count; + self.bytes += other.bytes; + } +} + +#[derive(Default, Clone, Serialize)] +struct KindMetrics { + slice: Metric, + medium: Metric, + blob: Metric, +} + +impl KindMetrics { + fn get_mut(&mut self, kind: SampleKind) -> &mut Metric { + match kind { + SampleKind::Slice => &mut self.slice, + SampleKind::Medium => &mut self.medium, + SampleKind::Blob => &mut self.blob, + } + } + + fn merge(&mut self, other: &Self) { + self.slice.merge(&other.slice); + self.medium.merge(&other.medium); + self.blob.merge(&other.blob); + } + + fn total(&self) -> Metric { + Metric { + count: self.slice.count + self.medium.count + self.blob.count, + bytes: self.slice.bytes + self.medium.bytes + self.blob.bytes, + } + } +} + +#[derive(Default, Clone, Serialize)] +struct CompressionMetric { + input_bytes: u64, + raw_compressed_bytes: u64, + estimated_stored_bytes: u64, + raw_compression_ratio: Option, + encode_ns: u64, + decode_ns: u64, +} + +impl CompressionMetric { + fn merge(&mut self, other: &Self) { + self.input_bytes += other.input_bytes; + self.raw_compressed_bytes += other.raw_compressed_bytes; + self.estimated_stored_bytes += other.estimated_stored_bytes; + self.encode_ns += other.encode_ns; + self.decode_ns += other.decode_ns; + } + + fn finalize(&mut self) { + self.raw_compression_ratio = (self.input_bytes > 0) + .then(|| self.raw_compressed_bytes as f64 / self.input_bytes as f64); + } +} + +#[derive(Default, Clone, Serialize)] +struct CompressionByKind { + slice: CompressionMetric, + medium: CompressionMetric, + blob: CompressionMetric, +} + +impl CompressionByKind { + fn get_mut(&mut self, kind: SampleKind) -> &mut CompressionMetric { + match kind { + SampleKind::Slice => &mut self.slice, + SampleKind::Medium => &mut self.medium, + SampleKind::Blob => &mut self.blob, + } + } + + fn merge(&mut self, other: &Self) { + self.slice.merge(&other.slice); + self.medium.merge(&other.medium); + self.blob.merge(&other.blob); + } + + fn finalize(&mut self) { + self.slice.finalize(); + self.medium.finalize(); + self.blob.finalize(); + } + + fn total(&self) -> CompressionMetric { + let mut total = CompressionMetric::default(); + total.merge(&self.slice); + total.merge(&self.medium); + total.merge(&self.blob); + total.finalize(); + total + } +} + +#[derive(Clone, Serialize)] +struct CandidateResult { + dictionary: DictionaryInfo, + by_kind: CompressionByKind, + combined: CompressionMetric, + setup_ns: u64, +} + +#[derive(Serialize)] +struct CacheReport { + path: PathBuf, + family: u32, + active_ssts: u64, + recorded_codecs: BTreeSet, + samples: KindMetrics, + duplicate_blob_references: u64, + candidates: Vec, +} + +#[derive(Serialize)] +struct EvaluationReport { + schema_version: u32, + family: u32, + timing_note: &'static str, + threshold_note: &'static str, + caches: Vec, + combined_samples: KindMetrics, + combined_candidates: Vec, +} + +#[derive(Serialize)] +struct TrainingCacheReport { + path: PathBuf, + samples: KindMetrics, +} + +#[derive(Serialize)] +struct TrainingReport { + schema_version: u32, + family: u32, + zstd_version: &'static str, + dictionary_size: usize, + sample_byte_target: usize, + selected: KindMetrics, + selected_bytes: u64, + inputs_exhausted: bool, + caches: Vec, + dictionary: DictionaryInfo, +} + +struct CacheSampleIter { + path: PathBuf, + pending: VecDeque, + current: Option, + current_compression: Option, + seen_blobs: HashSet, + active_ssts: u64, + recorded_codecs: BTreeSet, + duplicate_blob_references: u64, +} + +impl CacheSampleIter { + fn open(path: PathBuf, family: u32) -> Result { + let mut families = collect_sst_info(&path) + .with_context(|| format!("Failed to inspect cache {}", path.display()))?; + let mut ssts = families.remove(&family).with_context(|| { + format!( + "Cache {} has no active SSTs for family {family}", + path.display() + ) + })?; + ssts.sort_by_key(|sst| sst.sequence_number); + let recorded_codecs = ssts + .iter() + .map(|sst| format!("{:?}", sst.compression)) + .collect(); + Ok(Self { + path, + active_ssts: ssts.len() as u64, + pending: ssts.into(), + current: None, + current_compression: None, + seen_blobs: HashSet::new(), + recorded_codecs, + duplicate_blob_references: 0, + }) + } + + fn open_next_sst(&mut self) -> Result { + let Some(sst) = self.pending.pop_front() else { + return Ok(false); + }; + self.current = Some( + StaticSortedFileIter::open( + &self.path, + StaticSortedFileMetaData { + sequence_number: sst.sequence_number, + block_count: sst.block_count, + }, + sst.compression, + ) + .with_context(|| format!("Failed to open {:08}.sst", sst.sequence_number))?, + ); + self.current_compression = Some(sst.compression); + Ok(true) + } + + fn next_sample(&mut self) -> Result> { + loop { + if self.current.is_none() && !self.open_next_sst()? { + return Ok(None); + } + let compression = self.current_compression.expect("set with current SST"); + let entry = match self.current.as_mut().unwrap().next() { + Some(entry) => entry?, + None => { + self.current = None; + self.current_compression = None; + continue; + } + }; + if let Some(sample) = self.sample_from_value(entry.value, compression)? { + return Ok(Some(sample)); + } + } + } + + fn sample_from_value( + &mut self, + value: IterValue, + compression: Compression, + ) -> Result> { + match value { + IterValue::Slice { value } if value.len() > MAX_INLINE_VALUE_SIZE => Ok(Some(Sample { + kind: SampleKind::Slice, + data: Arc::from(value.as_ref()), + })), + IterValue::Medium { + uncompressed_size, + checksum, + block, + } => { + let value = decode_medium(compression, uncompressed_size, checksum, &block) + .with_context(|| { + format!("Failed to read medium value in {}", self.path.display()) + })?; + Ok(Some(Sample { + kind: SampleKind::Medium, + data: value, + })) + } + IterValue::Blob { sequence_number } => { + if !self.seen_blobs.insert(sequence_number) { + self.duplicate_blob_references += 1; + return Ok(None); + } + let value = read_blob(&self.path, sequence_number, compression)?; + Ok(Some(Sample { + kind: SampleKind::Blob, + data: value, + })) + } + IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. } | IterValue::Slice { .. } => { + Ok(None) + } + } + } +} + +fn dictionary_info( + path: Option<&Path>, + dictionary: &[u8], + baseline: bool, +) -> Result { + let name = if baseline { + "zstd3 (no dictionary)".to_owned() + } else { + path.and_then(Path::file_name) + .context("Dictionary path has no filename")? + .to_str() + .context("Dictionary filename must be UTF-8")? + .to_owned() + }; + Ok(DictionaryInfo { + name, + path: path.map(Path::to_path_buf), + bytes: dictionary.len(), + dictionary_id: zstd::zstd_safe::get_dict_id_from_dict(dictionary).map(|id| id.get()), + xxh3_64: (!baseline).then(|| format!("{:016x}", xxh3_64(dictionary))), + }) +} + +fn make_candidates(paths: &[PathBuf]) -> Result> { + let mut result = Vec::with_capacity(paths.len() + 1); + let started = Instant::now(); + result.push(Candidate { + info: dictionary_info(None, &[], true)?, + compressor: zstd::bulk::Compressor::new(3)?, + decompressor: zstd::bulk::Decompressor::new()?, + setup_ns: started.elapsed().as_nanos() as u64, + }); + let mut names = BTreeSet::new(); + for path in paths { + let dictionary = fs::read(path) + .with_context(|| format!("Failed to read dictionary {}", path.display()))?; + ensure!( + !dictionary.is_empty(), + "Dictionary {} is empty", + path.display() + ); + let info = dictionary_info(Some(path), &dictionary, false)?; + ensure!( + names.insert(info.name.clone()), + "Duplicate dictionary name {}", + info.name + ); + let started = Instant::now(); + result.push(Candidate { + info, + compressor: zstd::bulk::Compressor::with_dictionary(3, &dictionary)?, + decompressor: zstd::bulk::Decompressor::with_dictionary(&dictionary)?, + setup_ns: started.elapsed().as_nanos() as u64, + }); + } + Ok(result) +} + +/// Evaluates all dictionary candidates against one logical value. +fn evaluate_sample( + sample: &Sample, + candidates: &mut [Candidate], + results: &mut [CandidateResult], +) -> Result<()> { + for (candidate, result) in candidates.iter_mut().zip(results) { + let started = Instant::now(); + let compressed = candidate + .compressor + .compress(&sample.data) + .with_context(|| format!("Failed to compress with {}", candidate.info.name))?; + let encode_ns = started.elapsed().as_nanos() as u64; + let started = Instant::now(); + let decompressed = candidate + .decompressor + .decompress(&compressed, sample.data.len()) + .with_context(|| format!("Failed to decompress with {}", candidate.info.name))?; + let decode_ns = started.elapsed().as_nanos() as u64; + ensure!( + decompressed.as_slice() == sample.data.as_ref(), + "Round-trip mismatch with {}", + candidate.info.name + ); + + let metric = result.by_kind.get_mut(sample.kind); + metric.input_bytes += sample.data.len() as u64; + metric.raw_compressed_bytes += compressed.len() as u64; + metric.encode_ns += encode_ns; + metric.decode_ns += decode_ns; + metric.estimated_stored_bytes += if matches!(sample.kind, SampleKind::Blob) { + (compressed.len() + BLOB_HEADER_SIZE) as u64 + } else { + estimated_value_bytes(sample.data.len(), compressed.len()) as u64 + }; + } + Ok(()) +} + +/// Applies the writer's 12.5% minimum-savings rule as a per-value estimate. +/// +/// Small values are grouped into physical blocks in production, so this is a comparative proxy, +/// not exact SST-size modeling. See `write_block_to_file` for the production block-level rule. +fn estimated_value_bytes(original_len: usize, compressed_len: usize) -> usize { + if compressed_len < original_len - original_len / 8 { + compressed_len + } else { + original_len + } +} + +fn empty_results(candidates: &[Candidate]) -> Vec { + candidates + .iter() + .map(|candidate| CandidateResult { + dictionary: candidate.info.clone(), + by_kind: CompressionByKind::default(), + combined: CompressionMetric::default(), + setup_ns: candidate.setup_ns, + }) + .collect() +} + +fn finalize_results(results: &mut [CandidateResult]) { + for result in results { + result.by_kind.finalize(); + result.combined = result.by_kind.total(); + } +} + +fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Result { + let mut iter = CacheSampleIter::open(path.to_path_buf(), family)?; + let mut samples = KindMetrics::default(); + let mut results = empty_results(candidates); + while let Some(sample) = iter.next_sample()? { + samples.get_mut(sample.kind).add(sample.data.len()); + evaluate_sample(&sample, candidates, &mut results)?; + } + finalize_results(&mut results); + Ok(CacheReport { + path: path.to_path_buf(), + family, + active_ssts: iter.active_ssts, + recorded_codecs: iter.recorded_codecs, + samples, + duplicate_blob_references: iter.duplicate_blob_references, + candidates: results, + }) +} + +fn combine_evaluation( + family: u32, + caches: Vec, + candidates: &[Candidate], +) -> EvaluationReport { + let mut combined_samples = KindMetrics::default(); + let mut combined_candidates = empty_results(candidates); + for cache in &caches { + combined_samples.merge(&cache.samples); + for (combined, current) in combined_candidates.iter_mut().zip(&cache.candidates) { + combined.by_kind.merge(¤t.by_kind); + } + } + finalize_results(&mut combined_candidates); + EvaluationReport { + schema_version: SCHEMA_VERSION, + family, + timing_note: "Single-pass wall-clock diagnostics; byte/count fields are the comparison \ + contract.", + threshold_note: "Slice/medium stored bytes apply the 12.5% rule per logical value and are \ + a proxy for grouped small-value blocks. Blob bytes include the fixed \ + 8-byte header.", + caches, + combined_samples, + combined_candidates, + } +} + +struct TrainingSelection { + samples: Vec>, + caches: Vec, + inputs_exhausted: bool, +} + +fn select_training_samples( + paths: &[PathBuf], + family: u32, + byte_budget: usize, +) -> Result { + let mut paths = paths.to_vec(); + paths.sort(); + let mut iterators = paths + .into_iter() + .map(|path| CacheSampleIter::open(path, family)) + .collect::>>()?; + let mut per_cache = iterators + .iter() + .map(|iter| TrainingCacheReport { + path: iter.path.clone(), + samples: KindMetrics::default(), + }) + .collect::>(); + let mut samples = Vec::new(); + let mut selected_bytes = 0_usize; + let mut active = vec![true; iterators.len()]; + let mut active_count = iterators.len(); + + while active_count > 0 && selected_bytes < byte_budget { + for index in 0..iterators.len() { + if !active[index] { + continue; + } + match iterators[index].next_sample()? { + Some(sample) => { + selected_bytes += sample.data.len(); + per_cache[index] + .samples + .get_mut(sample.kind) + .add(sample.data.len()); + samples.push(Box::from(sample.data.as_ref())); + if selected_bytes >= byte_budget { + break; + } + } + None => { + active[index] = false; + active_count -= 1; + } + } + } + } + Ok(TrainingSelection { + samples, + caches: per_cache, + inputs_exhausted: active_count == 0, + }) +} + +fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent).with_context(|| format!("Failed to create {}", parent.display()))?; + let filename = path + .file_name() + .and_then(|name| name.to_str()) + .context("Output filename must be UTF-8")?; + let temporary = parent.join(format!(".{filename}.{}.tmp", std::process::id())); + let result = (|| -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&temporary)?; + file.write_all(bytes)?; + file.sync_all()?; + fs::rename(&temporary, path).with_context(|| { + format!( + "Failed to replace {} with {}", + path.display(), + temporary.display() + ) + })?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result +} + +fn train(source: &Source, output: &Path) -> Result { + let TrainingSelection { + samples, + caches, + inputs_exhausted, + } = select_training_samples(&source.caches, source.family, SAMPLE_BYTE_BUDGET)?; + ensure!(!samples.is_empty(), "No eligible values found for training"); + let selected = caches + .iter() + .fold(KindMetrics::default(), |mut total, cache| { + total.merge(&cache.samples); + total + }); + let selected_bytes = selected.total().bytes; + let dictionary = zstd::dict::from_samples(&samples, DICTIONARY_SIZE).with_context(|| { + format!( + "Failed to train a {DICTIONARY_SIZE}-byte dictionary from {} values ({selected_bytes} \ + bytes)", + samples.len() + ) + })?; + write_atomic(output, &dictionary)?; + Ok(TrainingReport { + schema_version: SCHEMA_VERSION, + family: source.family, + zstd_version: zstd::zstd_safe::version_string(), + dictionary_size: DICTIONARY_SIZE, + sample_byte_target: SAMPLE_BYTE_BUDGET, + selected, + selected_bytes, + inputs_exhausted, + caches, + dictionary: dictionary_info(Some(output), &dictionary, false)?, + }) +} + +fn print_training(report: &TrainingReport) { + println!( + "Trained {} ({} bytes, id {:?}, xxh3 {}) from {} values / {} bytes (target {}, exhausted: \ + {})", + report.dictionary.path.as_ref().unwrap().display(), + report.dictionary.bytes, + report.dictionary.dictionary_id, + report.dictionary.xxh3_64.as_deref().unwrap_or("none"), + report.selected.total().count, + report.selected_bytes, + report.sample_byte_target, + report.inputs_exhausted, + ); + for cache in &report.caches { + let total = cache.samples.total(); + println!( + " {}: {} values / {} bytes", + cache.path.display(), + total.count, + total.bytes + ); + } +} + +fn print_evaluation(report: &EvaluationReport) { + let samples = report.combined_samples.total(); + println!( + "Evaluated family {}: {} caches, {} logical values / {} bytes", + report.family, + report.caches.len(), + samples.count, + samples.bytes + ); + println!( + "{:<28} {:>15} {:>9} {:>18} {:>12} {:>12}", + "Candidate", "Raw compressed", "ratio", "Estimated stored", "encode ms", "decode ms" + ); + for result in &report.combined_candidates { + println!( + "{:<28} {:>15} {:>8.2}% {:>18} {:>12.3} {:>12.3}", + result.dictionary.name, + result.combined.raw_compressed_bytes, + result.combined.raw_compression_ratio.unwrap_or_default() * 100.0, + result.combined.estimated_stored_bytes, + result.combined.encode_ns as f64 / 1_000_000.0, + result.combined.decode_ns as f64 / 1_000_000.0, + ); + } + println!("Note: {}", report.threshold_note); + println!("Note: {}", report.timing_note); +} + +fn write_json(path: Option<&Path>, report: &impl Serialize) -> Result<()> { + if let Some(path) = path { + let file = File::create(path) + .with_context(|| format!("Failed to create JSON report {}", path.display()))?; + serde_json::to_writer_pretty(BufWriter::new(file), report)?; + } + Ok(()) +} + +fn run(cli: Cli) -> Result<()> { + match cli.command { + Command::Train { + source, + output, + json, + } => { + let report = train(&source, &output)?; + print_training(&report); + write_json(json.as_deref(), &report) + } + Command::Evaluate { + source, + dictionary, + json, + } => { + let mut candidates = make_candidates(&dictionary)?; + let caches = source + .caches + .iter() + .map(|path| evaluate_cache(path, source.family, &mut candidates)) + .collect::>>()?; + let report = combine_evaluation(source.family, caches, &candidates); + print_evaluation(&report); + write_json(json.as_deref(), &report) + } + } +} + +fn main() { + if let Err(error) = run(Cli::parse()) { + eprintln!("Error: {error:#}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use std::{ffi::OsString, fs, path::PathBuf}; + + use anyhow::Result; + use byteorder::{BE, WriteBytesExt}; + use clap::Parser; + use tempfile::TempDir; + use turbo_persistence::{Compression, DbConfig, SerialScheduler, TurboPersistence}; + + use super::{ + CacheSampleIter, Cli, DICTIONARY_SIZE, SAMPLE_BYTE_BUDGET, Source, empty_results, + estimated_value_bytes, evaluate_cache, evaluate_sample, finalize_results, make_candidates, + select_training_samples, train, + }; + + fn make_cache(family: usize, compression: Compression, values: usize) -> Result { + let tempdir = tempfile::tempdir()?; + let mut config = DbConfig::<8>::default(); + config.family_configs[family].compression = compression; + let db = TurboPersistence::::open_with_config( + tempdir.path().to_path_buf(), + config, + )?; + let batch = db.write_batch()?; + for index in 0..values { + let key = format!("key-{index:06}").into_bytes(); + let value = if index.is_multiple_of(10) { + vec![b'a' + (index % 26) as u8; 5000] + } else { + format!("function component{index}() {{ return null; }}") + .repeat(64) + .into_bytes() + }; + batch.put(family as u32, key, value.into())?; + } + let mut state = 0x1234_5678_u32; + let incompressible = (0..5000) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect::>(); + batch.put( + family as u32, + b"incompressible-medium".to_vec(), + incompressible.into(), + )?; + db.commit_write_batch(batch)?; + db.shutdown()?; + Ok(tempdir) + } + + #[test] + fn clap_requires_family_output_and_cache() { + assert!(Cli::try_parse_from(["tool", "train"]).is_err()); + assert!(Cli::try_parse_from(["tool", "train", "--family", "2", "cache"]).is_err()); + assert!( + Cli::try_parse_from([ + "tool", "train", "--family", "2", "--output", "dict", "cache" + ]) + .is_ok() + ); + assert_eq!(DICTIONARY_SIZE, 64 * 1024); + assert_eq!(SAMPLE_BYTE_BUDGET, 64 * 1024 * 1000); + } + + #[test] + fn threshold_proxy_is_strict() { + assert_eq!(estimated_value_bytes(800, 699), 699); + assert_eq!(estimated_value_bytes(800, 700), 800); + } + + #[test] + fn round_robin_samples_multiple_caches() -> Result<()> { + let first = make_cache(2, Compression::Zstd3, 20)?; + let second = make_cache(2, Compression::Lz4, 20)?; + let paths = [first.path().to_path_buf(), second.path().to_path_buf()]; + let selection = select_training_samples(&paths, 2, 10_000)?; + assert!(!selection.samples.is_empty()); + assert!(!selection.inputs_exhausted); + assert_eq!(selection.caches.len(), 2); + assert!( + selection + .caches + .iter() + .all(|report| report.samples.total().count > 0) + ); + Ok(()) + } + + #[test] + fn insufficient_samples_fail_with_context() -> Result<()> { + let cache = make_cache(2, Compression::Zstd3, 1)?; + let output_dir = tempfile::tempdir()?; + let source = Source { + family: 2, + caches: vec![cache.path().to_path_buf()], + }; + let error = train(&source, &output_dir.path().join("dictionary.zdict")) + .err() + .expect("one small cache should not train a 64 KiB dictionary"); + assert!(format!("{error:#}").contains("Failed to train a 65536-byte dictionary")); + Ok(()) + } + + #[test] + fn trains_replaces_output_and_evaluates() -> Result<()> { + let cache = make_cache(2, Compression::Zstd3, 3000)?; + let output_dir = tempfile::tempdir()?; + let output = output_dir.path().join("dictionary.zdict"); + fs::write(&output, b"old")?; + let source = Source { + family: 2, + caches: vec![cache.path().to_path_buf()], + }; + let report = train(&source, &output)?; + assert_eq!(report.dictionary.bytes, DICTIONARY_SIZE); + assert_ne!(fs::read(&output)?, b"old"); + + let mut candidates = make_candidates(&[output])?; + let evaluation = evaluate_cache(cache.path(), 2, &mut candidates)?; + assert_eq!(evaluation.candidates.len(), 2); + assert!(evaluation.samples.slice.count > 0); + assert!(evaluation.samples.medium.count > 0); + Ok(()) + } + + #[test] + fn reads_and_deduplicates_blob_samples() -> Result<()> { + let cache = make_cache(2, Compression::Zstd3, 20)?; + let value = b"export default function BlobComponent() {}".repeat(100); + let compressed = zstd::bulk::compress(&value, 3)?; + let mut blob = Vec::new(); + blob.write_u32::(value.len() as u32)?; + blob.write_u32::(turbo_persistence::checksum_block(&compressed))?; + blob.extend_from_slice(&compressed); + fs::write(cache.path().join("00000042.blob"), blob)?; + + let mut iter = CacheSampleIter::open(cache.path().to_path_buf(), 2)?; + let sample = iter + .sample_from_value( + turbo_persistence::IterValue::Blob { + sequence_number: 42, + }, + Compression::Zstd3, + )? + .unwrap(); + assert_eq!(sample.data.as_ref(), value); + let mut candidates = make_candidates(&[])?; + let mut results = empty_results(&candidates); + evaluate_sample(&sample, &mut candidates, &mut results)?; + finalize_results(&mut results); + assert_eq!( + results[0].by_kind.blob.estimated_stored_bytes, + results[0].by_kind.blob.raw_compressed_bytes + 8 + ); + assert!( + iter.sample_from_value( + turbo_persistence::IterValue::Blob { + sequence_number: 42, + }, + Compression::Zstd3, + )? + .is_none() + ); + assert_eq!(iter.duplicate_blob_references, 1); + assert!( + iter.sample_from_value( + turbo_persistence::IterValue::Blob { + sequence_number: 43, + }, + Compression::Zstd3, + ) + .is_err() + ); + Ok(()) + } + + #[test] + fn active_ssts_follow_current_deletions_and_supersession() -> Result<()> { + let cache = make_cache(2, Compression::Zstd3, 20)?; + let original = turbo_persistence::offline::collect_sst_info(cache.path())?; + assert_eq!(original[&2].len(), 1); + + fs::copy( + cache.path().join("00000002.meta"), + cache.path().join("00000003.meta"), + )?; + let mut current: serde_json::Value = + serde_json::from_slice(&fs::read(cache.path().join("CURRENT"))?)?; + current["max_sequence_number"] = 3.into(); + fs::write(cache.path().join("CURRENT"), serde_json::to_vec(¤t)?)?; + let superseded = turbo_persistence::offline::collect_sst_info(cache.path())?; + assert_eq!(superseded[&2].len(), 1); + + let mut deletion = Vec::new(); + deletion.write_u32::(3)?; + fs::write(cache.path().join("00000004.del"), deletion)?; + let deleted = turbo_persistence::offline::collect_sst_info(cache.path())?; + assert_eq!(deleted[&2].len(), 1); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn rejects_non_utf8_dictionary_name() { + use std::os::unix::ffi::OsStringExt; + + let path = PathBuf::from(OsString::from_vec(vec![0xff])); + assert!(super::dictionary_info(Some(&path), b"data", false).is_err()); + } +} diff --git a/turbopack/crates/turbo-persistence/src/lib.rs b/turbopack/crates/turbo-persistence/src/lib.rs index 1d693f64a5fe..c0255a0f4d79 100644 --- a/turbopack/crates/turbo-persistence/src/lib.rs +++ b/turbopack/crates/turbo-persistence/src/lib.rs @@ -120,11 +120,12 @@ impl Default for DbConfig { } } pub use key::{KeyBase, QueryKey, StoreKey, hash_key}; +pub use lookup_entry::{IterValue, LookupEntry}; pub use meta_file::MetaEntryFlags; pub use parallel_scheduler::{ParallelScheduler, SerialScheduler}; pub use static_sorted_file::{ BlockCache, BlockCacheLifecycle, BlockWeighter, KeyBlockLayout, SstLookupResult, - StaticSortedFile, StaticSortedFileMetaData, + StaticSortedFile, StaticSortedFileIter, StaticSortedFileMetaData, }; pub use static_sorted_file_builder::{ BLOCK_HEADER_SIZE, Entry, EntryValue, StreamingSstWriter, write_static_stored_file, diff --git a/turbopack/crates/turbo-persistence/src/offline.rs b/turbopack/crates/turbo-persistence/src/offline.rs index 25fd880bf86c..a519e05947c7 100644 --- a/turbopack/crates/turbo-persistence/src/offline.rs +++ b/turbopack/crates/turbo-persistence/src/offline.rs @@ -1,38 +1,21 @@ -//! Shared helpers for offline inspection of persistence SST files. +//! Shared helpers for offline inspection of persistence databases. use std::{ collections::{BTreeMap, HashSet}, - mem::size_of, path::Path, + sync::Arc, }; use anyhow::{Context, Result, bail, ensure}; use byteorder::{BE, ReadBytesExt}; -use fs_err::{self as fs, File}; -use lzzzz::lz4::decompress; -use memmap2::Mmap; +use fs_err as fs; use crate::{ - BLOCK_HEADER_SIZE, Compression, checksum_block, - meta_file::MetaFile, - mmap_helper::advise_mmap_for_persistence, - read_current_version, - sst_filter::SstFilter, - static_sorted_file::{ - BLOB_VALUE_REF_SIZE, BLOCK_TYPE_FIXED_KEY_NO_HASH, BLOCK_TYPE_FIXED_KEY_WITH_HASH, - BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY_NO_HASH, BLOCK_TYPE_KEY_WITH_HASH, - FIXED_KEY_BLOCK_MIXED_VALUE_TYPE, KEY_BLOCK_ENTRY_TYPE_BLOB, - KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, KEY_BLOCK_ENTRY_TYPE_KEY_DELETED, - KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN, KEY_BLOCK_ENTRY_TYPE_MEDIUM, - KEY_BLOCK_ENTRY_TYPE_SMALL, KEY_DELETED_REF_SIZE, MEDIUM_VALUE_REF_SIZE, - SMALL_VALUE_REF_SIZE, - }, + Compression, checksum_block, compression::decompress_into_arc, meta_file::MetaFile, + read_current_version, sst_filter::SstFilter, }; -const KEY_BLOCK_HEADER_SIZE: usize = 4; -pub const MIN_KEY_SIZE_FOR_COMPRESSION: usize = 16; - -/// Information about an active SST file recorded by a meta file. +/// Information about an active SST recorded by a meta file. #[derive(Clone, Copy, Debug)] pub struct SstInfo { pub sequence_number: u32, @@ -51,7 +34,7 @@ pub fn collect_sst_info(db_path: &Path) -> Result>> { .with_context(|| format!("Failed to read database directory {}", db_path.display()))? { let path = entry?.path(); - if path.extension().and_then(|s| s.to_str()) == Some("del") { + if path.extension().and_then(|extension| extension.to_str()) == Some("del") { let content = fs::read(&path) .with_context(|| format!("Failed to read deletion file {}", path.display()))?; let mut cursor: &[u8] = &content; @@ -69,14 +52,13 @@ pub fn collect_sst_info(db_path: &Path) -> Result>> { .filter_map(|entry| entry.ok()) .filter_map(|entry| { let path = entry.path(); - if path.extension().and_then(|s| s.to_str()) != Some("meta") { + if path.extension().and_then(|extension| extension.to_str()) != Some("meta") { return None; } let sequence: u32 = path.file_stem()?.to_str()?.parse().ok()?; (sequence <= current && !deleted_seqs.contains(&sequence)).then_some(sequence) }) .collect(); - if meta_seqs.is_empty() { bail!("No active .meta files found in {}", db_path.display()); } @@ -89,7 +71,6 @@ pub fn collect_sst_info(db_path: &Path) -> Result>> { .with_context(|| format!("Failed to open {sequence:08}.meta")) }) .collect::>()?; - let mut sst_filter = SstFilter::new(); for meta in meta_files.iter_mut().rev() { sst_filter.apply_filter(meta); @@ -108,341 +89,104 @@ pub fn collect_sst_info(db_path: &Path) -> Result>> { Ok(families) } -/// A checksummed block reconstructed to its original bytes. -pub struct RawBlock { - pub data: Box<[u8]>, - pub stored_size: u64, - pub actual_size: u64, - pub was_compressed: bool, +/// Verifies and reconstructs a raw medium-value block from an SST iterator. +pub fn decode_medium( + compression: Compression, + uncompressed_length: u32, + expected_checksum: u32, + stored: &[u8], +) -> Result> { + verify_checksum(stored, expected_checksum, "medium value")?; + if uncompressed_length > 0 { + decompress_into_arc(compression, uncompressed_length, stored) + .context("Failed to decompress medium value") + } else { + Ok(Arc::from(stored)) + } } -/// Reads, checksums, and decompresses a single SST block. -pub fn read_block( - mmap: &Mmap, - block_offsets_start: usize, - block_index: u16, +/// Reads, verifies, and decompresses one blob file. +pub fn read_blob( + db_path: &Path, sequence_number: u32, compression: Compression, -) -> Result { - let offset = block_offsets_start - .checked_add(block_index as usize * size_of::()) - .context("Block offset overflow")?; - let end_bytes = mmap - .get(offset..offset + size_of::()) - .with_context(|| { - format!( - "Block {block_index} directory entry is out of bounds in {sequence_number:08}.sst" - ) - })?; - let block_end = (&end_bytes[..]).read_u32::()? as usize; - let block_start = if block_index == 0 { - 0 - } else { - let start_bytes = mmap - .get(offset - size_of::()..offset) - .with_context(|| format!("Block {block_index} start offset is out of bounds"))?; - (&start_bytes[..]).read_u32::()? as usize - }; +) -> Result> { + let path = db_path.join(format!("{sequence_number:08}.blob")); + let content = fs::read(&path).with_context(|| format!("Failed to read {}", path.display()))?; ensure!( - block_end >= block_start + BLOCK_HEADER_SIZE && block_end <= block_offsets_start, - "Invalid bounds {block_start}..{block_end} for block {block_index} in \ - {sequence_number:08}.sst" + content.len() >= 8, + "Blob file {} is truncated", + path.display() ); - - let header = mmap - .get(block_start..block_start + BLOCK_HEADER_SIZE) - .with_context(|| format!("Truncated header for block {block_index}"))?; - let uncompressed_length = (&header[..4]).read_u32::()?; - let expected_checksum = (&header[4..]).read_u32::()?; - let stored_data = mmap - .get(block_start + BLOCK_HEADER_SIZE..block_end) - .with_context(|| format!("Truncated payload for block {block_index}"))?; - let actual_checksum = checksum_block(stored_data); + let mut reader: &[u8] = &content; + let uncompressed_length = reader.read_u32::()?; + let expected_checksum = reader.read_u32::()?; + verify_checksum( + reader, + expected_checksum, + &format!("blob file {}", path.display()), + )?; ensure!( - actual_checksum == expected_checksum, - "Checksum mismatch in block {block_index} of {sequence_number:08}.sst (expected \ - {expected_checksum:08x}, got {actual_checksum:08x})" + uncompressed_length > 0, + "Blob file {} has an invalid uncompressed length of zero", + path.display() ); - - let was_compressed = uncompressed_length > 0; - let data = if was_compressed { - let mut output = vec![0; uncompressed_length as usize]; - let written = match compression { - Compression::Lz4 => decompress(stored_data, &mut output) - .map_err(anyhow::Error::from) - .context("LZ4 decompression failed"), - Compression::Zstd3 => zstd::bulk::decompress_to_buffer(stored_data, &mut output) - .map_err(anyhow::Error::from) - .context("zstd decompression failed"), - } - .with_context(|| { - format!("Failed to decompress block {block_index} of {sequence_number:08}.sst") - })?; - ensure!( - written == uncompressed_length as usize, - "Decompressed block {block_index} of {sequence_number:08}.sst to {written} bytes, \ - expected {uncompressed_length}" - ); - output.into_boxed_slice() - } else { - Box::from(stored_data) - }; - - Ok(RawBlock { - actual_size: data.len() as u64, - stored_size: stored_data.len() as u64, - data, - was_compressed, - }) + decompress_into_arc(compression, uncompressed_length, reader) + .with_context(|| format!("Failed to decompress {}", path.display())) } -/// Parses an index block and returns all key-block indices. -pub fn parse_key_block_indices(index_block: &[u8]) -> Result> { - ensure!(index_block.len() >= 3, "Index block is too small"); +fn verify_checksum(data: &[u8], expected: u32, description: &str) -> Result<()> { + let actual = checksum_block(data); ensure!( - index_block[0] == BLOCK_TYPE_INDEX, - "Invalid index block type" + actual == expected, + "Checksum mismatch in {description} (expected {expected:08x}, got {actual:08x})" ); - let mut data = &index_block[1..]; - let first_block = data.read_u16::()?; - let mut indices = HashSet::from([first_block]); - const ENTRY_SIZE: usize = size_of::() + size_of::(); - let (entries, remainder) = data.as_chunks::(); - ensure!(remainder.is_empty(), "Index block has a truncated entry"); - for entry in entries { - indices.insert((&entry[size_of::()..]).read_u16::()?); - } - Ok(indices) -} - -/// Parsed key-block layout used by both offline tools. -#[derive(Clone, Copy)] -pub enum KeyBlockHeader { - Variable { - entry_count: u32, - hash_len: usize, - }, - Fixed { - entry_count: u32, - hash_len: usize, - key_size: usize, - value_type: u8, - }, - FixedMixedType { - entry_count: u32, - hash_len: usize, - key_size: usize, - stride: usize, - }, -} - -impl KeyBlockHeader { - pub fn entry_count(self) -> u32 { - match self { - Self::Variable { entry_count, .. } - | Self::Fixed { entry_count, .. } - | Self::FixedMixedType { entry_count, .. } => entry_count, - } - } -} - -/// Parses a key-block header. -pub fn parse_key_block_header(block: &[u8]) -> Result { - ensure!( - block.len() >= KEY_BLOCK_HEADER_SIZE, - "Key block is too small" - ); - let block_type = block[0]; - let entry_count = ((block[1] as u32) << 16) | ((block[2] as u32) << 8) | block[3] as u32; - let hash_len = match block_type { - BLOCK_TYPE_KEY_WITH_HASH | BLOCK_TYPE_FIXED_KEY_WITH_HASH => size_of::(), - BLOCK_TYPE_KEY_NO_HASH | BLOCK_TYPE_FIXED_KEY_NO_HASH => 0, - _ => bail!("Invalid key block type {block_type}"), - }; - match block_type { - BLOCK_TYPE_KEY_WITH_HASH | BLOCK_TYPE_KEY_NO_HASH => Ok(KeyBlockHeader::Variable { - entry_count, - hash_len, - }), - BLOCK_TYPE_FIXED_KEY_WITH_HASH | BLOCK_TYPE_FIXED_KEY_NO_HASH => { - ensure!(block.len() >= 6, "Fixed key block header is too small"); - let key_size = block[4] as usize; - if block[5] == FIXED_KEY_BLOCK_MIXED_VALUE_TYPE { - ensure!(block.len() >= 7, "Mixed key block header is too small"); - Ok(KeyBlockHeader::FixedMixedType { - entry_count, - hash_len, - key_size, - stride: hash_len + key_size + block[6] as usize + 1, - }) - } else { - Ok(KeyBlockHeader::Fixed { - entry_count, - hash_len, - key_size, - value_type: block[5], - }) - } - } - _ => unreachable!(), - } -} - -/// Returns the entry type bytes from a key block after validating its layout. -pub fn key_block_entry_types(header: KeyBlockHeader, block: &[u8]) -> Result> { - let count = header.entry_count() as usize; - match header { - KeyBlockHeader::Variable { .. } => { - let end = KEY_BLOCK_HEADER_SIZE + count * size_of::(); - let offsets = block - .get(KEY_BLOCK_HEADER_SIZE..end) - .context("Variable key block offset table is truncated")?; - Ok(offsets - .as_chunks::<4>() - .0 - .iter() - .map(|entry| entry[0]) - .collect()) - } - KeyBlockHeader::Fixed { - hash_len, - key_size, - value_type, - .. - } => { - let stride = hash_len + key_size + entry_value_size(value_type)?; - ensure!( - block.len() == 6 + count * stride, - "Fixed key block has an invalid length" - ); - Ok(vec![value_type; count]) - } - KeyBlockHeader::FixedMixedType { - hash_len, - key_size, - stride, - .. - } => { - ensure!( - block.len() == 7 + count * stride, - "Mixed key block has an invalid length" - ); - Ok((0..count) - .map(|index| block[7 + index * stride + hash_len + key_size]) - .collect()) - } - } -} - -fn entry_value_size(entry_type: u8) -> Result { - match entry_type { - KEY_BLOCK_ENTRY_TYPE_SMALL => Ok(SMALL_VALUE_REF_SIZE), - KEY_BLOCK_ENTRY_TYPE_MEDIUM => Ok(MEDIUM_VALUE_REF_SIZE), - KEY_BLOCK_ENTRY_TYPE_BLOB => Ok(BLOB_VALUE_REF_SIZE), - KEY_BLOCK_ENTRY_TYPE_KEY_DELETED => Ok(KEY_DELETED_REF_SIZE), - value if value >= KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN => { - Ok((value - KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN) as usize) - } - value if value >= KEY_BLOCK_ENTRY_TYPE_INLINE_MIN => { - Ok((value - KEY_BLOCK_ENTRY_TYPE_INLINE_MIN) as usize) - } - value => bail!("Invalid key block entry type {value}"), - } -} - -/// Returns the maximum stored key length in a parsed key block. -pub fn max_key_length(header: KeyBlockHeader, block: &[u8]) -> Result { - match header { - KeyBlockHeader::Fixed { key_size, .. } - | KeyBlockHeader::FixedMixedType { key_size, .. } => Ok(key_size), - KeyBlockHeader::Variable { - entry_count, - hash_len, - } => { - let entry_count = entry_count as usize; - let header_size = KEY_BLOCK_HEADER_SIZE + entry_count * size_of::(); - ensure!( - header_size <= block.len(), - "Variable key block header is truncated" - ); - let offsets = &block[KEY_BLOCK_HEADER_SIZE..header_size]; - let mut max_key = 0; - for index in 0..entry_count { - let word = (&offsets[index * 4..]).read_u32::()?; - let entry_type = (word >> 24) as u8; - let start = header_size + (word & 0x00ff_ffff) as usize; - let end = if index + 1 < entry_count { - let next = (&offsets[(index + 1) * 4..]).read_u32::()?; - header_size + (next & 0x00ff_ffff) as usize - } else { - block.len() - }; - let overhead = hash_len + entry_value_size(entry_type)?; - ensure!( - end >= start + overhead && end <= block.len(), - "Invalid entry bounds in variable key block" - ); - max_key = max_key.max(end - start - overhead); - } - Ok(max_key) - } - } -} - -/// Opens and mmaps an SST for offline analysis. -pub fn open_sst(db_path: &Path, info: &SstInfo) -> Result<(Mmap, u64, usize)> { - let path = db_path.join(format!("{:08}.sst", info.sequence_number)); - let file = File::open(&path).with_context(|| format!("Failed to open {}", path.display()))?; - let file_size = file.metadata()?.len(); - let mmap = unsafe { Mmap::map(file.file()) } - .with_context(|| format!("Failed to mmap {}", path.display()))?; - advise_mmap_for_persistence(&mmap)?; - let directory_size = info.block_count as usize * size_of::(); - ensure!( - mmap.len() >= directory_size, - "SST block directory is truncated" - ); - let block_offsets_start = mmap.len() - directory_size; - Ok((mmap, file_size, block_offsets_start)) + Ok(()) } #[cfg(test)] mod tests { use byteorder::{BE, WriteBytesExt}; - use super::{max_key_length, parse_key_block_header, parse_key_block_indices}; - use crate::static_sorted_file::{ - BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY_NO_HASH, KEY_BLOCK_ENTRY_TYPE_INLINE_MIN, - }; + use super::{decode_medium, read_blob}; + use crate::{Compression, checksum_block, compression::Compressor}; #[test] - fn parses_index_block_indices() { - let mut block = vec![BLOCK_TYPE_INDEX]; - block.write_u16::(3).unwrap(); - block.write_u64::(42).unwrap(); - block.write_u16::(7).unwrap(); - assert_eq!(parse_key_block_indices(&block).unwrap(), [3, 7].into()); + fn decodes_compressed_and_uncompressed_medium_values() -> anyhow::Result<()> { + let value = b"function component() { return null; }".repeat(100); + let mut compressed = Vec::new(); + Compressor::new(Compression::Zstd3)?.compress_into_buffer(&value, &mut compressed)?; + let decoded = decode_medium( + Compression::Zstd3, + value.len() as u32, + checksum_block(&compressed), + &compressed, + )?; + assert_eq!(decoded.as_ref(), value); + + let decoded = decode_medium(Compression::Zstd3, 0, checksum_block(&value), &value)?; + assert_eq!(decoded.as_ref(), value); + Ok(()) } #[test] - fn finds_maximum_variable_key_length() { - let mut block = vec![BLOCK_TYPE_KEY_NO_HASH, 0, 0, 2]; - block - .write_u32::((KEY_BLOCK_ENTRY_TYPE_INLINE_MIN as u32) << 24) - .unwrap(); - block - .write_u32::(((KEY_BLOCK_ENTRY_TYPE_INLINE_MIN as u32) << 24) | 3) - .unwrap(); - block.extend_from_slice(b"abc"); - block.extend_from_slice(b"a-much-longer-key"); - let header = parse_key_block_header(&block).unwrap(); - assert_eq!(max_key_length(header, &block).unwrap(), 17); - } + fn reads_blob_and_rejects_bad_checksum() -> anyhow::Result<()> { + let directory = tempfile::tempdir()?; + let value = b"blob data".repeat(100); + let compressed = zstd::bulk::compress(&value, 3)?; + let mut file = Vec::new(); + file.write_u32::(value.len() as u32)?; + file.write_u32::(checksum_block(&compressed))?; + file.extend_from_slice(&compressed); + fs_err::write(directory.path().join("00000001.blob"), &file)?; + assert_eq!( + read_blob(directory.path(), 1, Compression::Zstd3)?.as_ref(), + value + ); - #[test] - fn rejects_truncated_variable_key_table() { - let block = [BLOCK_TYPE_KEY_NO_HASH, 0, 0, 2, 0, 0, 0, 0]; - let header = parse_key_block_header(&block).unwrap(); - assert!(max_key_length(header, &block).is_err()); + file[4] ^= 1; + fs_err::write(directory.path().join("00000001.blob"), file)?; + assert!(read_blob(directory.path(), 1, Compression::Zstd3).is_err()); + Ok(()) } } diff --git a/turbopack/crates/turbo-persistence/src/static_sorted_file.rs b/turbopack/crates/turbo-persistence/src/static_sorted_file.rs index 9438ea64abf2..8ba5605488db 100644 --- a/turbopack/crates/turbo-persistence/src/static_sorted_file.rs +++ b/turbopack/crates/turbo-persistence/src/static_sorted_file.rs @@ -32,7 +32,7 @@ use crate::{ }; /// The block header for an index block. -pub const BLOCK_TYPE_INDEX: u8 = 0; +pub(crate) const BLOCK_TYPE_INDEX: u8 = 0; /// The block header for a key block with 8-byte hash per entry. pub const BLOCK_TYPE_KEY_WITH_HASH: u8 = 1; /// The block header for a key block without hash. Entries are ordered by key. diff --git a/turbopack/crates/turbo-tasks-backend/README.md b/turbopack/crates/turbo-tasks-backend/README.md new file mode 100644 index 000000000000..2f7fe5abb4ce --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/README.md @@ -0,0 +1,24 @@ +# turbo-tasks-backend + +## Training a TaskData zstd dictionary + +TaskData is persistence family `2`. After producing copied Turbopack cache database directories, +train and evaluate a dictionary without rebuilding the applications: + +```sh +cargo run -p turbo-persistence --bin zstd_dictionary -- train \ + --family 2 --output taskdata.zdict \ + path/to/database-a path/to/database-b + +cargo run -p turbo-persistence --bin zstd_dictionary -- evaluate \ + --family 2 --dictionary taskdata.zdict --json report.json \ + path/to/holdout-database-a path/to/holdout-database-b +``` + +Training and evaluation inputs should be disjoint. A dictionary evaluated against its own training +caches is useful only as a tool smoke test and overstates its real benefit. + +The intended corpus sources are the public application matrices in `vercel/next-benchmarks` and +`vercel-labs/next-npm-stability-tests`. Record the resolved revision for each corpus run and exclude +private Vercel projects rather than requiring credentials. Corpus collection and selecting or +embedding a production dictionary are separate follow-ups. From dd8f9d04f54374870d6ec1fdb6081f1432e851a7 Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:53 +0000 Subject: [PATCH 04/13] Tighten zstd dictionary sampling Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- .../src/bin/zstd_dictionary.rs | 90 ++++++++++--------- 1 file changed, 49 insertions(+), 41 deletions(-) diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs index 103090962bcc..642498005e30 100644 --- a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -219,7 +219,6 @@ struct CacheReport { path: PathBuf, family: u32, active_ssts: u64, - recorded_codecs: BTreeSet, samples: KindMetrics, duplicate_blob_references: u64, candidates: Vec, @@ -260,10 +259,8 @@ struct CacheSampleIter { path: PathBuf, pending: VecDeque, current: Option, - current_compression: Option, seen_blobs: HashSet, active_ssts: u64, - recorded_codecs: BTreeSet, duplicate_blob_references: u64, } @@ -277,19 +274,23 @@ impl CacheSampleIter { path.display() ) })?; + for sst in &ssts { + ensure!( + sst.compression == Compression::Zstd3, + "Cache {} family {family} SST {:08}.sst uses {:?}; zstd_dictionary requires Zstd3", + path.display(), + sst.sequence_number, + sst.compression + ); + } + // A stable SST order makes repeated runs against one unchanged cache snapshot comparable. ssts.sort_by_key(|sst| sst.sequence_number); - let recorded_codecs = ssts - .iter() - .map(|sst| format!("{:?}", sst.compression)) - .collect(); Ok(Self { path, active_ssts: ssts.len() as u64, pending: ssts.into(), current: None, - current_compression: None, seen_blobs: HashSet::new(), - recorded_codecs, duplicate_blob_references: 0, }) } @@ -305,11 +306,10 @@ impl CacheSampleIter { sequence_number: sst.sequence_number, block_count: sst.block_count, }, - sst.compression, + Compression::Zstd3, ) .with_context(|| format!("Failed to open {:08}.sst", sst.sequence_number))?, ); - self.current_compression = Some(sst.compression); Ok(true) } @@ -318,26 +318,20 @@ impl CacheSampleIter { if self.current.is_none() && !self.open_next_sst()? { return Ok(None); } - let compression = self.current_compression.expect("set with current SST"); let entry = match self.current.as_mut().unwrap().next() { Some(entry) => entry?, None => { self.current = None; - self.current_compression = None; continue; } }; - if let Some(sample) = self.sample_from_value(entry.value, compression)? { + if let Some(sample) = self.sample_from_value(entry.value)? { return Ok(Some(sample)); } } } - fn sample_from_value( - &mut self, - value: IterValue, - compression: Compression, - ) -> Result> { + fn sample_from_value(&mut self, value: IterValue) -> Result> { match value { IterValue::Slice { value } if value.len() > MAX_INLINE_VALUE_SIZE => Ok(Some(Sample { kind: SampleKind::Slice, @@ -348,7 +342,7 @@ impl CacheSampleIter { checksum, block, } => { - let value = decode_medium(compression, uncompressed_size, checksum, &block) + let value = decode_medium(Compression::Zstd3, uncompressed_size, checksum, &block) .with_context(|| { format!("Failed to read medium value in {}", self.path.display()) })?; @@ -362,12 +356,13 @@ impl CacheSampleIter { self.duplicate_blob_references += 1; return Ok(None); } - let value = read_blob(&self.path, sequence_number, compression)?; + let value = read_blob(&self.path, sequence_number, Compression::Zstd3)?; Ok(Some(Sample { kind: SampleKind::Blob, data: value, })) } + // Inline values live in key blocks and are not independently compressed. IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. } | IterValue::Slice { .. } => { Ok(None) } @@ -516,7 +511,6 @@ fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Res path: path.to_path_buf(), family, active_ssts: iter.active_ssts, - recorded_codecs: iter.recorded_codecs, samples, duplicate_blob_references: iter.duplicate_blob_references, candidates: results, @@ -555,6 +549,8 @@ struct TrainingSelection { samples: Vec>, caches: Vec, inputs_exhausted: bool, + #[cfg(test)] + selected_cache_indices: Vec, } fn select_training_samples( @@ -579,6 +575,8 @@ fn select_training_samples( let mut selected_bytes = 0_usize; let mut active = vec![true; iterators.len()]; let mut active_count = iterators.len(); + #[cfg(test)] + let mut selected_cache_indices = Vec::new(); while active_count > 0 && selected_bytes < byte_budget { for index in 0..iterators.len() { @@ -593,6 +591,8 @@ fn select_training_samples( .get_mut(sample.kind) .add(sample.data.len()); samples.push(Box::from(sample.data.as_ref())); + #[cfg(test)] + selected_cache_indices.push(index); if selected_bytes >= byte_budget { break; } @@ -608,6 +608,8 @@ fn select_training_samples( samples, caches: per_cache, inputs_exhausted: active_count == 0, + #[cfg(test)] + selected_cache_indices, }) } @@ -647,6 +649,7 @@ fn train(source: &Source, output: &Path) -> Result { samples, caches, inputs_exhausted, + .. } = select_training_samples(&source.caches, source.family, SAMPLE_BYTE_BUDGET)?; ensure!(!samples.is_empty(), "No eligible values found for training"); let selected = caches @@ -853,12 +856,13 @@ mod tests { #[test] fn round_robin_samples_multiple_caches() -> Result<()> { let first = make_cache(2, Compression::Zstd3, 20)?; - let second = make_cache(2, Compression::Lz4, 20)?; + let second = make_cache(2, Compression::Zstd3, 20)?; let paths = [first.path().to_path_buf(), second.path().to_path_buf()]; let selection = select_training_samples(&paths, 2, 10_000)?; assert!(!selection.samples.is_empty()); assert!(!selection.inputs_exhausted); assert_eq!(selection.caches.len(), 2); + assert_eq!(&selection.selected_cache_indices[..2], &[0, 1]); assert!( selection .caches @@ -868,6 +872,19 @@ mod tests { Ok(()) } + #[test] + fn rejects_non_zstd_families_before_sampling() -> Result<()> { + let cache = make_cache(2, Compression::Lz4, 20)?; + let error = CacheSampleIter::open(cache.path().to_path_buf(), 2) + .err() + .expect("LZ4 family must be rejected"); + let message = format!("{error:#}"); + assert!(message.contains("00000001.sst")); + assert!(message.contains("Lz4")); + assert!(message.contains("requires Zstd3")); + Ok(()) + } + #[test] fn insufficient_samples_fail_with_context() -> Result<()> { let cache = make_cache(2, Compression::Zstd3, 1)?; @@ -918,12 +935,9 @@ mod tests { let mut iter = CacheSampleIter::open(cache.path().to_path_buf(), 2)?; let sample = iter - .sample_from_value( - turbo_persistence::IterValue::Blob { - sequence_number: 42, - }, - Compression::Zstd3, - )? + .sample_from_value(turbo_persistence::IterValue::Blob { + sequence_number: 42, + })? .unwrap(); assert_eq!(sample.data.as_ref(), value); let mut candidates = make_candidates(&[])?; @@ -935,22 +949,16 @@ mod tests { results[0].by_kind.blob.raw_compressed_bytes + 8 ); assert!( - iter.sample_from_value( - turbo_persistence::IterValue::Blob { - sequence_number: 42, - }, - Compression::Zstd3, - )? + iter.sample_from_value(turbo_persistence::IterValue::Blob { + sequence_number: 42, + })? .is_none() ); assert_eq!(iter.duplicate_blob_references, 1); assert!( - iter.sample_from_value( - turbo_persistence::IterValue::Blob { - sequence_number: 43, - }, - Compression::Zstd3, - ) + iter.sample_from_value(turbo_persistence::IterValue::Blob { + sequence_number: 43, + }) .is_err() ); Ok(()) From 69586b8d0698608b9afddf9df4c61fad6745addb Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:53 +0000 Subject: [PATCH 05/13] Enable a baseline TaskData zstd dictionary Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- turbopack/crates/turbo-persistence/README.md | 12 +- .../crates/turbo-persistence/benches/mod.rs | 10 +- .../crates/turbo-persistence/src/arc_bytes.rs | 4 +- .../turbo-persistence/src/bin/sst_inspect.rs | 59 +++- .../src/bin/zstd_dictionary.rs | 275 ++++++++---------- .../turbo-persistence/src/compression.rs | 116 ++++++-- turbopack/crates/turbo-persistence/src/db.rs | 11 +- turbopack/crates/turbo-persistence/src/lib.rs | 6 +- .../crates/turbo-persistence/src/meta_file.rs | 30 +- .../src/meta_file_builder.rs | 9 +- .../crates/turbo-persistence/src/offline.rs | 22 +- .../crates/turbo-persistence/src/rc_bytes.rs | 4 +- .../turbo-persistence/src/shared_bytes.rs | 4 +- .../src/static_sorted_file.rs | 26 +- .../src/static_sorted_file_builder.rs | 46 +-- .../crates/turbo-persistence/src/tests.rs | 58 +++- .../crates/turbo-tasks-backend/README.md | 19 +- .../scripts/train-taskdata-dictionary.sh | 123 ++++++++ .../src/database/key_value_database.rs | 10 +- .../src/database/taskdata-dictionary.md | 36 +++ .../src/database/taskdata.zdict | Bin 0 -> 65536 bytes 21 files changed, 609 insertions(+), 271 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh create mode 100644 turbopack/crates/turbo-tasks-backend/src/database/taskdata-dictionary.md create mode 100644 turbopack/crates/turbo-tasks-backend/src/database/taskdata.zdict diff --git a/turbopack/crates/turbo-persistence/README.md b/turbopack/crates/turbo-persistence/README.md index 22a453494bad..0d0692d0007e 100644 --- a/turbopack/crates/turbo-persistence/README.md +++ b/turbopack/crates/turbo-persistence/README.md @@ -54,9 +54,10 @@ Small value blocks are emitted once they accumulate at least `MIN_SMALL_VALUE_BL A meta file can contain metadata about multiple SST files. The metadata is stored in a single file to avoid having too many small files. - Header - - 4 bytes magic number (0xFE4ADA4A) + - 4 bytes magic number (0xFE4ADA4B) - 4 bytes key family - 1 byte compression algorithm, which must match the configuration used to open the database + - 4 bytes zstd dictionary ID (zero when no dictionary is configured) - 4 bytes count of obsolete SST files - foreach obsolete SST file - 4 bytes sequence number of the obsolete SST file @@ -368,11 +369,11 @@ Configuration options for compactions are: copies without modifying them or running the application that created them: ```sh -cargo run -p turbo-persistence --bin zstd_dictionary -- train \ +cargo run -p turbo-persistence --release --bin zstd_dictionary -- train \ --family --output candidate.zdict \ path/to/database-a path/to/database-b -cargo run -p turbo-persistence --bin zstd_dictionary -- evaluate \ +cargo run -p turbo-persistence --release --bin zstd_dictionary -- evaluate \ --family --dictionary candidate.zdict --json report.json \ path/to/database-a path/to/database-b ``` @@ -381,9 +382,10 @@ Training produces a 64 KiB dictionary from up to approximately 64 MiB of samples hash-ordered logical value from each cache in turn, so one large cache cannot monopolize the sample. The output path is replaced atomically. -The no-dictionary zstd level 3 baseline is always included during evaluation. The tool follows +The no-dictionary zstd level 3 baseline is always included during evaluation. Pass +`--source-dictionary ` when the input caches were written with a dictionary. The tool follows `CURRENT`, deletion files, and meta-file supersession, and uses `StaticSortedFileIter` to read slice, -medium, and blob values. Checksums and decompressed lengths are verified. +medium, and blob values. Checksums, dictionary IDs, and decompressed lengths are verified. Small values are grouped into physical blocks in production, so the report's per-value 12.5% minimum-savings calculation is a comparative estimate, not exact SST-size modeling. Blob estimates diff --git a/turbopack/crates/turbo-persistence/benches/mod.rs b/turbopack/crates/turbo-persistence/benches/mod.rs index 1f5e839e9bcc..6d671f6cd8da 100644 --- a/turbopack/crates/turbo-persistence/benches/mod.rs +++ b/turbopack/crates/turbo-persistence/benches/mod.rs @@ -622,7 +622,7 @@ fn prefill_multi_value_database( family_configs: [FamilyConfig { name: "test", kind: FamilyKind::MultiValue, - compression: Compression::Lz4, + compression: Compression::Lz4.into(), }], ..TpDbConfig::new() }; @@ -698,7 +698,7 @@ fn open_multi_value_db(path: &Path) -> TurboPersistence { family_configs: [FamilyConfig { name: "test", kind: FamilyKind::MultiValue, - compression: Compression::Lz4, + compression: Compression::Lz4.into(), }], ..TpDbConfig::new() }; @@ -968,7 +968,7 @@ fn bench_write_multi_value(c: &mut Criterion) { family_configs: [FamilyConfig { name: "test", kind: FamilyKind::MultiValue, - compression: Compression::Lz4, + compression: Compression::Lz4.into(), }], ..TpDbConfig::new() }; @@ -1211,7 +1211,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) { &entries, &sst_path, MetaEntryFlags::FRESH, - Compression::Lz4, + Compression::Lz4.into(), ) .unwrap(); @@ -1223,7 +1223,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) { let sst = StaticSortedFile::open( tempdir.path(), sst_meta, - Compression::Lz4, + Compression::Lz4.into(), turbo_persistence::AccessMode::Mmap, ) .unwrap(); diff --git a/turbopack/crates/turbo-persistence/src/arc_bytes.rs b/turbopack/crates/turbo-persistence/src/arc_bytes.rs index 09e189e9a307..bd7dbd1b916b 100644 --- a/turbopack/crates/turbo-persistence/src/arc_bytes.rs +++ b/turbopack/crates/turbo-persistence/src/arc_bytes.rs @@ -9,7 +9,7 @@ use std::{ use memmap2::Mmap; use crate::{ - Compression, + CompressionConfig, compression::decompress_into_arc, shared_bytes::{SharedBytes, is_subslice_of}, }; @@ -146,7 +146,7 @@ impl SharedBytes for ArcBytes { } fn from_decompressed( - compression: Compression, + compression: CompressionConfig, uncompressed_length: u32, block: &[u8], ) -> anyhow::Result { diff --git a/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs b/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs index caa0b9ed555d..7014d0e7e96b 100644 --- a/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs +++ b/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs @@ -20,7 +20,7 @@ use fs_err::File; use lzzzz::lz4::decompress; use memmap2::Mmap; use turbo_persistence::{ - BLOCK_HEADER_SIZE, Compression, MAX_INLINE_VALUE_SIZE, checksum_block, + BLOCK_HEADER_SIZE, Compression, CompressionConfig, MAX_INLINE_VALUE_SIZE, checksum_block, mmap_helper::advise_mmap_for_persistence, offline::{SstInfo, collect_sst_info}, static_sorted_file::{ @@ -225,7 +225,7 @@ fn read_block( block_offsets_start: usize, block_index: u16, sequence_number: u32, - compression: Compression, + compression: CompressionConfig, ) -> Result { let offset = block_offsets_start + block_index as usize * size_of::(); @@ -266,11 +266,18 @@ fn read_block( let data = if was_compressed { let mut buffer = vec![0u8; uncompressed_length as usize]; let bytes_written = match compression { - Compression::Lz4 => { + CompressionConfig::Lz4 => { decompress(compressed_data, &mut buffer).context("LZ4 decompression failed")? } - Compression::Zstd3 => zstd::bulk::decompress_to_buffer(compressed_data, &mut buffer) - .context("zstd decompression failed")?, + CompressionConfig::Zstd3 => { + zstd::bulk::decompress_to_buffer(compressed_data, &mut buffer) + .context("zstd decompression failed")? + } + CompressionConfig::Zstd3WithDictionary(dictionary) => { + zstd::bulk::Decompressor::with_dictionary(dictionary)? + .decompress_to_buffer(compressed_data, &mut buffer) + .context("zstd dictionary decompression failed")? + } }; assert_eq!( bytes_written, uncompressed_length as usize, @@ -398,8 +405,11 @@ fn iter_key_block_entry_types( } /// Analyze an SST file and return entry type statistics -fn analyze_sst_file(db_path: &Path, info: &SstInfo) -> Result { - let compression = info.compression; +fn analyze_sst_file( + db_path: &Path, + info: &SstInfo, + compression: CompressionConfig, +) -> Result { let filename = format!("{:08}.sst", info.sequence_number); let path = db_path.join(&filename); @@ -788,11 +798,18 @@ fn main() -> Result<()> { // Parse arguments let mut db_path: Option = None; let mut verbose = false; + let mut source_dictionary: Option = None; let mut i = 1; while i < args.len() { match args[i].as_str() { "--verbose" | "-v" => verbose = true, + "--source-dictionary" => { + i += 1; + source_dictionary = Some(PathBuf::from( + args.get(i).context("--source-dictionary requires a path")?, + )); + } arg if !arg.starts_with('-') => { if db_path.is_none() { db_path = Some(PathBuf::from(arg)); @@ -815,6 +832,7 @@ fn main() -> Result<()> { eprintln!(); eprintln!("Options:"); eprintln!(" -v, --verbose Show per-SST file details (default: family totals only)"); + eprintln!(" --source-dictionary Dictionary used by zstd input caches"); eprintln!(); eprintln!("Entry types:"); eprintln!( @@ -851,6 +869,13 @@ fn main() -> Result<()> { bail!("Not a directory: {}", db_path.display()); } + let source_dictionary = source_dictionary + .map(|path| { + fs_err::read(&path).with_context(|| format!("Failed to read {}", path.display())) + }) + .transpose()? + .map(|bytes| Box::leak(bytes.into_boxed_slice()) as &'static [u8]); + // Collect SST info grouped by family let family_sst_info = collect_sst_info(&db_path)?; @@ -867,7 +892,25 @@ fn main() -> Result<()> { let mut sst_stats_list: Vec<(u32, SstStats)> = Vec::new(); for info in sst_list { - match analyze_sst_file(&db_path, info) { + let compression = match (info.compression, info.dictionary_id, source_dictionary) { + (Compression::Lz4, 0, _) => CompressionConfig::Lz4, + (Compression::Zstd3, 0, _) => CompressionConfig::Zstd3, + (Compression::Zstd3, id, Some(dictionary)) + if Some(id) + == CompressionConfig::Zstd3WithDictionary(dictionary).dictionary_id() => + { + CompressionConfig::Zstd3WithDictionary(dictionary) + } + (_, id, _) => { + eprintln!( + "Warning: Missing or wrong source dictionary for {:08}.sst (dictionary ID \ + {id})", + info.sequence_number + ); + continue; + } + }; + match analyze_sst_file(&db_path, info, compression) { Ok(stats) => { family_stats.merge(&stats); if verbose { diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs index 642498005e30..2ad0887486c0 100644 --- a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -2,7 +2,7 @@ use std::{ collections::{BTreeSet, HashSet, VecDeque}, - fs::{self, File, OpenOptions}, + fs::{self, File}, io::{BufWriter, Write}, path::{Path, PathBuf}, sync::Arc, @@ -13,7 +13,8 @@ use anyhow::{Context, Result, ensure}; use clap::{Args, Parser, Subcommand}; use serde::Serialize; use turbo_persistence::{ - Compression, IterValue, MAX_INLINE_VALUE_SIZE, StaticSortedFileIter, StaticSortedFileMetaData, + Compression, CompressionConfig, IterValue, MAX_INLINE_VALUE_SIZE, StaticSortedFileIter, + StaticSortedFileMetaData, offline::{SstInfo, collect_sst_info, decode_medium, read_blob}, }; use xxhash_rust::xxh3::xxh3_64; @@ -62,6 +63,9 @@ struct Source { /// Persistence family ID to inspect. #[arg(long)] family: u32, + /// Dictionary used to compress the input caches. + #[arg(long)] + source_dictionary: Option, /// Database directories containing CURRENT, meta, SST, and blob files. #[arg(required = true)] caches: Vec, @@ -113,36 +117,6 @@ impl Metric { } } -#[derive(Default, Clone, Serialize)] -struct KindMetrics { - slice: Metric, - medium: Metric, - blob: Metric, -} - -impl KindMetrics { - fn get_mut(&mut self, kind: SampleKind) -> &mut Metric { - match kind { - SampleKind::Slice => &mut self.slice, - SampleKind::Medium => &mut self.medium, - SampleKind::Blob => &mut self.blob, - } - } - - fn merge(&mut self, other: &Self) { - self.slice.merge(&other.slice); - self.medium.merge(&other.medium); - self.blob.merge(&other.blob); - } - - fn total(&self) -> Metric { - Metric { - count: self.slice.count + self.medium.count + self.blob.count, - bytes: self.slice.bytes + self.medium.bytes + self.blob.bytes, - } - } -} - #[derive(Default, Clone, Serialize)] struct CompressionMetric { input_bytes: u64, @@ -168,48 +142,9 @@ impl CompressionMetric { } } -#[derive(Default, Clone, Serialize)] -struct CompressionByKind { - slice: CompressionMetric, - medium: CompressionMetric, - blob: CompressionMetric, -} - -impl CompressionByKind { - fn get_mut(&mut self, kind: SampleKind) -> &mut CompressionMetric { - match kind { - SampleKind::Slice => &mut self.slice, - SampleKind::Medium => &mut self.medium, - SampleKind::Blob => &mut self.blob, - } - } - - fn merge(&mut self, other: &Self) { - self.slice.merge(&other.slice); - self.medium.merge(&other.medium); - self.blob.merge(&other.blob); - } - - fn finalize(&mut self) { - self.slice.finalize(); - self.medium.finalize(); - self.blob.finalize(); - } - - fn total(&self) -> CompressionMetric { - let mut total = CompressionMetric::default(); - total.merge(&self.slice); - total.merge(&self.medium); - total.merge(&self.blob); - total.finalize(); - total - } -} - #[derive(Clone, Serialize)] struct CandidateResult { dictionary: DictionaryInfo, - by_kind: CompressionByKind, combined: CompressionMetric, setup_ns: u64, } @@ -219,7 +154,7 @@ struct CacheReport { path: PathBuf, family: u32, active_ssts: u64, - samples: KindMetrics, + samples: Metric, duplicate_blob_references: u64, candidates: Vec, } @@ -231,14 +166,14 @@ struct EvaluationReport { timing_note: &'static str, threshold_note: &'static str, caches: Vec, - combined_samples: KindMetrics, + combined_samples: Metric, combined_candidates: Vec, } #[derive(Serialize)] struct TrainingCacheReport { path: PathBuf, - samples: KindMetrics, + samples: Metric, } #[derive(Serialize)] @@ -248,7 +183,7 @@ struct TrainingReport { zstd_version: &'static str, dictionary_size: usize, sample_byte_target: usize, - selected: KindMetrics, + selected: Metric, selected_bytes: u64, inputs_exhausted: bool, caches: Vec, @@ -259,13 +194,14 @@ struct CacheSampleIter { path: PathBuf, pending: VecDeque, current: Option, + compression: CompressionConfig, seen_blobs: HashSet, active_ssts: u64, duplicate_blob_references: u64, } impl CacheSampleIter { - fn open(path: PathBuf, family: u32) -> Result { + fn open(path: PathBuf, family: u32, compression: CompressionConfig) -> Result { let mut families = collect_sst_info(&path) .with_context(|| format!("Failed to inspect cache {}", path.display()))?; let mut ssts = families.remove(&family).with_context(|| { @@ -276,11 +212,15 @@ impl CacheSampleIter { })?; for sst in &ssts { ensure!( - sst.compression == Compression::Zstd3, - "Cache {} family {family} SST {:08}.sst uses {:?}; zstd_dictionary requires Zstd3", + sst.compression == Compression::Zstd3 + && sst.dictionary_id == compression.dictionary_id().unwrap_or(0), + "Cache {} family {family} SST {:08}.sst uses {:?} dictionary {}, but source \ + config uses {:?}", path.display(), sst.sequence_number, - sst.compression + sst.compression, + sst.dictionary_id, + compression ); } // A stable SST order makes repeated runs against one unchanged cache snapshot comparable. @@ -290,6 +230,7 @@ impl CacheSampleIter { active_ssts: ssts.len() as u64, pending: ssts.into(), current: None, + compression, seen_blobs: HashSet::new(), duplicate_blob_references: 0, }) @@ -306,7 +247,7 @@ impl CacheSampleIter { sequence_number: sst.sequence_number, block_count: sst.block_count, }, - Compression::Zstd3, + self.compression, ) .with_context(|| format!("Failed to open {:08}.sst", sst.sequence_number))?, ); @@ -342,7 +283,7 @@ impl CacheSampleIter { checksum, block, } => { - let value = decode_medium(Compression::Zstd3, uncompressed_size, checksum, &block) + let value = decode_medium(self.compression, uncompressed_size, checksum, &block) .with_context(|| { format!("Failed to read medium value in {}", self.path.display()) })?; @@ -356,7 +297,7 @@ impl CacheSampleIter { self.duplicate_blob_references += 1; return Ok(None); } - let value = read_blob(&self.path, sequence_number, Compression::Zstd3)?; + let value = read_blob(&self.path, sequence_number, self.compression)?; Ok(Some(Sample { kind: SampleKind::Blob, data: value, @@ -453,7 +394,7 @@ fn evaluate_sample( candidate.info.name ); - let metric = result.by_kind.get_mut(sample.kind); + let metric = &mut result.combined; metric.input_bytes += sample.data.len() as u64; metric.raw_compressed_bytes += compressed.len() as u64; metric.encode_ns += encode_ns; @@ -484,7 +425,6 @@ fn empty_results(candidates: &[Candidate]) -> Vec { .iter() .map(|candidate| CandidateResult { dictionary: candidate.info.clone(), - by_kind: CompressionByKind::default(), combined: CompressionMetric::default(), setup_ns: candidate.setup_ns, }) @@ -493,17 +433,21 @@ fn empty_results(candidates: &[Candidate]) -> Vec { fn finalize_results(results: &mut [CandidateResult]) { for result in results { - result.by_kind.finalize(); - result.combined = result.by_kind.total(); + result.combined.finalize(); } } -fn evaluate_cache(path: &Path, family: u32, candidates: &mut [Candidate]) -> Result { - let mut iter = CacheSampleIter::open(path.to_path_buf(), family)?; - let mut samples = KindMetrics::default(); +fn evaluate_cache( + path: &Path, + family: u32, + compression: CompressionConfig, + candidates: &mut [Candidate], +) -> Result { + let mut iter = CacheSampleIter::open(path.to_path_buf(), family, compression)?; + let mut samples = Metric::default(); let mut results = empty_results(candidates); while let Some(sample) = iter.next_sample()? { - samples.get_mut(sample.kind).add(sample.data.len()); + samples.add(sample.data.len()); evaluate_sample(&sample, candidates, &mut results)?; } finalize_results(&mut results); @@ -522,12 +466,12 @@ fn combine_evaluation( caches: Vec, candidates: &[Candidate], ) -> EvaluationReport { - let mut combined_samples = KindMetrics::default(); + let mut combined_samples = Metric::default(); let mut combined_candidates = empty_results(candidates); for cache in &caches { combined_samples.merge(&cache.samples); for (combined, current) in combined_candidates.iter_mut().zip(&cache.candidates) { - combined.by_kind.merge(¤t.by_kind); + combined.combined.merge(¤t.combined); } } finalize_results(&mut combined_candidates); @@ -556,19 +500,20 @@ struct TrainingSelection { fn select_training_samples( paths: &[PathBuf], family: u32, + compression: CompressionConfig, byte_budget: usize, ) -> Result { let mut paths = paths.to_vec(); paths.sort(); let mut iterators = paths .into_iter() - .map(|path| CacheSampleIter::open(path, family)) + .map(|path| CacheSampleIter::open(path, family, compression)) .collect::>>()?; let mut per_cache = iterators .iter() .map(|iter| TrainingCacheReport { path: iter.path.clone(), - samples: KindMetrics::default(), + samples: Metric::default(), }) .collect::>(); let mut samples = Vec::new(); @@ -586,10 +531,7 @@ fn select_training_samples( match iterators[index].next_sample()? { Some(sample) => { selected_bytes += sample.data.len(); - per_cache[index] - .samples - .get_mut(sample.kind) - .add(sample.data.len()); + per_cache[index].samples.add(sample.data.len()); samples.push(Box::from(sample.data.as_ref())); #[cfg(test)] selected_cache_indices.push(index); @@ -613,52 +555,44 @@ fn select_training_samples( }) } -fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> { - let parent = path.parent().unwrap_or_else(|| Path::new(".")); - fs::create_dir_all(parent).with_context(|| format!("Failed to create {}", parent.display()))?; - let filename = path - .file_name() - .and_then(|name| name.to_str()) - .context("Output filename must be UTF-8")?; - let temporary = parent.join(format!(".{filename}.{}.tmp", std::process::id())); - let result = (|| -> Result<()> { - let mut file = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(&temporary)?; - file.write_all(bytes)?; - file.sync_all()?; - fs::rename(&temporary, path).with_context(|| { - format!( - "Failed to replace {} with {}", - path.display(), - temporary.display() - ) - })?; - Ok(()) - })(); - if result.is_err() { - let _ = fs::remove_file(&temporary); +fn write_dictionary(path: &Path, bytes: &[u8]) -> Result<()> { + let mut file = File::create(path) + .with_context(|| format!("Failed to create dictionary {}", path.display()))?; + file.write_all(bytes) + .with_context(|| format!("Failed to write dictionary {}", path.display())) +} + +fn source_compression(source: &Source) -> Result { + match &source.source_dictionary { + Some(path) => { + let dictionary = fs::read(path) + .with_context(|| format!("Failed to read source dictionary {}", path.display()))?; + let dictionary = Box::leak(dictionary.into_boxed_slice()); + Ok(CompressionConfig::Zstd3WithDictionary(dictionary)) + } + None => Ok(CompressionConfig::Zstd3), } - result } fn train(source: &Source, output: &Path) -> Result { + let compression = source_compression(source)?; let TrainingSelection { samples, caches, inputs_exhausted, .. - } = select_training_samples(&source.caches, source.family, SAMPLE_BYTE_BUDGET)?; + } = select_training_samples( + &source.caches, + source.family, + compression, + SAMPLE_BYTE_BUDGET, + )?; ensure!(!samples.is_empty(), "No eligible values found for training"); - let selected = caches - .iter() - .fold(KindMetrics::default(), |mut total, cache| { - total.merge(&cache.samples); - total - }); - let selected_bytes = selected.total().bytes; + let selected = caches.iter().fold(Metric::default(), |mut total, cache| { + total.merge(&cache.samples); + total + }); + let selected_bytes = selected.bytes; let dictionary = zstd::dict::from_samples(&samples, DICTIONARY_SIZE).with_context(|| { format!( "Failed to train a {DICTIONARY_SIZE}-byte dictionary from {} values ({selected_bytes} \ @@ -666,7 +600,7 @@ fn train(source: &Source, output: &Path) -> Result { samples.len() ) })?; - write_atomic(output, &dictionary)?; + write_dictionary(output, &dictionary)?; Ok(TrainingReport { schema_version: SCHEMA_VERSION, family: source.family, @@ -689,24 +623,23 @@ fn print_training(report: &TrainingReport) { report.dictionary.bytes, report.dictionary.dictionary_id, report.dictionary.xxh3_64.as_deref().unwrap_or("none"), - report.selected.total().count, + report.selected.count, report.selected_bytes, report.sample_byte_target, report.inputs_exhausted, ); for cache in &report.caches { - let total = cache.samples.total(); println!( " {}: {} values / {} bytes", cache.path.display(), - total.count, - total.bytes + cache.samples.count, + cache.samples.bytes ); } } fn print_evaluation(report: &EvaluationReport) { - let samples = report.combined_samples.total(); + let samples = &report.combined_samples; println!( "Evaluated family {}: {} caches, {} logical values / {} bytes", report.family, @@ -759,10 +692,13 @@ fn run(cli: Cli) -> Result<()> { json, } => { let mut candidates = make_candidates(&dictionary)?; + let source_compression = source_compression(&source)?; let caches = source .caches .iter() - .map(|path| evaluate_cache(path, source.family, &mut candidates)) + .map(|path| { + evaluate_cache(path, source.family, source_compression, &mut candidates) + }) .collect::>>()?; let report = combine_evaluation(source.family, caches, &candidates); print_evaluation(&report); @@ -786,7 +722,9 @@ mod tests { use byteorder::{BE, WriteBytesExt}; use clap::Parser; use tempfile::TempDir; - use turbo_persistence::{Compression, DbConfig, SerialScheduler, TurboPersistence}; + use turbo_persistence::{ + Compression, CompressionConfig, DbConfig, SerialScheduler, TurboPersistence, + }; use super::{ CacheSampleIter, Cli, DICTIONARY_SIZE, SAMPLE_BYTE_BUDGET, Source, empty_results, @@ -795,6 +733,14 @@ mod tests { }; fn make_cache(family: usize, compression: Compression, values: usize) -> Result { + make_cache_with_config(family, compression.into(), values) + } + + fn make_cache_with_config( + family: usize, + compression: CompressionConfig, + values: usize, + ) -> Result { let tempdir = tempfile::tempdir()?; let mut config = DbConfig::<8>::default(); config.family_configs[family].compression = compression; @@ -858,7 +804,7 @@ mod tests { let first = make_cache(2, Compression::Zstd3, 20)?; let second = make_cache(2, Compression::Zstd3, 20)?; let paths = [first.path().to_path_buf(), second.path().to_path_buf()]; - let selection = select_training_samples(&paths, 2, 10_000)?; + let selection = select_training_samples(&paths, 2, CompressionConfig::Zstd3, 10_000)?; assert!(!selection.samples.is_empty()); assert!(!selection.inputs_exhausted); assert_eq!(selection.caches.len(), 2); @@ -867,7 +813,7 @@ mod tests { selection .caches .iter() - .all(|report| report.samples.total().count > 0) + .all(|report| report.samples.count > 0) ); Ok(()) } @@ -875,13 +821,35 @@ mod tests { #[test] fn rejects_non_zstd_families_before_sampling() -> Result<()> { let cache = make_cache(2, Compression::Lz4, 20)?; - let error = CacheSampleIter::open(cache.path().to_path_buf(), 2) + let error = CacheSampleIter::open(cache.path().to_path_buf(), 2, CompressionConfig::Zstd3) .err() .expect("LZ4 family must be rejected"); let message = format!("{error:#}"); assert!(message.contains("00000001.sst")); assert!(message.contains("Lz4")); - assert!(message.contains("requires Zstd3")); + assert!(message.contains("source config uses Zstd3")); + Ok(()) + } + + #[test] + fn source_dictionary_opens_dictionary_cache_and_plain_config_rejects_it() -> Result<()> { + let samples = (0..100) + .map(|index| format!("export function Component{index}() {{ return null }}")) + .collect::>(); + let dictionary = zstd::dict::from_samples(&samples, 1024)?; + let dictionary = Box::leak(dictionary.into_boxed_slice()); + let cache = + make_cache_with_config(2, CompressionConfig::Zstd3WithDictionary(dictionary), 20)?; + assert!( + CacheSampleIter::open(cache.path().to_path_buf(), 2, CompressionConfig::Zstd3,) + .is_err() + ); + let mut iter = CacheSampleIter::open( + cache.path().to_path_buf(), + 2, + CompressionConfig::Zstd3WithDictionary(dictionary), + )?; + assert!(iter.next_sample()?.is_some()); Ok(()) } @@ -891,6 +859,7 @@ mod tests { let output_dir = tempfile::tempdir()?; let source = Source { family: 2, + source_dictionary: None, caches: vec![cache.path().to_path_buf()], }; let error = train(&source, &output_dir.path().join("dictionary.zdict")) @@ -908,6 +877,7 @@ mod tests { fs::write(&output, b"old")?; let source = Source { family: 2, + source_dictionary: None, caches: vec![cache.path().to_path_buf()], }; let report = train(&source, &output)?; @@ -915,10 +885,10 @@ mod tests { assert_ne!(fs::read(&output)?, b"old"); let mut candidates = make_candidates(&[output])?; - let evaluation = evaluate_cache(cache.path(), 2, &mut candidates)?; + let evaluation = + evaluate_cache(cache.path(), 2, CompressionConfig::Zstd3, &mut candidates)?; assert_eq!(evaluation.candidates.len(), 2); - assert!(evaluation.samples.slice.count > 0); - assert!(evaluation.samples.medium.count > 0); + assert!(evaluation.samples.count > 0); Ok(()) } @@ -933,7 +903,8 @@ mod tests { blob.extend_from_slice(&compressed); fs::write(cache.path().join("00000042.blob"), blob)?; - let mut iter = CacheSampleIter::open(cache.path().to_path_buf(), 2)?; + let mut iter = + CacheSampleIter::open(cache.path().to_path_buf(), 2, CompressionConfig::Zstd3)?; let sample = iter .sample_from_value(turbo_persistence::IterValue::Blob { sequence_number: 42, @@ -945,8 +916,8 @@ mod tests { evaluate_sample(&sample, &mut candidates, &mut results)?; finalize_results(&mut results); assert_eq!( - results[0].by_kind.blob.estimated_stored_bytes, - results[0].by_kind.blob.raw_compressed_bytes + 8 + results[0].combined.estimated_stored_bytes, + results[0].combined.raw_compressed_bytes + 8 ); assert!( iter.sample_from_value(turbo_persistence::IterValue::Blob { diff --git a/turbopack/crates/turbo-persistence/src/compression.rs b/turbopack/crates/turbo-persistence/src/compression.rs index 69d645f23d61..15859588b690 100644 --- a/turbopack/crates/turbo-persistence/src/compression.rs +++ b/turbopack/crates/turbo-persistence/src/compression.rs @@ -14,17 +14,57 @@ pub enum Compression { Zstd3 = 1, } +/// Runtime compression configuration for a persistence family. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum CompressionConfig { + #[default] + Lz4, + Zstd3, + Zstd3WithDictionary(&'static [u8]), +} + +impl CompressionConfig { + pub fn algorithm(self) -> Compression { + match self { + Self::Lz4 => Compression::Lz4, + Self::Zstd3 | Self::Zstd3WithDictionary(_) => Compression::Zstd3, + } + } + + pub fn dictionary(self) -> Option<&'static [u8]> { + match self { + Self::Zstd3WithDictionary(dictionary) => Some(dictionary), + Self::Lz4 | Self::Zstd3 => None, + } + } + + pub fn dictionary_id(self) -> Option { + self.dictionary() + .and_then(zstd::zstd_safe::get_dict_id_from_dict) + .map(|id| id.get()) + } +} + +impl From for CompressionConfig { + fn from(value: Compression) -> Self { + match value { + Compression::Lz4 => Self::Lz4, + Compression::Zstd3 => Self::Zstd3, + } + } +} + thread_local! { /// Zstd decompression contexts are reusable and relatively expensive to create. Keep one per /// worker thread to avoid allocation on every block read without a global lock. - static ZSTD_DECOMPRESSOR: RefCell> = RefCell::new( - zstd::bulk::Decompressor::new().expect("zstd decompressor initialization should succeed") + static ZSTD_DECOMPRESSOR: RefCell<(Option, zstd::bulk::Decompressor<'static>)> = RefCell::new( + (None, zstd::bulk::Decompressor::new().expect("zstd decompressor initialization should succeed")) ); } /// Decompresses `block` into `dest`, verifying the output length matches `expected_len`. fn decompress_block( - compression: Compression, + compression: CompressionConfig, block: &[u8], dest: &mut [u8], expected_len: u32, @@ -35,12 +75,25 @@ fn decompress_block( zero-copy mmap path" ); let bytes_written = match compression { - Compression::Lz4 => decompress(block, dest).map_err(anyhow::Error::from), - Compression::Zstd3 => ZSTD_DECOMPRESSOR.with_borrow_mut(|decompressor| { - decompressor - .decompress_to_buffer(block, dest) - .map_err(anyhow::Error::from) - }), + CompressionConfig::Lz4 => decompress(block, dest).map_err(anyhow::Error::from), + CompressionConfig::Zstd3 | CompressionConfig::Zstd3WithDictionary(_) => ZSTD_DECOMPRESSOR + .with_borrow_mut(|state| { + let dictionary = compression.dictionary().unwrap_or_default(); + let key = compression + .dictionary() + .map(|dictionary| dictionary.as_ptr() as usize); + if state.0 != key { + state + .1 + .set_dictionary(dictionary) + .map_err(anyhow::Error::from)?; + state.0 = key; + } + state + .1 + .decompress_to_buffer(block, dest) + .map_err(anyhow::Error::from) + }), } .with_context(|| { format!( @@ -63,7 +116,7 @@ fn decompress_block( /// The caller must ensure `uncompressed_length > 0` (i.e., the block is actually compressed). /// Uncompressed blocks should be handled via zero-copy mmap slices before calling this. pub(crate) fn decompress_into_arc( - compression: Compression, + compression: CompressionConfig, uncompressed_length: u32, block: &[u8], ) -> Result> { @@ -81,7 +134,7 @@ pub(crate) fn decompress_into_arc( /// Like [`decompress_into_arc`] but returns an `Rc<[u8]>` for thread-local use. pub(crate) fn decompress_into_rc( - compression: Compression, + compression: CompressionConfig, uncompressed_length: u32, block: &[u8], ) -> Result> { @@ -101,17 +154,21 @@ pub fn checksum_block(data: &[u8]) -> u32 { /// Reusable compressor for a stream of blocks using the same family configuration. pub(crate) struct Compressor { - compression: Compression, + compression: CompressionConfig, zstd: Option>, } impl Compressor { - pub(crate) fn new(compression: Compression) -> Result { + pub(crate) fn new(compression: CompressionConfig) -> Result { let zstd = match compression { - Compression::Zstd3 => { + CompressionConfig::Zstd3 => { Some(zstd::bulk::Compressor::new(3).context("Failed to create zstd compressor")?) } - Compression::Lz4 => None, + CompressionConfig::Zstd3WithDictionary(dictionary) => Some( + zstd::bulk::Compressor::with_dictionary(3, dictionary) + .context("Failed to create zstd dictionary compressor")?, + ), + CompressionConfig::Lz4 => None, }; Ok(Self { compression, zstd }) } @@ -123,11 +180,11 @@ impl Compressor { buffer: &mut Vec, ) -> Result<()> { match self.compression { - Compression::Lz4 => { + CompressionConfig::Lz4 => { lz4::compress_to_vec(block, buffer, lz4::ACC_LEVEL_DEFAULT) .context("LZ4 compression failed")?; } - Compression::Zstd3 => { + CompressionConfig::Zstd3 | CompressionConfig::Zstd3WithDictionary(_) => { buffer.reserve(zstd::zstd_safe::compress_bound(block.len())); self.zstd .as_mut() @@ -144,10 +201,33 @@ impl Compressor { mod tests { use super::*; + #[test] + fn dictionary_compression_round_trips() { + let samples = (0..100) + .map(|index| format!("export default function Component{index}() {{ return null }}")) + .collect::>(); + let dictionary = zstd::dict::from_samples(&samples, 1024).unwrap(); + let dictionary = Box::leak(dictionary.into_boxed_slice()); + let config = CompressionConfig::Zstd3WithDictionary(dictionary); + assert_eq!(config.algorithm(), Compression::Zstd3); + assert!(config.dictionary_id().is_some()); + let input = samples.concat(); + let mut compressor = Compressor::new(config).unwrap(); + let mut compressed = Vec::new(); + compressor + .compress_into_buffer(input.as_bytes(), &mut compressed) + .unwrap(); + let output = decompress_into_arc(config, input.len() as u32, &compressed).unwrap(); + assert_eq!(&*output, input.as_bytes()); + assert!( + decompress_into_arc(CompressionConfig::Zstd3, input.len() as u32, &compressed).is_err() + ); + } + #[test] fn compression_round_trips() { let input = b"turbo persistence compression ".repeat(1024); - for compression in [Compression::Lz4, Compression::Zstd3] { + for compression in [CompressionConfig::Lz4, CompressionConfig::Zstd3] { let mut compressor = Compressor::new(compression).unwrap(); let mut compressed = Vec::new(); compressor diff --git a/turbopack/crates/turbo-persistence/src/db.rs b/turbopack/crates/turbo-persistence/src/db.rs index 72d5a3a3acf3..b4ba2e8f2f9b 100644 --- a/turbopack/crates/turbo-persistence/src/db.rs +++ b/turbopack/crates/turbo-persistence/src/db.rs @@ -30,7 +30,7 @@ use tracing::span::EnteredSpan; pub use crate::compaction::selector::CompactConfig; use crate::{ - AccessMode, DbConfig, FamilyKind, QueryKey, + AccessMode, CompressionConfig, DbConfig, FamilyKind, QueryKey, arc_bytes::ArcBytes, compaction::selector::{Compactable, get_merge_segments}, compression::{Compression, checksum_block, decompress_into_arc}, @@ -659,7 +659,7 @@ impl TurboPersistence /// Reads and decompresses a blob file. This is not backed by any cache. #[tracing::instrument(level = "info", name = "reading database blob", skip_all)] - fn read_blob(&self, seq: u32, compression: Compression) -> Result { + fn read_blob(&self, seq: u32, compression: CompressionConfig) -> Result { let path = self.path.join(format!("{seq:08}.blob")); let file = File::open(&path)?; let data: Either> = match self.config.access_mode { @@ -1636,7 +1636,7 @@ impl TurboPersistence /// used set). writer: Option<(u32, StreamingSstWriter)>, flags: MetaEntryFlags, - compression: Compression, + compression: CompressionConfig, new_sst_files: Vec<(u32, File, StaticSortedFileBuilderMeta<'static>)>, /// Hash of the last key added. Used to ensure we only split @@ -1644,7 +1644,10 @@ impl TurboPersistence last_hash: Option, } impl Collector { - fn new(flags: MetaEntryFlags, compression: Compression) -> Self { + fn new( + flags: MetaEntryFlags, + compression: CompressionConfig, + ) -> Self { Self { writer: None, flags, diff --git a/turbopack/crates/turbo-persistence/src/lib.rs b/turbopack/crates/turbo-persistence/src/lib.rs index c0255a0f4d79..97e2629ebf0d 100644 --- a/turbopack/crates/turbo-persistence/src/lib.rs +++ b/turbopack/crates/turbo-persistence/src/lib.rs @@ -32,7 +32,7 @@ mod write_batch; mod tests; pub use arc_bytes::ArcBytes; -pub use compression::{Compression, checksum_block}; +pub use compression::{Compression, CompressionConfig, checksum_block}; pub use db::{ CommitStats, CompactConfig, CurrentDbVersion, MetaFileEntryInfo, MetaFileInfo, TurboPersistence, read_current_version, @@ -66,7 +66,7 @@ pub enum FamilyKind { pub struct FamilyConfig { pub name: &'static str, pub kind: FamilyKind, - pub compression: Compression, + pub compression: CompressionConfig, } /// Database-wide configuration with per-family storage settings. @@ -104,7 +104,7 @@ impl DbConfig { family_configs: [FamilyConfig { name: "unknown", kind: FamilyKind::SingleValue, - compression: Compression::Lz4, + compression: CompressionConfig::Lz4, }; FAMILIES], access_mode: access_mode_env_var(), } diff --git a/turbopack/crates/turbo-persistence/src/meta_file.rs b/turbopack/crates/turbo-persistence/src/meta_file.rs index 0a288ceb284f..c126c00c9950 100644 --- a/turbopack/crates/turbo-persistence/src/meta_file.rs +++ b/turbopack/crates/turbo-persistence/src/meta_file.rs @@ -15,7 +15,7 @@ use smallvec::SmallVec; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Ref, big_endian as be}; use crate::{ - AccessMode, Compression, FamilyConfig, QueryKey, + AccessMode, Compression, CompressionConfig, FamilyConfig, QueryKey, lookup_entry::LookupValue, mmap_helper::advise_mmap_for_persistence, static_sorted_file::{BlockCache, SstLookupResult, StaticSortedFile, StaticSortedFileMetaData}, @@ -51,7 +51,7 @@ impl Display for MetaEntryFlags { } /// Magic number identifying a `.meta` file. -pub(crate) const META_FILE_MAGIC: u32 = 0xFE4ADA4A; +pub(crate) const META_FILE_MAGIC: u32 = 0xFE4ADA4B; /// On-disk layout of a single entry header in the `.meta` file. /// @@ -119,7 +119,7 @@ pub struct MetaEntry { /// The `'static` lifetime is transmuted — the actual borrow is from `MetaFile::backing`. amqf: qfilter::FilterRef<'static>, /// Compression recorded in this entry's meta file. - compression: Compression, + compression: CompressionConfig, /// The static sorted file that is lazily loaded sst: OnceLock, } @@ -268,8 +268,10 @@ pub struct MetaFile { sequence_number: u32, /// The key family of the SST files in this meta file. family: u32, - /// Compression recorded for this family. + /// Compression algorithm recorded for this family. compression: Compression, + /// Zstd dictionary ID recorded for this family, or zero without a dictionary. + dictionary_id: u32, /// The entries of the file. Dropped before `backing` (field declaration order). entries: Vec, /// The entries that have been marked as obsolete. @@ -341,17 +343,22 @@ impl MetaFile { value if value == Compression::Zstd3 as u8 => Compression::Zstd3, value => bail!("Invalid compression algorithm {value}"), }; - if let Some(configs) = family_configs { + let dictionary_id = reader.read_u32::()?; + let compression_config = if let Some(configs) = family_configs { let configured = configs .get(family as usize) .with_context(|| format!("No configuration for family {family}"))? .compression; ensure!( - compression == configured, + compression == configured.algorithm() + && dictionary_id == configured.dictionary_id().unwrap_or(0), "Compression configuration mismatch for family {family}: meta file uses \ - {compression:?}, runtime config uses {configured:?}" + {compression:?} dictionary {dictionary_id}, runtime config uses {configured:?}" ); - } + configured + } else { + CompressionConfig::from(compression) + }; let obsolete_count = reader.read_u32::()?; let mut obsolete_sst_files = Vec::with_capacity(obsolete_count as usize); for _ in 0..obsolete_count { @@ -409,7 +416,7 @@ impl MetaFile { flags, amqf_data_offset: start_of_amqf_data_offset..end_of_amqf_data_offset, amqf, - compression, + compression: compression_config, sst: OnceLock::new(), }); start_of_amqf_data_offset = end_of_amqf_data_offset; @@ -423,6 +430,7 @@ impl MetaFile { sequence_number, family, compression, + dictionary_id, entries, obsolete_entries: Vec::new(), obsolete_sst_files, @@ -458,6 +466,10 @@ impl MetaFile { self.compression } + pub fn dictionary_id(&self) -> u32 { + self.dictionary_id + } + /// The on-disk size of this meta file in bytes (the length of its memory map). pub fn byte_size(&self) -> u64 { self.backing.len() as u64 diff --git a/turbopack/crates/turbo-persistence/src/meta_file_builder.rs b/turbopack/crates/turbo-persistence/src/meta_file_builder.rs index 0b4dadd26292..3d74ba1f721d 100644 --- a/turbopack/crates/turbo-persistence/src/meta_file_builder.rs +++ b/turbopack/crates/turbo-persistence/src/meta_file_builder.rs @@ -10,14 +10,14 @@ use qfilter::Filter; use zerocopy::IntoBytes; use crate::{ - Compression, + CompressionConfig, meta_file::{EntryHeader, META_FILE_MAGIC}, static_sorted_file_builder::StaticSortedFileBuilderMeta, }; pub struct MetaFileBuilder<'a> { family: u32, - compression: Compression, + compression: CompressionConfig, /// Entries in the meta file, tuples of (sequence_number, StaticSortedFileBuilderMetaResult) entries: Vec<(u32, StaticSortedFileBuilderMeta<'a>)>, /// Obsolete SST files, represented by their sequence numbers @@ -27,7 +27,7 @@ pub struct MetaFileBuilder<'a> { } impl<'a> MetaFileBuilder<'a> { - pub fn new(family: u32, compression: Compression) -> Self { + pub fn new(family: u32, compression: CompressionConfig) -> Self { Self { family, compression, @@ -62,7 +62,8 @@ impl<'a> MetaFileBuilder<'a> { let mut file = CountingWriter::new(BufWriter::new(File::create(file)?)); file.write_u32::(META_FILE_MAGIC)?; // Magic number file.write_u32::(self.family)?; - file.write_u8(self.compression as u8)?; + file.write_u8(self.compression.algorithm() as u8)?; + file.write_u32::(self.compression.dictionary_id().unwrap_or(0))?; self.obsolete_sst_files.sort(); file.write_u32::(self.obsolete_sst_files.len() as u32)?; diff --git a/turbopack/crates/turbo-persistence/src/offline.rs b/turbopack/crates/turbo-persistence/src/offline.rs index a519e05947c7..f8dbbda4b826 100644 --- a/turbopack/crates/turbo-persistence/src/offline.rs +++ b/turbopack/crates/turbo-persistence/src/offline.rs @@ -11,8 +11,8 @@ use byteorder::{BE, ReadBytesExt}; use fs_err as fs; use crate::{ - Compression, checksum_block, compression::decompress_into_arc, meta_file::MetaFile, - read_current_version, sst_filter::SstFilter, + Compression, CompressionConfig, checksum_block, compression::decompress_into_arc, + meta_file::MetaFile, read_current_version, sst_filter::SstFilter, }; /// Information about an active SST recorded by a meta file. @@ -21,6 +21,7 @@ pub struct SstInfo { pub sequence_number: u32, pub block_count: u16, pub compression: Compression, + pub dictionary_id: u32, } /// Collects active SSTs by family, mirroring database open logic. @@ -83,6 +84,7 @@ pub fn collect_sst_info(db_path: &Path) -> Result>> { sequence_number: entry.sequence_number(), block_count: entry.block_count(), compression: meta.compression(), + dictionary_id: meta.dictionary_id(), }); } } @@ -91,7 +93,7 @@ pub fn collect_sst_info(db_path: &Path) -> Result>> { /// Verifies and reconstructs a raw medium-value block from an SST iterator. pub fn decode_medium( - compression: Compression, + compression: CompressionConfig, uncompressed_length: u32, expected_checksum: u32, stored: &[u8], @@ -109,7 +111,7 @@ pub fn decode_medium( pub fn read_blob( db_path: &Path, sequence_number: u32, - compression: Compression, + compression: CompressionConfig, ) -> Result> { let path = db_path.join(format!("{sequence_number:08}.blob")); let content = fs::read(&path).with_context(|| format!("Failed to read {}", path.display()))?; @@ -149,22 +151,22 @@ mod tests { use byteorder::{BE, WriteBytesExt}; use super::{decode_medium, read_blob}; - use crate::{Compression, checksum_block, compression::Compressor}; + use crate::{CompressionConfig, checksum_block, compression::Compressor}; #[test] fn decodes_compressed_and_uncompressed_medium_values() -> anyhow::Result<()> { let value = b"function component() { return null; }".repeat(100); let mut compressed = Vec::new(); - Compressor::new(Compression::Zstd3)?.compress_into_buffer(&value, &mut compressed)?; + Compressor::new(CompressionConfig::Zstd3)?.compress_into_buffer(&value, &mut compressed)?; let decoded = decode_medium( - Compression::Zstd3, + CompressionConfig::Zstd3, value.len() as u32, checksum_block(&compressed), &compressed, )?; assert_eq!(decoded.as_ref(), value); - let decoded = decode_medium(Compression::Zstd3, 0, checksum_block(&value), &value)?; + let decoded = decode_medium(CompressionConfig::Zstd3, 0, checksum_block(&value), &value)?; assert_eq!(decoded.as_ref(), value); Ok(()) } @@ -180,13 +182,13 @@ mod tests { file.extend_from_slice(&compressed); fs_err::write(directory.path().join("00000001.blob"), &file)?; assert_eq!( - read_blob(directory.path(), 1, Compression::Zstd3)?.as_ref(), + read_blob(directory.path(), 1, CompressionConfig::Zstd3)?.as_ref(), value ); file[4] ^= 1; fs_err::write(directory.path().join("00000001.blob"), file)?; - assert!(read_blob(directory.path(), 1, Compression::Zstd3).is_err()); + assert!(read_blob(directory.path(), 1, CompressionConfig::Zstd3).is_err()); Ok(()) } } diff --git a/turbopack/crates/turbo-persistence/src/rc_bytes.rs b/turbopack/crates/turbo-persistence/src/rc_bytes.rs index c4c1007cab32..e5d9d019769e 100644 --- a/turbopack/crates/turbo-persistence/src/rc_bytes.rs +++ b/turbopack/crates/turbo-persistence/src/rc_bytes.rs @@ -9,7 +9,7 @@ use std::{ use memmap2::Mmap; use crate::{ - Compression, + CompressionConfig, compression::decompress_into_rc, shared_bytes::{SharedBytes, is_subslice_of}, }; @@ -126,7 +126,7 @@ impl SharedBytes for RcBytes { } fn from_decompressed( - compression: Compression, + compression: CompressionConfig, uncompressed_length: u32, block: &[u8], ) -> anyhow::Result { diff --git a/turbopack/crates/turbo-persistence/src/shared_bytes.rs b/turbopack/crates/turbo-persistence/src/shared_bytes.rs index 6dc0f8407d8f..8fab799fcb1e 100644 --- a/turbopack/crates/turbo-persistence/src/shared_bytes.rs +++ b/turbopack/crates/turbo-persistence/src/shared_bytes.rs @@ -2,7 +2,7 @@ use std::ops::{Deref, Range}; use memmap2::Mmap; -use crate::Compression; +use crate::CompressionConfig; /// Trait abstracting over `ArcBytes` and `RcBytes`. /// @@ -39,7 +39,7 @@ pub trait SharedBytes: Clone + Deref + Sized { /// Creates an instance from a decompressed block. fn from_decompressed( - compression: Compression, + compression: CompressionConfig, uncompressed_length: u32, block: &[u8], ) -> anyhow::Result; diff --git a/turbopack/crates/turbo-persistence/src/static_sorted_file.rs b/turbopack/crates/turbo-persistence/src/static_sorted_file.rs index 8ba5605488db..57691ba33f06 100644 --- a/turbopack/crates/turbo-persistence/src/static_sorted_file.rs +++ b/turbopack/crates/turbo-persistence/src/static_sorted_file.rs @@ -19,7 +19,7 @@ use rustc_hash::FxHasher; use smallvec::SmallVec; use crate::{ - AccessMode, Compression, QueryKey, + AccessMode, CompressionConfig, QueryKey, arc_bytes::ArcBytes, be, compression::checksum_block, @@ -201,7 +201,7 @@ trait ValueBlockCache { self, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result; } @@ -218,7 +218,7 @@ impl ValueBlockCache for ArcBlockCacheReader<'_> { self, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result { get_or_cache_block( self.backing, @@ -251,7 +251,7 @@ impl ValueBlockCache for RcBlockCacheReader<'_> { self, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result { if let Some((idx, block)) = self.cache.as_ref() && *idx == block_index @@ -307,7 +307,7 @@ pub struct StaticSortedFile { /// bitmap the CRC would be re-computed on every access. `Relaxed` ordering /// suffices: racing first-time verifications are idempotent. verified_blocks: Box<[AtomicU64]>, - compression: Compression, + compression: CompressionConfig, } impl StaticSortedFile { @@ -315,7 +315,7 @@ impl StaticSortedFile { pub fn open( db_path: &Path, meta: StaticSortedFileMetaData, - compression: Compression, + compression: CompressionConfig, access_mode: AccessMode, ) -> Result { let filename = format!("{:08}.sst", meta.sequence_number); @@ -631,7 +631,7 @@ fn get_or_cache_block( block_index: u16, cache: &BlockCache, verified_blocks: &[AtomicU64], - compression: Compression, + compression: CompressionConfig, ) -> Result { let mmap_block = if let StaticSortedFileBacking::Mmap(mmap) = backing { let (uncompressed_length, checksum, block_data) = @@ -882,7 +882,7 @@ fn read_block_lookup( backing: &StaticSortedFileBacking, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result { let (uncompressed_length, checksum, block) = get_raw_block(backing, meta, block_index)?; verify_checksum(meta, &block, checksum, block_index)?; @@ -982,7 +982,7 @@ fn handle_key_match_generic( ty: u8, val: &[u8], key_block: &B, - compression: Compression, + compression: CompressionConfig, reader: impl ValueBlockCache, ) -> Result> { Ok(match ty { @@ -1048,7 +1048,7 @@ pub struct StaticSortedFileIter { /// value blocks sequentially and don't revisit earlier blocks, so caching /// just the current one avoids redundant decompression. value_block_cache: Option<(u16, RcBytes)>, - compression: Compression, + compression: CompressionConfig, } enum CurrentKeyBlockKind { @@ -1120,7 +1120,7 @@ impl StaticSortedFileIter { pub fn open( db_path: &Path, meta: StaticSortedFileMetaData, - compression: Compression, + compression: CompressionConfig, access_mode: AccessMode, ) -> Result { let filename = format!("{:08}.sst", meta.sequence_number); @@ -1166,7 +1166,7 @@ impl StaticSortedFileIter { fn new( backing: StaticSortedFileIterBacking, meta: StaticSortedFileMetaData, - compression: Compression, + compression: CompressionConfig, ) -> Result { let root_block_index = meta.block_count - 1; let block = read_block_iter(&backing, &meta, root_block_index, compression)?; @@ -1206,7 +1206,7 @@ impl StaticSortedFileIter { backing: &StaticSortedFileIterBacking, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result { let block = read_block_iter(backing, meta, block_index, compression)?; let data = &*block; diff --git a/turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs b/turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs index c112788ed0f5..e79c7f0d58cf 100644 --- a/turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs +++ b/turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs @@ -10,7 +10,7 @@ use byteorder::{BE, ByteOrder, WriteBytesExt}; use fs_err::File; use crate::{ - Compression, + CompressionConfig, compression::{Compressor, checksum_block}, constants::{MAX_INLINE_VALUE_SIZE, MAX_SMALL_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE}, meta_file::MetaEntryFlags, @@ -340,7 +340,7 @@ pub fn write_static_stored_file( entries: &[E], file: &Path, flags: MetaEntryFlags, - compression: Compression, + compression: CompressionConfig, ) -> Result<(StaticSortedFileBuilderMeta<'static>, File)> { debug_assert!(entries.iter().map(|e| e.key_hash()).is_sorted()); let mut writer = StreamingSstWriter::new(file, flags, entries.len() as u64, compression)?; @@ -616,7 +616,7 @@ impl StreamingSstWriter { file: &Path, flags: MetaEntryFlags, max_entry_count: u64, - compression: Compression, + compression: CompressionConfig, ) -> Result { let file = BufWriter::new(File::create(file)?); let compressor = Compressor::new(compression)?; @@ -1508,7 +1508,7 @@ mod tests { sequence_number: seq, block_count: meta.block_count, }, - Compression::Lz4, + CompressionConfig::Lz4, AccessMode::Mmap, ) } @@ -1521,8 +1521,12 @@ mod tests { flags: MetaEntryFlags, ) -> Result> { let sst_path = dir.join(format!("{seq:08}.sst")); - let mut writer = - StreamingSstWriter::new(&sst_path, flags, entries.len() as u64, Compression::Lz4)?; + let mut writer = StreamingSstWriter::new( + &sst_path, + flags, + entries.len() as u64, + CompressionConfig::Lz4, + )?; for entry in entries { writer.add(entry)?; } @@ -1757,9 +1761,13 @@ mod tests { fn is_full_entry_count_limit() { let dir = tempfile::tempdir().unwrap(); let sst_path = dir.path().join("test.sst"); - let mut writer = - StreamingSstWriter::new(&sst_path, MetaEntryFlags::default(), 100, Compression::Lz4) - .unwrap(); + let mut writer = StreamingSstWriter::new( + &sst_path, + MetaEntryFlags::default(), + 100, + CompressionConfig::Lz4, + ) + .unwrap(); let max_entries = 50; for i in 0..max_entries { @@ -1784,9 +1792,13 @@ mod tests { fn is_full_data_size_limit() { let dir = tempfile::tempdir().unwrap(); let sst_path = dir.path().join("test.sst"); - let mut writer = - StreamingSstWriter::new(&sst_path, MetaEntryFlags::default(), 100, Compression::Lz4) - .unwrap(); + let mut writer = StreamingSstWriter::new( + &sst_path, + MetaEntryFlags::default(), + 100, + CompressionConfig::Lz4, + ) + .unwrap(); let value = vec![0u8; 1000]; for i in 0..10 { @@ -1826,7 +1838,7 @@ mod tests { &entries, &batch_path, MetaEntryFlags::default(), - Compression::Lz4, + CompressionConfig::Lz4, )?; // Write via streaming API @@ -1835,7 +1847,7 @@ mod tests { &streaming_path, MetaEntryFlags::default(), entries.len() as u64, - Compression::Lz4, + CompressionConfig::Lz4, )?; for entry in &entries { writer.add(entry)?; @@ -1855,7 +1867,7 @@ mod tests { sequence_number: 1, block_count: meta1.block_count, }, - Compression::Lz4, + CompressionConfig::Lz4, AccessMode::Mmap, )?; let sst2 = StaticSortedFile::open( @@ -1864,7 +1876,7 @@ mod tests { sequence_number: 2, block_count: meta2.block_count, }, - Compression::Lz4, + CompressionConfig::Lz4, AccessMode::Mmap, )?; let kc = make_cache(); @@ -1924,7 +1936,7 @@ mod tests { &sst_path, MetaEntryFlags::default(), 0, - Compression::Lz4, + CompressionConfig::Lz4, ) .unwrap(); writer.close().unwrap(); diff --git a/turbopack/crates/turbo-persistence/src/tests.rs b/turbopack/crates/turbo-persistence/src/tests.rs index a3f9c6595de4..f68581eaa322 100644 --- a/turbopack/crates/turbo-persistence/src/tests.rs +++ b/turbopack/crates/turbo-persistence/src/tests.rs @@ -5,7 +5,8 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; use rstest::rstest; use crate::{ - AccessMode, Compression, DbConfig, FamilyConfig, FamilyKind, + AccessMode, Compression, CompressionConfig, DbConfig, FamilyConfig, FamilyKind, + SerialScheduler, constants::{MAX_INLINE_VALUE_SIZE, MAX_MEDIUM_VALUE_SIZE, MAX_SMALL_VALUE_SIZE}, db::{CompactConfig, TurboPersistence, read_current_version}, lookup_entry::IterValue, @@ -1054,7 +1055,7 @@ fn batch_get_different_sizes(#[case] mmap: bool) -> Result<()> { let path = tempdir.path(); let mut config = config_with_mmap(mmap); - config.family_configs[0].compression = Compression::Zstd3; + config.family_configs[0].compression = Compression::Zstd3.into(); let db = open_db_with_config::<16>(path, config)?; // Write values of different sizes @@ -1109,8 +1110,8 @@ fn batch_get_across_families(#[case] mmap: bool) -> Result<()> { let path = tempdir.path(); let mut config = config_with_mmap(mmap); - // Set zstd on an arbitrary family; lz4 is used by default. - config.family_configs[2].compression = Compression::Zstd3; + // Set zstd on an arbitrary family; LZ4 is used by default. + config.family_configs[2].compression = Compression::Zstd3.into(); let db = open_db_with_config::<16>(path, config.clone())?; // Write compressible values to multiple families so every configured codec is exercised. @@ -1188,7 +1189,7 @@ fn batch_get_after_compaction(#[case] mmap: bool) -> Result<()> { let path = tempdir.path(); let mut config = config_with_mmap(mmap); - config.family_configs[0].compression = Compression::Zstd3; + config.family_configs[0].compression = Compression::Zstd3.into(); let db = open_db_with_config::<16>(path, config)?; // Write data across multiple batches to create multiple SST files @@ -1561,7 +1562,7 @@ fn multi_value_config() -> DbConfig<1> { config.family_configs[0] = FamilyConfig { name: "test", kind: FamilyKind::MultiValue, - compression: Compression::Lz4, + compression: Compression::Lz4.into(), }; config } @@ -2456,7 +2457,12 @@ fn count_tombstones( sequence_number: entry.sequence_number, block_count: entry.block_count, }; - for item in StaticSortedFileIter::open(path, sst, Compression::Lz4, AccessMode::Mmap)? { + for item in StaticSortedFileIter::open( + path, + sst, + Compression::Lz4.into(), + AccessMode::Mmap, + )? { if matches!( item?.value, IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. } @@ -2812,3 +2818,41 @@ fn valued_tombstone_rejects_single_value_families() -> Result<()> { db.shutdown()?; Ok(()) } + +#[test] +fn dictionary_config_round_trips_and_rejects_mismatch() -> Result<()> { + let samples = (0..100) + .map(|index| format!("export default function Component{index}() {{ return null }}")) + .collect::>(); + let dictionary = zstd::dict::from_samples(&samples, 1024)?; + let dictionary = Box::leak(dictionary.into_boxed_slice()); + let tempdir = tempfile::tempdir()?; + let mut config = DbConfig::<1>::default(); + config.family_configs[0].compression = CompressionConfig::Zstd3WithDictionary(dictionary); + let db = TurboPersistence::::open_with_config( + tempdir.path().to_path_buf(), + config.clone(), + )?; + let batch = db.write_batch()?; + batch.put(0, b"key".to_vec(), samples.concat().into_bytes().into())?; + db.commit_write_batch(batch)?; + db.shutdown()?; + + let db = TurboPersistence::::open_with_config( + tempdir.path().to_path_buf(), + config, + )?; + assert!(db.get(0, &b"key".to_vec())?.is_some()); + db.shutdown()?; + + let mut plain_zstd = DbConfig::<1>::default(); + plain_zstd.family_configs[0].compression = CompressionConfig::Zstd3; + let error = TurboPersistence::::open_with_config( + tempdir.path().to_path_buf(), + plain_zstd, + ) + .err() + .expect("plain zstd config must reject dictionary metadata"); + assert!(format!("{error:#}").contains("Compression configuration mismatch")); + Ok(()) +} diff --git a/turbopack/crates/turbo-tasks-backend/README.md b/turbopack/crates/turbo-tasks-backend/README.md index 2f7fe5abb4ce..ce746fc5d533 100644 --- a/turbopack/crates/turbo-tasks-backend/README.md +++ b/turbopack/crates/turbo-tasks-backend/README.md @@ -6,19 +6,26 @@ TaskData is persistence family `2`. After producing copied Turbopack cache datab train and evaluate a dictionary without rebuilding the applications: ```sh -cargo run -p turbo-persistence --bin zstd_dictionary -- train \ +cargo run -p turbo-persistence --release --bin zstd_dictionary -- train \ --family 2 --output taskdata.zdict \ path/to/database-a path/to/database-b -cargo run -p turbo-persistence --bin zstd_dictionary -- evaluate \ +cargo run -p turbo-persistence --release --bin zstd_dictionary -- evaluate \ --family 2 --dictionary taskdata.zdict --json report.json \ path/to/holdout-database-a path/to/holdout-database-b + +# For caches produced after the dictionary was enabled: +cargo run -p turbo-persistence --release --bin zstd_dictionary -- evaluate \ + --family 2 --source-dictionary src/database/taskdata.zdict \ + --dictionary candidate.zdict path/to/database-a ``` Training and evaluation inputs should be disjoint. A dictionary evaluated against its own training caches is useful only as a tool smoke test and overstates its real benefit. -The intended corpus sources are the public application matrices in `vercel/next-benchmarks` and -`vercel-labs/next-npm-stability-tests`. Record the resolved revision for each corpus run and exclude -private Vercel projects rather than requiring credentials. Corpus collection and selecting or -embedding a production dictionary are separate follow-ups. +The checked-in baseline is produced by +[`scripts/train-taskdata-dictionary.sh`](./scripts/train-taskdata-dictionary.sh) from preserved +`test/production` filesystem caches. Its corpus and held-out receipts are recorded in +[`src/database/taskdata-dictionary.md`](./src/database/taskdata-dictionary.md). The script is +resumable; set `CORPUS_JOBS` for bounded parallelism and pass an output directory plus dictionary +path. diff --git a/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh b/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh new file mode 100644 index 000000000000..cc80ff8ebbd7 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +set -u -o pipefail + +repo_root=$(git rev-parse --show-toplevel) +output_root=${1:-/tmp/taskdata-dictionary-corpus} +dictionary=${2:-$repo_root/turbopack/crates/turbo-tasks-backend/src/database/taskdata.zdict} +shift $(( $# >= 2 ? 2 : $# )) +jobs=${CORPUS_JOBS:-4} + +mkdir -p "$output_root"/{train,holdout,logs,tmp,reports,state} +manifest="$output_root/manifest.tsv" + +if (( $# )); then + tests=("$@") +else + mapfile -t tests < <(find "$repo_root/test/production" -name '*.test.ts' -print | sort) +fi +if [[ -n ${CORPUS_LIMIT:-} ]]; then + tests=("${tests[@]:0:$CORPUS_LIMIT}") +fi + +digest_for() { + printf '%s' "$1" | sha256sum | cut -d' ' -f1 +} + +# Migrate output from the earlier sequential collector once so interrupted runs can resume. +if [[ -f $manifest ]] && ! compgen -G "$output_root/state/*.tsv" >/dev/null; then + while IFS=$'\t' read -r test_path split status cache; do + [[ $test_path == test || -z $test_path ]] && continue + digest=$(digest_for "$test_path") + printf '%s\t%s\t%s\t%s\n' "$test_path" "$split" "$status" "$cache" \ + >> "$output_root/state/$digest.tsv" + done < "$manifest" +fi +rm -f "$manifest" + +run_case() { + local test_path=$1 relative digest split case_root log status cache_index current cache destination state_tmp + relative=${test_path#"$repo_root/"} + digest=$(digest_for "$relative") + [[ -f $output_root/state/$digest.tsv ]] && return 0 + if (( 16#${digest:0:2} % 5 == 0 )); then split=holdout; else split=train; fi + case_root="$output_root/tmp/$digest" + rm -rf "$case_root" + mkdir -p "$case_root" + log="$output_root/logs/$digest.log" + echo "[$split] $relative" + ( + cd "$repo_root" + TMPDIR="$case_root" NEXT_TEST_SKIP_CLEANUP=1 \ + pnpm test-start-turbo "$relative" + ) >"$log" 2>&1 + status=$? + cache_index=0 + state_tmp="$output_root/state/$digest.tmp" + : > "$state_tmp" + while IFS= read -r current; do + cache=$(dirname "$current") + compgen -G "$cache/*.meta" >/dev/null || continue + compgen -G "$cache/*.sst" >/dev/null || continue + destination="$output_root/$split/${digest}-${cache_index}" + rm -rf "$destination" + cp -a "$cache" "$destination" + printf '%s\t%s\t%s\t%s\n' "$relative" "$split" "$status" "$destination" >> "$state_tmp" + cache_index=$((cache_index + 1)) + done < <(find "$case_root" -type f -name CURRENT -path '*/.next/cache/turbopack/*' | sort) + if (( cache_index == 0 )); then + printf '%s\t%s\t%s\t\n' "$relative" "$split" "$status" >> "$state_tmp" + fi + mv "$state_tmp" "$output_root/state/$digest.tsv" + rm -rf "$case_root" +} + +for test_path in "${tests[@]}"; do + run_case "$test_path" & + while (( $(jobs -rp | wc -l) >= jobs )); do + wait -n || true + done +done +wait + +printf 'test\tsplit\tstatus\tcache\n' > "$manifest" +while IFS= read -r state; do + cat "$state" >> "$manifest" +done < <(find "$output_root/state" -name '*.tsv' | sort) + +mapfile -t train_caches < <( + for cache in "$output_root"/train/*; do + [[ -d $cache ]] || continue + compgen -G "$cache/*.meta" >/dev/null && compgen -G "$cache/*.sst" >/dev/null && printf '%s\n' "$cache" + done | sort +) +mapfile -t holdout_caches < <( + for cache in "$output_root"/holdout/*; do + [[ -d $cache ]] || continue + compgen -G "$cache/*.meta" >/dev/null && compgen -G "$cache/*.sst" >/dev/null && printf '%s\n' "$cache" + done | sort +) +if (( ${#train_caches[@]} == 0 || ${#holdout_caches[@]} == 0 )); then + echo "Need at least one train and one holdout cache; see $manifest" >&2 + exit 1 +fi + +cd "$repo_root" +source_args=() +source_dictionary=${SOURCE_DICTIONARY:-$dictionary} +if [[ -f $source_dictionary ]]; then + source_copy="$output_root/source-dictionary.zdict" + cp "$source_dictionary" "$source_copy" + source_args=(--source-dictionary "$source_copy") +fi +cargo run -p turbo-persistence --release --bin zstd_dictionary -- train \ + --family 2 "${source_args[@]}" --output "$dictionary" "${train_caches[@]}" +for run in 1 2 3 4 5; do + cargo run -p turbo-persistence --release --bin zstd_dictionary -- evaluate \ + --family 2 "${source_args[@]}" --dictionary "$dictionary" \ + --json "$output_root/reports/holdout-$run.json" \ + "${holdout_caches[@]}" +done + +echo "Dictionary: $dictionary" +echo "Manifest: $manifest" +echo "Reports: $output_root/reports" diff --git a/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs b/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs index 926374691b61..df88d0fd4bae 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs @@ -1,4 +1,4 @@ -use turbo_persistence::{Compression, FamilyConfig, FamilyKind}; +use turbo_persistence::{CompressionConfig, FamilyConfig, FamilyKind}; #[derive(Debug, Clone, Copy)] pub enum KeySpace { @@ -38,18 +38,20 @@ impl KeySpace { KeySpace::Infra | KeySpace::TaskMeta => FamilyConfig { name: self.name(), kind: FamilyKind::SingleValue, - compression: Compression::Lz4, + compression: CompressionConfig::Lz4, }, KeySpace::TaskData => FamilyConfig { name: self.name(), kind: FamilyKind::SingleValue, - compression: Compression::Zstd3, + compression: CompressionConfig::Zstd3WithDictionary(include_bytes!( + "taskdata.zdict" + )), }, KeySpace::TaskCache => FamilyConfig { name: self.name(), // TaskCache uses hash-based lookups with potential collisions. kind: FamilyKind::MultiValue, - compression: Compression::Lz4, + compression: CompressionConfig::Lz4, }, } } diff --git a/turbopack/crates/turbo-tasks-backend/src/database/taskdata-dictionary.md b/turbopack/crates/turbo-tasks-backend/src/database/taskdata-dictionary.md new file mode 100644 index 000000000000..c2fe7d55bc9b --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/database/taskdata-dictionary.md @@ -0,0 +1,36 @@ +# TaskData zstd dictionary provenance + +The adjacent `taskdata.zdict` is a 64 KiB dictionary trained from filesystem caches produced by +Next.js `test/production` fixtures. + +## Corpus + +- Next.js revision: `45b39da5ba613553540c4d7fb9196e79a58bbe61` +- Split: SHA-256 of test path, 80% train / 20% holdout +- Completed test files: 194 of 362 attempted before the sandbox disk budget stopped collection +- Valid caches: 204 train, 38 holdout +- Training policy: round-robin logical TaskData values across caches to a 64 MiB target +- Dictionary size: 65,536 bytes +- Dictionary ID: `1166300072` +- Dictionary xxh3-64: `2d9e019e3c1071f1` + +The incomplete tail is a limitation of this baseline, but 242 independent caches provide broad +coverage. The checked-in script is resumable and can complete/refresh the corpus in a larger local +environment. + +## Held-out results + +The 38 holdout caches contained 1,462,462,144 uncompressed logical-value bytes. + +| Metric | zstd3 | Dictionary | Delta | +| --------------------------- | ----------: | ----------: | ----------: | +| Raw compressed bytes | 674,350,217 | 460,786,575 | **-31.67%** | +| Median encode time (5 runs) | 18.646 s | 9.132 s | **-51.02%** | +| Median decode time (5 runs) | 5.822 s | 3.030 s | **-47.91%** | + +The copied holdout cache directories occupied 728,480,057 bytes. The raw compressed-byte delta is +213,563,642 bytes, or 29.32% of that directory total; this is a directional total-cache estimate, +not an exact rewritten-cache measurement. + +Timing is machine-specific single-process diagnostic data. The stable byte result is the primary +receipt. diff --git a/turbopack/crates/turbo-tasks-backend/src/database/taskdata.zdict b/turbopack/crates/turbo-tasks-backend/src/database/taskdata.zdict new file mode 100644 index 0000000000000000000000000000000000000000..24e85081f33c1c7f5d58d65617734232bf73b769 GIT binary patch literal 65536 zcmdSC34B~vc{e;Gc~uf;Asb}jT3IwoMl+JvSdJZyq_M5oS|Z7@gKWm5nJa0c(TqHI zWLa?x3c(bTkT|@dv<)q#K+DpWKvM{Tf=i)!q0myc77AT>%Nl6Y0&ghS)4u=z^PF>M zMz%w~_xpX{kNTHr?mhRM=j_jU_Otn&?LU0?$ZL~*7w!6Cl+q#HC!Cp z^g!c*XB$(Yi$jZB{`5#`?d|ux^@k^3Q91h8Q(HH@Z2kXO*|O%nH#~9SB~Sj{uEb|H zJ@K0de)!JHSN^lj9a$E-a!u2x|Ljlq_x|qpf3RlRKPE1D*JHo)_(Pxm>)FVv*Id8j zj>kIw{L1cC;l+y@&hN;Et`3DlPxd$6e*fFAyza7|6)$~z7zy~tpJn)e75>i@XJ(59 zw@~)Nq3|V{lAA8OJ$YxF6FE?DoZ@UbS1fq9bvivkzGPu4UC6kl1BF&++jgh7n3L3wyDz@ zb~D9N_Es$&WnnbLac0w#u7%0>!*gzFq0<>kmpnJSx0Jtw@BA=6m&<4SvbA~Bp68Z_ zO72ANC|awKnH9Y&C#8c(9xoR2ZW>*7ob+6|c((MMw$Z%=YKHIV4XYH%jq@~~UTx+OO|lrB%9dN+dUk|}C+ zjy1*OPTz!6o^l;erq4kSCW@sQH|wM^uk)o`*^SEgSySAZ!%#crBEAwXzYz*&#+lFM z^Uk>IWOEY}sIpLYCa|>Xi{}dRUUD&aUfJ0@yeH=LF)img8LX`+p|L@>nF=*m$mHj; zE?PC^rnTO5*_q58aXpipZ7sV6v_$(EYsz3DmQ644g#&oA%iDp(%5v{Adx&c3-YMh01s2bCXl${&Z>DEkz3MJO?M3tr#-5w#IDft+tj( zd(=ryrM+>C9Y4nK%st9hwMK0AR_8WHa+chgqK?b4V5FSmXv-}35DQ6})~&j}Ojj%z zXB)aXD|^Vf_uc?N4C}L0_SlVx`FgX{5^o_C+4AxDb!MsdW1xW$(6em>+J@G)X_467 zUTh{k+w6}mZ_WYH0pVy9ccneuwrHoUrvwD~4yC)Pv(>x5Z=IJl5a zn{s7cOB{nEt}Gk#Wqt}p@Yy|@^U9v1YgC|dh3T5YGDU`hH-`npyff)?W(tq)(R8Mq zUjS&CLiq7>cr}>e?>!hWmsPPe>*RJljC~% zT%p{S&3WnZyz6ukNqL!4ZnoT}pW=mL+gyPdrR@j?$+;=+#IgNvMN5f4GTPN`&9QiC zu22SQZR0YJmAz&szP$-E-h>Iq@s*v+@bGg6fFZGt9Fsf7&wD*gQa`ffT`thxT*=kU zoDB~v4K16I&zN^CWYm)uV0J=x^$$DV+$=|k0NF%SpeKzLAZttk?7&JWmU5^sjp@v~ znLK_m<-i5?%~u=;jEDbljMrtEE9deax`++K6`{vGAlgQ}Y_s2**^pJlO`yS?Qx zrZ2~9u@DYbR?DhbOoYPcGd-n);WdUv?qtnlr z4y7G`A~y@-g5uJ<>Aw7hET}w z0)DeC+}S;yHG!R#Dk^KKs6SBpg0{87c{dY?XV~ zv&!yOp~|%prgxpFL~oV6^FYDnmrB|JVBm*W`5##oY6zcx`KXSt7j8Ix1)rH9|HA$* zz_CqlJx9}pAdaK47!?d`dCIE{Nb630@GR52{Txs{9A$1cp3jZPa~-!7;#s#0WZIT3 za+kK{i&@@qHhz?yC{>384F&$0#{TJE2}BJBUmhw4|r?R5iSHZ?jwiw(VgV{Ai8&$64D zt!v$;SjQQAoS9`^V?q4&2eiJ*yVLT31>ryUAwHswTUUlcm&RmF&2VCPGZ-k0%Rhw8 zfjWQoEEvI)EB%XBq08t1*VFbV2^H%pX8MTr^in}vU0hF3uJpgRG8As`J3j1h{xI;` z<_~l5Fk}A1EBrOUGGDyLF3rZPWf`8Rthtad7r8Xw@B=;ys5b%{vNoLF=3qJY;18B5 ze=NY|EBzBI!~Ta4g#9MW?S`kMXx{bW^?Tmzs(E)ju4f+?kl?$nD!!8w$DS-N1^`Cy zFqc^PzDECz%lyiKf7^K2pFtmhEpi^1C|xJ+r_%?;6ATbT=0H z!Fn8C?Ly6-m;gR+%jAooILd8@=YSK+3z$e}OwrmwCsM`ntr-0MB$h&*n3w&snZthn zTf_bbutJC4$~G>Z#G>ziD>9ub=mAg&SAIvE|Lf@BN5f&P{~=~rJZXL(V?ivdAE2jS zy_U~d48wcnAzPV{htLxOZSjF>4*Ma<3h*~I2221LeGqF~3@h%8s5Sc|Oylb?74=85yPe~#L z(F<^W1x_UBMBtTzM`$BbK4R)*9!&wu3v#J7T7?CguZI)sl+%+S&A_hZ!CBzRgtH?Eeoq}dU7WNrfD2UBaS<~V4$BtwIzy@f)?nV2gmkxFhNSD174JhPGq z@I!h4k=@cK+wfy{1Z#T(7I#GW`#y+fN@X|8fr?T(P@mjy&~y~U>ukQbAi|=oa~xc) zHWtG$0vUpn_MAiYO$(ZIC<^II8Dm5GRjLE=3WhA3h0v*3Nar0Tasq%DG9IHP(C{*e zx>-1qIIgfNQj}V>Kx8wUE)3wYW>z@SE!>JReoT>$MMnNC$rOvljC?GXE<+@R7P{og z%)jG?-Q}xcdD%MxCb=CUbW0sWigaf>KL&v!Aok;7p?G$Ik zK=hO2NMI&0^~fo3qzhJBr^O6tuDWyv=UuWuektqRc%xZqy>8x}gtU)9qK=%id{ik} zwLld7L@|$i5cXhINbArZ4_VPBGljO#{LQ;%9pq8)RyAW_^H9zqf{?+{vfZsjENwz7G&ll(SfS#iuBEpZY!RoPK?XS!)s|J74VkA$L zSGHh5IFtF}csf5yxe%bY%(!W?-tx(06)sK1E)=f>dj=eJZeoGMUM#RBZow^i5%6Ys zAHac48BO&Kbyr_vd7FQ06A(lMT&a^oj`pqixwS5%TbL|QA^GN;TPazo!WhdPY>hxj zRMR5;$UW>1gE4?aHd4%ImPh9gvwKn8<|mLh_p)VbWEcD~aIeW(kNh zm-BWlWJ{p<3X@D?Up%k0L+~D2f^N+hOVfP?l$ivf<_%*(#}qB%vr4%r({d=>sx2u4 z*l9rCK4N5@rIg~)6kyHeDILzt0Vft992pi_A=4B!YP4-SHk}kIqs*t?IBW=0DL=k5)PNpjwa5Bm1dquE%r|A=MIYdw3^n31)nWvr*4f|h>BX#jn9Opj-t>eFTg})5v z>^u8OQ2i?aycAD7j=>K6o;>0W^bwBxxgQer;-i3R>9EENEgiP7#?WCAX@JmSk3G%; zLWg}V>^C%!4y#-O+H3Vogbw2mn@GY=*qmj4-<$oONBvVz`Hy@q?4OV3d|7CYuHJxm z*ic9GRw*1ZW`;7vf~;DHf-5kSIc|Si+Fx ze-%UivGD0|2pmVqfAw?p=e+v5#g64o^1|pU66!^QJ6meZ%2JG6W~qYz`_C179IPG>)`U ztuhjn!4|3>KYCgiu{;*Ac?S!HZ4f(qd4sJXUL`hF;U}BAx8P0XCLlga6R}td$&e-3 znIig+p##9HB5lM$2fd9Rz4kl1qFN9@OqWXO1qaFn?IwnBf}@D?6kVqC6ky?dK_y`V z6Nf!Vx4s2B9+HlYjT!-xnFM~!*+&^HdyW%WTgfWe`7E$2u#m}aT4A7BfvM!vMod78 zFMx}HUOMYUvdJQn@x zW`#&Av@DQdf`ThgLCwNhD&o);0S9NrLI(+4w)z^NcsoVdPE;s5cNfR0qMHM~J}>fi zAp>O#D!on|!_Ak$kU5Y{Obf805wjy>3*_q@bka0Vnln3}E@izK@-eHfVi2PsAZH;s z%sS%>(pnMgiY88o1haHV67MOB`UoroG+pTo)QLoAUS~W$30iG#JceT=K3Pi7P94t2 zvuWsYX42E{-U4=RW?BfV^o$q5*p)z5Wr*MKZq+4rH^w1}F0_$Agsh$!L?8_Dli_42 zdd7$pT%sJR(lSs;ltgKP+-=~IF~;}xUR+tTrdeJJ8z}O4&WCU_?OJMS(Un{MvX0Ahr=w+WY7~~$2 zjOhyMDVK_QNL9I<*cVWVRPGeLS@LyI>DaypT#@d6p*I1wA2SJ2K1y;3i-nf5GglY~ zIXx{}8;~lx-sk0$Xh|sH7x6sz0Qh^bFACgEPjkOfW$6gx;l*}>2GF97uI1dUg-)*pccV!B7}xkW{eDw$Ui7MtuZA<>4O#ZyBEsA43hk^m2YjXoqIJg!Q~N#(+@ zAS*JkHba(CBF!papfaTs1VP~s{%bxM3eS!;8|>R0ZFUPs@K^QTdk04Q`jdFoCh@DN z2t7QRM&T)ylHAGt;0%#{U^KBOHJnWJjN;qCU~e*&9JmvYqkD(D2Zu1Ms4^7(_vQrt zMf(zozC`qnMBc-C5=myy%+_QdSem_G60`z0k#Jd}zf@CVXlaST=tv?l5=G`Dn{iL~p6FmA zG1w!2rR6%!|Bwk+=qH$HIw zvjVp9fz>A)8y^8wfL>l(nS2hRNW*uRg?`Y`e%@sbmp26UHL<>kA`(@XSp496dp&F% z91J*`ht*)?vNIa6%vKe8;F5J=0B;O+@JH*o@Umd`fop-mpbZC68(w_|oaGNJ(*bB; zgD%%vZBu$xg`B;B#p1R7uMXmx(77r*dnq$4rq)%g^%@d}gQXt)K-i+AA!Hem#==Es zRL*%=GN{YK;WMjiFf-lIzDoNTW^L!2-X#*5sE6h6nVrD(70obJ*ESc?)Gq!ti_yc9 z_qwC$L?X@9c#rH2KHjd&l6xsSjla`93W#f6c9`rFUH?*zY&Gu1=p?dF_C)t45_<`0 zb07W=?$g!ZjtbkmwHP;=RIQ+F{*I(Hf*ZwU`3Y@_bc67jR#Fgr+#|sF)EmtKWKMS1 z-O1=QUZ&+GpNvl8Wl~!il@$>;{WY>=0Y$>=&%u}(^^@71pMCldXNuw($`bzdT}PjL~zZPtm2%9m-3 z;k3b1_}0;_$x_rzC_31qsi-&^?e}l|pJD%xz_+fVN-flNBGl;kKcLQ{Yk%dtzqkIn zum8^voz{fVT}u*>3QEKJbcd^Kla-@ zw|_NAcgWbEoT3ARRH5w@3ULKvN`eT z0LCBw;j#DJu>8ai0~qxQfN|pI|N6h5pV+)5fbsE$pB6v!)`en_aQPqn=6Ov!{_|IY zg!8r?x})up?H>th`RjcTeK6Ph!MlQ5>Jw1Qm-C`MKmNz6pYN{@^5LhK zKlalb{@?>aj+tjZpZLkl)1M0x-Z-|R()=4MFAEYnAA7QC_Tsz#+$6|CPe$9ZOxk6A zV)t#r%O-i*oHXc!*~;dmn#A&(?AGNf>ud%IGu^t9CBg2}SpkyJoo`N8vp{88%6TYH zJgivr2;_m8N4HdYT8{L z;M%dQodRtJfo$2SLoZFwb(7*Eb3@*qvJ|EIL_!p&(ni>&qYH+MBvB1U45vFd$pu(} zsg0!wVKNYutGOKsS7To#Rs>!RQ)TGmIu=>U;vwOCssHG8kL&{<4N3xhGb!i%0;vii zo;4TvIq-?XBSwYhh5l6D0>1hTQu!BEe2$s|fITR|xwb6E3LL6oy2(Tv(me*k zJ!Z*mvCwR(F}rHfDlzwz(18h?0VnQEV8Us_Y{UmpLy1^{0tsQkie!jAh(uMACd?xy zO$9s@%c)eOI8Lt1X*Pz<%`s;LhPuMVVG*LeWYLAOqmBR#fbBZ%*n|OxeIK7>wb`X9 zVk40`6(G1Mu`OT)DVh=c)VsrblWqw5a8nDR_l= zg4BcO7}AhGoFbCU-TLYC*mp}(Z0L&gLm z64#@UM9g9X^Z`L&neyIhMvwEPYavE^OQ0Fd1<(w|&Xk)~B~{RqiLmcmi;uzB9+r#( zn@dZlfsiO2VyOAJ*$Z;dnB98_E>Yc%!u)?lUPCB|uzY~#fD7qt#Gw@{H z1Mgw@42NqL`jRXW-DRc}WG6h)IJg{zxiS>k!|tSeGzN8kIYQZpAQ?0)GhNj?V@*NY zqyj%HbJcZe#M{7V_%Ewn6+>^l77I8Ni5IDu@QN^Do00`bp$$HZfrBlg=d2DSpGD4s zYKIkh$#TXROynkc;9^5SH;F`mq3kj@k;|x<40WqgM1*F91R%%-`8JR-_LY~%)}Ij< z0USMY%46Fy6{1qDA?6HX_mt)-fhV3fPz2o@J&=(g%0aGECRJjg&=Z=sYXUmRUt8&43DdbRKNt4zUl9sbE_{wvz$)r} z((Y5QM-L7$35-`w+9Q(f)E$*4P4}R#T+P-mj`;6c;z+m(HZh)y>#3S;qn(o2o{6=pm#H^C3s-C}FfzAu3PK3ew+_r$V5UeyUC*7Ru4Ke0!!!Lt^+fArs> zN0jl>h>>=?+L?O$^oudPIB9_cWfn1sn7lBon(MmyS9M+Gl&B2bNUiJE?Hn|6w}>jZ z4GJ$DZCt$oS6}Z+E-jd%gfG?OqU!7DjEoF(C06a#?}1@$FzV;BXb!~bf|j`=-8PUJ zXd!%zWG$f?#R33dZS4+zrN%$t|0URvTGrnW+lTvfe-)S%_P;nChJ8f1GWavDp2gcL zViL~HiS75rY1yZ;{X(72#`W2ax2&6(*wm59ZcJw;+-y2~OULH)mT`B(mI*gaYgX*? z!JiQ&-R8ekjZRSuEMxz4kN>qj{>uKa|G&|OKib3EXrST`HNv*yZA89Zuk*#eb8$Zs z@qP*o^dDU2pI#RB$B(Ies_7mr7($_^k4fwN&+-?B8Gj+~C&SC|#s9=Hq$6qd3V#Re zLCnT`QG2Zh;ejOp(btxR4zf*%+PM5}^QdLSdz3?&OHmg#9%>aIb&C)&9H6VSf^{_`a)TG7ezPZnB3@f2eZl)l%X% z4EwvT;yi9gv3FhNUwSo+pDwtXlhqFx^2?l9g_kyjE59P=z~Te2OnTjF|MArgr`yki zZPCp)&^+mRRCoNP%nStg9l>;=C*Zwu`c~#i;*bBm@1DMcFC+Z34}YvhQiFfumHs#G z4*M6^IMaOAOc>Qu(PkQKdCi?;&E0e5aQOv>Tcwu zL|X)6QV37GMd8pBO+!r~kCZ6x?IlnG=-C1lMR(5*R|Sm#P^L({C#ls`tA=(^s)BT{ zKcW1G&bH`&sam%>RJak%V$M6A7PUld14DDP#n?_@CruXm%I*x@%Ud!SShr+9-8}~t z^u$6BXQ)s<VG^nBPDf4p1b~cS8R0|6hH7>u?=x^V!lpve4JHSF?}9`yI~ouppRYb2nALz^2kGmXIc^qS^;p^GJqVA=0KckFs{##>)jz zC0Kp5LTKX>Y_U*~6 ziD@_>!aCl~woud{Y>62r2nx!@8GNptfV?}wIcYbYF}4glY3!TEIbdaKPNYV`4yMs? zWXi~jVF(R6j^v8)uP2U8a>sQBX?+Et8ZjgScWytMX0db*0Il$)Fj@0~M%Oj3z&9CV z#IF`IXdjp}_`B;$0Lv&L8fqFVBY+CP>09|LI5Aoo=-!RxJcw3BvLKKwAV{PDJOa{O9o#Q-M=#{#I@pfSsUZ1I2 z!60~dyA)<-a^+nxMokKfOM~!!EctR|0|Y@ZorZl?b|1vgOQdBToiP63W`jg0zW{*- z1UnwAw}B!dx!xlui$e4w77Puj>w(fZTZHPYp=xPu3ZhD5BF)uM(-xQ(nm2;PN0TiE zDz)qmKy0d3I?Yys-7L$%2%Kum(wt3YBE7k+^t)B;Xka>2Q^T3L#DELP^qfcm*06Dr?e zIgYd~6=OSa#6bu|m!Vql%&J`^CSjLKv~OZZtFWqLOQ_Wr%TgVYz$NTJY*?)dm%=}r z(2gzA{?ba=#0c^=IaF3PUeo|7955w2#?+Z-3B-jL{LXsHKyIf=920+$M* zFNu8vcP94q^`=IK5(BB8!GTf4JF5FIIt+@pueUCZnN3p86X7}?tNM71yEybE1qV{b z4U+i1N2`^ul2aFvWW*L0e;+p{uM7#Hjb=f9)Zju=j7gOA>q{E(F);7F5K@aZGMfN` zxG~|8juXT%b?NeMR^2G(x<5TT;!-(tmJd6tAJF(Ymqn`}h% zGys33n9IWVOU@1+!(gN`nfY08a{-qZ+1S3JwT4>4salR@=MYuoC=mTB>2q9U9g7V1uYp}=BS{sVUnuOump~Qgf62K?M82_ z35J+GZd%InV|`fk!vl#uBQ#_VYR`yw5O^Cl&~#ZnhF+*z^eLd1(HoNK;yWNJaAS=+ zFqrD^>+Ri>+?NH=EOUlGd>b2C zm>DnT=_40uakIeu=FovIQEbp-8E}%tnU>arFe+wW%giP;f_>1ux;k z1dA0n1FYazZ?NDI-vqg?tN{;N9ekn0)sd!L9J6^5xdVAG&}yj-RA9wxZHC(gSZ56| zH5Zmc3N4P8sRwMUPvlq$v8K@4K4Cj>RI1x$dMV-wFEFN#!;K*e>0UNo@-lH1-K!Xx z5_~narkDjtoc1gV`;(Vi*coYQYfVe#EI!9EF5@r-6_JkYqtdV&>l>z1s$CQEnMT=K z?M}o%(USK{weZ$B5+BzY>} za@w251C)MXn+)9zJ*BP{j5$b1EKk_hn*+3l2?NA51Pg%37+X$7^hW2w8?? zurLC}Gwe7DkOTCp9vJ=s>4aD-L}cp0%^W4s$a9eTb7QFy7%Blt*cfE0n8gHH{!Ep~ zK^cylkT8Rj2P2!QAqk?Pm5alM8m-DA;6^~Y&}oUq;`k4x z!0v&3707YX3D&_FC|cVYVcnJ#2ZWk<7KqM81v14%O3lWAC53FO0|}RA2}gw$Yzc|_ zMggj-@qv}B84y~krDVs>d2OWI>%iu`Zt%WmK&okBr2 zD&&s7ct|J)EREoFm@gEp19_IfG%p^o7N*8b!rHWXjB7*iCiX)HGl!stZW#tml*wZ8 zM&MUf$_(MmFnmv}>9{e5qp6HIO^TZwP#feDY7NA_HUUqm;=F8XvUxC6gU=M)Y>fSb zT2=+&<7iP}?QY;MCSe@75v~ICs5Ci;O2tG%w-^UwstBo% zi910XICo|iCQP95iAVxpDFH#cmo&*<>LU|$eN{f$)aJA~gx`O-&Dd+-P z1aeneGm$HyYN!+ge^(MhQ2FsjKB3uP%d850ucb_m!O zS!FaOl-}7Rude4r>5$JlQyO-mSg`(M@tdworH1x`9!(*Bpty@6o}MKh@z@$^W|Ny~ zSq7v7qFhLCBn`V0(I$o; z_$pd%q(61E5#CaB`oY}UB(i3n`1#S~gqM||ARKVlSZETgGYI_n2%qp>izw(G_NtHwX0 zuJXH$4~ryX1k`7R(1*1H;(kVF5}3+MCiaNm*Q*&lA9n?QCsmJ<>XQtku_qS@G*oA? zS7;3~hSaWQv;xWF(mhAoBmOuwBgm2)jX}P2SWUm9W)=d^Iq};G8WuOsI0C$g87zu5 zI!x9DhFY!UX2^-q_Hc%bNS%!f1FN0>Vh2>VAY^+^pMHfrQDv;DOjzD<`gu)9jxUUv zyio5RUG9HndHAxhf5S2ay&x40uQnu5wfvIR;Q@`UwfGW*hoWsj*M@MTH+KvL-@4kr zx+(16_mi-HAym%`f60=kUSIi}f8_&o&@g_uPwd1#4CVGWtU<&!1?j7TDr5%^o1>eJ!FH6qQ z$xy{+t$bFZ*j49CE+;JBpOH?+DP!Ulle?G(#@IlZ7fsI7-J z$oU#*IBV)OX0>vFW`}WFh*@ncF*Jl2(>s%A0CEtYUG4|?OK*k61Esg39p&!V6aEL)Q zAV`vk7oUM<-e=BSi-LUOU>f?w0SM&b2JJq|K?k zY*k8!0P~<}KwH*jh;h>yndg{p`-8NWrzDsjMx?~25Z?-_vza{H| zX2J)ZhowQ?G?*mFN?-(yWQ8knnQxJoCD3* zAH%Gb-Qx$EJ4L{ZqfR(9DDxl~$vU(;ay_+FXpSCJe7Ls#cx%)+G$`edWg~73k&?AU z>KhrB3Q@@TcECoVb8UMRE4R~jF4B6;G_ZIaEDU!_VH?69MAomnWlMYOR-4Z$Mvl4E zL3g^xTaV=?B5MmVyOwLWMXN#+K;>;`&r`c~%(f`sX7&PIYG9mZL_*D-_3uV&c1ALT z`ehcHJJo^6z41owV<<9W(&PvNP`XMNA35R?_K9~BRov^k<~ z402tfHP0{tRT{-2j*XTudXY{YwByGkRXc=j+suhviW&9Kz4ta3#SN!fzo}Qf@jC<- zRe4p^85$ZkYgL>h(vEo(l7QTo&Dts+K%NUHGG0%CBT*z?L+J(TM!h}+1C`TN7<+Q0 zJ8FZZM5+7D!>GfE*8{MgfCUmlkP*mF#sAE~Vhk`Ayn}MyUX9DZ;lej7N)w=D*}&*a zD+~_9J}^^c;+cx@Urh_QQm_jnK;#iDiJI9nti25>&8+>!b1=m9a7Bt$*p>Of4J%|XH(IW%`jN0kGE ztfcv{onhobK+x^FTX{1G)XNf2o5D{C7!g2)r2{mE!3IE|*i$faTmV*H zJ;dLr$!$=FhQxv490>yBVb_ZTXh*Rmr6RC%AyHX?!4SO0bXYW;ER;lu_y$`aj4_3C zVwC|In|&9O)+Va#6O263L&oh-7jhF&(ZF&8K+3MkD++w1k@ln!5yVE)3OMA6mzjC- zXpO;f0UfFXWDjy1#1up~B7KP{v8E(lH|vI^R%xjtku7SsYf1&MYv~NT#;v%|mQxZ{ zVA#2|<;{r8$-?B{4XXb@bcBe@dqiOR&?y`6Y#p9}<^GhHI5~tQ>uyBP3GQViBp`iM~Au z(3Bb-O!bTmtJEwwW+1CZm8fyFh>y}lT1`=IMEc6$M5HKxq6o2tj1JRytBTk+;eBnS zMXQuZr-VTuX8eTIVAbXwpMwSh#+gk72OI)V+1}r<(vJ_5k9-|o@Ox_1|ATA!e$sDv zIer%(_1`V;51RMK^tbCF@Z8&3^3;h?=zg&^uf+Z5<6-~(2u$&LVPT zE@~h<+68tr1it{E(J>Kg`}}rR*Y95-zqf(W_2Wo{5qd>(oCI*@L@0Rwf))N75uN=X zj@L(L|A67uIq8@gi~`KlD9cpuRf(P(!$yHN0&7CtU2#I?;W7jrT{8{)#E}?iNV5uY z0%0`{aR)Zl^l6rSdM*dqbY1Ao*MDMKuohr>!!yXrj&TtcmK`jTC`-ns^kqS~%Y{YU83o@66iCOer8^_!me{tCoLUP0i*4t7>X1^affI zu!LnW7IR06({Ly-Zs6OTwH-Bfc3hCQSSt|p(!4ND{f*tde zd?~X!h-!WWQO&mp`3z4&l|dCD-}*Fm6!z6yOcE~wxFQx`VhF|fedQ&00h}urWKHHy zi=SLX5m9ZJ^*=a^e2c^MOXWyMyE{1mG0^{kE?{)#aYF}tBO{H8`$s!*Fc)wzalDH{ zidaiK4Bsgd&mncyWu ze8AEOg|E|iXd0PsB)LLL&-Qr$sJ~BrM1PNcU4GH-&)IgvLG32A`>bnD z>=VdVmEZ&?HaHlFjI?I4orwIcL%^gT51(!bE$;9y86xm95)oB*@h*!m8~nm2{qKBS zgZw^?kZaN>>wfs>ztrpC%x$RF1WWAt;Nta=*o|OQU-?S*5Lff?#;RBR8DFsUJ&1kp zPxF~u&TfmbySOb9HMbA^i5U)|N%#kJGVNKjcT$aJr4RgIY?}B;ju}KBcvo7yYRFUw zhH%4TUFpCm3p8{BzLLdB(a?}6Im93qhTes;7Z}qGgwB|u?J1y1cnxJNv@%Li zNj`fL&w@4}ib&-qFEkBRMcmkA49SvDkT5*h!O;M$y1>svkmtxBLQ)+4YsGlma04K)HY6HvMk zmGG)FoP9Jj*UY%jip@nUtlCQ`F|=5M$e6l?tRjIXjneQQI63W%p)WEw0yu)ff~aL! z+F<4pA`?!fAq7l!rs0E#Q_OkpmMR~;nd=Bk?NxGx)Z)_fqpA?N6@upK9i(i~G9sKR;K@onP1Icu$d@1|=`U@2B$O^&+J_<4MF^{ zYy7v+i4(!EHviVmKhHn7`U^qABcJ(Se{=WGzW!^PfPi90v(Ycz_$x0t<^P5hxvSFp zSJ$puS3drZAjcUn8t;6*>Eyx}{2<5spW60J^x@0j8^HMad*1x^XP@qB4XO=t*oK`^ z?XSP*zjX(Y8{QGv_9Mtly=%3tr;#+H;*91;Uc?l)) zG92HqG!Qcv3DT*y)mRdlSNh^D(Kxhp(1m9Oml?&+g}y- zUkRFI&#TBXbiD~J+w&^4j0YzEA*B6^(5lK|iS-U&&zHT1sd#t;^mg*_xP4fCuSvO9 ziXzYRlCt==%EaCJDSUZnJFDTphR{X$drj!t(96vwXSx5+>p{nVFBqEIyWH5GCb7Ce z>}hwVvT9UlkOQ@8t4&(?s_JDvdfzNi8H6(#-20|@h$u?2j9^ySyGf)bUU@sL0ak_c z6H8L^f|_@2ZDwQB+5@d~5_*~bp}_&rxFgINAUj~;QJ`zFwWM?fXM%(-AvT7aG{j2` zU=qk?m_-CmJ(M(59q2v7j04>KR1Vk}a40h;5{A*B2x{b9py6vE){0Q>!_y+DzU0EF zkWJ9;0NenEh2r7FP@iXy4uf|g(*?1^oYtyIOPSAls0K(~g&MGfqYa=58`B374Qy+0 zg{hVVP6tvK7u*Og3YDx&El{y@)QYWD{6tiRs8)6_qKw)CvmR}a%H6#|8pHF<0iUJa zf?h$+7{@f2y8LQ5SvSzuPxVA+psAmccyI+{S|Nr2)@ULd&A?!Q<+IUE$ht|gZZ=s( zr!RkOekl6I&Oq!j_(hA(z{Frij0p-$0&T8?K|+0SO9?_3KxX-S4A3Ak2ZPdsVI8a!(f#UTk9q*+39xdnHomUiQ{C#SEQuCISu8861}kc{ zLg<<%jpFOB`e>#LsB1y$x~JEOCW$W$(Pgx>qj38Pq(i}m=hBl+&m9^lb3h-_-jsp- zgdfU^(jJ7ANVAIpIcIbUu*e00#gN!FR=7YC;t5+-5z%Wf=Lt!Jb*=IzyQ(eqWSVa% zD%&z4O(}i{(FZZZf*=#x7mG=8AT2>FaAaoSg9M6NFbU-#kZn`3tTh2*5P(i8R#Ae3 zriKmQ+_*=JcRHL3;NYGDhE|e*-)fLxV<7@qP`8Oc4x*j+)iAC&#P3vez*(kdU}=#I z@h>bwMdWTS#^4la?0B}xh6w}Sa|7x)Xnr4waSTXMGiY&XofF|*S=piv4nwp9B|&UY zkOf!(LcLA-uhdWC5>g^cbGR<#>L>;@H+m9El&I)%A%-?RPr`x$xERvpS`CP?__On5 zW%>r)Q$}p>aj?1zEDM1^gK_jvT1STg;5FB++E0q(fi78RtVzBZ;8BR&g0DSB9u;k( zI7cfoVI|el3Wx4Rj_sGjok-w5CF)n!AGcrSq_Jg+f>K7~PPf zQ0l>j#f4FX#>0fchXho#22+@Be1L~cjE^>1 zps4GV@ungHfrJQi@g6I2p~3P}Frps@9EeTT&_&wBHMyYc+5$ugqCU-UFH)yLsH7M{ zuAo6F31{W^kfYaMH%;3$`MeV;2(5YDMB}HZ=mO;B&BHv~^s#+Ab~x ztP1oW+fCh;_treCngeDV62Ua+caT@QNt!1}QJvAS2zJL{oaARpcQRiv~BmsUhM?XsI?FAI;GxVZ7Q z2T5u#_E)~kUy1P*_HTO-EX-oRBxts)zQq@0yZ;GE?F9SauUh5LB1+mAB>qhhlHsl? z{DY8eRBoe!AT*4aZ7+xHWdaf|iPp9=RLM)O#kW_E+J{%$hZ_{Y>sa|+xROc1oPb>G z24Q-8{dGpZwIF(chVX@c>;gZ20lxd$27jW#&o}so8~k#Ef3(5Bx50mPgZ~;tr@^PV zzYvG?b&Lfm5W_#U0K9&Y&w&MC6WF029%j;6*&!67Fm{8A>G1cQJ_j^*$hZ=pQr)wb zNNWGDK!7JgyW`=?XXZ>3B|(_D^ORFOlZmsslGy$A0K7k{|X-*Cd;biyAy0l~dv0$Xi5fsCP(2;kSj zCq%1~UtA-W*LPNrww#CRtN``nW6 z&VzhYfUur`VS$|Ium%DlW4!a~l*K%rP?<)wJygq;OWW((%VQQ7O(Hr0NCtI97b?kP z5LZzz3eOEJu5AZgoB>eOPoz=oEX^8Mr|1`I*NQU+^_tp2y)!x7gG)QRhjCx$F8aYP z_pe?$ra0mpZvics0hMti2lqFIx-kQ)rVk$W^_Adft&4+bHJvSHhYj}cS@}iag>mZo?-kbc-|GnStt6=ilhgl4JxN*5Zu?qQd zDZ!_D`er)sd~YF0=sf?m7yir2C+`gs9{k zy0i%pymoc^ujuqY|F9o@*pEGo9R8kF{?TRrJC*^S#Gz~Odx1WNaRPPV_htCM8zI$J z`d^`j8sYD{I@uAzz%Ot_jK5HvzhfDH8?;h{$HRBt>#=e*vpp&rt4Dzvp__R$T=~8V z(jMiv#~Lc1*N#6X4=lxOXHAEy=P|#sHHVGTw zmZN)YfPS9-rADJh4|VoFV2$4|2S|_q2L6U<#Be{{4jMpvtiGEmNAW)o4w4bq1Vdt= z!mCKQ3~@5V_|@F<{=#`fIBFYCkDZSr^NAHaZ>hEX!jD)L2(qg$1-87o0_}9NRNm|4kCM#CmdE*xf$P-)eyr|$uR=$8l;R2 zfmrtuJVh=Od`QgBnu5ZBGH*2r6H8lQj5%Z;{Imx~lYya|CqaLNQ?aJIK}D)4>@_ZU zEIL5JE`$WTq}>Q=s1=b{5fCQbi@_5NqDNfsMZumtLL!DIk)(-Sj~_NAqsECQctxGiCCCi*BcFP8%ZRjV)=txiU@cz85LnGi^*4djlsVR^2R>tHLf<2 zK2&?MTl;TrXkwFg$w$h1d9M?0Ct_T-XF( zGq}#NF2dNP5>rHmq-Yq8U@!S0Df5tss`?59CPJNRl)X43xg)HCTX_ID2Z!Z+WtGwJ zD|LT_N0H(l{Dv>$InpVzztj{E7U&h|z8d_$1>FH7AEHd~D_;uxZQxCOF;Y8qBshgW z_TS+YGNP=u9FPOdW7K4;_AGtJe}i6B-f2ihoRJrQ2}j4`91f(f8mky8v%|hmg6b3| z*k2*G0<@4ubR3{ZTIjl2jk;%qccNM|(k}SfsGZr^LFZnvaua11MY)hA=LH!-P-S!B zS_1oMvR4&$Yb{8kc>8RfziAgoS)>YftxyS4OeTocfJo+HK#UNhM$CzOMd**Be&7b+ zR-ygneqtKsN+VUq17R(5BQO9oZz`oEY$H)i&yN@jY@_Y6g>4U2p%_%H?PRzc+Y=Z< z+e8e30`tb~gY9RV(}8Tv2+2~PTAUcGD!qXKTo;7r3@<>tXqByD7%2G6K!C~12&g+O zyP2tixn1Z$vuCF9Kr_7lm?a(9){CYLb*UVbEFLrk5dVsr&M;8C)WbO>3h+}0f*pM7 zmwN5fI&A`u6E2-js*&$yI$?!#))?nav;tF7F??l@Mra8RXs27_$WVDNq0as3mHx9U zSNK<@{qm@vn)biH&;QGB`1`)8)Zwi{6M{P2w1(JJs6%EJiV!9AHzg6pJ(Sh=zxsy$ zX1*!EODM-HDehQecmN@92DD)1h*Sd_5$HF&rlIkRYO1zr4FQ#rgkP1+g-@5DNmr|b zBcfBM)xl{#B99pAom}ZZzp}yq_M80k-mB8bCv11VChSz#0lXXWqX`&9VU|~o7KD*z z$Xm}`O~6on2sGhlaf$s*9yk~bZ6ma_^DHo$GhG%cho94Xr)h6f5jAjHrB$7QHXewqc|SGeR_v z0hlovxe>wYA<8QJzZm4C;6cJMgI+L`){(JATh1&x?~a?ip1A@*%S_lZ&2?qYabZAZ z%4WZuR>Bag!?m5;kh=;IXUy~hbmekcs~|A0TE%&#yfgfg-sFzN-aVr!YwNi?c^8)Q zRaqQQUsORTj;CX*{5PzE_<7v#{62!NLmK+h?V->LaLC{dU%LG?pZuMV$>s?!_owgn zyDsymFY`AuRn6#^`)_RbpKAA;-s6Aaeg4eTYENH;T?`gK?CGCrls3J+34$jR{#R^z z)SSM_e+z{Z5K6pR%;@|5-<03m{AXyZwfJ`b)-b=n!(S%9os{At-SK}--VgZiX1>tx z_#K)L^3k`n0~o$9XVducB-*FHP4Cg)FT78GXP=f|RvVq5n3?F#oxg{z~W(ejnEctb#c8-y`xJWkC@9zDQDv$P0rc0TC*$sa)48Ssnu} z358ag^Cw*S6z*yPJREC#owF!xXjvD*$gs2c&MWq2j5Bj}y1&_kjRgkluI*g*hL_X-0hT7oj# z8D|R4>7+>j&h!9xHmT|y9W+)CHvE8^i2M(n$x^&|Ph+yPY z9j%wDA!Nj{CLSDEQdOrIY%pe`jyYoTfkqAc&;2BoYA%PzXKK;=Fwjz$1z20*qI zroqr;#sv(c0KX)LBeC1nbO6~nh}g11Dm&`p(>*-6ZzMUKvVLwz9T^k4Dad%!;_VH~bvw4m5cek)o5a-L6o!;DS6|eT`vD3eSoH*FK-gG*n zUqO*Kp8ku3ih4Uwe^H+n-*WmZCJop6==ZnXhD`FEpUm|>eIS~as8v={W24HEt~Uh@ zs-(KuGLT|l(k3Dh&b8~1qMBUjKd2oEk(V&K5Vmvib@KOQLuK0r7C-gtmA9e??0|Vd zNE@cGuspE5&GV4WgYcdq`;1GN!uN&!S%m8OA_i>zT0mp7=9~F|x&5tA@Zr?1$`{18 zKLj4E@-uvFLZuk^DI0s|H=vyJpPp@0C7Q8pu5zv0>ZZENPf>kKVwFcM57M|5rpFoi z4R++EX(5V3;mdXEu_ldMF;FPNGNZF{M;1=n@I@mcG_IQXR+&a|6KnJzdB6YAd;Mp? zE1r*j{29ltHYQTy1zM?Gh)~rk7>*m$MdMcwd`lX;xPve;f~sqT zW6G%tDtrGESTJh?D*K*g{?9=z?!>)D|BB}Ih^|3yr@9@Fy!R;VA1Q+VhK0)x`3TdM zKW2TQ#YAsRM%6S^Y`Vb&dZqXn1I)D*C)$92OuY1isZ_^YVax2s^3lop?Bp%66q?Qm z=%|2Ih3{+dH+|5L{8iXrQ*&8SYu)g4ykV@Q~63nzgBhXSZZW%fWz|ILf9OhnrV42?pD7TgU%_W_!0kK~JXKxky z(YoeT*OVCoCRF3YR~j#nahk6@EO3nfR5yC7mwPSqKh@%2w%-5a{r=hsMIWDRVG$~8 zhy6Q^7s<>Q-q-$L&%Ej7{$b-Ia>Ew;Q7eDzZspXoaETu{p=e>Qi^vzz|y=RrbnKfHzU^Eq*AR+&$zxj6V*_~Gh2~%JDm&Lb#Zfxp=L^hUZLu61G+fWS7>JlM<+-vR(qw=VFfi*yp z=sOb($s|k2(yXrzT(y4S7-QlNYFHesAi=yU^HK+fx1$TZvYMsL z1-hL9%|;HT?h;#y73*))9n`ygp*+Ryw^cxZI4dslLgpakgsBzzKw*H`VFtOeSi{YN z4-f}Ca2cpVB$o(bPV2#G+Cyr@i=140oDOWJjYY?}OKTF!H^FQa)XM^MRlOTlI2G7o z(;m@Mv>>h+rv#SJWNAn!*(NHHHK(Fd@s>l_4Y?eh7{h{)#p_dKq4@mBoR+;q-?QUi|di(K=ovt+4$*(Q4t4yp-IHe! z7|%t0Yy-^=T#1uL_*i>OWo6^FL{KEuBFuDqVI~Ze-b0mE0o!(B4iJJ zS8&jlXvdLa6c#WL#G&*!Ga>vTpI8Uznv{wz_oh^;fJ}Q+AX5T|DIX`rWg->n%CBzZ z{kZ-I@&A($a0sf*3-KauBnS;S(6De$nufsz~M z?Erf0kBD?H>Wpb_mcZie7BQg1JKPtr3zeNu%XJFUBuy#SSZY;^DWtopG}^qrHQmzONp}7U8UcYw2B4 ztwhuJBFavY5|y>vfI=kTB2b|XzxINcWfSWN$`sZW*3=kF$Cd;W2d)uU$T46#%&dU} zc{jH8DYn;Y^A=t>5a28DFAU-3ZW#gw%MCvd$O&MEmu)h%BAl#EI6>ZOp3slI8h#xI zn?!Cm#}Wk%5_5v*7d<^pHlZDPdYuWcJlqZ3g0_NR3qo)h-;4SVz1_6U;gdfC4^}5(FTummY35;rJ!49*_(|MjdG0O3NH~ z+p#Sj>pCNDv>YulT4RW>(+JH*CJjwY_M;+Z3l4iC;D=ft5s-)?6t|>VTO+xs8!g6o;SJgwT%&>8$&T|p z1Px|O%Px+k(Do(pTVpooak_x&yhULNhS*8k(#a(KGel}7YY`g@o7ebSDD?uJ7ST7B z4Kj*a+YqOL%ZkE4@OnfnPYJCVGv7pZB=oA}X;p^{EJ1>lB)B_?bCczO?*z(W5Pz=R zxqP0vT2m1+zG47*|zq$SoN?`4HJ-I7I=vA$@^o%yt=eC4L-nM}5nQ zi|X?PxCo9!ac8JFJBQ%5>IX%tv7owpX*q+2(~66dE`bk|W2nJNuLKJ1{DMYk8^xim z!w5Rn2Gb1!H-HM?s`!}qlj3wo2m^$BFqjn=6i}Ya_5g2@tb~cLy1pP6-(|;>C_s!X zh|yzB;%A2wS%Rj@On|y9N2CYxI*O6!g@bkucnJan6fqm=El^u?^b`QCbCnoAb^J`@q|#jJD_0KL47F zB9T-o5Q$c=%Z0u{1eX$b7iSA3x(KuZp)M(L#YW9ro2-MEO2j2-lG=C6wx+5*@V_!k zL)j)eV8xxYqb(zzk2*ETOCq$3LASlW6?x>!h?ZN|wzV;8WU6~`??7*2_%7&d z2S)n_l6#!Cw(XSlvV2Y8mYRBmMza3d8oim9R)OT@spl839;3-5Bm2q5=$93U%aY}d zx1-@1qcj}&N}$uA3p5q<9j->hGi^alWYHWZcTw5E2g4UvvI8ZE*3LbGYvT}Q2iidQJ zZW7rCaecu6u2tK?;Z6-RBoaPi6l@esi&7XTG};U<>ta#R)`+M!*YTNE5f#OrVk8dh zKSWywZq=KgN>nvaj5sYqE^v1LqB?u3>euuQ?C2Zl8%1CQhDQJ@`Y!yE{C~FcLt)bx z{{2wZi|AP><5x8LKi=&BB<_FwYic9&_g3!tUMTVLuZ(xN1pRn3ahZpl`up*($uH2> z=d6+%(qEjOXOHEC`)EYuCVeRB{W;(&+iznPY9RCd?eIA`WxUMw0<^5nNX5GzY@oD4tZl`F zl~$oJ2kkj(?q#4Ms#=v!!;lnqyj914#v@?XyL+{~d|`ej!ZTm2Z;g3r&o~nkV&SUb zYV3-m9zc6)k;-gwal&76*~BgZz&};b{zUW^*tOmw@e{TrqZ{zDL0&c{qwDapPF~h0 zqaI#7c_}BO?RaUImyTp~0WS;ka!)eKzzD426;YShKS-Pe+o8=Sc!ES>knv*R0_KJx zv%JgDVc{dp0)VOwFHM4Xim9#kS~BWLvM4TGmjO0|AS8TJTWj1k( zggQuQa}5xwdmTn}ooPpgK^-L3jI%t;Y|E3>B~Q{}dAbGVZ|RQe$-?6RuM@#~3#T2M*ahnk|4MHJ{XKlGwB`*%IXA z*&)oPT1mp#3DCTKq?_plQ)?kae$XB_EN0>}xQ-%ct1{&Q0IHED6CyJdCp2?Gw9;%U z^i$&qhW0uGsK^g-EZL^VoLyk3LDYd}1I<@Q+u$?@Q)$GY%8na_?-^iWk^uklqQoAP zQ$?WFw}w%<)CV3l708M*(~QHIl@ep-45)08kpERGmjYJ^C!~z%oYQ8E;ZEMZUso)d=aT;^uTLSvDNahk2*tXbGXvoy}y zI&G}Me5Q%#pkCr`A#Qy zBq86a0>~VAw5xS`VP)lNRWSUz$q9=D7NL4T9vbXJqkULrAC}_*RMMvicJZY7eT>zi zev-3a+E03wu?+m5uJY$rul9Fr^gnsMA4&U}l>eGJ|81WCci;0@eBb|*A1HzQ50=3F zGUz}2gNxu2D)!}ExmG$!Km!?TZ;{w9<-!EoSC0zGk3&Qo&hH(9|P?%+ZKjTL) z-nE?V;m~<#l>SE2`kP7V?`!7t_idj3{{8p#_q^}x??3)Pepw&-3?ItIWua{*w4UG+ z)q~;Wggqt%DCxXLjg3Rd@4-kyDI> zv$3`@GDo!^bvvlQdTGN=L}T2EsbYJWp_^7$uPK$=BYBjP(UtSo42+;ViF{9_PfK+< zfE9$QF>yK@3zT_1%z%(@;&CJfo8}S)bK1*6Z^Y#gE5M?bgb5BX5v0(mA37yC;frto zEU$MLKYF$dt8xx^16aBykeBm_xk;)79U?Z70vN5GNyFxhEyhhWMi&gF1kNlh6dsJ) z3DNj!I8;(k&fG7==e4n^TdY}*G10e!+u3Aty%74Y2^B8Ld9)}Z&C=W~4kTk3tU`xCXH34!*Q%8NN(9UD9g2R zm|%r3vsR;5$9m5?2M%3-tlIPA2M%d$BVLySns}lpG~J81IL9nGWA&m){t!-+V2tDt zgq+=w0Hoj$+FaXi3h#{?R@Aa);8n>QkxP=h3YV~j!ZwjPQ)UdjQ$j-a^^qy+k*gMG zYVH;kR!R6e<+%8`OL;-zq{!yV=N{`DE6FZ6Mka%$Q?FDB5NNXSL%vs2B)8=>kJ{YK zfuMb+XOc%fH~C@8+yN=csKP2CCaGD(E2# zjX7QQMD0MN4$dNmO(OKQbdi4~jC4z|Z`$Ik=A(g?b&Gu&|QYA=6L07AAYfPSuS z%NQ+f%DFPHVYOE=)JZjtNl*z!X+`D1#~E`N0Zp#L^z~?rHzD6dH+d(_TX6bbN-$T~ zcE9m~^Pe?CeaX6T`^x2w7>kC>8!)*F^IF2ZrV7*0Ee0y&Bq07o7(!EcY$w^I_D&G- z6M*3^k2%k{=F*Jf&>Tqcjd`dtwl6Jyv zQP2fs;dgV$!mCEYvoJ)>0g)COMm1|IPD8oCFA_(sak%%kq|^X0*gfjTE9H1e6STk) zTYZ%MBN(_b2d(5g6-lHT>S{Hm?RRpFl;Z%&l@_)(fh4ro7#X0nA+hjr_Xj!j0H6gD z^g~qGv!-fXf&xk#*0{L6n#H6Fk@`_ipm%t}-J8@UXeH?y`k>i@KD9cxit{%VX4HJc z7(@H2O!+y$vQyPF!W<33Q33rP!=IL{gf-q>f9%N-CTEl!zI@{025~nJCq60(veo?C znYg1jpga2XiASnR!uiB9Sv|$7=UcTJIT__uzDl%Qd0mo4+Eh)GC#gt^$*UDu=XdH! zEa$r=C-!R-B%>Uu0?=G)Ty60@1jF-&S`tb^vZ)kZMBMDotjNU^T1&Nl$tth5--wDF z2)TpO<4cB88zux?wSO`W0x)QW%tOsx)_lJ+S{GrtD!jMd#>7>D%4dOhwlpnl=nK%? zH3xFt*+v(d>x`XMKUO5@0~i8qv(6wiBX9QK(cB#Hs>4R8ml%gK-OW-~13L;4Pf1b_ zc8aQK1I9BJhcqCL3p_JzTo6eX_C^G|lnlvvGT@37C@@p< z#mKrMK$z-<2Ma~Dih|uCLbYPTS{=7*-KJPatUaZIn{08$+*7Ry_NDHv-gSu6rcV4y zDdMX+Rxqbj#+K>aAr2m-99eB!H49`a5Fv@55rs4eR{4LbyY}Fysx!WK^MX`?R3(Ty z#8f0qk_`cZA?mOsJgW!-qp1m9ATL57K{g?vV@<7~qcaZGs?&#BJ5|SV+RjwcsI^n7 z{)3%5)t0Gsus+%XRZ6wAYI}Cu{(j$g&b@axo1mR`re&Dyz2~0yJ?Gr-`TJh}K;$(r zmldfEu|nT}gGtc;rlwH>1-E9(orn`E&d8rblyP%cp_zo+3CN-0v3He9-o5EtMCK;p zv2Qym6~QaW%q}qN3lJVVb@*J^H2=+`@2~t=k&!`=?0rR^xvNgSMqc&(jLpQ^tR><- z0~wk~#3D7m_Va}diQ)g61$36&QePOBn05UDszGNT)#T9SzIeY3vI~x1EjO#<)_s`t zX6Kl6U;cUBNubAW+Qz-G79;{hxI?Bg6B2Ea91TUOP zXG7Tsg`+L+X5vB|!;mI!+fxllO=KrmX@!C=%S>BWmO$3;nw`_h!8o)}7mQnVWHPu2 zF>!S5oEfSB{hc1V)o+EK9vn-t4E46Cc`j<6kNQrV(QKzptUHI8`Mb=O~HZwmPvCJ!_iZeBYxJOj?QYP!WoeJ zcTBI)!m&8QlAF9+!KjjH1M?aM^BRS7-;nnOFX7F}o9`_w0M_*#`IFAcX@cnW7qfgG z81(3|{Kz{|$MsD~AJc){U6h?+{HpTDsETtSRfRK*HK++}vmlTfU;un6)+;tY{fzVt zUa^1Bi$1@%av}4r{fAd9FFd-(@;U=l%L@z>j+<7G# z2|S^*lD5K{PzG^u5JHy!L~R&0fJU)J0hC^t=b6{f01#Wqn{{Q54_UH*4~T+lsNmyu zpf(p+axx@l3w$zQLZWswAjK_A#d-qpWE#|{Neu}79Z7*3p_>YAUui*mp@dA0fh=aI zRJ!=tOry)K$g2_O@24n1m&jKu1$0et1ri40Mya3f_2 zhzCSDbvYqJf(HHOTrZHIPqo~Q0&Y=7(GkKTm93>}H-<0UDuCC(l|#&@U2POwT&UVe zp8;X7wI)XntiU%0H9;f;926LGpuekk!sK2eKdixfveVHE(Fmco=ajFpIa6&{HQj_#* zXjN*my#i#gKIbFg^|B4--H(Rk7Xt=8*VkxHHD%IseL422e%Qxg0y9ZZKdaE4{+-p4 z#TeWxhMi|2r|`~TYueX{F2Q&GRFnK-QVn1R$iwkC_RK&PySxamZ=ebWree2>_!5VL z14eCXEN43)&@{^s|D8HcpS7;Aq@CL5tn9_Q>cRqqDACUXsUo;mE;ytrb$8_^M9~nV z8F&7dY5l7^I|=^P@o^dmRTXN2vOBC^X|u_~`A~@Ix)2L6eyE(G4o8HXJkALFV#h-V zLzOVcwqplUZmU_m1~0B>p^kTMgKb9H#bvc~>&1J{&X`1==rhAI~vfqD@rFI0BRjJN(SCG9&0K);fCgM$!CneZuW4*Vw@O zip(D&LaA%?rg-UBb?CL}-!JAA0M_Pg;jiOM(9biLUe&aE$^^2FU^W(ro#l!jF z@<@7}kWFC4Tir#wyQq?j_&)TCabTu2O6_}z^n%-)7q47VwE*JZ1&twi=0Hb2&w@vb zP*w!S{9~<_{lum~@a+qHw^*gMh=%=~q6E$^zFdko1whm*R63j?1l~&f@ueEqZ)H^v zOtRarWVSX`+0-r)Vw!m`y-vzrVU_MHMpaD#@~-fx+MFe|?Up~vRSss^mu@e1DI!T` z#;ZiEtu_E0Bx|pwCLcYez8PQIyt-~ignhXrHd^(HXrR!YjBqf&*tED z&5_hvYboyR`qS^TZN~j--0;C~@lRdq|cR=XQ|Fs)H&pzZRhR@}q#*&>{aXTMZ1C5NMcc zWV)%jZU%m9mllt55zERes-)Rb;0y>z82-Me?j`y+$2q_wc;@^$l+ezc+Zc{ZmMW?{VQ@s_^BSmB`gZ!(arg+dmc#KH{$COAl9qDiESg02ugeES$8Ok& z!4)uvk0R3PI2Q4)D2Khz;;I2`!2!{=?XsR3+>=Ph4gqvMZ@B8W0)$?@BrGyMcj~e# z)q0hCc>rtXp%^iTaR!6BI`tCFE?;{QCuV$xDJPIn{0<2=A}2Gx8c&7zxt5I8B5wk= z>WED&g9^V2YmDgFdkXcei(+qAb$=h+mg}K;kf<#&z6x{={I_F8#>)vF z{q{UF0nk^6hcgjB=4HH7)lwuowxfyoC9(QgUFJab2~2m74zrI32Vt#zc$j_Pjn2Gj zI}_RMMW)2y^{>XVknUi#{(F`4z?zl2Xji9eF{>`j+)1{Infp*$XSkG;!??l920+bj zh_oNVKpE5P=_lAzaD3?ML(DvM|CrIe-jz{FbT9BAC#35AsQSdPImW65H6?1^88#0B z&Ml7G-D3T&oyMP4$YiidY#wT?3KgDjN&3im(d}+>Xnwo^z4CCH7Z9511qR@#n%}mv zZslOT)KLheyc3?^5C7sOGY@FwMWh@r_=V@N)&hkp;kvn^ik0yNec&0_SQwa*o z=Wg_|y~|uZ!HHq@gCaE-2L`-g@FBpCwswRAI!;p-;wVF*+BHpLWv z6sv`p!Vf*5vn`myj~4LdW?~9Q)5Ai$I?Uk11uOspKK)Jp9_&*n;i>%2eUhok#_|Tw z{N^B<2hY6W>+*}yXY$Mkq_DfLPnSmopBA5K4 z%CIBq!r+Bb@`#1guBv(a`e^K%td@@#W415?8b}nu{?7B#4Z84Kv6I#eu=;+six&~z zH$VP=Sf=bFLCsw>v%q8_8S&EZ2*5Uk(^twA10u?*prhtMpBd^@fUE+EI1+MP{=OYK z#Q}-&U2p2=T(Yn>k$|8eC#ewYu8FaRf8p~qULGtPK5Dmm-o!JfVcn6r=LbG4`}SE+_RttV$MM{i#}H#!cE33h6u5IM_Ydlq~y+nLr5gqK-i>k64$SOKrLkgYwCAA zd@g66#PTF}^;o@=7#K6(oHJr-o=W}tYMize?UW!8@U>)woxtZOuh=|d5gUD!LrDn3 z9=>ED9KZVb{`(S(-+$0A;v*%xBAE&C=JcwbqW2En)!J-a9TN^*vtZksw|aXVT9Sd zvD9v&OU!q=L{D{zkx{xtTYQfdtE)gg33((|Iqk)-?by__4ycC^ol_WZWuu=3pfQAe zl(2Vq?y2b8ytS&VB0)WdI?IHm_v+Cp%&0ocmLWT_un8>a;AGe$Wu;=&DM2X9TFYCS zx_7q00dMDq)^gPZP;h6LvQ|jh<1DRj7G;f^>Y_8-L6zYjOR^E7<^X{NoN=bQ#tv*& z&?a1%SWqJ7=!KMrBabbw&#junmY16Oz~XcSUQA6}I_Tx>JYi}1+66SEUfX555sNv! zu#e(5R3`q#gmB#4LC|aQHrjzX$J7CB2IjZG?13u`9tfKV$5z4F*Xe_FoPd(F;MI^c zw6fg@(dNeXZhT_c10LxMKtFie0gvWnn!0m?NaI_@N#hz$AUgWD-C4a!nVj zs-aytT7K?mctqK}JS-`|b%?p?-i|71?a|o{)+DFT8MuV#r%^wpnvjJ~IbW!@$Zqo| zWqKe9Wg2cl)6K9?K_i&SML>~=n)2gj$#Ks}7(iA71m5TWr>O-osS@sNJmbdnmp02_ z2GiZ2k&EnE4?-x0$YYA-t0g2E{KM2s0ED{uB4VCTPG?P+XYW&H&J1(e4D)u8Iay?u zVa}jIC9VOGcrb;;@qsoFm9o&}L6VzT?U{E$QGdB!n%)a~ zIk8#}!-=5CmOrdS*LHS6$802U|hiK32?p4R|EK z6TRcMmWH13fb7Q*KuFv+%Pafqb8E7!Wz?WHz;?TB`z9!Mph}&M4qrol0!|3LV(;+b YYX&(xyqwoVPhwO|dL|?n$w|ci3l@+K`~Uy| literal 0 HcmV?d00001 From 14a3dcb204ce3955ab7cf1f76211e05300897d4d Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:54 +0000 Subject: [PATCH 06/13] Make TaskData dictionary trainer executable Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- .../turbo-tasks-backend/scripts/train-taskdata-dictionary.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh diff --git a/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh b/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh old mode 100644 new mode 100755 From eb4e3f38db6054c8e40c05038a3cc059aee0344c Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:54 +0000 Subject: [PATCH 07/13] Allow zstd dictionary training from LZ4 caches Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- turbopack/crates/turbo-persistence/README.md | 11 +- .../src/bin/zstd_dictionary.rs | 239 +++++++++++------- .../turbo-persistence/src/compression.rs | 25 +- 3 files changed, 177 insertions(+), 98 deletions(-) diff --git a/turbopack/crates/turbo-persistence/README.md b/turbopack/crates/turbo-persistence/README.md index 0d0692d0007e..d552f4db381d 100644 --- a/turbopack/crates/turbo-persistence/README.md +++ b/turbopack/crates/turbo-persistence/README.md @@ -380,12 +380,13 @@ cargo run -p turbo-persistence --release --bin zstd_dictionary -- evaluate \ Training produces a 64 KiB dictionary from up to approximately 64 MiB of samples. It takes one hash-ordered logical value from each cache in turn, so one large cache cannot monopolize the sample. -The output path is replaced atomically. +The output path is overwritten directly. -The no-dictionary zstd level 3 baseline is always included during evaluation. Pass -`--source-dictionary ` when the input caches were written with a dictionary. The tool follows -`CURRENT`, deletion files, and meta-file supersession, and uses `StaticSortedFileIter` to read slice, -medium, and blob values. Checksums, dictionary IDs, and decompressed lengths are verified. +The no-dictionary zstd level 3 baseline is always included during evaluation. Source SSTs may use +LZ4 or plain zstd without extra options. Pass `--source-dictionary ` when any input SST records +a nonzero dictionary ID; it is ignored for LZ4 and plain-zstd SSTs. The tool follows `CURRENT`, +deletion files, and meta-file supersession, and uses `StaticSortedFileIter` to read slice, medium, and +blob values. Checksums, dictionary IDs, and decompressed lengths are verified. Small values are grouped into physical blocks in production, so the report's per-value 12.5% minimum-savings calculation is a comparative estimate, not exact SST-size modeling. Blob estimates diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs index 2ad0887486c0..51b4605d105b 100644 --- a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -154,6 +154,7 @@ struct CacheReport { path: PathBuf, family: u32, active_ssts: u64, + source_codecs: BTreeSet, samples: Metric, duplicate_blob_references: u64, candidates: Vec, @@ -173,6 +174,7 @@ struct EvaluationReport { #[derive(Serialize)] struct TrainingCacheReport { path: PathBuf, + source_codecs: BTreeSet, samples: Metric, } @@ -192,16 +194,52 @@ struct TrainingReport { struct CacheSampleIter { path: PathBuf, - pending: VecDeque, - current: Option, - compression: CompressionConfig, + pending: VecDeque<(SstInfo, CompressionConfig)>, + current: Option<(StaticSortedFileIter, CompressionConfig)>, + source_codecs: BTreeSet, seen_blobs: HashSet, active_ssts: u64, duplicate_blob_references: u64, } +fn source_compression_for_sst( + path: &Path, + family: u32, + sst: &SstInfo, + source_dictionary: Option<&'static [u8]>, +) -> Result { + match (sst.compression, sst.dictionary_id) { + (Compression::Lz4, 0) => Ok(CompressionConfig::Lz4), + (Compression::Lz4, id) => anyhow::bail!( + "Cache {} family {family} SST {:08}.sst records LZ4 with unexpected dictionary ID {id}", + path.display(), + sst.sequence_number + ), + (Compression::Zstd3, 0) => Ok(CompressionConfig::Zstd3), + (Compression::Zstd3, id) => { + let dictionary = source_dictionary.with_context(|| { + format!( + "Cache {} family {family} SST {:08}.sst requires source dictionary ID {id}", + path.display(), + sst.sequence_number + ) + })?; + let configured = CompressionConfig::Zstd3WithDictionary(dictionary); + ensure!( + configured.dictionary_id() == Some(id), + "Cache {} family {family} SST {:08}.sst requires source dictionary ID {id}, but \ + supplied dictionary has ID {:?}", + path.display(), + sst.sequence_number, + configured.dictionary_id() + ); + Ok(configured) + } + } +} + impl CacheSampleIter { - fn open(path: PathBuf, family: u32, compression: CompressionConfig) -> Result { + fn open(path: PathBuf, family: u32, source_dictionary: Option<&'static [u8]>) -> Result { let mut families = collect_sst_info(&path) .with_context(|| format!("Failed to inspect cache {}", path.display()))?; let mut ssts = families.remove(&family).with_context(|| { @@ -210,47 +248,43 @@ impl CacheSampleIter { path.display() ) })?; - for sst in &ssts { - ensure!( - sst.compression == Compression::Zstd3 - && sst.dictionary_id == compression.dictionary_id().unwrap_or(0), - "Cache {} family {family} SST {:08}.sst uses {:?} dictionary {}, but source \ - config uses {:?}", - path.display(), - sst.sequence_number, - sst.compression, - sst.dictionary_id, - compression - ); - } // A stable SST order makes repeated runs against one unchanged cache snapshot comparable. ssts.sort_by_key(|sst| sst.sequence_number); + let mut source_codecs = BTreeSet::new(); + let pending = ssts + .into_iter() + .map(|sst| { + let compression = + source_compression_for_sst(&path, family, &sst, source_dictionary)?; + source_codecs.insert(format!("{compression:?}")); + Ok((sst, compression)) + }) + .collect::>>()?; Ok(Self { path, - active_ssts: ssts.len() as u64, - pending: ssts.into(), + active_ssts: pending.len() as u64, + pending, current: None, - compression, + source_codecs, seen_blobs: HashSet::new(), duplicate_blob_references: 0, }) } fn open_next_sst(&mut self) -> Result { - let Some(sst) = self.pending.pop_front() else { + let Some((sst, compression)) = self.pending.pop_front() else { return Ok(false); }; - self.current = Some( - StaticSortedFileIter::open( - &self.path, - StaticSortedFileMetaData { - sequence_number: sst.sequence_number, - block_count: sst.block_count, - }, - self.compression, - ) - .with_context(|| format!("Failed to open {:08}.sst", sst.sequence_number))?, - ); + let iter = StaticSortedFileIter::open( + &self.path, + StaticSortedFileMetaData { + sequence_number: sst.sequence_number, + block_count: sst.block_count, + }, + compression, + ) + .with_context(|| format!("Failed to open {:08}.sst", sst.sequence_number))?; + self.current = Some((iter, compression)); Ok(true) } @@ -259,20 +293,26 @@ impl CacheSampleIter { if self.current.is_none() && !self.open_next_sst()? { return Ok(None); } - let entry = match self.current.as_mut().unwrap().next() { + let (iter, compression) = self.current.as_mut().unwrap(); + let compression = *compression; + let entry = match iter.next() { Some(entry) => entry?, None => { self.current = None; continue; } }; - if let Some(sample) = self.sample_from_value(entry.value)? { + if let Some(sample) = self.sample_from_value(entry.value, compression)? { return Ok(Some(sample)); } } } - fn sample_from_value(&mut self, value: IterValue) -> Result> { + fn sample_from_value( + &mut self, + value: IterValue, + compression: CompressionConfig, + ) -> Result> { match value { IterValue::Slice { value } if value.len() > MAX_INLINE_VALUE_SIZE => Ok(Some(Sample { kind: SampleKind::Slice, @@ -283,7 +323,7 @@ impl CacheSampleIter { checksum, block, } => { - let value = decode_medium(self.compression, uncompressed_size, checksum, &block) + let value = decode_medium(compression, uncompressed_size, checksum, &block) .with_context(|| { format!("Failed to read medium value in {}", self.path.display()) })?; @@ -297,7 +337,7 @@ impl CacheSampleIter { self.duplicate_blob_references += 1; return Ok(None); } - let value = read_blob(&self.path, sequence_number, self.compression)?; + let value = read_blob(&self.path, sequence_number, compression)?; Ok(Some(Sample { kind: SampleKind::Blob, data: value, @@ -440,10 +480,10 @@ fn finalize_results(results: &mut [CandidateResult]) { fn evaluate_cache( path: &Path, family: u32, - compression: CompressionConfig, + source_dictionary: Option<&'static [u8]>, candidates: &mut [Candidate], ) -> Result { - let mut iter = CacheSampleIter::open(path.to_path_buf(), family, compression)?; + let mut iter = CacheSampleIter::open(path.to_path_buf(), family, source_dictionary)?; let mut samples = Metric::default(); let mut results = empty_results(candidates); while let Some(sample) = iter.next_sample()? { @@ -455,6 +495,7 @@ fn evaluate_cache( path: path.to_path_buf(), family, active_ssts: iter.active_ssts, + source_codecs: iter.source_codecs, samples, duplicate_blob_references: iter.duplicate_blob_references, candidates: results, @@ -500,19 +541,20 @@ struct TrainingSelection { fn select_training_samples( paths: &[PathBuf], family: u32, - compression: CompressionConfig, + source_dictionary: Option<&'static [u8]>, byte_budget: usize, ) -> Result { let mut paths = paths.to_vec(); paths.sort(); let mut iterators = paths .into_iter() - .map(|path| CacheSampleIter::open(path, family, compression)) + .map(|path| CacheSampleIter::open(path, family, source_dictionary)) .collect::>>()?; let mut per_cache = iterators .iter() .map(|iter| TrainingCacheReport { path: iter.path.clone(), + source_codecs: iter.source_codecs.clone(), samples: Metric::default(), }) .collect::>(); @@ -562,20 +604,20 @@ fn write_dictionary(path: &Path, bytes: &[u8]) -> Result<()> { .with_context(|| format!("Failed to write dictionary {}", path.display())) } -fn source_compression(source: &Source) -> Result { - match &source.source_dictionary { - Some(path) => { - let dictionary = fs::read(path) - .with_context(|| format!("Failed to read source dictionary {}", path.display()))?; - let dictionary = Box::leak(dictionary.into_boxed_slice()); - Ok(CompressionConfig::Zstd3WithDictionary(dictionary)) - } - None => Ok(CompressionConfig::Zstd3), - } +fn source_dictionary(source: &Source) -> Result> { + source + .source_dictionary + .as_ref() + .map(|path| { + fs::read(path) + .with_context(|| format!("Failed to read source dictionary {}", path.display())) + .map(|bytes| Box::leak(bytes.into_boxed_slice()) as &'static [u8]) + }) + .transpose() } fn train(source: &Source, output: &Path) -> Result { - let compression = source_compression(source)?; + let source_dictionary = source_dictionary(source)?; let TrainingSelection { samples, caches, @@ -584,7 +626,7 @@ fn train(source: &Source, output: &Path) -> Result { } = select_training_samples( &source.caches, source.family, - compression, + source_dictionary, SAMPLE_BYTE_BUDGET, )?; ensure!(!samples.is_empty(), "No eligible values found for training"); @@ -630,8 +672,9 @@ fn print_training(report: &TrainingReport) { ); for cache in &report.caches { println!( - " {}: {} values / {} bytes", + " {} ({:?}): {} values / {} bytes", cache.path.display(), + cache.source_codecs, cache.samples.count, cache.samples.bytes ); @@ -647,6 +690,9 @@ fn print_evaluation(report: &EvaluationReport) { samples.count, samples.bytes ); + for cache in &report.caches { + println!(" {}: {:?}", cache.path.display(), cache.source_codecs); + } println!( "{:<28} {:>15} {:>9} {:>18} {:>12} {:>12}", "Candidate", "Raw compressed", "ratio", "Estimated stored", "encode ms", "decode ms" @@ -692,13 +738,11 @@ fn run(cli: Cli) -> Result<()> { json, } => { let mut candidates = make_candidates(&dictionary)?; - let source_compression = source_compression(&source)?; + let source_dictionary = source_dictionary(&source)?; let caches = source .caches .iter() - .map(|path| { - evaluate_cache(path, source.family, source_compression, &mut candidates) - }) + .map(|path| evaluate_cache(path, source.family, source_dictionary, &mut candidates)) .collect::>>()?; let report = combine_evaluation(source.family, caches, &candidates); print_evaluation(&report); @@ -716,7 +760,7 @@ fn main() { #[cfg(test)] mod tests { - use std::{ffi::OsString, fs, path::PathBuf}; + use std::{collections::BTreeSet, ffi::OsString, fs, path::PathBuf}; use anyhow::Result; use byteorder::{BE, WriteBytesExt}; @@ -802,13 +846,20 @@ mod tests { #[test] fn round_robin_samples_multiple_caches() -> Result<()> { let first = make_cache(2, Compression::Zstd3, 20)?; - let second = make_cache(2, Compression::Zstd3, 20)?; + let second = make_cache(2, Compression::Lz4, 20)?; let paths = [first.path().to_path_buf(), second.path().to_path_buf()]; - let selection = select_training_samples(&paths, 2, CompressionConfig::Zstd3, 10_000)?; + let selection = select_training_samples(&paths, 2, None, 10_000)?; assert!(!selection.samples.is_empty()); assert!(!selection.inputs_exhausted); assert_eq!(selection.caches.len(), 2); assert_eq!(&selection.selected_cache_indices[..2], &[0, 1]); + let source_codecs = selection + .caches + .iter() + .flat_map(|cache| cache.source_codecs.iter()) + .collect::>(); + assert!(source_codecs.contains(&"Zstd3".to_owned())); + assert!(source_codecs.contains(&"Lz4".to_owned())); assert!( selection .caches @@ -819,15 +870,21 @@ mod tests { } #[test] - fn rejects_non_zstd_families_before_sampling() -> Result<()> { + fn accepts_lz4_source_cache_and_ignores_source_dictionary() -> Result<()> { let cache = make_cache(2, Compression::Lz4, 20)?; - let error = CacheSampleIter::open(cache.path().to_path_buf(), 2, CompressionConfig::Zstd3) - .err() - .expect("LZ4 family must be rejected"); - let message = format!("{error:#}"); - assert!(message.contains("00000001.sst")); - assert!(message.contains("Lz4")); - assert!(message.contains("source config uses Zstd3")); + let mut iter = + CacheSampleIter::open(cache.path().to_path_buf(), 2, Some(b"irrelevant for LZ4"))?; + assert!(iter.source_codecs.contains("Lz4")); + assert!(iter.next_sample()?.is_some()); + + let cache = make_cache(2, Compression::Zstd3, 20)?; + let mut iter = CacheSampleIter::open( + cache.path().to_path_buf(), + 2, + Some(b"irrelevant for plain zstd"), + )?; + assert!(iter.source_codecs.contains("Zstd3")); + assert!(iter.next_sample()?.is_some()); Ok(()) } @@ -840,15 +897,8 @@ mod tests { let dictionary = Box::leak(dictionary.into_boxed_slice()); let cache = make_cache_with_config(2, CompressionConfig::Zstd3WithDictionary(dictionary), 20)?; - assert!( - CacheSampleIter::open(cache.path().to_path_buf(), 2, CompressionConfig::Zstd3,) - .is_err() - ); - let mut iter = CacheSampleIter::open( - cache.path().to_path_buf(), - 2, - CompressionConfig::Zstd3WithDictionary(dictionary), - )?; + assert!(CacheSampleIter::open(cache.path().to_path_buf(), 2, None).is_err()); + let mut iter = CacheSampleIter::open(cache.path().to_path_buf(), 2, Some(dictionary))?; assert!(iter.next_sample()?.is_some()); Ok(()) } @@ -885,8 +935,7 @@ mod tests { assert_ne!(fs::read(&output)?, b"old"); let mut candidates = make_candidates(&[output])?; - let evaluation = - evaluate_cache(cache.path(), 2, CompressionConfig::Zstd3, &mut candidates)?; + let evaluation = evaluate_cache(cache.path(), 2, None, &mut candidates)?; assert_eq!(evaluation.candidates.len(), 2); assert!(evaluation.samples.count > 0); Ok(()) @@ -903,12 +952,14 @@ mod tests { blob.extend_from_slice(&compressed); fs::write(cache.path().join("00000042.blob"), blob)?; - let mut iter = - CacheSampleIter::open(cache.path().to_path_buf(), 2, CompressionConfig::Zstd3)?; + let mut iter = CacheSampleIter::open(cache.path().to_path_buf(), 2, None)?; let sample = iter - .sample_from_value(turbo_persistence::IterValue::Blob { - sequence_number: 42, - })? + .sample_from_value( + turbo_persistence::IterValue::Blob { + sequence_number: 42, + }, + CompressionConfig::Zstd3, + )? .unwrap(); assert_eq!(sample.data.as_ref(), value); let mut candidates = make_candidates(&[])?; @@ -920,16 +971,22 @@ mod tests { results[0].combined.raw_compressed_bytes + 8 ); assert!( - iter.sample_from_value(turbo_persistence::IterValue::Blob { - sequence_number: 42, - })? + iter.sample_from_value( + turbo_persistence::IterValue::Blob { + sequence_number: 42 + }, + CompressionConfig::Zstd3, + )? .is_none() ); assert_eq!(iter.duplicate_blob_references, 1); assert!( - iter.sample_from_value(turbo_persistence::IterValue::Blob { - sequence_number: 43, - }) + iter.sample_from_value( + turbo_persistence::IterValue::Blob { + sequence_number: 43 + }, + CompressionConfig::Zstd3, + ) .is_err() ); Ok(()) diff --git a/turbopack/crates/turbo-persistence/src/compression.rs b/turbopack/crates/turbo-persistence/src/compression.rs index 15859588b690..6634c53b9037 100644 --- a/turbopack/crates/turbo-persistence/src/compression.rs +++ b/turbopack/crates/turbo-persistence/src/compression.rs @@ -1,4 +1,4 @@ -use std::{cell::RefCell, mem::MaybeUninit, rc::Rc, sync::Arc}; +use std::{cell::RefCell, fmt, mem::MaybeUninit, rc::Rc, sync::Arc}; use anyhow::{Context, Result, ensure}; use lzzzz::lz4::{self, decompress}; @@ -15,7 +15,7 @@ pub enum Compression { } /// Runtime compression configuration for a persistence family. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Copy, Default, PartialEq, Eq)] pub enum CompressionConfig { #[default] Lz4, @@ -23,6 +23,19 @@ pub enum CompressionConfig { Zstd3WithDictionary(&'static [u8]), } +impl fmt::Debug for CompressionConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Lz4 => formatter.write_str("Lz4"), + Self::Zstd3 => formatter.write_str("Zstd3"), + Self::Zstd3WithDictionary(_) => formatter + .debug_struct("Zstd3WithDictionary") + .field("dictionary_id", &self.dictionary_id()) + .finish(), + } + } +} + impl CompressionConfig { pub fn algorithm(self) -> Compression { match self { @@ -201,6 +214,14 @@ impl Compressor { mod tests { use super::*; + #[test] + fn dictionary_debug_does_not_include_bytes() { + let dictionary = Box::leak(vec![42; 64 * 1024].into_boxed_slice()); + let debug = format!("{:?}", CompressionConfig::Zstd3WithDictionary(dictionary)); + assert!(debug.starts_with("Zstd3WithDictionary")); + assert!(debug.len() < 100); + } + #[test] fn dictionary_compression_round_trips() { let samples = (0..100) From bffedcdcd921d79dc37a22abcd5d28a157b4c33d Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:55 +0000 Subject: [PATCH 08/13] Remove dictionary sample kind tracking Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- turbopack/crates/turbo-persistence/README.md | 6 ++-- .../src/bin/zstd_dictionary.rs | 35 +++++-------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/turbopack/crates/turbo-persistence/README.md b/turbopack/crates/turbo-persistence/README.md index d552f4db381d..0eea00ce6a60 100644 --- a/turbopack/crates/turbo-persistence/README.md +++ b/turbopack/crates/turbo-persistence/README.md @@ -389,9 +389,9 @@ deletion files, and meta-file supersession, and uses `StaticSortedFileIter` to r blob values. Checksums, dictionary IDs, and decompressed lengths are verified. Small values are grouped into physical blocks in production, so the report's per-value 12.5% -minimum-savings calculation is a comparative estimate, not exact SST-size modeling. Blob estimates -include their fixed 8-byte headers. Timing fields are single-pass diagnostics; use byte/count fields -for repeatable comparisons of one copied cache snapshot. +minimum-savings calculation is a comparative estimate, not exact SST-size modeling. Estimated stored +bytes exclude fixed container headers. Timing fields are single-pass diagnostics; use byte/count +fields for repeatable comparisons of one copied cache snapshot. ## Opening diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs index 51b4605d105b..810de14ed1f7 100644 --- a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -23,7 +23,6 @@ const SCHEMA_VERSION: u32 = 2; const DICTIONARY_SIZE: usize = 64 * 1024; const SAMPLE_BUDGET_MULTIPLIER: usize = 1000; const SAMPLE_BYTE_BUDGET: usize = DICTIONARY_SIZE * SAMPLE_BUDGET_MULTIPLIER; -const BLOB_HEADER_SIZE: usize = 8; #[derive(Parser)] #[command(about = "Train and evaluate zstd dictionaries from persistence caches")] @@ -87,15 +86,7 @@ struct Candidate { setup_ns: u64, } -#[derive(Clone, Copy)] -enum SampleKind { - Slice, - Medium, - Blob, -} - struct Sample { - kind: SampleKind, data: Arc<[u8]>, } @@ -315,7 +306,6 @@ impl CacheSampleIter { ) -> Result> { match value { IterValue::Slice { value } if value.len() > MAX_INLINE_VALUE_SIZE => Ok(Some(Sample { - kind: SampleKind::Slice, data: Arc::from(value.as_ref()), })), IterValue::Medium { @@ -327,10 +317,7 @@ impl CacheSampleIter { .with_context(|| { format!("Failed to read medium value in {}", self.path.display()) })?; - Ok(Some(Sample { - kind: SampleKind::Medium, - data: value, - })) + Ok(Some(Sample { data: value })) } IterValue::Blob { sequence_number } => { if !self.seen_blobs.insert(sequence_number) { @@ -338,10 +325,7 @@ impl CacheSampleIter { return Ok(None); } let value = read_blob(&self.path, sequence_number, compression)?; - Ok(Some(Sample { - kind: SampleKind::Blob, - data: value, - })) + Ok(Some(Sample { data: value })) } // Inline values live in key blocks and are not independently compressed. IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. } | IterValue::Slice { .. } => { @@ -439,11 +423,8 @@ fn evaluate_sample( metric.raw_compressed_bytes += compressed.len() as u64; metric.encode_ns += encode_ns; metric.decode_ns += decode_ns; - metric.estimated_stored_bytes += if matches!(sample.kind, SampleKind::Blob) { - (compressed.len() + BLOB_HEADER_SIZE) as u64 - } else { - estimated_value_bytes(sample.data.len(), compressed.len()) as u64 - }; + metric.estimated_stored_bytes += + estimated_value_bytes(sample.data.len(), compressed.len()) as u64; } Ok(()) } @@ -521,9 +502,9 @@ fn combine_evaluation( family, timing_note: "Single-pass wall-clock diagnostics; byte/count fields are the comparison \ contract.", - threshold_note: "Slice/medium stored bytes apply the 12.5% rule per logical value and are \ - a proxy for grouped small-value blocks. Blob bytes include the fixed \ - 8-byte header.", + threshold_note: "Estimated stored bytes apply the 12.5% rule per logical value and \ + exclude fixed container headers; they are a proxy for grouped \ + small-value blocks.", caches, combined_samples, combined_candidates, @@ -968,7 +949,7 @@ mod tests { finalize_results(&mut results); assert_eq!( results[0].combined.estimated_stored_bytes, - results[0].combined.raw_compressed_bytes + 8 + results[0].combined.raw_compressed_bytes ); assert!( iter.sample_from_value( From 776c6a92ac140258bc1c88d340422c98d8b86d6e Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:56 +0000 Subject: [PATCH 09/13] Use typed dictionary timing and context keys Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- .../src/bin/zstd_dictionary.rs | 34 +++++++++---------- .../turbo-persistence/src/compression.rs | 26 ++++++-------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs index 810de14ed1f7..f8190fa9545e 100644 --- a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -6,7 +6,7 @@ use std::{ io::{BufWriter, Write}, path::{Path, PathBuf}, sync::Arc, - time::Instant, + time::{Duration, Instant}, }; use anyhow::{Context, Result, ensure}; @@ -19,7 +19,7 @@ use turbo_persistence::{ }; use xxhash_rust::xxh3::xxh3_64; -const SCHEMA_VERSION: u32 = 2; +const SCHEMA_VERSION: u32 = 3; const DICTIONARY_SIZE: usize = 64 * 1024; const SAMPLE_BUDGET_MULTIPLIER: usize = 1000; const SAMPLE_BYTE_BUDGET: usize = DICTIONARY_SIZE * SAMPLE_BUDGET_MULTIPLIER; @@ -83,7 +83,7 @@ struct Candidate { info: DictionaryInfo, compressor: zstd::bulk::Compressor<'static>, decompressor: zstd::bulk::Decompressor<'static>, - setup_ns: u64, + setup_duration: Duration, } struct Sample { @@ -114,8 +114,8 @@ struct CompressionMetric { raw_compressed_bytes: u64, estimated_stored_bytes: u64, raw_compression_ratio: Option, - encode_ns: u64, - decode_ns: u64, + encode_duration: Duration, + decode_duration: Duration, } impl CompressionMetric { @@ -123,8 +123,8 @@ impl CompressionMetric { self.input_bytes += other.input_bytes; self.raw_compressed_bytes += other.raw_compressed_bytes; self.estimated_stored_bytes += other.estimated_stored_bytes; - self.encode_ns += other.encode_ns; - self.decode_ns += other.decode_ns; + self.encode_duration += other.encode_duration; + self.decode_duration += other.decode_duration; } fn finalize(&mut self) { @@ -137,7 +137,7 @@ impl CompressionMetric { struct CandidateResult { dictionary: DictionaryInfo, combined: CompressionMetric, - setup_ns: u64, + setup_duration: Duration, } #[derive(Serialize)] @@ -365,7 +365,7 @@ fn make_candidates(paths: &[PathBuf]) -> Result> { info: dictionary_info(None, &[], true)?, compressor: zstd::bulk::Compressor::new(3)?, decompressor: zstd::bulk::Decompressor::new()?, - setup_ns: started.elapsed().as_nanos() as u64, + setup_duration: started.elapsed(), }); let mut names = BTreeSet::new(); for path in paths { @@ -387,7 +387,7 @@ fn make_candidates(paths: &[PathBuf]) -> Result> { info, compressor: zstd::bulk::Compressor::with_dictionary(3, &dictionary)?, decompressor: zstd::bulk::Decompressor::with_dictionary(&dictionary)?, - setup_ns: started.elapsed().as_nanos() as u64, + setup_duration: started.elapsed(), }); } Ok(result) @@ -405,13 +405,13 @@ fn evaluate_sample( .compressor .compress(&sample.data) .with_context(|| format!("Failed to compress with {}", candidate.info.name))?; - let encode_ns = started.elapsed().as_nanos() as u64; + let encode_duration = started.elapsed(); let started = Instant::now(); let decompressed = candidate .decompressor .decompress(&compressed, sample.data.len()) .with_context(|| format!("Failed to decompress with {}", candidate.info.name))?; - let decode_ns = started.elapsed().as_nanos() as u64; + let decode_duration = started.elapsed(); ensure!( decompressed.as_slice() == sample.data.as_ref(), "Round-trip mismatch with {}", @@ -421,8 +421,8 @@ fn evaluate_sample( let metric = &mut result.combined; metric.input_bytes += sample.data.len() as u64; metric.raw_compressed_bytes += compressed.len() as u64; - metric.encode_ns += encode_ns; - metric.decode_ns += decode_ns; + metric.encode_duration += encode_duration; + metric.decode_duration += decode_duration; metric.estimated_stored_bytes += estimated_value_bytes(sample.data.len(), compressed.len()) as u64; } @@ -447,7 +447,7 @@ fn empty_results(candidates: &[Candidate]) -> Vec { .map(|candidate| CandidateResult { dictionary: candidate.info.clone(), combined: CompressionMetric::default(), - setup_ns: candidate.setup_ns, + setup_duration: candidate.setup_duration, }) .collect() } @@ -685,8 +685,8 @@ fn print_evaluation(report: &EvaluationReport) { result.combined.raw_compressed_bytes, result.combined.raw_compression_ratio.unwrap_or_default() * 100.0, result.combined.estimated_stored_bytes, - result.combined.encode_ns as f64 / 1_000_000.0, - result.combined.decode_ns as f64 / 1_000_000.0, + result.combined.encode_duration.as_secs_f64() * 1_000.0, + result.combined.decode_duration.as_secs_f64() * 1_000.0, ); } println!("Note: {}", report.threshold_note); diff --git a/turbopack/crates/turbo-persistence/src/compression.rs b/turbopack/crates/turbo-persistence/src/compression.rs index 6634c53b9037..fa517b383504 100644 --- a/turbopack/crates/turbo-persistence/src/compression.rs +++ b/turbopack/crates/turbo-persistence/src/compression.rs @@ -70,7 +70,7 @@ impl From for CompressionConfig { thread_local! { /// Zstd decompression contexts are reusable and relatively expensive to create. Keep one per /// worker thread to avoid allocation on every block read without a global lock. - static ZSTD_DECOMPRESSOR: RefCell<(Option, zstd::bulk::Decompressor<'static>)> = RefCell::new( + static ZSTD_DECOMPRESSOR: RefCell<(Option<&'static [u8]>, zstd::bulk::Decompressor<'static>)> = RefCell::new( (None, zstd::bulk::Decompressor::new().expect("zstd decompressor initialization should succeed")) ); } @@ -91,16 +91,18 @@ fn decompress_block( CompressionConfig::Lz4 => decompress(block, dest).map_err(anyhow::Error::from), CompressionConfig::Zstd3 | CompressionConfig::Zstd3WithDictionary(_) => ZSTD_DECOMPRESSOR .with_borrow_mut(|state| { - let dictionary = compression.dictionary().unwrap_or_default(); - let key = compression - .dictionary() - .map(|dictionary| dictionary.as_ptr() as usize); - if state.0 != key { + let dictionary = compression.dictionary(); + let same_dictionary = match (state.0, dictionary) { + (Some(current), Some(next)) => std::ptr::eq(current, next), + (None, None) => true, + _ => false, + }; + if !same_dictionary { state .1 - .set_dictionary(dictionary) + .set_dictionary(dictionary.unwrap_or_default()) .map_err(anyhow::Error::from)?; - state.0 = key; + state.0 = dictionary; } state .1 @@ -214,14 +216,6 @@ impl Compressor { mod tests { use super::*; - #[test] - fn dictionary_debug_does_not_include_bytes() { - let dictionary = Box::leak(vec![42; 64 * 1024].into_boxed_slice()); - let debug = format!("{:?}", CompressionConfig::Zstd3WithDictionary(dictionary)); - assert!(debug.starts_with("Zstd3WithDictionary")); - assert!(debug.len() < 100); - } - #[test] fn dictionary_compression_round_trips() { let samples = (0..100) From b98de5c2980eff83dec3d6d14f0c7bc91adab9f5 Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:56 +0000 Subject: [PATCH 10/13] Support keyed dictionaries in SST inspector Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- .../turbo-persistence/src/bin/sst_inspect.rs | 188 +++++++++--------- .../crates/turbo-persistence/src/offline.rs | 2 + 2 files changed, 101 insertions(+), 89 deletions(-) diff --git a/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs b/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs index 7014d0e7e96b..1e48fd8f4d9b 100644 --- a/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs +++ b/turbopack/crates/turbo-persistence/src/bin/sst_inspect.rs @@ -16,6 +16,7 @@ use std::{ use anyhow::{Context, Result, bail}; use byteorder::{BE, ReadBytesExt}; +use clap::Parser; use fs_err::File; use lzzzz::lz4::decompress; use memmap2::Mmap; @@ -792,89 +793,60 @@ fn print_family_summary(family: u32, sst_count: usize, stats: &SstStats) { println!(); } +fn entry_type_help() -> String { + format!( + "Entry types:\n {KEY_BLOCK_ENTRY_TYPE_SMALL}: Small value (stored in separate value \ + block)\n {KEY_BLOCK_ENTRY_TYPE_BLOB}: Blob reference\n \ + {KEY_BLOCK_ENTRY_TYPE_KEY_DELETED}: Key tombstone (deletes all values for the key)\n \ + {KEY_BLOCK_ENTRY_TYPE_MEDIUM}: Medium value\n {KEY_BLOCK_ENTRY_TYPE_INLINE_MIN}-{}: \ + Inline value (size = type - {KEY_BLOCK_ENTRY_TYPE_INLINE_MIN})\n \ + {KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN}-{}: Key-value tombstone (deleted value size \ + = type - {KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN})\n\nFor TaskCache (family 3), \ + values are 4-byte TaskIds. Expected entry type is {} ({KEY_BLOCK_ENTRY_TYPE_INLINE_MIN} \ + + 4) for inline optimization.", + KEY_BLOCK_ENTRY_TYPE_INLINE_MIN + MAX_INLINE_VALUE_SIZE as u8, + KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN + MAX_INLINE_VALUE_SIZE as u8, + KEY_BLOCK_ENTRY_TYPE_INLINE_MIN + 4, + ) +} + +#[derive(Parser)] +#[command(about = "Inspect turbo-persistence SST files", after_long_help = entry_type_help())] +struct Cli { + /// Show per-SST file details (default: family totals only). + #[arg(short, long)] + verbose: bool, + /// Dictionary used by zstd input SSTs. May be supplied multiple times; IDs are read from the + /// dictionaries. + #[arg(long)] + source_dictionary: Vec, + /// Database directory containing CURRENT, meta, SST, and blob files. + db_path: PathBuf, +} + fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - - // Parse arguments - let mut db_path: Option = None; - let mut verbose = false; - let mut source_dictionary: Option = None; - - let mut i = 1; - while i < args.len() { - match args[i].as_str() { - "--verbose" | "-v" => verbose = true, - "--source-dictionary" => { - i += 1; - source_dictionary = Some(PathBuf::from( - args.get(i).context("--source-dictionary requires a path")?, - )); - } - arg if !arg.starts_with('-') => { - if db_path.is_none() { - db_path = Some(PathBuf::from(arg)); - } - } - _ => { - eprintln!("Unknown option: {}", args[i]); - std::process::exit(1); - } - } - i += 1; - } - - let db_path = match db_path { - Some(p) => p, - None => { - eprintln!("Usage: {} [OPTIONS] ", args[0]); - eprintln!(); - eprintln!("Inspects turbo-persistence SST files to report entry type statistics."); - eprintln!(); - eprintln!("Options:"); - eprintln!(" -v, --verbose Show per-SST file details (default: family totals only)"); - eprintln!(" --source-dictionary Dictionary used by zstd input caches"); - eprintln!(); - eprintln!("Entry types:"); - eprintln!( - " {KEY_BLOCK_ENTRY_TYPE_SMALL}: Small value (stored in separate value block)" - ); - eprintln!(" {KEY_BLOCK_ENTRY_TYPE_BLOB}: Blob reference"); - eprintln!( - " {KEY_BLOCK_ENTRY_TYPE_KEY_DELETED}: Key tombstone (deletes all values for the \ - key)" - ); - eprintln!(" {KEY_BLOCK_ENTRY_TYPE_MEDIUM}: Medium value"); - eprintln!( - " {KEY_BLOCK_ENTRY_TYPE_INLINE_MIN}-{}: Inline value (size = type - \ - {KEY_BLOCK_ENTRY_TYPE_INLINE_MIN})", - KEY_BLOCK_ENTRY_TYPE_INLINE_MIN + MAX_INLINE_VALUE_SIZE as u8 - ); - eprintln!( - " {KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN}-{}: Key-value tombstone (deleted \ - value size = type - {KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN})", - KEY_BLOCK_ENTRY_TYPE_KEY_VALUE_DELETED_MIN + MAX_INLINE_VALUE_SIZE as u8 - ); - eprintln!(); - eprintln!("For TaskCache (family 3), values are 4-byte TaskIds."); - eprintln!( - "Expected entry type is {} ({KEY_BLOCK_ENTRY_TYPE_INLINE_MIN} + 4) for inline \ - optimization.", - KEY_BLOCK_ENTRY_TYPE_INLINE_MIN + 4 - ); - std::process::exit(1); - } - }; + let Cli { + verbose, + source_dictionary, + db_path, + } = Cli::parse(); if !db_path.is_dir() { bail!("Not a directory: {}", db_path.display()); } - let source_dictionary = source_dictionary - .map(|path| { - fs_err::read(&path).with_context(|| format!("Failed to read {}", path.display())) - }) - .transpose()? - .map(|bytes| Box::leak(bytes.into_boxed_slice()) as &'static [u8]); + let mut source_dictionaries = BTreeMap::new(); + for path in source_dictionary { + let bytes = + fs_err::read(&path).with_context(|| format!("Failed to read {}", path.display()))?; + let dictionary = Box::leak(bytes.into_boxed_slice()) as &'static [u8]; + let id = CompressionConfig::Zstd3WithDictionary(dictionary) + .dictionary_id() + .with_context(|| format!("Dictionary {} has no zstd dictionary ID", path.display()))?; + if source_dictionaries.insert(id, dictionary).is_some() { + bail!("Duplicate source dictionary ID {id}"); + } + } // Collect SST info grouped by family let family_sst_info = collect_sst_info(&db_path)?; @@ -892,19 +864,22 @@ fn main() -> Result<()> { let mut sst_stats_list: Vec<(u32, SstStats)> = Vec::new(); for info in sst_list { - let compression = match (info.compression, info.dictionary_id, source_dictionary) { - (Compression::Lz4, 0, _) => CompressionConfig::Lz4, - (Compression::Zstd3, 0, _) => CompressionConfig::Zstd3, - (Compression::Zstd3, id, Some(dictionary)) - if Some(id) - == CompressionConfig::Zstd3WithDictionary(dictionary).dictionary_id() => - { - CompressionConfig::Zstd3WithDictionary(dictionary) - } - (_, id, _) => { + let compression = match (info.compression, info.dictionary_id) { + (Compression::Lz4, 0) => CompressionConfig::Lz4, + (Compression::Zstd3, 0) => CompressionConfig::Zstd3, + (Compression::Zstd3, id) => match source_dictionaries.get(&id) { + Some(dictionary) => CompressionConfig::Zstd3WithDictionary(dictionary), + None => { + eprintln!( + "Warning: Missing source dictionary ID {id} for {:08}.sst", + info.sequence_number + ); + continue; + } + }, + (Compression::Lz4, id) => { eprintln!( - "Warning: Missing or wrong source dictionary for {:08}.sst (dictionary ID \ - {id})", + "Warning: LZ4 SST {:08}.sst has unexpected dictionary ID {id}", info.sequence_number ); continue; @@ -941,3 +916,38 @@ fn main() -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clap_help_keeps_entry_type_reference() { + let help = Cli::try_parse_from(["sst_inspect", "--help"]) + .err() + .expect("--help should exit through clap") + .to_string(); + assert!(help.contains("Entry types:")); + assert!(help.contains("For TaskCache (family 3)")); + assert!(help.contains(&format!( + "{} ({} + 4)", + KEY_BLOCK_ENTRY_TYPE_INLINE_MIN + 4, + KEY_BLOCK_ENTRY_TYPE_INLINE_MIN + ))); + } + + #[test] + fn clap_accepts_multiple_source_dictionaries() { + let cli = Cli::try_parse_from([ + "sst_inspect", + "--source-dictionary", + "first.zdict", + "--source-dictionary", + "second.zdict", + "database", + ]) + .unwrap(); + assert_eq!(cli.source_dictionary.len(), 2); + assert_eq!(cli.db_path, PathBuf::from("database")); + } +} diff --git a/turbopack/crates/turbo-persistence/src/offline.rs b/turbopack/crates/turbo-persistence/src/offline.rs index f8dbbda4b826..69e50585dbb9 100644 --- a/turbopack/crates/turbo-persistence/src/offline.rs +++ b/turbopack/crates/turbo-persistence/src/offline.rs @@ -128,6 +128,8 @@ pub fn read_blob( expected_checksum, &format!("blob file {}", path.display()), )?; + // Blob writers always compress payloads; unlike SST blocks, the blob format currently has no + // zero-length sentinel for an uncompressed payload. ensure!( uncompressed_length > 0, "Blob file {} has an invalid uncompressed length of zero", From d46c6a851e19ec71917228254a9149b029957d5b Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:57 +0000 Subject: [PATCH 11/13] Model production blocks in dictionary evaluation Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- turbopack/crates/turbo-persistence/README.md | 11 +- .../src/bin/zstd_dictionary.rs | 209 ++++++++++-------- turbopack/crates/turbo-persistence/src/lib.rs | 2 +- .../crates/turbo-persistence/src/meta_file.rs | 2 +- .../scripts/train-taskdata-dictionary.sh | 4 +- .../src/database/taskdata-dictionary.md | 29 ++- 6 files changed, 147 insertions(+), 110 deletions(-) diff --git a/turbopack/crates/turbo-persistence/README.md b/turbopack/crates/turbo-persistence/README.md index 0eea00ce6a60..d9d8f50179ab 100644 --- a/turbopack/crates/turbo-persistence/README.md +++ b/turbopack/crates/turbo-persistence/README.md @@ -54,7 +54,7 @@ Small value blocks are emitted once they accumulate at least `MIN_SMALL_VALUE_BL A meta file can contain metadata about multiple SST files. The metadata is stored in a single file to avoid having too many small files. - Header - - 4 bytes magic number (0xFE4ADA4B) + - 4 bytes magic number (0xFE4ADA4A) - 4 bytes key family - 1 byte compression algorithm, which must match the configuration used to open the database - 4 bytes zstd dictionary ID (zero when no dictionary is configured) @@ -388,10 +388,11 @@ a nonzero dictionary ID; it is ignored for LZ4 and plain-zstd SSTs. The tool fol deletion files, and meta-file supersession, and uses `StaticSortedFileIter` to read slice, medium, and blob values. Checksums, dictionary IDs, and decompressed lengths are verified. -Small values are grouped into physical blocks in production, so the report's per-value 12.5% -minimum-savings calculation is a comparative estimate, not exact SST-size modeling. Estimated stored -bytes exclude fixed container headers. Timing fields are single-pass diagnostics; use byte/count -fields for repeatable comparisons of one copied cache snapshot. +Evaluation groups small logical values into SST-local 8–12 KiB units, while medium values and blobs +remain independent. The 12.5% minimum-savings rule is applied per approximated unit, so this remains +comparative rather than exact SST-size modeling. Estimated stored bytes exclude fixed container +headers. Timing fields are single-pass diagnostics; use byte/count fields for repeatable comparisons +of one copied cache snapshot. ## Opening diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs index f8190fa9545e..bad9e162ea34 100644 --- a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -13,13 +13,12 @@ use anyhow::{Context, Result, ensure}; use clap::{Args, Parser, Subcommand}; use serde::Serialize; use turbo_persistence::{ - Compression, CompressionConfig, IterValue, MAX_INLINE_VALUE_SIZE, StaticSortedFileIter, - StaticSortedFileMetaData, + Compression, CompressionConfig, IterValue, MAX_INLINE_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE, + StaticSortedFileIter, StaticSortedFileMetaData, offline::{SstInfo, collect_sst_info, decode_medium, read_blob}, }; -use xxhash_rust::xxh3::xxh3_64; -const SCHEMA_VERSION: u32 = 3; +const SCHEMA_VERSION: u32 = 4; const DICTIONARY_SIZE: usize = 64 * 1024; const SAMPLE_BUDGET_MULTIPLIER: usize = 1000; const SAMPLE_BYTE_BUDGET: usize = DICTIONARY_SIZE * SAMPLE_BUDGET_MULTIPLIER; @@ -75,8 +74,6 @@ struct DictionaryInfo { name: String, path: Option, bytes: usize, - dictionary_id: Option, - xxh3_64: Option, } struct Candidate { @@ -88,6 +85,7 @@ struct Candidate { struct Sample { data: Arc<[u8]>, + small_value_sst: Option, } #[derive(Default, Clone, Serialize)] @@ -186,7 +184,7 @@ struct TrainingReport { struct CacheSampleIter { path: PathBuf, pending: VecDeque<(SstInfo, CompressionConfig)>, - current: Option<(StaticSortedFileIter, CompressionConfig)>, + current: Option<(StaticSortedFileIter, CompressionConfig, u32)>, source_codecs: BTreeSet, seen_blobs: HashSet, active_ssts: u64, @@ -275,7 +273,7 @@ impl CacheSampleIter { compression, ) .with_context(|| format!("Failed to open {:08}.sst", sst.sequence_number))?; - self.current = Some((iter, compression)); + self.current = Some((iter, compression, sst.sequence_number)); Ok(true) } @@ -284,8 +282,9 @@ impl CacheSampleIter { if self.current.is_none() && !self.open_next_sst()? { return Ok(None); } - let (iter, compression) = self.current.as_mut().unwrap(); + let (iter, compression, sequence_number) = self.current.as_mut().unwrap(); let compression = *compression; + let sequence_number = *sequence_number; let entry = match iter.next() { Some(entry) => entry?, None => { @@ -293,7 +292,9 @@ impl CacheSampleIter { continue; } }; - if let Some(sample) = self.sample_from_value(entry.value, compression)? { + if let Some(sample) = + self.sample_from_value(entry.value, compression, sequence_number)? + { return Ok(Some(sample)); } } @@ -303,10 +304,12 @@ impl CacheSampleIter { &mut self, value: IterValue, compression: CompressionConfig, + sequence_number: u32, ) -> Result> { match value { IterValue::Slice { value } if value.len() > MAX_INLINE_VALUE_SIZE => Ok(Some(Sample { data: Arc::from(value.as_ref()), + small_value_sst: Some(sequence_number), })), IterValue::Medium { uncompressed_size, @@ -317,7 +320,10 @@ impl CacheSampleIter { .with_context(|| { format!("Failed to read medium value in {}", self.path.display()) })?; - Ok(Some(Sample { data: value })) + Ok(Some(Sample { + data: value, + small_value_sst: None, + })) } IterValue::Blob { sequence_number } => { if !self.seen_blobs.insert(sequence_number) { @@ -325,7 +331,10 @@ impl CacheSampleIter { return Ok(None); } let value = read_blob(&self.path, sequence_number, compression)?; - Ok(Some(Sample { data: value })) + Ok(Some(Sample { + data: value, + small_value_sst: None, + })) } // Inline values live in key blocks and are not independently compressed. IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. } | IterValue::Slice { .. } => { @@ -335,6 +344,63 @@ impl CacheSampleIter { } } +/// Groups logical small values into production-like SST-local compression units for evaluation. +struct EvaluationSampleIter { + source: CacheSampleIter, + small_block: Vec, + small_block_sst: Option, +} + +impl EvaluationSampleIter { + fn new(source: CacheSampleIter) -> Self { + Self { + source, + small_block: Vec::with_capacity(MIN_SMALL_VALUE_BLOCK_SIZE), + small_block_sst: None, + } + } + + fn next_sample(&mut self) -> Result> { + loop { + match self.source.next_sample()? { + Some(sample) => match sample.small_value_sst { + None => return Ok(Some(sample)), + Some(sequence_number) => { + if self.small_block_sst != Some(sequence_number) + && !self.small_block.is_empty() + { + let completed = std::mem::take(&mut self.small_block); + self.small_block_sst = Some(sequence_number); + self.small_block.extend_from_slice(&sample.data); + return Ok(Some(Sample { + data: completed.into(), + small_value_sst: None, + })); + } + self.small_block_sst = Some(sequence_number); + self.small_block.extend_from_slice(&sample.data); + if self.small_block.len() >= MIN_SMALL_VALUE_BLOCK_SIZE { + let completed = std::mem::take(&mut self.small_block); + return Ok(Some(Sample { + data: completed.into(), + small_value_sst: None, + })); + } + } + }, + None if self.small_block.is_empty() => return Ok(None), + None => { + let completed = std::mem::take(&mut self.small_block); + return Ok(Some(Sample { + data: completed.into(), + small_value_sst: None, + })); + } + } + } + } +} + fn dictionary_info( path: Option<&Path>, dictionary: &[u8], @@ -353,8 +419,6 @@ fn dictionary_info( name, path: path.map(Path::to_path_buf), bytes: dictionary.len(), - dictionary_id: zstd::zstd_safe::get_dict_id_from_dict(dictionary).map(|id| id.get()), - xxh3_64: (!baseline).then(|| format!("{:016x}", xxh3_64(dictionary))), }) } @@ -464,7 +528,8 @@ fn evaluate_cache( source_dictionary: Option<&'static [u8]>, candidates: &mut [Candidate], ) -> Result { - let mut iter = CacheSampleIter::open(path.to_path_buf(), family, source_dictionary)?; + let source = CacheSampleIter::open(path.to_path_buf(), family, source_dictionary)?; + let mut iter = EvaluationSampleIter::new(source); let mut samples = Metric::default(); let mut results = empty_results(candidates); while let Some(sample) = iter.next_sample()? { @@ -475,10 +540,10 @@ fn evaluate_cache( Ok(CacheReport { path: path.to_path_buf(), family, - active_ssts: iter.active_ssts, - source_codecs: iter.source_codecs, + active_ssts: iter.source.active_ssts, + source_codecs: iter.source.source_codecs, samples, - duplicate_blob_references: iter.duplicate_blob_references, + duplicate_blob_references: iter.source.duplicate_blob_references, candidates: results, }) } @@ -502,9 +567,9 @@ fn combine_evaluation( family, timing_note: "Single-pass wall-clock diagnostics; byte/count fields are the comparison \ contract.", - threshold_note: "Estimated stored bytes apply the 12.5% rule per logical value and \ - exclude fixed container headers; they are a proxy for grouped \ - small-value blocks.", + threshold_note: "Small logical values are grouped into SST-local 8-12 KiB evaluation \ + units; medium values and blobs remain independent. Estimated stored \ + bytes apply the 12.5% rule per unit and exclude fixed container headers.", caches, combined_samples, combined_candidates, @@ -640,12 +705,9 @@ fn train(source: &Source, output: &Path) -> Result { fn print_training(report: &TrainingReport) { println!( - "Trained {} ({} bytes, id {:?}, xxh3 {}) from {} values / {} bytes (target {}, exhausted: \ - {})", + "Trained {} ({} bytes) from {} values / {} bytes (target {}, exhausted: {})", report.dictionary.path.as_ref().unwrap().display(), report.dictionary.bytes, - report.dictionary.dictionary_id, - report.dictionary.xxh3_64.as_deref().unwrap_or("none"), report.selected.count, report.selected_bytes, report.sample_byte_target, @@ -665,7 +727,7 @@ fn print_training(report: &TrainingReport) { fn print_evaluation(report: &EvaluationReport) { let samples = &report.combined_samples; println!( - "Evaluated family {}: {} caches, {} logical values / {} bytes", + "Evaluated family {}: {} caches, {} compression units / {} bytes", report.family, report.caches.len(), samples.count, @@ -741,19 +803,19 @@ fn main() { #[cfg(test)] mod tests { - use std::{collections::BTreeSet, ffi::OsString, fs, path::PathBuf}; + use std::{collections::BTreeSet, fs}; use anyhow::Result; use byteorder::{BE, WriteBytesExt}; - use clap::Parser; use tempfile::TempDir; use turbo_persistence::{ - Compression, CompressionConfig, DbConfig, SerialScheduler, TurboPersistence, + Compression, CompressionConfig, DbConfig, MIN_SMALL_VALUE_BLOCK_SIZE, SerialScheduler, + TurboPersistence, }; use super::{ - CacheSampleIter, Cli, DICTIONARY_SIZE, SAMPLE_BYTE_BUDGET, Source, empty_results, - estimated_value_bytes, evaluate_cache, evaluate_sample, finalize_results, make_candidates, + CacheSampleIter, DICTIONARY_SIZE, EvaluationSampleIter, Source, empty_results, + evaluate_cache, evaluate_sample, finalize_results, make_candidates, select_training_samples, train, }; @@ -804,26 +866,6 @@ mod tests { Ok(tempdir) } - #[test] - fn clap_requires_family_output_and_cache() { - assert!(Cli::try_parse_from(["tool", "train"]).is_err()); - assert!(Cli::try_parse_from(["tool", "train", "--family", "2", "cache"]).is_err()); - assert!( - Cli::try_parse_from([ - "tool", "train", "--family", "2", "--output", "dict", "cache" - ]) - .is_ok() - ); - assert_eq!(DICTIONARY_SIZE, 64 * 1024); - assert_eq!(SAMPLE_BYTE_BUDGET, 64 * 1024 * 1000); - } - - #[test] - fn threshold_proxy_is_strict() { - assert_eq!(estimated_value_bytes(800, 699), 699); - assert_eq!(estimated_value_bytes(800, 700), 800); - } - #[test] fn round_robin_samples_multiple_caches() -> Result<()> { let first = make_cache(2, Compression::Zstd3, 20)?; @@ -850,6 +892,28 @@ mod tests { Ok(()) } + #[test] + fn evaluation_groups_small_values_into_production_sized_blocks() -> Result<()> { + let cache = make_cache(2, Compression::Zstd3, 20)?; + let mut raw = CacheSampleIter::open(cache.path().to_path_buf(), 2, None)?; + let mut raw_count = 0; + while raw.next_sample()?.is_some() { + raw_count += 1; + } + + let source = CacheSampleIter::open(cache.path().to_path_buf(), 2, None)?; + let mut grouped = EvaluationSampleIter::new(source); + let mut sizes = Vec::new(); + while let Some(sample) = grouped.next_sample()? { + sizes.push(sample.data.len()); + } + assert!(sizes.len() < raw_count); + assert!(sizes.iter().any(|&size| { + (MIN_SMALL_VALUE_BLOCK_SIZE..MIN_SMALL_VALUE_BLOCK_SIZE + 4096).contains(&size) + })); + Ok(()) + } + #[test] fn accepts_lz4_source_cache_and_ignores_source_dictionary() -> Result<()> { let cache = make_cache(2, Compression::Lz4, 20)?; @@ -869,37 +933,6 @@ mod tests { Ok(()) } - #[test] - fn source_dictionary_opens_dictionary_cache_and_plain_config_rejects_it() -> Result<()> { - let samples = (0..100) - .map(|index| format!("export function Component{index}() {{ return null }}")) - .collect::>(); - let dictionary = zstd::dict::from_samples(&samples, 1024)?; - let dictionary = Box::leak(dictionary.into_boxed_slice()); - let cache = - make_cache_with_config(2, CompressionConfig::Zstd3WithDictionary(dictionary), 20)?; - assert!(CacheSampleIter::open(cache.path().to_path_buf(), 2, None).is_err()); - let mut iter = CacheSampleIter::open(cache.path().to_path_buf(), 2, Some(dictionary))?; - assert!(iter.next_sample()?.is_some()); - Ok(()) - } - - #[test] - fn insufficient_samples_fail_with_context() -> Result<()> { - let cache = make_cache(2, Compression::Zstd3, 1)?; - let output_dir = tempfile::tempdir()?; - let source = Source { - family: 2, - source_dictionary: None, - caches: vec![cache.path().to_path_buf()], - }; - let error = train(&source, &output_dir.path().join("dictionary.zdict")) - .err() - .expect("one small cache should not train a 64 KiB dictionary"); - assert!(format!("{error:#}").contains("Failed to train a 65536-byte dictionary")); - Ok(()) - } - #[test] fn trains_replaces_output_and_evaluates() -> Result<()> { let cache = make_cache(2, Compression::Zstd3, 3000)?; @@ -940,6 +973,7 @@ mod tests { sequence_number: 42, }, CompressionConfig::Zstd3, + 1, )? .unwrap(); assert_eq!(sample.data.as_ref(), value); @@ -957,6 +991,7 @@ mod tests { sequence_number: 42 }, CompressionConfig::Zstd3, + 1, )? .is_none() ); @@ -967,6 +1002,7 @@ mod tests { sequence_number: 43 }, CompressionConfig::Zstd3, + 1, ) .is_err() ); @@ -997,13 +1033,4 @@ mod tests { assert_eq!(deleted[&2].len(), 1); Ok(()) } - - #[cfg(unix)] - #[test] - fn rejects_non_utf8_dictionary_name() { - use std::os::unix::ffi::OsStringExt; - - let path = PathBuf::from(OsString::from_vec(vec![0xff])); - assert!(super::dictionary_info(Some(&path), b"data", false).is_err()); - } } diff --git a/turbopack/crates/turbo-persistence/src/lib.rs b/turbopack/crates/turbo-persistence/src/lib.rs index 97e2629ebf0d..fde00029ed4d 100644 --- a/turbopack/crates/turbo-persistence/src/lib.rs +++ b/turbopack/crates/turbo-persistence/src/lib.rs @@ -112,7 +112,7 @@ impl DbConfig { } /// The largest value that [`WriteBatch::delete_value`] can delete, since the tombstone stores /// a copy of the value inline. -pub use constants::MAX_INLINE_VALUE_SIZE; +pub use constants::{MAX_INLINE_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE}; impl Default for DbConfig { fn default() -> Self { diff --git a/turbopack/crates/turbo-persistence/src/meta_file.rs b/turbopack/crates/turbo-persistence/src/meta_file.rs index c126c00c9950..a1c0d810a2a6 100644 --- a/turbopack/crates/turbo-persistence/src/meta_file.rs +++ b/turbopack/crates/turbo-persistence/src/meta_file.rs @@ -51,7 +51,7 @@ impl Display for MetaEntryFlags { } /// Magic number identifying a `.meta` file. -pub(crate) const META_FILE_MAGIC: u32 = 0xFE4ADA4B; +pub(crate) const META_FILE_MAGIC: u32 = 0xFE4ADA4A; /// On-disk layout of a single entry header in the `.meta` file. /// diff --git a/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh b/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh index cc80ff8ebbd7..bed2654f686c 100755 --- a/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh +++ b/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -u -o pipefail +set -euo pipefail repo_root=$(git rev-parse --show-toplevel) output_root=${1:-/tmp/taskdata-dictionary-corpus} @@ -45,12 +45,14 @@ run_case() { mkdir -p "$case_root" log="$output_root/logs/$digest.log" echo "[$split] $relative" + set +e ( cd "$repo_root" TMPDIR="$case_root" NEXT_TEST_SKIP_CLEANUP=1 \ pnpm test-start-turbo "$relative" ) >"$log" 2>&1 status=$? + set -e cache_index=0 state_tmp="$output_root/state/$digest.tmp" : > "$state_tmp" diff --git a/turbopack/crates/turbo-tasks-backend/src/database/taskdata-dictionary.md b/turbopack/crates/turbo-tasks-backend/src/database/taskdata-dictionary.md index c2fe7d55bc9b..67bf0dac08de 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/taskdata-dictionary.md +++ b/turbopack/crates/turbo-tasks-backend/src/database/taskdata-dictionary.md @@ -20,17 +20,24 @@ environment. ## Held-out results -The 38 holdout caches contained 1,462,462,144 uncompressed logical-value bytes. - -| Metric | zstd3 | Dictionary | Delta | -| --------------------------- | ----------: | ----------: | ----------: | -| Raw compressed bytes | 674,350,217 | 460,786,575 | **-31.67%** | -| Median encode time (5 runs) | 18.646 s | 9.132 s | **-51.02%** | -| Median decode time (5 runs) | 5.822 s | 3.030 s | **-47.91%** | - -The copied holdout cache directories occupied 728,480,057 bytes. The raw compressed-byte delta is -213,563,642 bytes, or 29.32% of that directory total; this is a directional total-cache estimate, -not an exact rewritten-cache measurement. +Evaluation approximates production compression units: small values are accumulated into SST-local +8–12 KiB blocks, while medium values and blobs remain independent. The corrected evaluator was run +against 4 held-out caches from a fresh 20-test smoke corpus, containing 11,778 compression units and +176,698,226 uncompressed bytes. + +| Metric | zstd3 | Dictionary | Delta | +| --------------------------- | ---------: | ---------: | ----------: | +| Raw compressed bytes | 52,387,060 | 44,924,356 | **-14.25%** | +| Median encode time (5 runs) | 772.23 ms | 822.26 ms | **+6.48%** | +| Median decode time (5 runs) | 225.30 ms | 192.72 ms | **-14.46%** | + +The copied holdout cache directories occupied 77,370,864 bytes. The raw compressed-byte delta is +7,462,704 bytes, or 9.65% of that directory total; this is a directional total-cache estimate, not +an exact rewritten-cache measurement. + +The original 38-cache evaluation treated every logical value as a compression unit and overstated +the benefit, so those numbers are intentionally not retained here. The corrected result meets the +accepted ≥2% size / ≤10% encode-regression / no-decode-regression gate. Timing is machine-specific single-process diagnostic data. The stable byte result is the primary receipt. From 78cef82f148c9a2b8835db13f01bf360849466c7 Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:57 +0000 Subject: [PATCH 12/13] Compare dictionary results with LZ4 Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- turbopack/crates/turbo-persistence/README.md | 2 +- .../src/bin/zstd_dictionary.rs | 83 +++++++++++++++---- .../crates/turbo-tasks-backend/README.md | 4 +- .../scripts/train-taskdata-dictionary.sh | 5 +- 4 files changed, 73 insertions(+), 21 deletions(-) diff --git a/turbopack/crates/turbo-persistence/README.md b/turbopack/crates/turbo-persistence/README.md index d9d8f50179ab..05f7c9ce71fe 100644 --- a/turbopack/crates/turbo-persistence/README.md +++ b/turbopack/crates/turbo-persistence/README.md @@ -382,7 +382,7 @@ Training produces a 64 KiB dictionary from up to approximately 64 MiB of samples hash-ordered logical value from each cache in turn, so one large cache cannot monopolize the sample. The output path is overwritten directly. -The no-dictionary zstd level 3 baseline is always included during evaluation. Source SSTs may use +LZ4 and no-dictionary zstd level 3 baselines are always included during evaluation. Source SSTs may use LZ4 or plain zstd without extra options. Pass `--source-dictionary ` when any input SST records a nonzero dictionary ID; it is ignored for LZ4 and plain-zstd SSTs. The tool follows `CURRENT`, deletion files, and meta-file supersession, and uses `StaticSortedFileIter` to read slice, medium, and diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs index bad9e162ea34..cace68621230 100644 --- a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -11,6 +11,7 @@ use std::{ use anyhow::{Context, Result, ensure}; use clap::{Args, Parser, Subcommand}; +use lzzzz::lz4; use serde::Serialize; use turbo_persistence::{ Compression, CompressionConfig, IterValue, MAX_INLINE_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE, @@ -18,7 +19,7 @@ use turbo_persistence::{ offline::{SstInfo, collect_sst_info, decode_medium, read_blob}, }; -const SCHEMA_VERSION: u32 = 4; +const SCHEMA_VERSION: u32 = 5; const DICTIONARY_SIZE: usize = 64 * 1024; const SAMPLE_BUDGET_MULTIPLIER: usize = 1000; const SAMPLE_BYTE_BUDGET: usize = DICTIONARY_SIZE * SAMPLE_BUDGET_MULTIPLIER; @@ -43,7 +44,7 @@ enum Command { #[arg(long)] json: Option, }, - /// Compare dictionaries with zstd level 3 without a dictionary. + /// Compare dictionaries with LZ4 and zstd level 3 baselines. Evaluate { #[command(flatten)] source: Source, @@ -78,11 +79,18 @@ struct DictionaryInfo { struct Candidate { info: DictionaryInfo, - compressor: zstd::bulk::Compressor<'static>, - decompressor: zstd::bulk::Decompressor<'static>, + codec: CandidateCodec, setup_duration: Duration, } +enum CandidateCodec { + Lz4, + Zstd { + compressor: zstd::bulk::Compressor<'static>, + decompressor: zstd::bulk::Decompressor<'static>, + }, +} + struct Sample { data: Arc<[u8]>, small_value_sst: Option, @@ -423,12 +431,23 @@ fn dictionary_info( } fn make_candidates(paths: &[PathBuf]) -> Result> { - let mut result = Vec::with_capacity(paths.len() + 1); + let mut result = Vec::with_capacity(paths.len() + 2); + result.push(Candidate { + info: DictionaryInfo { + name: "lz4".to_owned(), + path: None, + bytes: 0, + }, + codec: CandidateCodec::Lz4, + setup_duration: Duration::ZERO, + }); let started = Instant::now(); result.push(Candidate { info: dictionary_info(None, &[], true)?, - compressor: zstd::bulk::Compressor::new(3)?, - decompressor: zstd::bulk::Decompressor::new()?, + codec: CandidateCodec::Zstd { + compressor: zstd::bulk::Compressor::new(3)?, + decompressor: zstd::bulk::Decompressor::new()?, + }, setup_duration: started.elapsed(), }); let mut names = BTreeSet::new(); @@ -449,15 +468,44 @@ fn make_candidates(paths: &[PathBuf]) -> Result> { let started = Instant::now(); result.push(Candidate { info, - compressor: zstd::bulk::Compressor::with_dictionary(3, &dictionary)?, - decompressor: zstd::bulk::Decompressor::with_dictionary(&dictionary)?, + codec: CandidateCodec::Zstd { + compressor: zstd::bulk::Compressor::with_dictionary(3, &dictionary)?, + decompressor: zstd::bulk::Decompressor::with_dictionary(&dictionary)?, + }, setup_duration: started.elapsed(), }); } Ok(result) } -/// Evaluates all dictionary candidates against one logical value. +impl Candidate { + fn compress(&mut self, input: &[u8]) -> Result> { + match &mut self.codec { + CandidateCodec::Lz4 => { + let mut output = Vec::new(); + lz4::compress_to_vec(input, &mut output, lz4::ACC_LEVEL_DEFAULT)?; + Ok(output) + } + CandidateCodec::Zstd { compressor, .. } => Ok(compressor.compress(input)?), + } + } + + fn decompress(&mut self, input: &[u8], output_len: usize) -> Result> { + match &mut self.codec { + CandidateCodec::Lz4 => { + let mut output = vec![0; output_len]; + let written = lz4::decompress(input, &mut output)?; + ensure!(written == output_len, "LZ4 decompressed length mismatch"); + Ok(output) + } + CandidateCodec::Zstd { decompressor, .. } => { + Ok(decompressor.decompress(input, output_len)?) + } + } + } +} + +/// Evaluates all compression candidates against one approximated compression unit. fn evaluate_sample( sample: &Sample, candidates: &mut [Candidate], @@ -466,13 +514,11 @@ fn evaluate_sample( for (candidate, result) in candidates.iter_mut().zip(results) { let started = Instant::now(); let compressed = candidate - .compressor .compress(&sample.data) .with_context(|| format!("Failed to compress with {}", candidate.info.name))?; let encode_duration = started.elapsed(); let started = Instant::now(); let decompressed = candidate - .decompressor .decompress(&compressed, sample.data.len()) .with_context(|| format!("Failed to decompress with {}", candidate.info.name))?; let decode_duration = started.elapsed(); @@ -493,10 +539,10 @@ fn evaluate_sample( Ok(()) } -/// Applies the writer's 12.5% minimum-savings rule as a per-value estimate. +/// Applies the writer's 12.5% minimum-savings rule to an approximated compression unit. /// -/// Small values are grouped into physical blocks in production, so this is a comparative proxy, -/// not exact SST-size modeling. See `write_block_to_file` for the production block-level rule. +/// Small values are grouped before this call; medium values and blobs arrive independently. See +/// `write_block_to_file` for the production block-level rule. fn estimated_value_bytes(original_len: usize, compressed_len: usize) -> usize { if compressed_len < original_len - original_len / 8 { compressed_len @@ -950,7 +996,12 @@ mod tests { let mut candidates = make_candidates(&[output])?; let evaluation = evaluate_cache(cache.path(), 2, None, &mut candidates)?; - assert_eq!(evaluation.candidates.len(), 2); + assert_eq!(evaluation.candidates.len(), 3); + assert_eq!(evaluation.candidates[0].dictionary.name, "lz4"); + assert_eq!( + evaluation.candidates[1].dictionary.name, + "zstd3 (no dictionary)" + ); assert!(evaluation.samples.count > 0); Ok(()) } diff --git a/turbopack/crates/turbo-tasks-backend/README.md b/turbopack/crates/turbo-tasks-backend/README.md index ce746fc5d533..7bb1e363e11c 100644 --- a/turbopack/crates/turbo-tasks-backend/README.md +++ b/turbopack/crates/turbo-tasks-backend/README.md @@ -27,5 +27,5 @@ The checked-in baseline is produced by [`scripts/train-taskdata-dictionary.sh`](./scripts/train-taskdata-dictionary.sh) from preserved `test/production` filesystem caches. Its corpus and held-out receipts are recorded in [`src/database/taskdata-dictionary.md`](./src/database/taskdata-dictionary.md). The script is -resumable; set `CORPUS_JOBS` for bounded parallelism and pass an output directory plus dictionary -path. +resumable; set `CORPUS_JOBS` for bounded parallelism, `CORPUS_FAMILY` to select a keyspace (defaults +to TaskData family 2), and pass an output directory plus dictionary path. diff --git a/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh b/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh index bed2654f686c..426dc82eba42 100755 --- a/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh +++ b/turbopack/crates/turbo-tasks-backend/scripts/train-taskdata-dictionary.sh @@ -6,6 +6,7 @@ output_root=${1:-/tmp/taskdata-dictionary-corpus} dictionary=${2:-$repo_root/turbopack/crates/turbo-tasks-backend/src/database/taskdata.zdict} shift $(( $# >= 2 ? 2 : $# )) jobs=${CORPUS_JOBS:-4} +family=${CORPUS_FAMILY:-2} mkdir -p "$output_root"/{train,holdout,logs,tmp,reports,state} manifest="$output_root/manifest.tsv" @@ -112,10 +113,10 @@ if [[ -f $source_dictionary ]]; then source_args=(--source-dictionary "$source_copy") fi cargo run -p turbo-persistence --release --bin zstd_dictionary -- train \ - --family 2 "${source_args[@]}" --output "$dictionary" "${train_caches[@]}" + --family "$family" "${source_args[@]}" --output "$dictionary" "${train_caches[@]}" for run in 1 2 3 4 5; do cargo run -p turbo-persistence --release --bin zstd_dictionary -- evaluate \ - --family 2 "${source_args[@]}" --dictionary "$dictionary" \ + --family "$family" "${source_args[@]}" --dictionary "$dictionary" \ --json "$output_root/reports/holdout-$run.json" \ "${holdout_caches[@]}" done From 0017e04fb8de876148ba8457a3ae8ed01c17cb0b Mon Sep 17 00:00:00 2001 From: "vercel-fleet-prod[bot]" <318278635+vercel-fleet-prod[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:48:58 +0000 Subject: [PATCH 13/13] Adapt persistence compression to access modes Co-authored-by: Luke Sandberg <210140+lukesandberg@users.noreply.github.com> --- .../turbo-persistence/src/bin/zstd_dictionary.rs | 5 +++-- turbopack/crates/turbo-persistence/src/db.rs | 5 +++-- turbopack/crates/turbo-persistence/src/offline.rs | 4 ++-- .../turbo-persistence/src/static_sorted_file.rs | 8 ++++---- turbopack/crates/turbo-persistence/src/tests.rs | 11 ++++------- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs index cace68621230..ed56f0c1feb1 100644 --- a/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs +++ b/turbopack/crates/turbo-persistence/src/bin/zstd_dictionary.rs @@ -14,8 +14,8 @@ use clap::{Args, Parser, Subcommand}; use lzzzz::lz4; use serde::Serialize; use turbo_persistence::{ - Compression, CompressionConfig, IterValue, MAX_INLINE_VALUE_SIZE, MIN_SMALL_VALUE_BLOCK_SIZE, - StaticSortedFileIter, StaticSortedFileMetaData, + AccessMode, Compression, CompressionConfig, IterValue, MAX_INLINE_VALUE_SIZE, + MIN_SMALL_VALUE_BLOCK_SIZE, StaticSortedFileIter, StaticSortedFileMetaData, offline::{SstInfo, collect_sst_info, decode_medium, read_blob}, }; @@ -279,6 +279,7 @@ impl CacheSampleIter { block_count: sst.block_count, }, compression, + AccessMode::Mmap, ) .with_context(|| format!("Failed to open {:08}.sst", sst.sequence_number))?; self.current = Some((iter, compression, sst.sequence_number)); diff --git a/turbopack/crates/turbo-persistence/src/db.rs b/turbopack/crates/turbo-persistence/src/db.rs index b4ba2e8f2f9b..4be939e6fa9c 100644 --- a/turbopack/crates/turbo-persistence/src/db.rs +++ b/turbopack/crates/turbo-persistence/src/db.rs @@ -33,7 +33,7 @@ use crate::{ AccessMode, CompressionConfig, DbConfig, FamilyKind, QueryKey, arc_bytes::ArcBytes, compaction::selector::{Compactable, get_merge_segments}, - compression::{Compression, checksum_block, decompress_into_arc}, + compression::{checksum_block, decompress_into_arc}, constants::{ DATA_THRESHOLD_PER_COMPACTED_FILE, KEY_BLOCK_AVG_SIZE, KEY_BLOCK_CACHE_SIZE, MAX_ENTRIES_PER_COMPACTED_FILE, VALUE_BLOCK_AVG_SIZE, VALUE_BLOCK_CACHE_SIZE, @@ -1531,6 +1531,7 @@ impl TurboPersistence "merge files", family = self.config.family_configs[family as usize].name ); + let compression = self.config.family_configs[family as usize].compression; enum PartialMergeResult<'l> { Merged { new_sst_files: Vec<(u32, File, StaticSortedFileBuilderMeta<'static>)>, @@ -1617,7 +1618,7 @@ impl TurboPersistence StaticSortedFileIter::open( path, entry.sst_metadata(), - meta_file.compression(), + compression, self.config.access_mode, ) }) diff --git a/turbopack/crates/turbo-persistence/src/offline.rs b/turbopack/crates/turbo-persistence/src/offline.rs index 69e50585dbb9..44daa0ae1643 100644 --- a/turbopack/crates/turbo-persistence/src/offline.rs +++ b/turbopack/crates/turbo-persistence/src/offline.rs @@ -11,7 +11,7 @@ use byteorder::{BE, ReadBytesExt}; use fs_err as fs; use crate::{ - Compression, CompressionConfig, checksum_block, compression::decompress_into_arc, + AccessMode, Compression, CompressionConfig, checksum_block, compression::decompress_into_arc, meta_file::MetaFile, read_current_version, sst_filter::SstFilter, }; @@ -68,7 +68,7 @@ pub fn collect_sst_info(db_path: &Path) -> Result>> { let mut meta_files: Vec = meta_seqs .iter() .map(|&sequence| { - MetaFile::open(db_path, sequence, None) + MetaFile::open(db_path, sequence, None, AccessMode::Mmap) .with_context(|| format!("Failed to open {sequence:08}.meta")) }) .collect::>()?; diff --git a/turbopack/crates/turbo-persistence/src/static_sorted_file.rs b/turbopack/crates/turbo-persistence/src/static_sorted_file.rs index 57691ba33f06..d5ca85a1a9fc 100644 --- a/turbopack/crates/turbo-persistence/src/static_sorted_file.rs +++ b/turbopack/crates/turbo-persistence/src/static_sorted_file.rs @@ -195,7 +195,7 @@ trait ValueBlockCache { self, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result; fn read_uncached( self, @@ -234,7 +234,7 @@ impl ValueBlockCache for ArcBlockCacheReader<'_> { self, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result { read_block_lookup(self.backing, meta, block_index, compression) } @@ -267,7 +267,7 @@ impl ValueBlockCache for RcBlockCacheReader<'_> { self, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result { read_block_iter(self.backing, meta, block_index, compression) } @@ -961,7 +961,7 @@ fn read_block_iter( backing: &StaticSortedFileIterBacking, meta: &StaticSortedFileMetaData, block_index: u16, - compression: Compression, + compression: CompressionConfig, ) -> Result { let (uncompressed_length, checksum, block) = get_raw_block_iter(backing, meta, block_index)?; verify_checksum(meta, &block, checksum, block_index)?; diff --git a/turbopack/crates/turbo-persistence/src/tests.rs b/turbopack/crates/turbo-persistence/src/tests.rs index f68581eaa322..789338bf59ff 100644 --- a/turbopack/crates/turbo-persistence/src/tests.rs +++ b/turbopack/crates/turbo-persistence/src/tests.rs @@ -149,7 +149,7 @@ fn multi_value_config_with_mmap(mmap: bool) -> DbConfig<1> { family_configs: [FamilyConfig { name: "test", kind: FamilyKind::MultiValue, - compression: Compression::Lz4, + compression: Compression::Lz4.into(), }], access_mode: if mmap { AccessMode::Mmap @@ -2457,12 +2457,9 @@ fn count_tombstones( sequence_number: entry.sequence_number, block_count: entry.block_count, }; - for item in StaticSortedFileIter::open( - path, - sst, - Compression::Lz4.into(), - AccessMode::Mmap, - )? { + for item in + StaticSortedFileIter::open(path, sst, Compression::Lz4.into(), AccessMode::Mmap)? + { if matches!( item?.value, IterValue::KeyDeleted | IterValue::KeyValueDeleted { .. }