Add offline zstd dictionary tooling

turbopack/crates/turbo-persistence/Cargo.toml

@@ -17,6 +17,7 @@ anyhow = { workspace = true }
 auto-hash-map = { workspace = true }
 bitfield = { workspace = true }
 byteorder = { workspace = true }
+clap = { workspace = true }
 crc32fast = { workspace = true }
 dashmap = { workspace = true}
 either = { workspace = true }
@@ -52,6 +53,10 @@ turbo-tasks-malloc = { workspace = true, features = ["custom_allocator"] }
 name = "sst_inspect"
 path = "src/bin/sst_inspect.rs"
 
+[[bin]]
+name = "zstd_dictionary"
+path = "src/bin/zstd_dictionary.rs"
+
 [lints]
 workspace = true
 

turbopack/crates/turbo-persistence/README.md

@@ -57,6 +57,7 @@ A meta file can contain metadata about multiple SST files. The metadata is store
   - 4 bytes magic number (0xFE4ADA4A)
   - 4 bytes key family
   - 1 byte compression algorithm, which must match the configuration used to open the database
+  - 4 bytes zstd dictionary ID (zero when no dictionary is configured)
   - 4 bytes count of obsolete SST files
   - foreach obsolete SST file
     - 4 bytes sequence number of the obsolete SST file
@@ -362,6 +363,37 @@ Configuration options for compactions are:
 - max number of SST files that are merged at once
 - coverage when compaction is triggered (otherwise calling compact is a noop)
 
+## Training and evaluating zstd dictionaries offline
+
+`zstd_dictionary` trains and compares zstd dictionaries from logical values in existing database
+copies without modifying them or running the application that created them:
+
+```sh
+cargo run -p turbo-persistence --release --bin zstd_dictionary -- train \
+  --family <id> --output candidate.zdict \
+  path/to/database-a path/to/database-b
+
+cargo run -p turbo-persistence --release --bin zstd_dictionary -- evaluate \
+  --family <id> --dictionary candidate.zdict --json report.json \
+  path/to/database-a path/to/database-b
+```
+
+Training produces a 64 KiB dictionary from up to approximately 64 MiB of samples. It takes one
+hash-ordered logical value from each cache in turn, so one large cache cannot monopolize the sample.
+The output path is overwritten directly.
+
+LZ4 and no-dictionary zstd level 3 baselines are always included during evaluation. Source SSTs may use
+LZ4 or plain zstd without extra options. Pass `--source-dictionary <path>` when any input SST records
+a nonzero dictionary ID; it is ignored for LZ4 and plain-zstd SSTs. The tool follows `CURRENT`,
+deletion files, and meta-file supersession, and uses `StaticSortedFileIter` to read slice, medium, and
+blob values. Checksums, dictionary IDs, and decompressed lengths are verified.
+
+Evaluation groups small logical values into SST-local 8–12 KiB units, while medium values and blobs
+remain independent. The 12.5% minimum-savings rule is applied per approximated unit, so this remains
+comparative rather than exact SST-size modeling. Estimated stored bytes exclude fixed container
+headers. Timing fields are single-pass diagnostics; use byte/count fields for repeatable comparisons
+of one copied cache snapshot.
+
 ## Opening
 
 - Read the `CURRENT` file

turbopack/crates/turbo-persistence/benches/mod.rs

@@ -622,7 +622,7 @@ fn prefill_multi_value_database(
         family_configs: [FamilyConfig {
             name: "test",
             kind: FamilyKind::MultiValue,
-            compression: Compression::Lz4,
+            compression: Compression::Lz4.into(),
         }],
         ..TpDbConfig::new()
     };
@@ -698,7 +698,7 @@ fn open_multi_value_db(path: &Path) -> TurboPersistence<SerialScheduler, 1> {
         family_configs: [FamilyConfig {
             name: "test",
             kind: FamilyKind::MultiValue,
-            compression: Compression::Lz4,
+            compression: Compression::Lz4.into(),
         }],
         ..TpDbConfig::new()
     };
@@ -968,7 +968,7 @@ fn bench_write_multi_value(c: &mut Criterion) {
                             family_configs: [FamilyConfig {
                                 name: "test",
                                 kind: FamilyKind::MultiValue,
-                                compression: Compression::Lz4,
+                                compression: Compression::Lz4.into(),
                             }],
                             ..TpDbConfig::new()
                         };
@@ -1211,7 +1211,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
                 &entries,
                 &sst_path,
                 MetaEntryFlags::FRESH,
-                Compression::Lz4,
+                Compression::Lz4.into(),
             )
             .unwrap();
 
@@ -1223,7 +1223,7 @@ fn bench_static_sorted_file_lookup(c: &mut Criterion) {
             let sst = StaticSortedFile::open(
                 tempdir.path(),
                 sst_meta,
-                Compression::Lz4,
+                Compression::Lz4.into(),
                 turbo_persistence::AccessMode::Mmap,
             )
             .unwrap();

turbopack/crates/turbo-persistence/src/arc_bytes.rs

@@ -9,7 +9,7 @@ use std::{
 use memmap2::Mmap;
 
 use crate::{
-    Compression,
+    CompressionConfig,
     compression::decompress_into_arc,
     shared_bytes::{SharedBytes, is_subslice_of},
 };
@@ -146,7 +146,7 @@ impl SharedBytes for ArcBytes {
     }
 
     fn from_decompressed(
-        compression: Compression,
+        compression: CompressionConfig,
         uncompressed_length: u32,
         block: &[u8],
     ) -> anyhow::Result<Self> {