diff --git a/Cargo.lock b/Cargo.lock index c103fb8..7c05ba9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -867,6 +867,7 @@ dependencies = [ "dotenvy", "feed-rs", "futures", + "isolang", "mockito", "regex", "reqwest", @@ -889,6 +890,7 @@ dependencies = [ "url", "urlencoding", "uuid", + "whatlang", ] [[package]] @@ -1613,6 +1615,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "isolang" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe50d48c77760c55188549098b9a7f6e37ae980c586a24693d6b01c3b2010c3c" +dependencies = [ + "phf", +] + [[package]] name = "itertools" version = "0.12.1" @@ -4101,6 +4112,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whatlang" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e8f38b596e2a359b755342473520a99421e43658548c79489ee221b728c107" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "whoami" version = "1.6.1" diff --git a/Cargo.toml b/Cargo.toml index 4d0175c..fb5f098 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,10 @@ sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "sqlite", "postgre # Cooklang parsing cooklang = { version = "0.17.1", default-features = false, features = ["aisle", "pantry"] } +# Language detection +whatlang = "0.18" +isolang = "2.4" + # Serialization serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/README.md b/README.md index 6dc2a81..4dc78e9 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,20 @@ cargo run -- download 123 --output ./recipes cargo run -- publish --input ./my-recipes --output feed.xml ``` +### Backfill recipe locales + +Detect and store the locale of recipes that don't have one yet (author-declared +`locale:` metadata wins, otherwise the language is detected from the recipe text). +Touched recipes are re-indexed automatically. + +```bash +# Tag only recipes that don't have a locale yet +cargo run -- backfill-locales + +# Recompute the locale for every recipe, including ones already tagged +cargo run -- backfill-locales --force +``` + ## API Endpoints ### Health & Status @@ -162,14 +176,19 @@ cargo run -- publish --input ./my-recipes --output feed.xml ### Search - `GET /api/search?q=` - Search recipes with unified query syntax + - `locale` (optional) - Filter to a single language, e.g. `locale=de`. Filtering + by a base language (`en`) also matches its regional variants (`en-US`). - Examples: - `/api/search?q=breakfast` - Basic search - `/api/search?q=tags:breakfast` - Field-specific search - `/api/search?q=pasta%20AND%20tags:italian` - Boolean search - `/api/search?q=total_time:[0%20TO%2030]` - Range search + - `/api/search?q=breakfast&locale=fr` - Search restricted to French recipes ### Recipes -- `GET /api/recipes/:id` - Get recipe details +- `GET /api/recipes/:id` - Get recipe details, including `locale` (e.g. `"de"`, + `"en-US"`) and `locale_source` (`"declared"` if set via a Cooklang `locale:` + key, or `"detected"` if inferred from the recipe text) - `GET /api/recipes/:id/download` - Download .cook file ### Feeds @@ -196,6 +215,27 @@ Environment variables (see `.env.example`): | `INDEX_PATH` | Search index directory | `./data/index` | | `RUST_LOG` | Logging level | `info,federation=debug` | +## Upgrading + +### Recipe locale field (search index rebuild required) + +This release adds a `locale` field to the Tantivy search schema (used to tag +and filter recipes by language). Tantivy pins field ids to the schema stored +on disk, so a search index built before this change is incompatible — **the +server now refuses to start** against a mismatched index, with an error +telling you what to do. + +Before deploying this version, delete the existing index and rebuild it: + +```bash +rm -rf data/index # or your configured INDEX_PATH +federation backfill-locales +``` + +`backfill-locales` runs pending database migrations and re-indexes every +recipe it touches, so this single command rebuilds the search index and +backfills locales in one step. Run it before starting `serve` again. + ## Production Build To build the project for production: diff --git a/docs/plans/2026-07-12-recipe-locale-design.md b/docs/plans/2026-07-12-recipe-locale-design.md new file mode 100644 index 0000000..7361d36 --- /dev/null +++ b/docs/plans/2026-07-12-recipe-locale-design.md @@ -0,0 +1,187 @@ +# Recipe Locale Detection — Design + +Date: 2026-07-12 + +## Goal + +Tag every recipe with a locale during indexing, so recipes can be filtered by +language in search and their language shown in the web UI and API. + +## Sources of locale + +Locale is **derived from recipe content**. It is recomputed whenever a recipe's +content is parsed (create or content update), so an author who later adds a +`locale:` key gets picked up on the next crawl. + +Resolution order: + +1. **Declared** — Cooklang's canonical `locale:` metadata key. `cooklang`'s + `Metadata::locale()` returns `(language, Option)`, e.g. + `("en", Some("US"))`. Normalized to `en` / `en-US` (lowercase language, + uppercase region). A declared locale always wins. +2. **Detected** — `whatlang::detect()` run over a plain-text rendering of the + recipe. Accepted only when `Info::is_reliable()` is true and the text clears + a minimum length. +3. **Neither** — `locale` stays NULL. We do not store a low-confidence guess. + +### Detection input + +The detector is fed text built from `ParsedRecipeData` (the output of +`src/indexer/cooklang_parser.rs`), never the raw `.cook` source. This keeps +Cooklang markup (`@flour{200%g}`, `#oven{}`, `~{20%minutes}`) out of the +trigram statistics. The text is the concatenation of: + +- metadata title and description +- every `StepItem::Text` value across all sections +- section notes +- ingredient names (strong language signal: "flour" vs "farine") + +Quantities, units and cookware are excluded. + +Guard: if the assembled text is shorter than 25 characters, return `None` +rather than detecting — there is not enough signal. + +### Code normalization + +`whatlang` returns ISO 639-3 (`eng`, `deu`). Cooklang's declared locale is +BCP-47-ish (`en`, `en-US`). Everything is normalized to a 2-letter BCP-47 +language code via the `isolang` crate; if a language has no 639-1 code, the +639-3 code is stored as-is. Region is only ever kept when it was *declared* — +detection never invents one. + +## Components + +### `src/indexer/locale.rs` (new) + +```rust +pub enum LocaleSource { Declared, Detected } + +pub struct RecipeLocale { + pub code: String, // "en", "en-US", "de" + pub source: LocaleSource, // declared | detected +} + +pub fn resolve_locale(parsed: &ParsedRecipeData) -> Option; +``` + +`LocaleSource` serializes to the strings `"declared"` / `"detected"` for +storage. + +New dependencies: `whatlang` (detection) and `isolang` (639-3 → 639-1 mapping +and English display names for the UI). + +### Database — migration `008_recipe_locale.sql` + +```sql +ALTER TABLE recipes ADD COLUMN locale TEXT; +ALTER TABLE recipes ADD COLUMN locale_source TEXT; -- 'declared' | 'detected' +CREATE INDEX IF NOT EXISTS idx_recipes_locale ON recipes(locale); +``` + +Portable across the SQLite and Postgres backends both used by sqlx here. + +`Recipe` and `NewRecipe` in `src/db/models.rs` gain +`locale: Option` and `locale_source: Option`. The INSERT in +`src/db/recipes.rs` and `update_recipe_with_content` carry both fields. + +Keeping `locale_source` separate from `locale` means the UI can distinguish a +stated language from our guess, and a future re-detection pass can refresh +detected values without clobbering declared ones. + +### Ingestion + +Both ingestion paths already parse Cooklang content: + +- `src/crawler/mod.rs` (feed crawler) — currently parses content only to fish + out a fallback image URL, and only on the create path. +- `src/github/indexer.rs` (GitHub indexer). + +Both are changed to parse content **once** per recipe and reuse the resulting +`ParsedRecipeData` for image extraction *and* locale resolution, on both the +create and the content-update path. + +### Search + +`RecipeSchema` (`src/indexer/schema.rs`) gains: + +```rust +let locale = schema_builder.add_text_field("locale", STRING | STORED); +``` + +`STRING` (untokenized) gives exact term matching on the code. + +The field is deliberately **excluded** from the QueryParser's default field +list, so a free-text search for "de" does not match every German recipe. +Filtering is explicit instead: + +- `SearchQuery` gains `locale: Option`. +- `SearchIndex::search()` ANDs a `TermQuery` on the locale field onto the + parsed free-text query (a `BooleanQuery` with both as MUST clauses). +- `SearchResult` gains `locale: Option`, read from the stored field, so + result cards can show the language without a database round-trip. + +`index_recipe()` writes the locale into the document. + +### API + +- `GET /api/search?q=…&locale=de` — new optional `locale` param on + `SearchParams`, threaded into `SearchQuery`. +- Recipe responses in `src/api/models.rs` gain `locale` and `locale_source`. + +### Web UI + +- `recipe.html` — language displayed alongside servings/time, using the English + language name from `isolang` (e.g. "German"), with a quiet "detected" + qualifier when `locale_source = detected`. +- `search.html` — a language dropdown that sets `?locale=`, threaded through the + existing web search handler. +- Search and browse cards — a small language chip. + +### Backfill — `federation backfill-locales [--force]` + +A new CLI subcommand alongside the existing `reindex`. Recipe content is already +stored in the `recipes` table, so backfill needs no network fetches. + +- Pages through `recipes WHERE locale IS NULL AND content IS NOT NULL` + (bounded batches, so memory stays flat on large databases). +- Resolves locale for each, updates the row. +- Re-indexes the recipe's Tantivy document (with its tags and ingredients from + the database) so the `locale` filter works over pre-existing data. +- `--force` recomputes every recipe with content, not just NULL-locale ones. + +New and updated recipes get their locale automatically via the crawler and +GitHub indexer, so this command is a one-off for existing data. + +## Error handling + +- Cooklang parse failure: recipe is stored as before with `locale = NULL`. + Locale is a nice-to-have; it never blocks ingestion. +- Unreliable or too-short detection: `locale = NULL`, no guess stored. +- Invalid declared locale (Cooklang's `locale()` returns `None` for malformed + values): falls through to detection. +- Backfill: a failure on one recipe is logged and skipped; the pass continues. + +## Testing + +Unit tests in `src/indexer/locale.rs`: + +- a declared `locale:` wins over what detection would have said +- English, German, Russian and French recipe fixtures each detect correctly +- region is preserved from a declared `en-US`, and never invented by detection +- too-short content and non-linguistic content (e.g. digits/symbols) → `None` +- ISO 639-3 → 639-1 mapping (`deu` → `de`) + +Integration tests: + +- a recipe created by the crawler carries a locale and a `locale_source` +- `locale=` search filter returns only recipes in that language, and combines + with a free-text query +- `backfill-locales` fills NULL-locale rows and leaves already-set rows + untouched (unless `--force`) + +## Out of scope + +- Per-language Tantivy tokenizers/stemmers. The locale is stored and filterable, + but text analysis stays language-agnostic for now. +- Translating UI strings, or serving different content per user locale. +- Storing a raw detection confidence value. diff --git a/docs/plans/2026-07-12-recipe-locale-implementation.md b/docs/plans/2026-07-12-recipe-locale-implementation.md new file mode 100644 index 0000000..17f4c65 --- /dev/null +++ b/docs/plans/2026-07-12-recipe-locale-implementation.md @@ -0,0 +1,1912 @@ +# Recipe Locale Detection Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Tag every recipe with a locale during indexing — from a declared Cooklang `locale:` key when present, otherwise by language detection — and make it storable, filterable in search, and visible in the API and web UI. + +**Architecture:** A new `src/indexer/locale.rs` resolves a locale from the already-parsed `ParsedRecipeData` (declared metadata wins; otherwise `whatlang` detection over plain recipe text with markup stripped). Both ingestion paths (feed crawler, GitHub indexer) call it and persist `locale` + `locale_source` on the `recipes` row. The Tantivy schema gains an untokenized `locale` field so search can filter by an explicit `?locale=` parameter. A `backfill-locales` CLI command fills in existing rows from content already stored in the database. + +**Tech Stack:** Rust, axum, sqlx (SQLite), Tantivy 0.22, cooklang 0.17, askama templates. New crates: `whatlang` 0.18 (detection), `isolang` 2.4 (ISO 639-3 → 639-1 mapping and English language names). + +**Spec:** `docs/superpowers/specs/2026-07-12-recipe-locale-design.md` + +**Conventions in this codebase:** +- Run all tests with `cargo test`. Run a single test with `cargo test -- --nocapture`. +- Format with `cargo fmt` and lint with `cargo clippy --all-targets -- -D warnings` before each commit. +- The database pool is `SqlitePool` (`DbPool`). Integration tests use `SqlitePool::connect("sqlite::memory:")` + `sqlx::migrate!("./migrations")`. +- Recipe locale codes are stored as BCP-47-style: lowercase language, optional uppercase region, e.g. `en`, `de`, `en-US`. Cooklang declares them with an underscore (`en_US`); we normalize to a hyphen. + +--- + +## File Structure + +**Created:** +- `src/indexer/locale.rs` — locale resolution: declared-vs-detected, detection-text assembly, code normalization, display names. Single responsibility, no I/O. +- `migrations/008_recipe_locale.sql` — `locale` + `locale_source` columns and an index. +- `tests/locale_test.rs` — integration tests for storage, search filtering, and backfill. + +**Modified:** +- `src/indexer/cooklang_parser.rs` — expose the declared `locale:` key on `RecipeMetadata`. +- `src/indexer/mod.rs` — register and re-export the new module. +- `src/indexer/schema.rs` — add the Tantivy `locale` field. +- `src/indexer/search.rs` — index locale, filter on it, return it in results. +- `src/db/models.rs` — `locale` / `locale_source` on `Recipe` and `NewRecipe`. +- `src/db/recipes.rs` — persist locale on insert/update; add `update_recipe_locale` and `list_locales`. +- `src/crawler/mod.rs` — resolve locale on recipe create and content update. +- `src/github/indexer.rs` — resolve locale on recipe create. +- `src/api/models.rs`, `src/api/handlers.rs` — `?locale=` search param, locale in responses. +- `src/web/handlers.rs`, `src/web/schema.rs`, `src/web/templates/search.html`, `src/web/templates/recipe.html` — language dropdown, card chip, detail pill, `inLanguage` in JSON-LD. +- `src/cli/mod.rs`, `src/cli/commands.rs`, `src/main.rs` — `backfill-locales` command. +- `Cargo.toml` — the two new dependencies. + +--- + +### Task 1: Expose the declared `locale:` metadata key + +Cooklang has a canonical `locale:` metadata key. `cooklang::Metadata::locale()` returns `Option<(&str, Option<&str>)>` — e.g. `("en", Some("US"))` for a frontmatter value of `en_US`. Cooklang validates it: language and region are each exactly two ASCII letters, or `locale()` returns `None`. + +Today `src/indexer/cooklang_parser.rs` never reads it, and because `"locale"` is missing from the skip-list in the `custom` loop, a declared locale silently lands in `RecipeMetadata::custom`. We surface it as a real field and stop duplicating it into `custom`. + +**Files:** +- Modify: `src/indexer/cooklang_parser.rs` (`RecipeMetadata` struct ~line 18; the `custom` skip-list ~line 150; the `RecipeMetadata { ... }` construction ~line 200) + +- [ ] **Step 1: Write the failing tests** + +Add to the `mod tests` block at the bottom of `src/indexer/cooklang_parser.rs`: + +```rust + #[test] + fn test_declared_locale_with_region() { + let content = r#"--- +locale: en_US +--- + +Mix @flour{2%cups} with @water{1%cup}. +"#; + + let parsed = parse_recipe(content).unwrap(); + let metadata = parsed.metadata.expect("metadata should be present"); + + assert_eq!(metadata.locale, Some("en-US".to_string())); + // The locale must not also leak into `custom`. + assert!(!metadata.custom.iter().any(|(k, _)| k == "locale")); + } + + #[test] + fn test_declared_locale_without_region() { + let content = r#"--- +locale: de +--- + +@Mehl{200%g} mit @Wasser{100%ml} verrühren. +"#; + + let parsed = parse_recipe(content).unwrap(); + let metadata = parsed.metadata.expect("metadata should be present"); + + assert_eq!(metadata.locale, Some("de".to_string())); + } + + #[test] + fn test_no_declared_locale() { + let content = "Mix @flour{2%cups} with @water{1%cup}.\n"; + + let parsed = parse_recipe(content).unwrap(); + + // No metadata at all, or metadata with no locale — both mean "not declared". + let locale = parsed.metadata.and_then(|m| m.locale); + assert_eq!(locale, None); + } +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test --lib cooklang_parser::tests::test_declared_locale -- --nocapture` +Expected: FAIL to compile — `no field 'locale' on type 'RecipeMetadata'`. + +- [ ] **Step 3: Add the field to `RecipeMetadata`** + +In the `RecipeMetadata` struct, add `locale` immediately after `title`: + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecipeMetadata { + pub tags: Vec, + pub title: Option, + pub locale: Option, + pub description: Option, + pub servings: Option, + pub time: Option, + pub difficulty: Option, + pub course: Option, + pub prep_time: Option, + pub cook_time: Option, + pub cuisine: Option, + pub diet: Option, + pub author: Option, + pub source: Option, + pub image: Option, + pub custom: Vec<(String, String)>, +} +``` + +- [ ] **Step 4: Stop leaking `locale` into `custom`** + +In `parse_recipe`, the loop over `&meta.map` skips standard keys with a `matches!`. Add `Some("locale")` to that list — it currently reads `Some("tags") | Some("title") | ...`: + +```rust + if !matches!( + key_str, + Some("tags") + | Some("title") + | Some("locale") + | Some("description") + | Some("servings") + | Some("time") + | Some("difficulty") + | Some("course") + | Some("prep time") + | Some("cook time") + | Some("cuisine") + | Some("diet") + | Some("author") + | Some("source") + | Some("image") + ) { +``` + +- [ ] **Step 5: Populate the field** + +In the `Some(RecipeMetadata { ... })` construction, add the `locale` field after `title`. `meta.locale()` yields the validated `(language, Option)` pair; normalize to lowercase language and uppercase region joined by a hyphen: + +```rust + locale: meta.locale().map(|(lang, region)| match region { + Some(region) => format!("{}-{}", lang.to_lowercase(), region.to_uppercase()), + None => lang.to_lowercase(), + }), +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cargo test --lib cooklang_parser` +Expected: PASS — including the three new tests and the pre-existing `test_parse_simple_recipe`. + +- [ ] **Step 7: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add src/indexer/cooklang_parser.rs +git commit -m "feat: expose declared cooklang locale metadata" +``` + +--- + +### Task 2: The locale resolution module + +The core unit. It takes a `ParsedRecipeData` and returns an optional `RecipeLocale`. No database, no network, no Tantivy — which is what makes it cheap to test. + +Detection runs over text assembled from the *parsed* recipe, never the raw `.cook` source, so Cooklang markup (`@flour{200%g}`, `#oven{}`, `~{20%minutes}`) can't skew the trigram statistics. + +**Files:** +- Modify: `Cargo.toml` +- Create: `src/indexer/locale.rs` +- Modify: `src/indexer/mod.rs` + +- [ ] **Step 1: Add the dependencies** + +In `Cargo.toml`, under `[dependencies]`, after the `# Cooklang parsing` block: + +```toml +# Language detection +whatlang = "0.18" +isolang = "2.4" +``` + +`isolang`'s default features include `english_names`, which is what gives us `Language::to_name()`. + +- [ ] **Step 2: Register the module** + +In `src/indexer/mod.rs`, add the module and re-exports: + +```rust +// Phase 3: Recipe indexing and search module +// This module handles Cooklang parsing and Tantivy search indexing + +pub mod cooklang_parser; +pub mod locale; +pub mod recipe; +pub mod schema; +pub mod search; + +// Re-exports +pub use cooklang_parser::{parse_recipe as parse_cooklang_full, ParsedRecipeData}; +pub use locale::{resolve_locale, LocaleSource, RecipeLocale}; +pub use recipe::{parse_cooklang, ParsedRecipe}; +pub use schema::RecipeSchema; +pub use search::{SearchIndex, SearchQuery, SearchResult, SearchResults}; +``` + +- [ ] **Step 3: Write the failing tests** + +Create `src/indexer/locale.rs` containing *only* the test module for now, so the tests fail against missing functions rather than a missing file: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::indexer::cooklang_parser::parse_recipe; + + const GERMAN: &str = "Den @Mehl{200%g} und das @Wasser{100%ml} in einer Schüssel \ +verrühren, bis ein glatter Teig entsteht. Den Teig ruhen lassen und anschließend \ +im #Ofen{} goldbraun backen."; + + const ENGLISH: &str = "Mix the @flour{200%g} and the @water{100%ml} in a bowl until \ +a smooth dough forms. Let the dough rest, then bake it in the #oven{} until golden brown."; + + #[test] + fn test_detects_german() { + let parsed = parse_recipe(GERMAN).unwrap(); + let locale = resolve_locale(&parsed).expect("should detect a locale"); + + assert_eq!(locale.code, "de"); + assert_eq!(locale.source, LocaleSource::Detected); + } + + #[test] + fn test_detects_english() { + let parsed = parse_recipe(ENGLISH).unwrap(); + let locale = resolve_locale(&parsed).expect("should detect a locale"); + + assert_eq!(locale.code, "en"); + assert_eq!(locale.source, LocaleSource::Detected); + } + + #[test] + fn test_declared_locale_beats_detection() { + // The body is unmistakably German, but the author declared French. + let content = format!("---\nlocale: fr\n---\n\n{GERMAN}"); + let parsed = parse_recipe(&content).unwrap(); + let locale = resolve_locale(&parsed).expect("should resolve a locale"); + + assert_eq!(locale.code, "fr"); + assert_eq!(locale.source, LocaleSource::Declared); + } + + #[test] + fn test_declared_region_is_preserved() { + let content = format!("---\nlocale: en_US\n---\n\n{ENGLISH}"); + let parsed = parse_recipe(&content).unwrap(); + let locale = resolve_locale(&parsed).unwrap(); + + assert_eq!(locale.code, "en-US"); + assert_eq!(locale.source, LocaleSource::Declared); + } + + #[test] + fn test_detection_never_invents_a_region() { + let parsed = parse_recipe(ENGLISH).unwrap(); + let locale = resolve_locale(&parsed).unwrap(); + + assert!(!locale.code.contains('-'), "detected code should be bare: {}", locale.code); + } + + #[test] + fn test_too_short_content_is_not_detected() { + let parsed = parse_recipe("Mix @salt{}.").unwrap(); + assert!(resolve_locale(&parsed).is_none()); + } + + #[test] + fn test_non_linguistic_content_is_not_detected() { + let parsed = parse_recipe("12345 67890 12345 67890 12345 67890 12345").unwrap(); + assert!(resolve_locale(&parsed).is_none()); + } + + #[test] + fn test_detection_text_excludes_markup() { + let parsed = parse_recipe(ENGLISH).unwrap(); + let text = detection_text(&parsed); + + assert!(text.contains("flour"), "ingredient names carry language signal"); + assert!(text.contains("smooth dough"), "step text must be present"); + assert!(!text.contains('@'), "cooklang markup must be stripped: {text}"); + assert!(!text.contains('{'), "cooklang markup must be stripped: {text}"); + assert!(!text.contains("200"), "quantities must not reach the detector: {text}"); + } + + #[test] + fn test_locale_source_as_str() { + assert_eq!(LocaleSource::Declared.as_str(), "declared"); + assert_eq!(LocaleSource::Detected.as_str(), "detected"); + } + + #[test] + fn test_display_name() { + assert_eq!(display_name("de").as_deref(), Some("German")); + assert_eq!(display_name("en-US").as_deref(), Some("English")); + assert_eq!(display_name("zzz"), None); + } +} +``` + +- [ ] **Step 4: Run tests to verify they fail** + +Run: `cargo test --lib locale` +Expected: FAIL to compile — `cannot find function 'resolve_locale' in this scope`. + +- [ ] **Step 5: Write the implementation** + +Prepend this above the test module in `src/indexer/locale.rs`: + +```rust +//! Recipe locale resolution. +//! +//! A declared Cooklang `locale:` key always wins. Otherwise the language is +//! detected from the recipe's plain text — assembled from the *parsed* recipe so +//! that Cooklang markup and quantities never reach the detector. When detection +//! is unreliable we return `None` rather than storing a guess. + +use serde::{Deserialize, Serialize}; + +use crate::indexer::cooklang_parser::{ParsedRecipeData, StepItem}; + +/// Below this many characters of plain text there is not enough signal to detect. +const MIN_DETECTION_CHARS: usize = 25; + +/// Where a recipe's locale came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum LocaleSource { + /// The recipe declared a `locale:` metadata key. + Declared, + /// We detected it from the recipe text. + Detected, +} + +impl LocaleSource { + /// The value stored in the `recipes.locale_source` column. + pub fn as_str(&self) -> &'static str { + match self { + LocaleSource::Declared => "declared", + LocaleSource::Detected => "detected", + } + } +} + +/// A resolved locale: a BCP-47-style code (`en`, `de`, `en-US`) and its provenance. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecipeLocale { + pub code: String, + pub source: LocaleSource, +} + +/// Resolve a recipe's locale: declared metadata first, then detection. +pub fn resolve_locale(parsed: &ParsedRecipeData) -> Option { + if let Some(code) = parsed.metadata.as_ref().and_then(|m| m.locale.clone()) { + return Some(RecipeLocale { + code, + source: LocaleSource::Declared, + }); + } + + detect(&detection_text(parsed)).map(|code| RecipeLocale { + code, + source: LocaleSource::Detected, + }) +} + +/// The plain text a recipe is detected from: title, description, section names, +/// step text, notes, and ingredient names. Quantities, units and cookware are +/// excluded — they are noise, not language signal. +pub(crate) fn detection_text(parsed: &ParsedRecipeData) -> String { + let mut parts: Vec<&str> = Vec::new(); + + if let Some(meta) = &parsed.metadata { + if let Some(title) = &meta.title { + parts.push(title); + } + if let Some(description) = &meta.description { + parts.push(description); + } + } + + for section in &parsed.sections { + if let Some(name) = §ion.name { + parts.push(name); + } + for step in §ion.steps { + for item in &step.items { + if let StepItem::Text { value } = item { + parts.push(value); + } + } + } + for note in §ion.notes { + parts.push(note); + } + } + + for ingredient in &parsed.ingredients { + parts.push(&ingredient.name); + } + + parts.join(" ") +} + +/// Detect a language code from plain text, or `None` if we can't trust the result. +fn detect(text: &str) -> Option { + if text.chars().count() < MIN_DETECTION_CHARS { + return None; + } + + let info = whatlang::detect(text)?; + if !info.is_reliable() { + return None; + } + + Some(to_bcp47(info.lang().code())) +} + +/// Map whatlang's ISO 639-3 code to a two-letter code where one exists. +/// Languages without a 639-1 code (e.g. Cebuano) keep their 639-3 code. +fn to_bcp47(code_639_3: &str) -> String { + isolang::Language::from_639_3(code_639_3) + .and_then(|lang| lang.to_639_1()) + .map(str::to_string) + .unwrap_or_else(|| code_639_3.to_string()) +} + +/// English display name for a stored code: `"de"` → `"German"`, `"en-US"` → `"English"`. +pub fn display_name(code: &str) -> Option { + let language = code.split('-').next()?; + isolang::Language::from_639_1(language).map(|lang| lang.to_name().to_string()) +} +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cargo test --lib locale` +Expected: PASS — all ten tests. + +If `test_non_linguistic_content_is_not_detected` fails, it means `whatlang` returned a reliable verdict for digits. Do not weaken the assertion: confirm the behavior by printing `whatlang::detect(...)` for that input, and if it genuinely is reported reliable, that is a real finding to raise rather than paper over. + +- [ ] **Step 7: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add Cargo.toml Cargo.lock src/indexer/locale.rs src/indexer/mod.rs +git commit -m "feat: add recipe locale resolution module" +``` + +--- + +### Task 3: Database column, model, and queries + +**Files:** +- Create: `migrations/008_recipe_locale.sql` +- Modify: `src/db/models.rs` (`Recipe` ~line 38, `NewRecipe` ~line 67) +- Modify: `src/db/recipes.rs` (`create_recipe` ~line 127, `update_recipe_with_content` ~line 416; new `update_recipe_locale` and `list_locales`) + +- [ ] **Step 1: Write the migration** + +Create `migrations/008_recipe_locale.sql`: + +```sql +-- Recipe locale: BCP-47-style code (e.g. 'en', 'de', 'en-US'). +-- locale_source records whether the author declared it or we detected it. +ALTER TABLE recipes ADD COLUMN locale TEXT; +ALTER TABLE recipes ADD COLUMN locale_source TEXT; + +CREATE INDEX IF NOT EXISTS idx_recipes_locale ON recipes(locale); +``` + +- [ ] **Step 2: Write the failing test** + +Add to the `mod tests` block at the bottom of `src/db/recipes.rs`. Look at the existing test there for the setup pattern and the full `NewRecipe` literal; this test follows it. + +```rust + #[tokio::test] + async fn test_create_and_update_recipe_locale() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + sqlx::migrate!("./migrations").run(&pool).await.unwrap(); + + let feed = crate::db::feeds::create_feed( + &pool, + &crate::db::models::NewFeed { + url: "https://example.com/feed.xml".to_string(), + title: Some("Test Feed".to_string()), + }, + ) + .await + .unwrap(); + + let new_recipe = NewRecipe { + feed_id: feed.id, + external_id: "recipe-1".to_string(), + title: "Pfannkuchen".to_string(), + source_url: None, + enclosure_url: "https://example.com/recipe.cook".to_string(), + content: Some("Mehl und Wasser verrühren.".to_string()), + summary: None, + servings: None, + total_time_minutes: None, + active_time_minutes: None, + difficulty: None, + image_url: None, + published_at: None, + content_hash: None, + content_etag: None, + content_last_modified: None, + feed_entry_updated: None, + locale: Some("de".to_string()), + locale_source: Some("detected".to_string()), + }; + + let recipe = create_recipe(&pool, &new_recipe).await.unwrap(); + assert_eq!(recipe.locale.as_deref(), Some("de")); + assert_eq!(recipe.locale_source.as_deref(), Some("detected")); + + // An author-declared locale overwrites the detected one. + update_recipe_locale(&pool, recipe.id, Some("fr"), Some("declared")) + .await + .unwrap(); + + let updated = get_recipe(&pool, recipe.id).await.unwrap(); + assert_eq!(updated.locale.as_deref(), Some("fr")); + assert_eq!(updated.locale_source.as_deref(), Some("declared")); + + // list_locales reports what is in the database. + let locales = list_locales(&pool).await.unwrap(); + assert_eq!(locales, vec![("fr".to_string(), 1)]); + } +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `cargo test --lib test_create_and_update_recipe_locale` +Expected: FAIL to compile — `struct 'NewRecipe' has no field named 'locale'`. + +- [ ] **Step 4: Add the model fields** + +In `src/db/models.rs`, add to the end of the `Recipe` struct (after `feed_entry_updated`): + +```rust + /// BCP-47-style locale code, e.g. "en", "de", "en-US". NULL when unknown. + pub locale: Option, + /// How the locale was obtained: "declared" or "detected". + pub locale_source: Option, +``` + +Add the same two fields to the end of the `NewRecipe` struct (without the doc comments duplicated — a single `/// Locale code and its provenance ("declared" | "detected")` above the pair is enough): + +```rust + pub locale: Option, + pub locale_source: Option, +``` + +- [ ] **Step 5: Persist locale on insert** + +In `src/db/recipes.rs`, `create_recipe`: add the two columns to the INSERT column list, two more `?` placeholders, and two `.bind(...)` calls in matching order (immediately after `.bind(new_recipe.feed_entry_updated)`): + +```rust + let recipe = sqlx::query_as::<_, Recipe>( + r#" + INSERT INTO recipes ( + feed_id, external_id, title, source_url, enclosure_url, + content, summary, servings, total_time_minutes, active_time_minutes, + difficulty, image_url, published_at, updated_at, created_at, content_hash, + content_etag, content_last_modified, feed_entry_updated, locale, locale_source + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + RETURNING * + "#, + ) +``` + +and at the end of the bind chain: + +```rust + .bind(new_recipe.feed_entry_updated) + .bind(&new_recipe.locale) + .bind(&new_recipe.locale_source) + .fetch_one(pool) + .await?; +``` + +- [ ] **Step 6: Carry locale through content updates** + +Still in `src/db/recipes.rs`, extend `update_recipe_with_content` with two more parameters and set the columns. Locale is derived from content, so it is recomputed whenever content changes: + +```rust +#[allow(clippy::too_many_arguments)] +pub async fn update_recipe_with_content( + pool: &DbPool, + recipe_id: i64, + content: &str, + content_hash: Option<&str>, + content_etag: Option<&str>, + content_last_modified: Option<&chrono::DateTime>, + feed_entry_updated: Option<&chrono::DateTime>, + locale: Option<&str>, + locale_source: Option<&str>, +) -> Result<()> { + let now = Utc::now(); + + sqlx::query( + r#" + UPDATE recipes + SET content = ?, content_hash = ?, content_etag = ?, + content_last_modified = ?, feed_entry_updated = ?, updated_at = ?, + locale = ?, locale_source = ? + WHERE id = ? + "#, + ) + .bind(content) + .bind(content_hash) + .bind(content_etag) + .bind(content_last_modified) + .bind(feed_entry_updated) + .bind(now) + .bind(locale) + .bind(locale_source) + .bind(recipe_id) + .execute(pool) + .await?; + + Ok(()) +} +``` + +- [ ] **Step 7: Add `update_recipe_locale` and `list_locales`** + +Append to `src/db/recipes.rs`, before the `#[cfg(test)]` module: + +```rust +/// Update only a recipe's locale columns. +pub async fn update_recipe_locale( + pool: &DbPool, + recipe_id: i64, + locale: Option<&str>, + locale_source: Option<&str>, +) -> Result<()> { + sqlx::query("UPDATE recipes SET locale = ?, locale_source = ? WHERE id = ?") + .bind(locale) + .bind(locale_source) + .bind(recipe_id) + .execute(pool) + .await?; + + Ok(()) +} + +/// Every distinct locale present in the database with its recipe count, +/// most common first. Used to populate the language filter. +pub async fn list_locales(pool: &DbPool) -> Result> { + let rows: Vec<(String, i64)> = sqlx::query_as( + r#" + SELECT locale, COUNT(*) as count + FROM recipes + WHERE locale IS NOT NULL + GROUP BY locale + ORDER BY count DESC, locale ASC + "#, + ) + .fetch_all(pool) + .await?; + + Ok(rows) +} +``` + +- [ ] **Step 8: Fix the other `NewRecipe` literals** + +Adding fields to `NewRecipe` breaks every struct literal. Add `locale: None, locale_source: None,` to each of: +- `src/db/ingredients.rs:212` (test) +- `src/db/tags.rs:208` (test) +- `src/crawler/mod.rs:394` — leave as `None` for now; Task 4 fills it in. +- `src/github/indexer.rs:373` — leave as `None` for now; Task 5 fills it in. +- `tests/reindex_test.rs` (test literal) +- Any other site `cargo build` points at. + +Also add `None, None` as the last two arguments to the `update_recipe_with_content` call at `src/crawler/mod.rs:359`; Task 4 replaces them with real values. + +- [ ] **Step 9: Run the full test suite** + +Run: `cargo test` +Expected: PASS, including the new `test_create_and_update_recipe_locale`. + +- [ ] **Step 10: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add migrations/008_recipe_locale.sql src/db tests/reindex_test.rs src/crawler/mod.rs src/github/indexer.rs +git commit -m "feat: add locale and locale_source columns to recipes" +``` + +--- + +### Task 4: Resolve locale in the feed crawler + +`process_entry` in `src/crawler/mod.rs` currently parses the Cooklang content only on the *create* path, and only to pull out a fallback image (`src/crawler/mod.rs:381`). We hoist that parse so it happens once for both paths and feeds both the image lookup and locale resolution. + +**Files:** +- Modify: `src/crawler/mod.rs` (`process_entry`, ~lines 350-425) + +- [ ] **Step 1: Parse once, before the create/update branch** + +Immediately after the `content_hash` is computed (~line 360) and *before* `let result = match existing_recipe {`, insert: + +```rust + // Parse the Cooklang content once: it feeds both the image fallback and + // locale resolution, on the create and the update path alike. + let parsed_content = content + .as_ref() + .and_then(|c| parse_cooklang_full(c).ok()); + + let locale = parsed_content + .as_ref() + .and_then(crate::indexer::resolve_locale); + let (locale_code, locale_source) = match &locale { + Some(l) => (Some(l.code.as_str()), Some(l.source.as_str())), + None => (None, None), + }; +``` + +- [ ] **Step 2: Pass locale into the update path** + +In the `Some(recipe) => { ... }` arm, the `update_recipe_with_content` call gained two `None, None` arguments in Task 3. Replace them: + +```rust + if let Some(ref content_str) = content { + db::recipes::update_recipe_with_content( + pool, + recipe.id, + content_str, + content_hash.as_deref(), + content_etag.as_deref(), + content_last_modified_dt.as_ref(), + entry.updated.as_ref(), + locale_code, + locale_source, + ) + .await?; + } +``` + +- [ ] **Step 3: Use the hoisted parse for the image, and set locale on create** + +In the `None => { ... }` arm, replace the local `metadata_image` re-parse with the hoisted `parsed_content`, and add the two locale fields to the `NewRecipe` literal (replacing the `None`s from Task 3): + +```rust + // Determine image URL: prefer feed entry image, fallback to Cooklang metadata + let metadata_image = parsed_content + .as_ref() + .and_then(|parsed| parsed.metadata.as_ref()) + .and_then(|m| m.image.clone()); + let image_url = entry + .image_url + .clone() + .or(metadata_image) + .and_then(|img| resolve_image_url(&img, enclosure_url)); + + // Create new recipe + let new_recipe = NewRecipe { + feed_id, + external_id: entry.id.clone(), + title: entry.title.clone(), + source_url: entry.source_url.clone(), + enclosure_url: enclosure_url.clone(), + content, + summary: entry.summary.clone(), + servings: entry.metadata.servings, + total_time_minutes: entry.metadata.total_time, + active_time_minutes: entry.metadata.active_time, + difficulty: entry.metadata.difficulty.clone(), + image_url, + published_at: entry.published, + content_hash, + content_etag, + content_last_modified: content_last_modified_dt, + feed_entry_updated: entry.updated, + locale: locale_code.map(str::to_string), + locale_source: locale_source.map(str::to_string), + }; +``` + +Note the `content` field moves the `Option` into `new_recipe`, which is why `parsed_content` must be computed *before* this literal. + +- [ ] **Step 4: Build** + +Run: `cargo build` +Expected: success. If `parse_cooklang_full` is not in scope, check the existing `use` at the top of `src/crawler/mod.rs` — it is already imported for the image lookup. + +- [ ] **Step 5: Run the crawler tests** + +Run: `cargo test --test crawler_tests` +Expected: PASS (no behavior change for feeds without parseable content). + +- [ ] **Step 6: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add src/crawler/mod.rs +git commit -m "feat: resolve recipe locale in the feed crawler" +``` + +--- + +### Task 5: Resolve locale in the GitHub indexer + +**Files:** +- Modify: `src/github/indexer.rs` (recipe create, ~lines 356-395) + +- [ ] **Step 1: Resolve the locale** + +`index_recipe` already parses the content into `let parsed = crate::indexer::parse_cooklang_full(&content);` (a `Result`, ~line 324). Reuse it — do not parse a second time. Directly after the `let (summary, servings, total_time, metadata_image) = ...` block that follows it, add: + +```rust + // Locale: declared `locale:` metadata wins, otherwise detected from text. + let locale = parsed.as_ref().ok().and_then(crate::indexer::resolve_locale); + let (locale_code, locale_source) = match &locale { + Some(l) => (Some(l.code.clone()), Some(l.source.as_str().to_string())), + None => (None, None), + }; +``` + +`parsed.as_ref().ok()` borrows, so `parsed` is still available to the ingredient/tag extraction further down the function. + +- [ ] **Step 2: Set it on the new recipe** + +In the `NewRecipe` literal (~line 373), replace the `locale: None, locale_source: None,` placeholders from Task 3: + +```rust + locale: locale_code.clone(), + locale_source: locale_source.clone(), +``` + +- [ ] **Step 3: Refresh locale on the update path** + +The `if let Some(existing)` arm currently only updates the GitHub SHA. Add a locale refresh right after `update_github_recipe_sha`, so a file whose content changed gets a re-resolved locale: + +```rust + db::github::update_github_recipe_sha(&self.pool, existing.id, file_sha).await?; + + db::recipes::update_recipe_locale( + &self.pool, + existing.recipe_id, + locale_code.as_deref(), + locale_source.as_deref(), + ) + .await?; + + recipe.id +``` + +- [ ] **Step 4: Build and test** + +Run: `cargo build && cargo test` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add src/github/indexer.rs +git commit -m "feat: resolve recipe locale in the github indexer" +``` + +--- + +### Task 6: Index and filter locale in Tantivy + +Two things to get right: + +1. The `locale` field is `STRING` (untokenized), so `de` matches only the exact term `de`. +2. A recipe declared `en_US` is stored as `en-US`, but a user filtering by `en` must still find it. So `index_recipe` writes **both** the full code and its base language when they differ. `locale=en` then matches both `en` and `en-US` recipes; `locale=en-US` matches only the regional ones. + +The field is deliberately **not** added to the `QueryParser`'s default field list — a free-text search for "de" must not match every German recipe. Filtering is only via the explicit parameter. + +**Files:** +- Modify: `src/indexer/schema.rs` +- Modify: `src/indexer/search.rs` (`SearchQuery` ~line 18, `SearchResult` ~line 25, `index_recipe` ~line 81, `search` ~line 182) + +- [ ] **Step 1: Write the failing test** + +Add to the `mod tests` block at the bottom of `src/indexer/search.rs`. Copy the `Recipe` literal shape from the existing `test_index_recipe_deletes_before_adding` test — it constructs a full `Recipe` struct; add `locale` / `locale_source` to it. + +```rust + fn test_recipe(id: i64, title: &str, locale: Option<&str>) -> Recipe { + Recipe { + id, + feed_id: 1, + external_id: format!("ext-{id}"), + title: title.to_string(), + source_url: None, + enclosure_url: format!("https://example.com/{id}.cook"), + content: Some("Mix the flour and the water.".to_string()), + summary: None, + servings: None, + total_time_minutes: None, + active_time_minutes: None, + difficulty: None, + image_url: None, + published_at: None, + updated_at: None, + indexed_at: None, + created_at: chrono::Utc::now(), + content_hash: None, + content_etag: None, + content_last_modified: None, + feed_entry_updated: None, + locale: locale.map(str::to_string), + locale_source: locale.map(|_| "detected".to_string()), + } + } + + #[test] + fn test_search_filters_by_locale() { + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + let mut writer = index.writer().unwrap(); + + index + .index_recipe(&mut writer, &test_recipe(1, "Pancakes", Some("en")), None, &[], &[]) + .unwrap(); + index + .index_recipe(&mut writer, &test_recipe(2, "Pfannkuchen", Some("de")), None, &[], &[]) + .unwrap(); + index + .index_recipe(&mut writer, &test_recipe(3, "Crepes", None), None, &[], &[]) + .unwrap(); + index.commit(&mut writer).unwrap(); + + // Filtering by locale returns only that language. + let results = index + .search( + &SearchQuery { + q: String::new(), + page: 1, + limit: 10, + locale: Some("de".to_string()), + }, + 10, + ) + .unwrap(); + assert_eq!(results.results.len(), 1); + assert_eq!(results.results[0].recipe_id, 2); + assert_eq!(results.results[0].locale.as_deref(), Some("de")); + + // No filter returns everything, including the recipe with no locale. + let all = index + .search( + &SearchQuery { + q: String::new(), + page: 1, + limit: 10, + locale: None, + }, + 10, + ) + .unwrap(); + assert_eq!(all.results.len(), 3); + } + + #[test] + fn test_locale_filter_combines_with_query() { + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + let mut writer = index.writer().unwrap(); + + index + .index_recipe(&mut writer, &test_recipe(1, "Pancakes", Some("en")), None, &[], &[]) + .unwrap(); + index + .index_recipe(&mut writer, &test_recipe(2, "Pancakes", Some("de")), None, &[], &[]) + .unwrap(); + index.commit(&mut writer).unwrap(); + + let results = index + .search( + &SearchQuery { + q: "pancakes".to_string(), + page: 1, + limit: 10, + locale: Some("en".to_string()), + }, + 10, + ) + .unwrap(); + + assert_eq!(results.results.len(), 1); + assert_eq!(results.results[0].recipe_id, 1); + } + + #[test] + fn test_regional_locale_matches_base_language_filter() { + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + let mut writer = index.writer().unwrap(); + + index + .index_recipe(&mut writer, &test_recipe(1, "Biscuits", Some("en-US")), None, &[], &[]) + .unwrap(); + index.commit(&mut writer).unwrap(); + + // Filtering by the base language finds the regional recipe... + let base = index + .search( + &SearchQuery { q: String::new(), page: 1, limit: 10, locale: Some("en".to_string()) }, + 10, + ) + .unwrap(); + assert_eq!(base.results.len(), 1); + + // ...and the stored code keeps its region. + assert_eq!(base.results[0].locale.as_deref(), Some("en-US")); + + // A different language does not match. + let other = index + .search( + &SearchQuery { q: String::new(), page: 1, limit: 10, locale: Some("de".to_string()) }, + 10, + ) + .unwrap(); + assert_eq!(other.results.len(), 0); + } +``` + +The existing tests in this module construct `SearchQuery` without a `locale` field — add `locale: None` to each of them. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test --lib search` +Expected: FAIL to compile — `struct 'SearchQuery' has no field named 'locale'`. + +- [ ] **Step 3: Add the schema field** + +In `src/indexer/schema.rs`, add `pub locale: Field,` to the `RecipeSchema` struct after `file_path`, build it in `new()`, and add it to the returned struct: + +```rust + // File path (searchable, stored) - for GitHub recipes + let file_path = schema_builder.add_text_field("file_path", TEXT | STORED); + + // Locale (exact-match filter, not tokenized, deliberately excluded from + // the default query-parser fields so free text can't match it) + let locale = schema_builder.add_text_field("locale", STRING | STORED); +``` + +- [ ] **Step 4: Write locale into the document** + +In `src/indexer/search.rs`, `index_recipe`, after the `file_path` block: + +```rust + // Add locale, plus its base language when the code carries a region, so a + // filter on "en" also matches an "en-US" recipe. + if let Some(locale) = &recipe.locale { + doc.add_text(self.schema.locale, locale); + + if let Some((language, _region)) = locale.split_once('-') { + doc.add_text(self.schema.locale, language); + } + } +``` + +- [ ] **Step 5: Extend `SearchQuery` and `SearchResult`** + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchQuery { + pub q: String, // Unified query string + pub page: usize, + pub limit: usize, + /// Optional exact-match language filter, e.g. "de" or "en-US". + pub locale: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchResult { + pub recipe_id: i64, + pub title: String, + pub summary: Option, + pub score: f32, + pub locale: Option, +} +``` + +- [ ] **Step 6: Apply the filter in `search`** + +Extend the imports at the top of `src/indexer/search.rs`: + +```rust +use tantivy::query::{BooleanQuery, Occur, TermQuery}; +use tantivy::schema::IndexRecordOption; +``` + +(keep the existing `Query`, `QueryParser`, `TopDocs`, `Term` imports; merge rather than duplicate.) + +Then, right after the existing `let tantivy_query = if query.q.is_empty() { ... };` block: + +```rust + // AND an exact locale term onto the parsed query when filtering. + let tantivy_query = match query.locale.as_deref().filter(|l| !l.is_empty()) { + Some(locale) => { + let term = Term::from_field_text(self.schema.locale, locale); + let locale_query: Box = + Box::new(TermQuery::new(term, IndexRecordOption::Basic)); + + Box::new(BooleanQuery::new(vec![ + (Occur::Must, tantivy_query), + (Occur::Must, locale_query), + ])) as Box + } + None => tantivy_query, + }; +``` + +- [ ] **Step 7: Return locale in results** + +In the `filter_map` that builds each `SearchResult`, after `summary`: + +```rust + let locale = doc.get_first(self.schema.locale).and_then(|v| match v { + tantivy::schema::OwnedValue::Str(s) => Some(s.to_string()), + _ => None, + }); + + Some(SearchResult { + recipe_id, + title, + summary, + score, + locale, + }) +``` + +`get_first` returns the first value written for the field, which is the full code (with region) because Step 4 writes it before the base language. + +- [ ] **Step 8: Fix the other `SearchQuery` literals** + +`cargo build` will point at `src/api/handlers.rs:27` and `src/web/handlers.rs:74`. Add `locale: None` to both for now — Tasks 7 and 8 wire the real values. + +- [ ] **Step 9: Run tests to verify they pass** + +Run: `cargo test --lib search` +Expected: PASS, including the three new tests. + +- [ ] **Step 10: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add src/indexer/schema.rs src/indexer/search.rs src/api/handlers.rs src/web/handlers.rs +git commit -m "feat: index and filter recipes by locale in search" +``` + +--- + +### Task 7: API — `?locale=` filter and locale in responses + +**Files:** +- Modify: `src/api/models.rs` (`SearchParams` ~line 5, `RecipeCard` ~line 31, `RecipeDetail` ~line 48) +- Modify: `src/api/handlers.rs` (`search_recipes` ~line 20, `get_recipe` ~line 67) + +- [ ] **Step 1: Extend the API models** + +In `src/api/models.rs`: + +```rust +pub struct SearchParams { + #[serde(default)] + pub q: String, // Unified query string + /// Optional language filter, e.g. "de". + #[serde(default)] + pub locale: Option, + #[serde(default = "default_page")] + pub page: usize, + #[serde(default = "default_limit")] + pub limit: usize, +} +``` + +Add `pub locale: Option,` to `RecipeCard`, and both fields to `RecipeDetail`: + +```rust +pub struct RecipeCard { + pub id: i64, + pub title: String, + pub summary: Option, + pub tags: Vec, + pub locale: Option, +} +``` + +```rust + pub enclosure_url: String, + pub locale: Option, + pub locale_source: Option, + pub feed: FeedInfo, +``` + +- [ ] **Step 2: Thread the filter through the search handler** + +In `src/api/handlers.rs`, `search_recipes`: + +```rust + let query = SearchQuery { + q: params.q, + page: params.page, + limit: params.limit.min(state.settings.pagination.api_max_limit), + locale: params.locale, + }; +``` + +and when building each card: + +```rust + recipe_cards.push(RecipeCard { + id: result.recipe_id, + title: result.title, + summary: result.summary, + tags, + locale: result.locale, + }); +``` + +- [ ] **Step 3: Return locale from the detail endpoint** + +In `get_recipe`, add to the `RecipeDetail` construction, after `enclosure_url`: + +```rust + enclosure_url: recipe.enclosure_url, + locale: recipe.locale, + locale_source: recipe.locale_source, + feed: FeedInfo { +``` + +- [ ] **Step 4: Build and test** + +Run: `cargo build && cargo test` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add src/api +git commit -m "feat: expose recipe locale in the API and filter search by it" +``` + +--- + +### Task 8: Web UI — language dropdown, card chip, detail pill + +**Files:** +- Modify: `src/web/handlers.rs` (`SearchParams` ~line 51, `SearchTemplate` ~line 27, `RecipeCardData` ~line 37, `index` ~line 63, `RecipeData` ~line 174, `recipe_detail` ~line 207) +- Modify: `src/web/schema.rs` (`recipe_to_schema_json`) +- Modify: `src/web/templates/search.html` +- Modify: `src/web/templates/recipe.html` + +- [ ] **Step 1: Extend the web view models** + +In `src/web/handlers.rs`: + +```rust +#[derive(Template)] +#[template(path = "search.html")] +struct SearchTemplate { + query: String, + locale: String, + locales: Vec, + results: Vec, + total: usize, + page: usize, + total_pages: usize, + recent_recipes: Vec, +} + +/// One entry in the language filter dropdown. +#[derive(Clone)] +#[allow(dead_code)] // Fields are used by Askama templates +struct LocaleOption { + code: String, + name: String, + count: i64, +} +``` + +Add `locale_name: String` to `RecipeCardData` (empty string = unknown, matching how the other optional fields on that struct are handled), and to `SearchParams`: + +```rust +#[derive(Deserialize)] +pub struct SearchParams { + #[serde(default, deserialize_with = "deserialize_optional_string")] + q: Option, + #[serde(default, deserialize_with = "deserialize_optional_string")] + locale: Option, + #[serde(default = "default_page")] + page: usize, +} +``` + +- [ ] **Step 2: Build the dropdown options and thread the filter** + +In `index`, at the top: + +```rust + let query = params.q.clone().unwrap_or_default(); + let locale = params.locale.clone().unwrap_or_default(); + + // Language filter options: distinct locales in the database, most common first. + // Regional codes ("en-US") are folded into their base language ("en") so the + // dropdown lists one entry per language. + let locales = { + let mut counts: Vec<(String, i64)> = Vec::new(); + for (code, count) in db::recipes::list_locales(&state.pool).await? { + let base = code.split('-').next().unwrap_or(&code).to_string(); + match counts.iter_mut().find(|(c, _)| *c == base) { + Some((_, existing)) => *existing += count, + None => counts.push((base, count)), + } + } + counts.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0))); + + counts + .into_iter() + .map(|(code, count)| LocaleOption { + name: crate::indexer::locale::display_name(&code).unwrap_or_else(|| code.clone()), + code, + count, + }) + .collect::>() + }; +``` + +The results branch currently runs only `if query.is_empty()` → no results. A language filter with no query should still return results, so change the guard to cover both, and pass the locale into the search: + +```rust + let (results, total, total_pages) = if query.is_empty() && locale.is_empty() { + (vec![], 0, 0) + } else { + // Build search query + let search_query = SearchQuery { + q: query.clone(), + page: params.page, + limit: state.settings.pagination.web_default_limit, + locale: params.locale.clone(), + }; +``` + +In the same branch, where each `RecipeCardData` is built from the database row `r`, add: + +```rust + locale_name: r + .locale + .as_deref() + .and_then(crate::indexer::locale::display_name) + .unwrap_or_default(), +``` + +Add the same line to the `recent_recipes` mapping further down (it builds `RecipeCardData` too), and update the `SearchTemplate` construction: + +```rust + let template = SearchTemplate { + query, + locale, + locales, + results, + total, + page: params.page, + total_pages, + recent_recipes, + }; +``` + +Note the `recent_recipes` block is gated on `query.is_empty()`; leave that gate alone — it should also require an empty locale so a filtered search doesn't show both. Change it to `if query.is_empty() && locale.is_empty()`. + +- [ ] **Step 3: Add the dropdown to `search.html`** + +In `src/web/templates/search.html`, inside the `
`, put the select between the text input and the submit button: + +```html +
+ + {% if !locales.is_empty() %} + + {% endif %} + +
+``` + +- [ ] **Step 4: Add the language chip to result cards** + +Still in `search.html`, the card body ends with a row of facts: + +```html +
+ {% if !recipe.total_time_minutes.is_empty() %} + ⏱️ {{ recipe.total_time_minutes }}m + {% endif %} + {% if !recipe.servings.is_empty() %} + 👥 {{ recipe.servings }} + {% endif %} + {% if !recipe.difficulty.is_empty() %} + 📊 {{ recipe.difficulty }} + {% endif %} +
+``` + +Add the language as one more fact, after the difficulty block and inside that same `
`: + +```html + {% if !recipe.locale_name.is_empty() %} + 🗣️ {{ recipe.locale_name }} + {% endif %} +``` + +This card markup is repeated for the `recent_recipes` grid further down the same template — add the chip there too, so the homepage cards match. + +- [ ] **Step 5: Add locale to the recipe detail page** + +In `src/web/handlers.rs`, add to `RecipeData`: + +```rust + pub locale: String, + pub locale_name: String, + pub locale_detected: bool, +``` + +and in `recipe_detail`, where `RecipeData` is constructed: + +```rust + locale: recipe.locale.clone().unwrap_or_default(), + locale_name: recipe + .locale + .as_deref() + .and_then(crate::indexer::locale::display_name) + .unwrap_or_default(), + locale_detected: recipe.locale_source.as_deref() == Some("detected"), +``` + +`recipe.locale` is moved by `recipe.title` earlier in the literal only if you reorder — keep these lines after the existing fields and use `.clone()` as shown. + +- [ ] **Step 6: Render the pill** + +In `src/web/templates/recipe.html`, in the second metadata-pill block (the one reading `recipe.*`, after the `recipe.difficulty` pill around line 144): + +```html + {% if !recipe.locale_name.is_empty() %} +
+ 🗣️ + {{ recipe.locale_name }}{% if recipe.locale_detected %} (detected){% endif %} +
+ {% endif %} +``` + +- [ ] **Step 7: Add `inLanguage` to the JSON-LD** + +`recipe_to_schema_json` builds a `serde_json::Value` and assigns optional properties by index (`schema["image"] = json!(recipe.image_url);`). Follow that style — add this after the `// URL (source)` block: + +```rust + // Language + if !recipe.locale.is_empty() { + schema["inLanguage"] = json!(recipe.locale); + } +``` + +- [ ] **Step 8: Build and test** + +Run: `cargo build && cargo test` +Expected: PASS. Askama compiles templates at build time, so a template typo shows up here as a compile error. + +- [ ] **Step 9: Verify in the browser** + +Run: `cargo run -- serve` +Open `http://localhost:3000`, confirm the language dropdown lists languages with counts, that selecting one filters results, and that a recipe detail page shows the language pill. + +- [ ] **Step 10: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add src/web +git commit -m "feat: show and filter recipe language in the web ui" +``` + +--- + +### Task 9: `backfill-locales` CLI command + +Recipe content is already in the `recipes` table, so backfill needs no network access. It walks recipes in batches keyed on `id` (never `OFFSET`, which would skip rows as we update them), resolves a locale, writes it, and re-indexes the recipe in Tantivy so the filter works over existing data. + +**Note — deliberate side effect:** re-indexing touches every recipe with content, and the feed crawler does not currently write to the Tantivy index at all (only the GitHub indexer does). Backfill will therefore *add* feed-crawled recipes to the search index for the first time. This is intended: it closes a pre-existing gap. Expect search result counts to rise after the first run. + +**Files:** +- Modify: `src/cli/mod.rs` (the `Commands` enum) +- Modify: `src/cli/commands.rs` +- Modify: `src/main.rs` + +- [ ] **Step 1: Add the subcommand** + +In `src/cli/mod.rs`, add to the end of the `Commands` enum, after `Reindex`: + +```rust + /// Detect and store the locale of recipes that don't have one + BackfillLocales { + /// Recompute the locale of every recipe, not just those without one + #[arg(long)] + force: bool, + }, +``` + +- [ ] **Step 2: Write the failing test** + +Create `tests/locale_test.rs`: + +```rust +use federation::cli::commands::backfill_locales; +use federation::db::models::{NewFeed, NewRecipe}; +use federation::db::{feeds, recipes}; +use federation::indexer::search::{SearchIndex, SearchQuery}; +use sqlx::SqlitePool; +use tempfile::tempdir; + +const GERMAN_RECIPE: &str = "Den Mehl und das Wasser in einer Schüssel verrühren, bis ein \ +glatter Teig entsteht. Den Teig ruhen lassen und anschließend goldbraun backen."; + +const ENGLISH_RECIPE: &str = "Mix the flour and the water in a bowl until a smooth dough \ +forms. Let the dough rest, then bake it until golden brown."; + +async fn setup() -> (SqlitePool, i64) { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + sqlx::migrate!("./migrations").run(&pool).await.unwrap(); + + let feed = feeds::create_feed( + &pool, + &NewFeed { + url: "https://example.com/feed.xml".to_string(), + title: Some("Test Feed".to_string()), + }, + ) + .await + .unwrap(); + + (pool, feed.id) +} + +fn new_recipe(feed_id: i64, external_id: &str, title: &str, content: &str) -> NewRecipe { + NewRecipe { + feed_id, + external_id: external_id.to_string(), + title: title.to_string(), + source_url: None, + enclosure_url: format!("https://example.com/{external_id}.cook"), + content: Some(content.to_string()), + summary: None, + servings: None, + total_time_minutes: None, + active_time_minutes: None, + difficulty: None, + image_url: None, + published_at: None, + content_hash: None, + content_etag: None, + content_last_modified: None, + feed_entry_updated: None, + locale: None, + locale_source: None, + } +} + +#[tokio::test] +async fn test_backfill_detects_locales_and_makes_them_searchable() { + let (pool, feed_id) = setup().await; + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + + let german = recipes::create_recipe(&pool, &new_recipe(feed_id, "de-1", "Pfannkuchen", GERMAN_RECIPE)) + .await + .unwrap(); + let english = recipes::create_recipe(&pool, &new_recipe(feed_id, "en-1", "Pancakes", ENGLISH_RECIPE)) + .await + .unwrap(); + + assert_eq!(german.locale, None, "precondition: no locale yet"); + + let stats = backfill_locales(&pool, &index, false).await.unwrap(); + assert_eq!(stats.scanned, 2); + assert_eq!(stats.updated, 2); + + let german = recipes::get_recipe(&pool, german.id).await.unwrap(); + assert_eq!(german.locale.as_deref(), Some("de")); + assert_eq!(german.locale_source.as_deref(), Some("detected")); + + let english = recipes::get_recipe(&pool, english.id).await.unwrap(); + assert_eq!(english.locale.as_deref(), Some("en")); + + // The backfilled recipes are searchable by locale. + let results = index + .search( + &SearchQuery { + q: String::new(), + page: 1, + limit: 10, + locale: Some("de".to_string()), + }, + 10, + ) + .unwrap(); + + assert_eq!(results.results.len(), 1); + assert_eq!(results.results[0].recipe_id, german.id); +} + +#[tokio::test] +async fn test_backfill_skips_recipes_that_already_have_a_locale() { + let (pool, feed_id) = setup().await; + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + + let recipe = recipes::create_recipe(&pool, &new_recipe(feed_id, "de-1", "Pfannkuchen", GERMAN_RECIPE)) + .await + .unwrap(); + + // Pretend the author declared French. + recipes::update_recipe_locale(&pool, recipe.id, Some("fr"), Some("declared")) + .await + .unwrap(); + + let stats = backfill_locales(&pool, &index, false).await.unwrap(); + assert_eq!(stats.scanned, 0, "already-tagged recipes are not scanned"); + assert_eq!(stats.updated, 0); + + let unchanged = recipes::get_recipe(&pool, recipe.id).await.unwrap(); + assert_eq!(unchanged.locale.as_deref(), Some("fr")); + assert_eq!(unchanged.locale_source.as_deref(), Some("declared")); + + // --force recomputes it, and detection overrides the stale value. + let stats = backfill_locales(&pool, &index, true).await.unwrap(); + assert_eq!(stats.scanned, 1); + assert_eq!(stats.updated, 1); + + let forced = recipes::get_recipe(&pool, recipe.id).await.unwrap(); + assert_eq!(forced.locale.as_deref(), Some("de")); + assert_eq!(forced.locale_source.as_deref(), Some("detected")); +} +``` + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `cargo test --test locale_test` +Expected: FAIL to compile — `cannot find function 'backfill_locales'`. + +- [ ] **Step 4: Implement the backfill** + +Append to `src/cli/commands.rs` (check the file's existing `use` block; add whatever is missing): + +```rust +/// What a backfill pass did. +#[derive(Debug, Default, Clone, Copy)] +pub struct BackfillStats { + /// Recipes considered (had content, and matched the locale predicate). + pub scanned: usize, + /// Recipes whose locale we resolved and stored. + pub updated: usize, + /// Recipes we could not resolve a locale for. + pub skipped: usize, +} + +/// Detect and store locales for recipes that don't have one. +/// +/// Recipe content is already in the database, so this makes no network calls. +/// Each touched recipe is re-indexed in Tantivy so the locale filter works over +/// existing data. With `force`, every recipe with content is recomputed. +pub async fn backfill_locales( + pool: &crate::db::DbPool, + search_index: &crate::indexer::search::SearchIndex, + force: bool, +) -> Result { + use crate::db::models::Recipe; + + /// Rows per batch. Keeps memory flat on large databases. + const BATCH_SIZE: i64 = 500; + + let mut stats = BackfillStats::default(); + let mut writer = search_index.writer()?; + let mut last_id: i64 = 0; + + loop { + // Keyset pagination on id: rows we update drop out of the unfiltered + // predicate, so an OFFSET would silently skip recipes. + let sql = if force { + "SELECT * FROM recipes WHERE content IS NOT NULL AND id > ? ORDER BY id LIMIT ?" + } else { + "SELECT * FROM recipes \ + WHERE content IS NOT NULL AND locale IS NULL AND id > ? ORDER BY id LIMIT ?" + }; + + let batch: Vec = sqlx::query_as::<_, Recipe>(sql) + .bind(last_id) + .bind(BATCH_SIZE) + .fetch_all(pool) + .await?; + + if batch.is_empty() { + break; + } + + for mut recipe in batch { + last_id = recipe.id; + stats.scanned += 1; + + let Some(content) = recipe.content.clone() else { + stats.skipped += 1; + continue; + }; + + let locale = match crate::indexer::parse_cooklang_full(&content) { + Ok(parsed) => crate::indexer::resolve_locale(&parsed), + Err(e) => { + warn!("Recipe {}: failed to parse content: {}", recipe.id, e); + None + } + }; + + let Some(locale) = locale else { + stats.skipped += 1; + continue; + }; + + crate::db::recipes::update_recipe_locale( + pool, + recipe.id, + Some(&locale.code), + Some(locale.source.as_str()), + ) + .await?; + + // Re-index with the locale so the search filter sees it. + recipe.locale = Some(locale.code.clone()); + recipe.locale_source = Some(locale.source.as_str().to_string()); + + let file_path = crate::db::github::get_github_recipe_by_recipe_id(pool, recipe.id) + .await? + .map(|gh| gh.file_path); + let tags = crate::db::tags::get_tags_for_recipe(pool, recipe.id).await?; + let ingredients = crate::db::ingredients::get_ingredients_for_recipe(pool, recipe.id) + .await? + .iter() + .map(|i| i.name.clone()) + .collect::>(); + + search_index.index_recipe( + &mut writer, + &recipe, + file_path.as_deref(), + &tags, + &ingredients, + )?; + + stats.updated += 1; + } + } + + search_index.commit(&mut writer)?; + + Ok(stats) +} +``` + +If `warn!` is not already imported in this file, add `use tracing::warn;`. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test --test locale_test` +Expected: PASS — both tests. + +- [ ] **Step 6: Wire the command into `main.rs`** + +In the `match cli.command` block, after the `Commands::Reindex` arm: + +```rust + Commands::BackfillLocales { force } => { + backfill_locales(settings, force).await?; + } +``` + +and add the function alongside `reindex_feed` (near line 276), following its shape: + +```rust +async fn backfill_locales(settings: Settings, force: bool) -> Result<()> { + info!("Backfilling recipe locales (force: {})", force); + + let pool = db::init_pool(&settings.database.url).await?; + db::run_migrations(&pool).await?; + + let index_path = std::path::PathBuf::from(&settings.search.index_path); + let search_index = SearchIndex::new(&index_path)?; + + let stats = federation::cli::commands::backfill_locales(&pool, &search_index, force).await?; + + println!( + "\x1b[32m\u{2713}\x1b[0m Backfill complete: {} scanned, {} tagged, {} left without a locale", + stats.scanned, stats.updated, stats.skipped + ); + + Ok(()) +} +``` + +- [ ] **Step 7: Run it against the real database** + +Run: `cargo run -- backfill-locales` +Expected: a summary line, e.g. `✓ Backfill complete: 412 scanned, 397 tagged, 15 left without a locale`. + +Then re-run it: `cargo run -- backfill-locales` +Expected: `0 scanned` — already-tagged recipes are not rescanned, which proves the run was persisted. + +- [ ] **Step 8: Commit** + +```bash +cargo fmt +cargo clippy --all-targets -- -D warnings +git add src/cli src/main.rs tests/locale_test.rs +git commit -m "feat: add backfill-locales command" +``` + +--- + +### Task 10: End-to-end verification + +- [ ] **Step 1: Full test suite** + +Run: `cargo test` +Expected: PASS, no warnings from `cargo clippy --all-targets -- -D warnings`. + +- [ ] **Step 2: Exercise the API** + +Run: `cargo run -- serve` in one terminal, then: + +```bash +curl -s 'http://localhost:3000/api/search?q=&locale=de&limit=3' | head -40 +curl -s 'http://localhost:3000/api/recipes/1' | grep -o '"locale[^,]*' +``` + +Expected: the search response contains only recipes whose `locale` is `de`; the detail response carries `locale` and `locale_source`. + +- [ ] **Step 3: Update the README** + +`README.md` documents the CLI commands and search syntax. Add `backfill-locales` to the command list and mention the `locale` search filter in whatever section covers the API's query parameters. Match the existing formatting. + +- [ ] **Step 4: Commit** + +```bash +git add README.md +git commit -m "docs: document locale filter and backfill-locales command" +``` diff --git a/migrations/008_recipe_locale.sql b/migrations/008_recipe_locale.sql new file mode 100644 index 0000000..4967e95 --- /dev/null +++ b/migrations/008_recipe_locale.sql @@ -0,0 +1,6 @@ +-- Recipe locale: BCP-47-style code (e.g. 'en', 'de', 'en-US'). +-- locale_source records whether the author declared it or we detected it. +ALTER TABLE recipes ADD COLUMN locale TEXT; +ALTER TABLE recipes ADD COLUMN locale_source TEXT; + +CREATE INDEX IF NOT EXISTS idx_recipes_locale ON recipes(locale); diff --git a/src/api/handlers.rs b/src/api/handlers.rs index 8ffbd34..d1fae3a 100644 --- a/src/api/handlers.rs +++ b/src/api/handlers.rs @@ -28,6 +28,7 @@ pub async fn search_recipes( q: params.q, page: params.page, limit: params.limit.min(state.settings.pagination.api_max_limit), + locale: params.locale, }; // Execute search @@ -49,6 +50,7 @@ pub async fn search_recipes( title: result.title, summary: result.summary, tags, + locale: result.locale, }); } @@ -104,6 +106,8 @@ pub async fn get_recipe( image_url: recipe.image_url, source_url: recipe.source_url, enclosure_url: recipe.enclosure_url, + locale: recipe.locale, + locale_source: recipe.locale_source, feed: FeedInfo { id: feed.id, title: feed.title, diff --git a/src/api/models.rs b/src/api/models.rs index bc8ef85..9eacb20 100644 --- a/src/api/models.rs +++ b/src/api/models.rs @@ -5,6 +5,9 @@ use serde::{Deserialize, Serialize}; pub struct SearchParams { #[serde(default)] pub q: String, // Unified query string + /// Optional language filter, e.g. "de". + #[serde(default)] + pub locale: Option, #[serde(default = "default_page")] pub page: usize, #[serde(default = "default_limit")] @@ -33,6 +36,7 @@ pub struct RecipeCard { pub title: String, pub summary: Option, pub tags: Vec, + pub locale: Option, } /// Pagination metadata @@ -60,6 +64,8 @@ pub struct RecipeDetail { pub image_url: Option, pub source_url: Option, pub enclosure_url: String, + pub locale: Option, + pub locale_source: Option, pub feed: FeedInfo, } diff --git a/src/api/routes.rs b/src/api/routes.rs index bdecb37..f51645f 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -173,10 +173,15 @@ mod tests { body::Body, http::{Request, StatusCode}, }; + use tempfile::TempDir; use tower::ServiceExt; - // Helper to create test app state - async fn create_test_state() -> AppState { + // Helper to create test app state. + // + // Returns the search index's `TempDir` guard alongside the state: the caller + // must keep it alive for the whole test, because dropping it deletes the + // index directory out from under Tantivy. + async fn create_test_state() -> (AppState, TempDir) { use std::sync::Arc; // Create in-memory database @@ -223,17 +228,21 @@ mod tests { }, }; - AppState { + let state = AppState { pool, search_index: Arc::new(search_index), github_indexer: None, settings, - } + }; + + (state, temp_dir) } #[tokio::test] async fn test_health_routes_exist() { - let state = create_test_state().await; + // The TempDir guard must stay alive for the whole test - dropping it deletes + // the search index directory out from under Tantivy. + let (state, _index_dir) = create_test_state().await; let app = create_router(state.clone(), &state.settings); // Test that API routes exist @@ -249,4 +258,117 @@ mod tests { assert_eq!(response.status(), StatusCode::OK); } + + async fn response_json(response: axum::response::Response) -> serde_json::Value { + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + serde_json::from_slice(&body).unwrap() + } + + #[tokio::test] + async fn test_search_locale_filter_and_recipe_detail_expose_locale() { + use crate::db::models::{NewFeed, NewRecipe}; + use crate::db::{feeds, recipes}; + + // The TempDir guard must stay alive for the whole test - dropping it deletes + // the search index directory out from under Tantivy. + let (state, _index_dir) = create_test_state().await; + + // Seed a feed and two recipes in different locales. + let feed = feeds::create_feed( + &state.pool, + &NewFeed { + url: "https://example.com/feed.xml".to_string(), + title: Some("Test Feed".to_string()), + }, + ) + .await + .unwrap(); + + let new_recipe = |external_id: &str, title: &str, locale: &str| NewRecipe { + feed_id: feed.id, + external_id: external_id.to_string(), + title: title.to_string(), + source_url: None, + enclosure_url: format!("https://example.com/{external_id}.cook"), + content: None, + summary: None, + servings: None, + total_time_minutes: None, + active_time_minutes: None, + difficulty: None, + image_url: None, + published_at: None, + content_hash: None, + content_etag: None, + content_last_modified: None, + feed_entry_updated: None, + locale: Some(locale.to_string()), + locale_source: Some("declared".to_string()), + }; + + let en_recipe = + recipes::create_recipe(&state.pool, &new_recipe("recipe-en", "Pancakes", "en")) + .await + .unwrap(); + + let de_recipe = + recipes::create_recipe(&state.pool, &new_recipe("recipe-de", "Pfannkuchen", "de")) + .await + .unwrap(); + + // Index both recipes into the search index and commit so they're + // immediately visible (SearchIndex::commit reloads the reader). + let mut writer = state.search_index.writer().unwrap(); + state + .search_index + .index_recipe(&mut writer, &en_recipe, None, &[], &[]) + .unwrap(); + state + .search_index + .index_recipe(&mut writer, &de_recipe, None, &[], &[]) + .unwrap(); + state.search_index.commit(&mut writer).unwrap(); + + // GET /api/search?locale=de returns only the German recipe. + let app = create_router(state.clone(), &state.settings); + let response = app + .oneshot( + Request::builder() + .uri("/api/search?locale=de") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let json = response_json(response).await; + let results = json["results"].as_array().unwrap(); + assert_eq!( + results.len(), + 1, + "expected only the German recipe: {json:?}" + ); + assert_eq!(results[0]["id"].as_i64().unwrap(), de_recipe.id); + assert_eq!(results[0]["locale"].as_str().unwrap(), "de"); + + // GET /api/recipes/:id includes locale and locale_source. + let app = create_router(state.clone(), &state.settings); + let response = app + .oneshot( + Request::builder() + .uri(format!("/api/recipes/{}", de_recipe.id)) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let json = response_json(response).await; + assert_eq!(json["locale"].as_str().unwrap(), "de"); + assert_eq!(json["locale_source"].as_str().unwrap(), "declared"); + } } diff --git a/src/cli/commands.rs b/src/cli/commands.rs index f7c8fec..e4d1257 100644 --- a/src/cli/commands.rs +++ b/src/cli/commands.rs @@ -395,6 +395,144 @@ pub async fn reindex_feed(pool: &crate::db::DbPool, url: &str) -> Result { Ok(deleted_count) } +/// What a backfill pass did. +#[derive(Debug, Default, Clone, Copy)] +pub struct BackfillStats { + /// Recipes considered (had content, and matched the locale predicate). + pub scanned: usize, + /// Recipes whose locale we resolved and stored. + pub updated: usize, + /// Recipes we could not resolve a locale for. + pub skipped: usize, +} + +/// Detect and store locales for recipes that don't have one. +/// +/// Recipe content is already in the database, so this makes no network calls. +/// Every recipe with content is re-indexed in Tantivy, whether or not a locale +/// could be resolved for it — a recipe must never fall out of the search index +/// just because we can't tell what language it's in. With `force`, every +/// recipe with content is recomputed. +/// +/// Per batch, the search index is committed *before* the DB locale rows are +/// written. If the process dies in between, those rows are left with +/// `locale IS NULL`, so a plain (non-`--force`) rerun picks them up again and +/// re-indexes them — `index_recipe` deletes-then-adds by recipe id, so +/// re-indexing is idempotent. The failure mode is "redo some work", not "lose +/// data". +pub async fn backfill_locales( + pool: &crate::db::DbPool, + search_index: &crate::indexer::search::SearchIndex, + force: bool, +) -> Result { + use crate::db::models::Recipe; + use crate::indexer::locale::RecipeLocale; + + /// Rows per batch. Keeps memory flat on large databases. + const BATCH_SIZE: i64 = 500; + + let mut stats = BackfillStats::default(); + let mut last_id: i64 = 0; + + loop { + // Keyset pagination on id: rows we update drop out of the unfiltered + // predicate, so an OFFSET would silently skip recipes. + let sql = if force { + "SELECT * FROM recipes WHERE content IS NOT NULL AND id > ? ORDER BY id LIMIT ?" + } else { + "SELECT * FROM recipes \ + WHERE content IS NOT NULL AND locale IS NULL AND id > ? ORDER BY id LIMIT ?" + }; + + let batch: Vec = sqlx::query_as::<_, Recipe>(sql) + .bind(last_id) + .bind(BATCH_SIZE) + .fetch_all(pool) + .await?; + + if batch.is_empty() { + break; + } + + let mut writer = search_index.writer()?; + // Locale resolved per recipe in this batch, to be written to the DB only + // after the batch's index writes are durably committed. + let mut resolved: Vec<(i64, Option)> = Vec::new(); + + for mut recipe in batch { + last_id = recipe.id; + stats.scanned += 1; + + let Some(content) = recipe.content.clone() else { + stats.skipped += 1; + continue; + }; + + let locale = match crate::indexer::parse_cooklang_full(&content) { + Ok(parsed) => crate::indexer::resolve_locale(&parsed), + Err(e) => { + warn!("Recipe {}: failed to parse content: {}", recipe.id, e); + None + } + }; + + if locale.is_none() { + stats.skipped += 1; + } + + // Apply the resolved locale (if any) to the in-memory recipe so the + // index reflects it, then index unconditionally: a recipe with + // content is always searchable, resolved locale or not. + if let Some(locale) = &locale { + recipe.locale = Some(locale.code.clone()); + recipe.locale_source = Some(locale.source.as_str().to_string()); + } + + let file_path = crate::db::github::get_github_recipe_by_recipe_id(pool, recipe.id) + .await? + .map(|gh| gh.file_path); + let tags = crate::db::tags::get_tags_for_recipe(pool, recipe.id).await?; + let ingredients = crate::db::ingredients::get_ingredients_for_recipe(pool, recipe.id) + .await? + .iter() + .map(|i| i.name.clone()) + .collect::>(); + + search_index.index_recipe( + &mut writer, + &recipe, + file_path.as_deref(), + &tags, + &ingredients, + )?; + + resolved.push((recipe.id, locale)); + } + + // Commit the index for this batch before touching the DB (Finding 2): + // see the doc comment above for why the ordering matters. + search_index.commit(&mut writer)?; + + for (recipe_id, locale) in resolved { + let Some(locale) = locale else { + continue; + }; + + crate::db::recipes::update_recipe_locale( + pool, + recipe_id, + Some(&locale.code), + Some(locale.source.as_str()), + ) + .await?; + + stats.updated += 1; + } + } + + Ok(stats) +} + // Response types (matching API models) #[derive(Debug, Deserialize)] diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 982412c..d484b14 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -75,4 +75,11 @@ pub enum Commands { /// Feed URL to reindex url: String, }, + + /// Detect and store the locale of recipes that don't have one + BackfillLocales { + /// Recompute the locale of every recipe, not just those without one + #[arg(long)] + force: bool, + }, } diff --git a/src/crawler/mod.rs b/src/crawler/mod.rs index 04406fc..b3ed18f 100644 --- a/src/crawler/mod.rs +++ b/src/crawler/mod.rs @@ -352,6 +352,18 @@ impl Crawler { .as_ref() .map(|c| db::recipes::calculate_content_hash(&entry.title, Some(c))); + // Parse the Cooklang content once: it feeds both the image fallback and + // locale resolution, on the create and the update path alike. + let parsed_content = content.as_ref().and_then(|c| parse_cooklang_full(c).ok()); + + let locale = parsed_content + .as_ref() + .and_then(crate::indexer::resolve_locale); + let (locale_code, locale_source) = match &locale { + Some(l) => (Some(l.code.as_str()), Some(l.source.as_str())), + None => (None, None), + }; + let result = match existing_recipe { Some(recipe) => { // Update existing recipe with new content @@ -364,6 +376,8 @@ impl Crawler { content_etag.as_deref(), content_last_modified_dt.as_ref(), entry.updated.as_ref(), + locale_code, + locale_source, ) .await?; } @@ -379,11 +393,10 @@ impl Crawler { } None => { // Determine image URL: prefer feed entry image, fallback to Cooklang metadata - let metadata_image = content.as_ref().and_then(|c| { - parse_cooklang_full(c) - .ok() - .and_then(|parsed| parsed.metadata.and_then(|m| m.image)) - }); + let metadata_image = parsed_content + .as_ref() + .and_then(|parsed| parsed.metadata.as_ref()) + .and_then(|m| m.image.clone()); let image_url = entry .image_url .clone() @@ -409,6 +422,8 @@ impl Crawler { content_etag, content_last_modified: content_last_modified_dt, feed_entry_updated: entry.updated, + locale: locale_code.map(str::to_string), + locale_source: locale_source.map(str::to_string), }; let (recipe, _) = db::recipes::get_or_create_recipe(pool, &new_recipe).await?; diff --git a/src/db/ingredients.rs b/src/db/ingredients.rs index b4cb0d8..6a61768 100644 --- a/src/db/ingredients.rs +++ b/src/db/ingredients.rs @@ -227,6 +227,8 @@ mod tests { content_etag: None, content_last_modified: None, feed_entry_updated: None, + locale: None, + locale_source: None, }, ) .await diff --git a/src/db/models.rs b/src/db/models.rs index 867538d..0844109 100644 --- a/src/db/models.rs +++ b/src/db/models.rs @@ -61,6 +61,10 @@ pub struct Recipe { pub content_last_modified: Option>, /// The timestamp from the feed entry for comparison pub feed_entry_updated: Option>, + /// BCP-47-style locale code, e.g. "en", "de", "en-US". NULL when unknown. + pub locale: Option, + /// How the locale was obtained: "declared" or "detected". + pub locale_source: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -85,6 +89,9 @@ pub struct NewRecipe { pub content_last_modified: Option>, /// The timestamp from the feed entry pub feed_entry_updated: Option>, + /// Locale code and its provenance ("declared" | "detected"). + pub locale: Option, + pub locale_source: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/db/recipes.rs b/src/db/recipes.rs index bb85981..120ad82 100644 --- a/src/db/recipes.rs +++ b/src/db/recipes.rs @@ -135,9 +135,9 @@ pub async fn create_recipe(pool: &DbPool, new_recipe: &NewRecipe) -> Result Result, content_last_modified: Option<&chrono::DateTime>, feed_entry_updated: Option<&chrono::DateTime>, + locale: Option<&str>, + locale_source: Option<&str>, ) -> Result<()> { let now = Utc::now(); @@ -428,7 +433,8 @@ pub async fn update_recipe_with_content( r#" UPDATE recipes SET content = ?, content_hash = ?, content_etag = ?, - content_last_modified = ?, feed_entry_updated = ?, updated_at = ? + content_last_modified = ?, feed_entry_updated = ?, updated_at = ?, + locale = ?, locale_source = ? WHERE id = ? "#, ) @@ -438,6 +444,8 @@ pub async fn update_recipe_with_content( .bind(content_last_modified) .bind(feed_entry_updated) .bind(now) + .bind(locale) + .bind(locale_source) .bind(recipe_id) .execute(pool) .await?; @@ -460,10 +468,46 @@ pub async fn update_feed_entry_timestamp( Ok(()) } +/// Update only a recipe's locale columns. +pub async fn update_recipe_locale( + pool: &DbPool, + recipe_id: i64, + locale: Option<&str>, + locale_source: Option<&str>, +) -> Result<()> { + sqlx::query("UPDATE recipes SET locale = ?, locale_source = ? WHERE id = ?") + .bind(locale) + .bind(locale_source) + .bind(recipe_id) + .execute(pool) + .await?; + + Ok(()) +} + +/// Every distinct locale present in the database with its recipe count, +/// most common first. Used to populate the language filter. +pub async fn list_locales(pool: &DbPool) -> Result> { + let rows: Vec<(String, i64)> = sqlx::query_as( + r#" + SELECT locale, COUNT(*) as count + FROM recipes + WHERE locale IS NOT NULL + GROUP BY locale + ORDER BY count DESC, locale ASC + "#, + ) + .fetch_all(pool) + .await?; + + Ok(rows) +} + #[cfg(test)] mod tests { use super::*; use crate::db::{feeds, init_pool, run_migrations}; + use sqlx::SqlitePool; #[tokio::test] async fn test_recipe_crud() { @@ -500,6 +544,8 @@ mod tests { content_etag: None, content_last_modified: None, feed_entry_updated: None, + locale: None, + locale_source: None, }; let recipe = create_recipe(&pool, &new_recipe).await.unwrap(); @@ -591,4 +637,204 @@ mod tests { assert_eq!(hash1, hash2); } + + #[tokio::test] + async fn test_create_and_update_recipe_locale() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + sqlx::migrate!("./migrations").run(&pool).await.unwrap(); + + let feed = crate::db::feeds::create_feed( + &pool, + &crate::db::models::NewFeed { + url: "https://example.com/feed.xml".to_string(), + title: Some("Test Feed".to_string()), + }, + ) + .await + .unwrap(); + + let new_recipe = NewRecipe { + feed_id: feed.id, + external_id: "recipe-1".to_string(), + title: "Pfannkuchen".to_string(), + source_url: None, + enclosure_url: "https://example.com/recipe.cook".to_string(), + content: Some("Mehl und Wasser verrühren.".to_string()), + summary: None, + servings: None, + total_time_minutes: None, + active_time_minutes: None, + difficulty: None, + image_url: None, + published_at: None, + content_hash: None, + content_etag: None, + content_last_modified: None, + feed_entry_updated: None, + locale: Some("de".to_string()), + locale_source: Some("detected".to_string()), + }; + + let recipe = create_recipe(&pool, &new_recipe).await.unwrap(); + assert_eq!(recipe.locale.as_deref(), Some("de")); + assert_eq!(recipe.locale_source.as_deref(), Some("detected")); + + // An author-declared locale overwrites the detected one. + update_recipe_locale(&pool, recipe.id, Some("fr"), Some("declared")) + .await + .unwrap(); + + let updated = get_recipe(&pool, recipe.id).await.unwrap(); + assert_eq!(updated.locale.as_deref(), Some("fr")); + assert_eq!(updated.locale_source.as_deref(), Some("declared")); + + // list_locales reports what is in the database. + let locales = list_locales(&pool).await.unwrap(); + assert_eq!(locales, vec![("fr".to_string(), 1)]); + } + + #[tokio::test] + async fn test_update_with_content_rederives_locale() { + // Locale is derived from content: a content update recomputes it, and an + // update that resolves to nothing clears it rather than leaving a stale value. + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + sqlx::migrate!("./migrations").run(&pool).await.unwrap(); + + let feed = feeds::create_feed( + &pool, + &NewFeed { + url: "https://example.com/feed.xml".to_string(), + title: Some("Test Feed".to_string()), + }, + ) + .await + .unwrap(); + + let recipe = create_recipe( + &pool, + &NewRecipe { + feed_id: feed.id, + external_id: "recipe-1".to_string(), + title: "Pfannkuchen".to_string(), + source_url: None, + enclosure_url: "https://example.com/recipe.cook".to_string(), + content: Some("Mehl und Wasser verrühren.".to_string()), + summary: None, + servings: None, + total_time_minutes: None, + active_time_minutes: None, + difficulty: None, + image_url: None, + published_at: None, + content_hash: None, + content_etag: None, + content_last_modified: None, + feed_entry_updated: None, + locale: Some("de".to_string()), + locale_source: Some("detected".to_string()), + }, + ) + .await + .unwrap(); + + assert_eq!(recipe.locale.as_deref(), Some("de")); + assert_eq!(recipe.locale_source.as_deref(), Some("detected")); + + // New content resolves to a new locale, which is carried through. + update_recipe_with_content( + &pool, + recipe.id, + "Mélanger la farine et l'eau.", + None, + None, + None, + None, + Some("fr"), + Some("declared"), + ) + .await + .unwrap(); + + let updated = get_recipe(&pool, recipe.id).await.unwrap(); + assert_eq!(updated.locale.as_deref(), Some("fr")); + assert_eq!(updated.locale_source.as_deref(), Some("declared")); + + // Content whose locale cannot be resolved clears the columns: we no longer + // know the locale, so NULL is recorded rather than a stale guess. + update_recipe_with_content(&pool, recipe.id, "???", None, None, None, None, None, None) + .await + .unwrap(); + + let cleared = get_recipe(&pool, recipe.id).await.unwrap(); + assert_eq!(cleared.locale, None); + assert_eq!(cleared.locale_source, None); + } + + #[tokio::test] + async fn test_list_locales_orders_by_count_then_alphabetically() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + sqlx::migrate!("./migrations").run(&pool).await.unwrap(); + + let feed = feeds::create_feed( + &pool, + &NewFeed { + url: "https://example.com/feed.xml".to_string(), + title: Some("Test Feed".to_string()), + }, + ) + .await + .unwrap(); + + // en x3, then de x2 and fr x2 (a tie), plus one recipe with no locale at all. + let locales = [ + Some("en"), + Some("en"), + Some("en"), + Some("fr"), + Some("fr"), + Some("de"), + Some("de"), + None, + ]; + + for (i, locale) in locales.iter().enumerate() { + create_recipe( + &pool, + &NewRecipe { + feed_id: feed.id, + external_id: format!("recipe-{i}"), + title: format!("Recipe {i}"), + source_url: None, + enclosure_url: format!("https://example.com/recipe-{i}.cook"), + content: None, + summary: None, + servings: None, + total_time_minutes: None, + active_time_minutes: None, + difficulty: None, + image_url: None, + published_at: None, + content_hash: None, + content_etag: None, + content_last_modified: None, + feed_entry_updated: None, + locale: locale.map(|l| l.to_string()), + locale_source: locale.map(|_| "detected".to_string()), + }, + ) + .await + .unwrap(); + } + + // Most common first; ties broken alphabetically; NULL locales excluded. + let listed = list_locales(&pool).await.unwrap(); + assert_eq!( + listed, + vec![ + ("en".to_string(), 3), + ("de".to_string(), 2), + ("fr".to_string(), 2), + ] + ); + } } diff --git a/src/db/tags.rs b/src/db/tags.rs index 18dc842..f4e600e 100644 --- a/src/db/tags.rs +++ b/src/db/tags.rs @@ -223,6 +223,8 @@ mod tests { content_etag: None, content_last_modified: None, feed_entry_updated: None, + locale: None, + locale_source: None, }, ) .await diff --git a/src/github/indexer.rs b/src/github/indexer.rs index e4866a9..0c84b41 100644 --- a/src/github/indexer.rs +++ b/src/github/indexer.rs @@ -333,6 +333,16 @@ impl GitHubIndexer { (None, None, None, None) }; + // Locale: declared `locale:` metadata wins, otherwise detected from text. + let locale = parsed + .as_ref() + .ok() + .and_then(crate::indexer::resolve_locale); + let (locale_code, locale_source) = match &locale { + Some(l) => (Some(l.code.clone()), Some(l.source.as_str().to_string())), + None => (None, None), + }; + // Look for image with the same name (sibling file) as fallback let sibling_image_url = Self::find_recipe_image( file_path, @@ -364,6 +374,14 @@ impl GitHubIndexer { // For now, we'll keep the existing recipe and just update the github_recipe SHA db::github::update_github_recipe_sha(&self.pool, existing.id, file_sha).await?; + db::recipes::update_recipe_locale( + &self.pool, + existing.recipe_id, + locale_code.as_deref(), + locale_source.as_deref(), + ) + .await?; + recipe.id } else { // Create new recipe @@ -388,6 +406,8 @@ impl GitHubIndexer { content_etag: None, content_last_modified: None, feed_entry_updated: None, + locale: locale_code.clone(), + locale_source: locale_source.clone(), }; let recipe = db::recipes::create_recipe(&self.pool, &new_recipe).await?; diff --git a/src/indexer/cooklang_parser.rs b/src/indexer/cooklang_parser.rs index cea6055..1ea2918 100644 --- a/src/indexer/cooklang_parser.rs +++ b/src/indexer/cooklang_parser.rs @@ -18,6 +18,7 @@ pub struct ParsedRecipeData { pub struct RecipeMetadata { pub tags: Vec, pub title: Option, + pub locale: Option, pub description: Option, pub servings: Option, pub time: Option, @@ -156,27 +157,45 @@ pub fn parse_recipe(content: &str) -> Result { .map(|tags_vec| tags_vec.iter().map(|t| t.to_string()).collect()) .unwrap_or_default(); + // Normalized `lang` / `lang-REGION` if cooklang could parse the declared locale. + // Cooklang rejects anything that isn't its strict format (e.g. `de`, `en_US`). + // Reuses the same canonicalization `SearchIndex::search` applies to incoming + // filters, so the two can't drift on what "canonical" means. + let declared_locale = meta.locale().map(|(lang, region)| { + let code = match region { + Some(region) => format!("{lang}-{region}"), + None => lang.to_string(), + }; + crate::indexer::locale::normalize_code(&code) + }); + let mut custom = Vec::new(); for (key, value) in &meta.map { // Skip standard metadata fields let key_str = key.as_str(); - if !matches!( - key_str, - Some("tags") - | Some("title") - | Some("description") - | Some("servings") - | Some("time") - | Some("difficulty") - | Some("course") - | Some("prep time") - | Some("cook time") - | Some("cuisine") - | Some("diet") - | Some("author") - | Some("source") - | Some("image") - ) { + // Only drop the raw `locale` key when we actually captured it as a typed + // locale; otherwise an unparseable value would vanish entirely, so let it + // fall through to `custom`. + let locale_captured = key_str == Some("locale") && declared_locale.is_some(); + if !locale_captured + && !matches!( + key_str, + Some("tags") + | Some("title") + | Some("description") + | Some("servings") + | Some("time") + | Some("difficulty") + | Some("course") + | Some("prep time") + | Some("cook time") + | Some("cuisine") + | Some("diet") + | Some("author") + | Some("source") + | Some("image") + ) + { if let (Some(k), Some(v)) = (key.as_str(), value.as_str()) { custom.push((k.to_string(), v.to_string())); } @@ -226,6 +245,7 @@ pub fn parse_recipe(content: &str) -> Result { .get("title") .and_then(|t| t.as_str()) .map(|t| t.to_string()), + locale: declared_locale, description: meta.description().map(|d| d.to_string()), servings: meta.servings().map(|s| format!("{s}")), time: time_str, @@ -453,4 +473,63 @@ Heat in #oven{} for ~{20%minutes}. let one_and_half = cooklang::Value::Number(cooklang::quantity::Number::Regular(1.5)); assert_eq!(format_quantity(&one_and_half), Some("1 ½".to_string())); } + + #[test] + fn test_declared_locale_with_region() { + let content = r#"--- +locale: en_US +--- + +Mix @flour{2%cups} with @water{1%cup}. +"#; + + let parsed = parse_recipe(content).unwrap(); + let metadata = parsed.metadata.expect("metadata should be present"); + + assert_eq!(metadata.locale, Some("en-US".to_string())); + // The locale must not also leak into `custom`. + assert!(!metadata.custom.iter().any(|(k, _)| k == "locale")); + } + + #[test] + fn test_declared_locale_without_region() { + let content = r#"--- +locale: de +--- + +@Mehl{200%g} mit @Wasser{100%ml} verrühren. +"#; + + let parsed = parse_recipe(content).unwrap(); + let metadata = parsed.metadata.expect("metadata should be present"); + + assert_eq!(metadata.locale, Some("de".to_string())); + } + + #[test] + fn test_unparseable_locale_falls_back_to_custom() { + // Cooklang only accepts `en_US`; a hyphenated value fails its validation. + // It must still survive as a custom metadata entry rather than vanishing. + let content = "---\nlocale: en-US\n---\n\nMix @flour{2%cups} with @water{1%cup}.\n"; + + let parsed = parse_recipe(content).unwrap(); + let metadata = parsed.metadata.expect("metadata should be present"); + + assert_eq!(metadata.locale, None); + assert!(metadata + .custom + .iter() + .any(|(k, v)| k == "locale" && v == "en-US")); + } + + #[test] + fn test_no_declared_locale() { + let content = "Mix @flour{2%cups} with @water{1%cup}.\n"; + + let parsed = parse_recipe(content).unwrap(); + + // No metadata at all, or metadata with no locale — both mean "not declared". + let locale = parsed.metadata.and_then(|m| m.locale); + assert_eq!(locale, None); + } } diff --git a/src/indexer/locale.rs b/src/indexer/locale.rs new file mode 100644 index 0000000..4d9ebf3 --- /dev/null +++ b/src/indexer/locale.rs @@ -0,0 +1,336 @@ +//! Recipe locale resolution. +//! +//! A declared Cooklang `locale:` key always wins. Otherwise the language is +//! detected from the recipe's plain text — assembled from the *parsed* recipe so +//! that Cooklang markup and quantities never reach the detector. When detection +//! is unreliable we return `None` rather than storing a guess. + +use serde::{Deserialize, Serialize}; + +use crate::indexer::cooklang_parser::{ParsedRecipeData, StepItem}; + +/// Below this many characters of plain text there is not enough signal to detect. +const MIN_DETECTION_CHARS: usize = 25; + +/// Where a recipe's locale came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum LocaleSource { + /// The recipe declared a `locale:` metadata key. + Declared, + /// We detected it from the recipe text. + Detected, +} + +impl LocaleSource { + /// The value stored in the `recipes.locale_source` column. + pub fn as_str(&self) -> &'static str { + match self { + LocaleSource::Declared => "declared", + LocaleSource::Detected => "detected", + } + } +} + +/// A resolved locale: a BCP-47-style code (`en`, `de`, `en-US`) and its provenance. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecipeLocale { + pub code: String, + pub source: LocaleSource, +} + +/// Normalize a locale code to the canonical stored form: lowercase language, +/// uppercase region (`EN` → `en`, `en-us` → `en-US`). +/// +/// Locale codes are stored canonically as lowercase language plus an optional +/// uppercase region. The Tantivy locale field is untokenized (exact match), so +/// any caller that compares a locale code against stored values — API filters +/// in particular — must normalize first or a differently-cased code will +/// silently match nothing. +pub fn normalize_code(code: &str) -> String { + match code.split_once('-') { + Some((language, region)) => { + format!( + "{}-{}", + language.to_ascii_lowercase(), + region.to_ascii_uppercase() + ) + } + None => code.to_ascii_lowercase(), + } +} + +/// Resolve a recipe's locale: declared metadata first, then detection. +pub fn resolve_locale(parsed: &ParsedRecipeData) -> Option { + if let Some(code) = parsed.metadata.as_ref().and_then(|m| m.locale.clone()) { + return Some(RecipeLocale { + code, + source: LocaleSource::Declared, + }); + } + + detect_language_code(parsed).map(|code| RecipeLocale { + code, + source: LocaleSource::Detected, + }) +} + +/// Detect the recipe's language code. +/// +/// Narrative prose is the most trustworthy signal. Ingredient names are often +/// culturally-marked proper nouns ("Prosciutto", "Focaccia") that can outvote a short +/// English method, so they only get a say when the prose alone can't decide. +fn detect_language_code(parsed: &ParsedRecipeData) -> Option { + detect_reliable(&narrative_text(parsed)).or_else(|| detect_reliable(&detection_text(parsed))) +} + +/// The recipe's prose: title, description, section names, step text and notes. +/// Quantities, units, cookware and ingredient names are excluded. +fn narrative_text(parsed: &ParsedRecipeData) -> String { + let mut parts: Vec<&str> = Vec::new(); + + if let Some(meta) = &parsed.metadata { + if let Some(title) = &meta.title { + parts.push(title); + } + if let Some(description) = &meta.description { + parts.push(description); + } + } + + for section in &parsed.sections { + if let Some(name) = §ion.name { + parts.push(name); + } + for step in §ion.steps { + for item in &step.items { + if let StepItem::Text { value } = item { + parts.push(value); + } + } + } + for note in §ion.notes { + parts.push(note); + } + } + + parts.join(" ") +} + +/// The recipe's prose plus its ingredient names — the fallback signal for recipes +/// that are ingredient-heavy but say little. Quantities, units and cookware stay +/// excluded: they are noise, not language signal. +pub(crate) fn detection_text(parsed: &ParsedRecipeData) -> String { + let mut parts = vec![narrative_text(parsed)]; + + for ingredient in &parsed.ingredients { + parts.push(ingredient.name.clone()); + } + + parts.join(" ") +} + +/// Detect a language code from plain text, or `None` if we can't trust the result. +fn detect_reliable(text: &str) -> Option { + if text.chars().count() < MIN_DETECTION_CHARS { + return None; + } + + let info = whatlang::detect(text)?; + if !info.is_reliable() { + return None; + } + + Some(to_bcp47(info.lang().code())) +} + +/// Map whatlang's ISO 639-3 code to a two-letter code where one exists. +/// Languages without a 639-1 code (e.g. Cebuano) keep their 639-3 code. +/// +/// whatlang reports individual-language codes for two macrolanguages, and ISO 639-1 +/// only has codes for their macrolanguage parents. isolang can't bridge this, so map +/// them explicitly. +fn to_bcp47(code_639_3: &str) -> String { + match code_639_3 { + "cmn" => return "zh".to_string(), + "pes" => return "fa".to_string(), + _ => {} + } + + isolang::Language::from_639_3(code_639_3) + .and_then(|lang| lang.to_639_1()) + .map(str::to_string) + .unwrap_or_else(|| code_639_3.to_string()) +} + +/// English display name for a stored code: `"de"` → `"German"`, `"en-US"` → `"English"`. +/// Falls back to 639-3 so the codes `to_bcp47` leaves as 639-3 (e.g. `"ceb"`) can be named. +pub fn display_name(code: &str) -> Option { + let language = code.split('-').next()?; + + isolang::Language::from_639_1(language) + .or_else(|| isolang::Language::from_639_3(language)) + .map(|lang| lang.to_name().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::indexer::cooklang_parser::parse_recipe; + + const GERMAN: &str = "Den @Mehl{200%g} und das @Wasser{100%ml} in einer Schüssel \ +verrühren, bis ein glatter Teig entsteht. Den Teig ruhen lassen und anschließend \ +im #Ofen{} goldbraun backen."; + + const ENGLISH: &str = "Mix the @flour{200%g} and the @water{100%ml} in a bowl until \ +a smooth dough forms. Let the dough rest, then bake it in the #oven{} until golden brown."; + + /// Mandarin. whatlang reports this as `Cmn` (ISO 639-3), which has no 639-1 code + /// of its own — only its macrolanguage parent `zho` does. + const CHINESE: &str = "将@面粉{200%克}和@水{100%毫升}放入碗中搅拌均匀，揉成光滑的面团。\ +让面团静置醒发，然后放入#烤箱{}中烤至金黄色。"; + + /// An English method with culturally-marked Italian ingredient names. The prose is + /// unambiguously English, but the ingredient names alone read as Italian. + const ENGLISH_WITH_ITALIAN_INGREDIENTS: &str = "Chop and mix everything well, then bake \ +it in the #oven{} until browned. Serve the dish hot. + +Add @Prosciutto{}, @Parmesan{}, @Chorizo{}, @Bruschetta{}, @Focaccia{}, @Crostini{}, \ +@Panzanella{}, @Mozzarella{}, @Gorgonzola{}, @Mascarpone{}."; + + #[test] + fn test_detects_german() { + let parsed = parse_recipe(GERMAN).unwrap(); + let locale = resolve_locale(&parsed).expect("should detect a locale"); + + assert_eq!(locale.code, "de"); + assert_eq!(locale.source, LocaleSource::Detected); + } + + #[test] + fn test_detects_english() { + let parsed = parse_recipe(ENGLISH).unwrap(); + let locale = resolve_locale(&parsed).expect("should detect a locale"); + + assert_eq!(locale.code, "en"); + assert_eq!(locale.source, LocaleSource::Detected); + } + + #[test] + fn test_declared_locale_beats_detection() { + // The body is unmistakably German, but the author declared French. + let content = format!("---\nlocale: fr\n---\n\n{GERMAN}"); + let parsed = parse_recipe(&content).unwrap(); + let locale = resolve_locale(&parsed).expect("should resolve a locale"); + + assert_eq!(locale.code, "fr"); + assert_eq!(locale.source, LocaleSource::Declared); + } + + #[test] + fn test_declared_region_is_preserved() { + let content = format!("---\nlocale: en_US\n---\n\n{ENGLISH}"); + let parsed = parse_recipe(&content).unwrap(); + let locale = resolve_locale(&parsed).unwrap(); + + assert_eq!(locale.code, "en-US"); + assert_eq!(locale.source, LocaleSource::Declared); + } + + #[test] + fn test_detection_never_invents_a_region() { + let parsed = parse_recipe(ENGLISH).unwrap(); + let locale = resolve_locale(&parsed).unwrap(); + + assert!( + !locale.code.contains('-'), + "detected code should be bare: {}", + locale.code + ); + } + + #[test] + fn test_too_short_content_is_not_detected() { + let parsed = parse_recipe("Mix @salt{}.").unwrap(); + assert!(resolve_locale(&parsed).is_none()); + } + + #[test] + fn test_non_linguistic_content_is_not_detected() { + let parsed = parse_recipe("12345 67890 12345 67890 12345 67890 12345").unwrap(); + assert!(resolve_locale(&parsed).is_none()); + } + + #[test] + fn test_detection_text_excludes_markup() { + let parsed = parse_recipe(ENGLISH).unwrap(); + let text = detection_text(&parsed); + + assert!( + text.contains("flour"), + "ingredient names carry language signal" + ); + assert!(text.contains("smooth dough"), "step text must be present"); + assert!( + !text.contains('@'), + "cooklang markup must be stripped: {text}" + ); + assert!( + !text.contains('{'), + "cooklang markup must be stripped: {text}" + ); + assert!( + !text.contains("200"), + "quantities must not reach the detector: {text}" + ); + } + + #[test] + fn test_locale_source_as_str() { + assert_eq!(LocaleSource::Declared.as_str(), "declared"); + assert_eq!(LocaleSource::Detected.as_str(), "detected"); + } + + #[test] + fn test_display_name() { + assert_eq!(display_name("de").as_deref(), Some("German")); + assert_eq!(display_name("en-US").as_deref(), Some("English")); + assert_eq!(display_name("zzz"), None); + } + + #[test] + fn test_macrolanguage_is_mapped_to_its_639_1_code() { + // whatlang detects Mandarin as `cmn`, an individual language with no 639-1 code. + // We must store the macrolanguage code `zh`, not the non-standard `cmn`. + let parsed = parse_recipe(CHINESE).unwrap(); + let locale = resolve_locale(&parsed).expect("should detect a locale"); + + assert_eq!(locale.code, "zh"); + assert_eq!(locale.source, LocaleSource::Detected); + } + + #[test] + fn test_display_name_for_language_without_a_639_1_code() { + // `to_bcp47` keeps the 639-3 code for languages that have no 639-1 code, so + // `display_name` must be able to name those stored values too. + assert_eq!(display_name("ceb").as_deref(), Some("Cebuano")); + } + + #[test] + fn test_normalize_code() { + assert_eq!(normalize_code("en"), "en"); + assert_eq!(normalize_code("EN"), "en"); + assert_eq!(normalize_code("en-US"), "en-US"); + assert_eq!(normalize_code("en-us"), "en-US"); + assert_eq!(normalize_code("EN-us"), "en-US"); + } + + #[test] + fn test_ingredient_names_do_not_outvote_the_narrative() { + // The method is plainly English; only the ingredient names look Italian. The + // narrative must win, otherwise an English recipe is filed as Italian. + let parsed = parse_recipe(ENGLISH_WITH_ITALIAN_INGREDIENTS).unwrap(); + let locale = resolve_locale(&parsed).expect("should detect a locale"); + + assert_eq!(locale.code, "en"); + } +} diff --git a/src/indexer/mod.rs b/src/indexer/mod.rs index a163ccc..7e76a0d 100644 --- a/src/indexer/mod.rs +++ b/src/indexer/mod.rs @@ -2,12 +2,14 @@ // This module handles Cooklang parsing and Tantivy search indexing pub mod cooklang_parser; +pub mod locale; pub mod recipe; pub mod schema; pub mod search; // Re-exports pub use cooklang_parser::{parse_recipe as parse_cooklang_full, ParsedRecipeData}; +pub use locale::{resolve_locale, LocaleSource, RecipeLocale}; pub use recipe::{parse_cooklang, ParsedRecipe}; pub use schema::RecipeSchema; pub use search::{SearchIndex, SearchQuery, SearchResult, SearchResults}; diff --git a/src/indexer/schema.rs b/src/indexer/schema.rs index 55685dc..ec656ac 100644 --- a/src/indexer/schema.rs +++ b/src/indexer/schema.rs @@ -14,6 +14,7 @@ pub struct RecipeSchema { pub servings: Field, pub total_time: Field, pub file_path: Field, + pub locale: Field, } impl RecipeSchema { @@ -50,6 +51,10 @@ impl RecipeSchema { // File path (searchable, stored) - for GitHub recipes let file_path = schema_builder.add_text_field("file_path", TEXT | STORED); + // Locale (exact-match filter, not tokenized, deliberately excluded from + // the default query-parser fields so free text can't match it) + let locale = schema_builder.add_text_field("locale", STRING | STORED); + let schema = schema_builder.build(); Self { @@ -64,6 +69,7 @@ impl RecipeSchema { servings, total_time, file_path, + locale, } } } diff --git a/src/indexer/search.rs b/src/indexer/search.rs index 7b3d058..d94db9d 100644 --- a/src/indexer/search.rs +++ b/src/indexer/search.rs @@ -1,10 +1,12 @@ use crate::db::models::Recipe; use crate::error::{Error, Result}; +use crate::indexer::locale::normalize_code; use crate::indexer::schema::RecipeSchema; use serde::{Deserialize, Serialize}; use std::path::Path; use tantivy::collector::TopDocs; -use tantivy::query::{Query, QueryParser}; +use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery}; +use tantivy::schema::IndexRecordOption; use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, Term}; use tracing::{debug, info}; @@ -19,6 +21,8 @@ pub struct SearchQuery { pub q: String, // Unified query string pub page: usize, pub limit: usize, + /// Optional exact-match language filter, e.g. "de" or "en-US". + pub locale: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -27,6 +31,7 @@ pub struct SearchResult { pub title: String, pub summary: Option, pub score: f32, + pub locale: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -48,8 +53,22 @@ impl SearchIndex { // Open or create index let index = if path.join("meta.json").exists() { - Index::open_in_dir(path) - .map_err(|e| Error::Search(format!("Failed to open index: {e}")))? + let index = Index::open_in_dir(path) + .map_err(|e| Error::Search(format!("Failed to open index: {e}")))?; + + // Tantivy pins field ids to the schema stored on disk. If our schema has + // changed since the index was written, every field id we hold is wrong and + // writing a document corrupts or panics. Refuse to open it. + if index.schema() != schema.schema { + return Err(Error::Search(format!( + "Search index at {} was built with a different schema and cannot be used. \ + Delete it and rebuild: rm -rf {} && federation backfill-locales", + path.display(), + path.display(), + ))); + } + + index } else { Index::create_in_dir(path, schema.schema.clone()) .map_err(|e| Error::Search(format!("Failed to create index: {e}")))? @@ -131,6 +150,16 @@ impl SearchIndex { doc.add_text(self.schema.file_path, path); } + // Add locale, plus its base language when the code carries a region, so a + // filter on "en" also matches an "en-US" recipe. + if let Some(locale) = &recipe.locale { + doc.add_text(self.schema.locale, locale); + + if let Some((language, _region)) = locale.split_once('-') { + doc.add_text(self.schema.locale, language); + } + } + // Add tags for tag in tags { doc.add_text(self.schema.tags, tag); @@ -205,6 +234,25 @@ impl SearchIndex { .map_err(|e| Error::Search(format!("Invalid query: {e}")))? }; + // AND an exact locale term onto the parsed query when filtering. The locale + // field is untokenized (exact match), so normalize the incoming filter to the + // canonical stored form first — otherwise `?locale=EN` or `?locale=en-us` + // would silently match nothing. + let tantivy_query = match query.locale.as_deref().filter(|l| !l.is_empty()) { + Some(locale) => { + let normalized = normalize_code(locale); + let term = Term::from_field_text(self.schema.locale, &normalized); + let locale_query: Box = + Box::new(TermQuery::new(term, IndexRecordOption::Basic)); + + Box::new(BooleanQuery::new(vec![ + (Occur::Must, tantivy_query), + (Occur::Must, locale_query), + ])) as Box + } + None => tantivy_query, + }; + // Calculate offset let offset = (query.page.saturating_sub(1)) * query.limit; let limit = query.limit.min(max_limit); @@ -240,11 +288,17 @@ impl SearchIndex { _ => None, }); + let locale = doc.get_first(self.schema.locale).and_then(|v| match v { + tantivy::schema::OwnedValue::Str(s) => Some(s.to_string()), + _ => None, + }); + Some(SearchResult { recipe_id, title, summary, score, + locale, }) }) .collect(); @@ -264,6 +318,15 @@ impl SearchIndex { writer .commit() .map_err(|e| Error::Search(format!("Failed to commit: {e}")))?; + + // `ReloadPolicy::OnCommitWithDelay` reloads the reader asynchronously via a + // filesystem watcher, so a `search()` call immediately after `commit()` can + // race ahead of that reload and observe a stale (empty) index. Reload + // explicitly so callers of this helper see their own writes right away. + self.reader + .reload() + .map_err(|e| Error::Search(format!("Failed to reload reader: {e}")))?; + Ok(()) } @@ -293,6 +356,33 @@ mod tests { assert!(index.is_ok()); } + #[test] + fn test_opening_an_index_with_a_stale_schema_is_refused() { + use tantivy::schema::{Schema, TEXT}; + + // An index written by an older build, whose schema no longer matches ours. + // Tantivy pins field ids to the on-disk schema, so using our field ids against + // it would panic or corrupt the index rather than fail cleanly. + let dir = tempdir().unwrap(); + let mut builder = Schema::builder(); + builder.add_text_field("title", TEXT); + Index::create_in_dir(dir.path(), builder.build()).unwrap(); + + let Err(err) = SearchIndex::new(dir.path()) else { + panic!("an index with a mismatched schema must not open"); + }; + let message = err.to_string(); + + assert!( + message.contains("different schema"), + "error should name the cause: {message}" + ); + assert!( + message.contains("rm -rf"), + "error should tell the operator how to rebuild: {message}" + ); + } + #[test] fn test_search_unified() { let dir = tempdir().unwrap(); @@ -303,6 +393,7 @@ mod tests { q: "chocolate".to_string(), page: 1, limit: 20, + locale: None, }; let result = index.search(&query, 1000); @@ -313,6 +404,7 @@ mod tests { q: "tags:dessert".to_string(), page: 1, limit: 20, + locale: None, }; let result = index.search(&query, 1000); @@ -323,6 +415,7 @@ mod tests { q: "chocolate tags:dessert total_time:[0 TO 60]".to_string(), page: 1, limit: 20, + locale: None, }; let result = index.search(&query, 1000); @@ -364,6 +457,8 @@ mod tests { content_etag: None, content_last_modified: None, feed_entry_updated: None, + locale: None, + locale_source: None, }; // Index recipe first time @@ -427,4 +522,223 @@ mod tests { panic!("ID field should be I64"); } } + + fn test_recipe(id: i64, title: &str, locale: Option<&str>) -> Recipe { + Recipe { + id, + feed_id: 1, + external_id: format!("ext-{id}"), + title: title.to_string(), + source_url: None, + enclosure_url: format!("https://example.com/{id}.cook"), + content: Some("Mix the flour and the water.".to_string()), + summary: None, + servings: None, + total_time_minutes: None, + active_time_minutes: None, + difficulty: None, + image_url: None, + published_at: None, + updated_at: None, + indexed_at: None, + created_at: chrono::Utc::now(), + content_hash: None, + content_etag: None, + content_last_modified: None, + feed_entry_updated: None, + locale: locale.map(str::to_string), + locale_source: locale.map(|_| "detected".to_string()), + } + } + + #[test] + fn test_search_filters_by_locale() { + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + let mut writer = index.writer().unwrap(); + + index + .index_recipe( + &mut writer, + &test_recipe(1, "Pancakes", Some("en")), + None, + &[], + &[], + ) + .unwrap(); + index + .index_recipe( + &mut writer, + &test_recipe(2, "Pfannkuchen", Some("de")), + None, + &[], + &[], + ) + .unwrap(); + index + .index_recipe(&mut writer, &test_recipe(3, "Crepes", None), None, &[], &[]) + .unwrap(); + index.commit(&mut writer).unwrap(); + + // Filtering by locale returns only that language. + let results = index + .search( + &SearchQuery { + q: String::new(), + page: 1, + limit: 10, + locale: Some("de".to_string()), + }, + 10, + ) + .unwrap(); + assert_eq!(results.results.len(), 1); + assert_eq!(results.results[0].recipe_id, 2); + assert_eq!(results.results[0].locale.as_deref(), Some("de")); + + // No filter returns everything, including the recipe with no locale. + let all = index + .search( + &SearchQuery { + q: String::new(), + page: 1, + limit: 10, + locale: None, + }, + 10, + ) + .unwrap(); + assert_eq!(all.results.len(), 3); + } + + #[test] + fn test_locale_filter_combines_with_query() { + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + let mut writer = index.writer().unwrap(); + + index + .index_recipe( + &mut writer, + &test_recipe(1, "Pancakes", Some("en")), + None, + &[], + &[], + ) + .unwrap(); + index + .index_recipe( + &mut writer, + &test_recipe(2, "Pancakes", Some("de")), + None, + &[], + &[], + ) + .unwrap(); + index.commit(&mut writer).unwrap(); + + let results = index + .search( + &SearchQuery { + q: "pancakes".to_string(), + page: 1, + limit: 10, + locale: Some("en".to_string()), + }, + 10, + ) + .unwrap(); + + assert_eq!(results.results.len(), 1); + assert_eq!(results.results[0].recipe_id, 1); + } + + #[test] + fn test_regional_locale_matches_base_language_filter() { + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + let mut writer = index.writer().unwrap(); + + index + .index_recipe( + &mut writer, + &test_recipe(1, "Biscuits", Some("en-US")), + None, + &[], + &[], + ) + .unwrap(); + index.commit(&mut writer).unwrap(); + + // Filtering by the base language finds the regional recipe... + let base = index + .search( + &SearchQuery { + q: String::new(), + page: 1, + limit: 10, + locale: Some("en".to_string()), + }, + 10, + ) + .unwrap(); + assert_eq!(base.results.len(), 1); + + // ...and the stored code keeps its region. + assert_eq!(base.results[0].locale.as_deref(), Some("en-US")); + + // A different language does not match. + let other = index + .search( + &SearchQuery { + q: String::new(), + page: 1, + limit: 10, + locale: Some("de".to_string()), + }, + 10, + ) + .unwrap(); + assert_eq!(other.results.len(), 0); + } + + #[test] + fn test_locale_filter_is_case_insensitive() { + // Locale codes are stored canonically as `en-US`. The field is untokenized + // (exact match), so a differently-cased filter must be normalized before + // comparison, or the API silently returns zero results. + let dir = tempdir().unwrap(); + let index = SearchIndex::new(dir.path()).unwrap(); + let mut writer = index.writer().unwrap(); + + index + .index_recipe( + &mut writer, + &test_recipe(1, "Biscuits", Some("en-US")), + None, + &[], + &[], + ) + .unwrap(); + index.commit(&mut writer).unwrap(); + + for filter in ["EN", "en-us", "En-Us"] { + let results = index + .search( + &SearchQuery { + q: String::new(), + page: 1, + limit: 10, + locale: Some(filter.to_string()), + }, + 10, + ) + .unwrap(); + assert_eq!( + results.results.len(), + 1, + "filter {filter:?} should match the recipe stored as en-US" + ); + } + } } diff --git a/src/main.rs b/src/main.rs index 93a9b49..cc48fef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -60,6 +60,9 @@ async fn main() -> Result<()> { Commands::Reindex { url } => { reindex_feed(settings, url).await?; } + Commands::BackfillLocales { force } => { + backfill_locales(settings, force).await?; + } } Ok(()) @@ -296,3 +299,22 @@ async fn reindex_feed(settings: Settings, url: String) -> Result<()> { Ok(()) } + +async fn backfill_locales(settings: Settings, force: bool) -> Result<()> { + info!("Backfilling recipe locales (force: {})", force); + + let pool = db::init_pool(&settings.database.url).await?; + db::run_migrations(&pool).await?; + + let index_path = std::path::PathBuf::from(&settings.search.index_path); + let search_index = SearchIndex::new(&index_path)?; + + let stats = federation::cli::commands::backfill_locales(&pool, &search_index, force).await?; + + println!( + "\x1b[32m\u{2713}\x1b[0m Backfill complete: {} scanned, {} tagged, {} left without a locale", + stats.scanned, stats.updated, stats.skipped + ); + + Ok(()) +} diff --git a/src/web/handlers.rs b/src/web/handlers.rs index 085e7c3..4903385 100644 --- a/src/web/handlers.rs +++ b/src/web/handlers.rs @@ -26,6 +26,8 @@ where #[template(path = "search.html")] struct SearchTemplate { query: String, + locale: String, + locales: Vec, results: Vec, total: usize, page: usize, @@ -33,6 +35,15 @@ struct SearchTemplate { recent_recipes: Vec, } +/// One entry in the language filter dropdown. +#[derive(Clone)] +#[allow(dead_code)] // Fields are used by Askama templates +struct LocaleOption { + code: String, + name: String, + count: i64, +} + #[derive(Clone)] #[allow(dead_code)] // Fields are used by Askama templates struct RecipeCardData { @@ -45,12 +56,15 @@ struct RecipeCardData { difficulty: String, image_url: String, source_url: String, + locale_name: String, } #[derive(Deserialize)] pub struct SearchParams { #[serde(default, deserialize_with = "deserialize_optional_string")] q: Option, + #[serde(default, deserialize_with = "deserialize_optional_string")] + locale: Option, #[serde(default = "default_page")] page: usize, } @@ -65,9 +79,34 @@ pub async fn index( Query(params): Query, ) -> Result { let query = params.q.clone().unwrap_or_default(); + let locale = params.locale.clone().unwrap_or_default(); + + // Language filter options: distinct locales in the database, most common first. + // Regional codes ("en-US") are folded into their base language ("en") so the + // dropdown lists one entry per language. + let locales = { + let mut counts: Vec<(String, i64)> = Vec::new(); + for (code, count) in db::recipes::list_locales(&state.pool).await? { + let base = code.split('-').next().unwrap_or(&code).to_string(); + match counts.iter_mut().find(|(c, _)| *c == base) { + Some((_, existing)) => *existing += count, + None => counts.push((base, count)), + } + } + counts.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0))); + + counts + .into_iter() + .map(|(code, count)| LocaleOption { + name: crate::indexer::locale::display_name(&code).unwrap_or_else(|| code.clone()), + code, + count, + }) + .collect::>() + }; - // If query is empty, show no results - let (results, total, total_pages) = if query.is_empty() { + // If query and locale filter are both empty, show no results + let (results, total, total_pages) = if query.is_empty() && locale.is_empty() { (vec![], 0, 0) } else { // Build search query @@ -75,6 +114,7 @@ pub async fn index( q: query.clone(), page: params.page, limit: state.settings.pagination.web_default_limit, + locale: params.locale.clone(), }; // Execute search @@ -111,6 +151,11 @@ pub async fn index( difficulty: r.difficulty.unwrap_or_default(), image_url: r.image_url.unwrap_or_default(), source_url: r.source_url.unwrap_or_default(), + locale_name: r + .locale + .as_deref() + .and_then(crate::indexer::locale::display_name) + .unwrap_or_default(), }); } } @@ -119,7 +164,7 @@ pub async fn index( }; // Fetch recently indexed recipes for the homepage - let recent_recipes = if query.is_empty() { + let recent_recipes = if query.is_empty() && locale.is_empty() { let recipes = db::recipes::list_recently_indexed(&state.pool, 6).await?; let recipe_ids: Vec = recipes.iter().map(|r| r.id).collect(); let tags_map = db::tags::get_tags_for_recipes(&state.pool, &recipe_ids).await?; @@ -141,6 +186,11 @@ pub async fn index( difficulty: r.difficulty.unwrap_or_default(), image_url: r.image_url.unwrap_or_default(), source_url: r.source_url.unwrap_or_default(), + locale_name: r + .locale + .as_deref() + .and_then(crate::indexer::locale::display_name) + .unwrap_or_default(), } }) .collect() @@ -150,6 +200,8 @@ pub async fn index( let template = SearchTemplate { query, + locale, + locales, results, total, page: params.page, @@ -187,6 +239,9 @@ pub struct RecipeData { pub source_url: String, pub feed: FeedData, pub metadata: Option, + pub locale: String, + pub locale_name: String, + pub locale_detected: bool, } #[derive(Clone)] @@ -274,6 +329,13 @@ pub async fn recipe_detail( author: feed.author.unwrap_or_default(), }, metadata, + locale: recipe.locale.clone().unwrap_or_default(), + locale_name: recipe + .locale + .as_deref() + .and_then(crate::indexer::locale::display_name) + .unwrap_or_default(), + locale_detected: recipe.locale_source.as_deref() == Some("detected"), }; // Generate Schema.org JSON-LD @@ -447,6 +509,11 @@ pub async fn feed_recipes_page( difficulty: recipe.difficulty.unwrap_or_default(), image_url: recipe.image_url.unwrap_or_default(), source_url: recipe.source_url.unwrap_or_default(), + locale_name: recipe + .locale + .as_deref() + .and_then(crate::indexer::locale::display_name) + .unwrap_or_default(), }); } @@ -530,6 +597,11 @@ pub async fn browse_page( difficulty: recipe.difficulty.unwrap_or_default(), image_url: recipe.image_url.unwrap_or_default(), source_url: recipe.source_url.unwrap_or_default(), + locale_name: recipe + .locale + .as_deref() + .and_then(crate::indexer::locale::display_name) + .unwrap_or_default(), } }) .collect(); diff --git a/src/web/schema.rs b/src/web/schema.rs index 22564e5..b0d611a 100644 --- a/src/web/schema.rs +++ b/src/web/schema.rs @@ -110,6 +110,11 @@ pub fn recipe_to_schema_json(recipe: &RecipeData) -> Value { schema["url"] = json!(recipe.source_url); } + // Language + if !recipe.locale.is_empty() { + schema["inLanguage"] = json!(recipe.locale); + } + // Keywords from tags if let Some(metadata) = &recipe.metadata { if !metadata.tags.is_empty() { @@ -293,8 +298,56 @@ fn step_to_text(step: &crate::indexer::cooklang_parser::StepData) -> String { #[cfg(test)] mod tests { + use super::super::handlers::FeedData; use super::*; + /// Minimal RecipeData for schema tests; only fields relevant to the test are varied. + fn sample_recipe_data() -> RecipeData { + RecipeData { + id: 1, + title: "Test Recipe".to_string(), + summary: String::new(), + parsed_sections: None, + ingredients: vec![], + cookware: vec![], + tags: vec![], + servings: String::new(), + total_time_minutes: String::new(), + active_time_minutes: String::new(), + difficulty: String::new(), + image_url: String::new(), + source_url: String::new(), + feed: FeedData { + id: 1, + title: "Test Feed".to_string(), + author: String::new(), + }, + metadata: None, + locale: String::new(), + locale_name: String::new(), + locale_detected: false, + } + } + + #[test] + fn test_schema_includes_in_language_when_locale_present() { + let mut recipe = sample_recipe_data(); + recipe.locale = "de".to_string(); + + let schema = recipe_to_schema_json(&recipe); + + assert_eq!(schema["inLanguage"], json!("de")); + } + + #[test] + fn test_schema_omits_in_language_when_locale_absent() { + let recipe = sample_recipe_data(); + + let schema = recipe_to_schema_json(&recipe); + + assert!(schema.get("inLanguage").is_none()); + } + #[test] fn test_format_iso_duration() { assert_eq!(format_iso_duration("30 minutes"), Some("PT30M".to_string())); diff --git a/src/web/templates/recipe.html b/src/web/templates/recipe.html index f5a3fd5..98147dc 100644 --- a/src/web/templates/recipe.html +++ b/src/web/templates/recipe.html @@ -106,6 +106,13 @@

{% if let Some(metadata) =

{% endif %} + {% if !recipe.locale_name.is_empty() %} +
+ 🗣️ + {{ recipe.locale_name }}{% if recipe.locale_detected %} (detected){% endif %} +
+ {% endif %} + {% for (key, value) in metadata.custom %}
@@ -147,6 +154,13 @@

{% if let Some(metadata) = {{ recipe.difficulty }}

{% endif %} + + {% if !recipe.locale_name.is_empty() %} +
+ 🗣️ + {{ recipe.locale_name }}{% if recipe.locale_detected %} (detected){% endif %} +
+ {% endif %} {% endif %} diff --git a/src/web/templates/search.html b/src/web/templates/search.html index 162db22..2ab25d1 100644 --- a/src/web/templates/search.html +++ b/src/web/templates/search.html @@ -22,6 +22,19 @@

Discover Recipes

placeholder="Search query" class="flex-1 px-4 py-3 rounded-lg border border-gray-300 focus:ring-2 focus:ring-orange-500 focus:border-transparent" > + {% if !locales.is_empty() %} + + {% endif %}