feat: rebuild the PostgreSQL query store from JSON source data

.changelog/NEXT.md

@@ -14,6 +14,7 @@
 
 ## Internal
 
+- **[issue-149] PostgreSQL JSON rebuild writer** — Family-tree JSON can now populate the staged PostgreSQL query store transactionally and idempotently while preserving canonical identities and leaving SQLite available during migration.
 - **[issue-148] PostgreSQL query-store foundation** — Added a packaged PostgreSQL schema and lazy async connection layer with transactional initialization, health checks, and named parameters, while leaving the current SQLite/JSON runtime unchanged until the staged migration completes.
 - Provider comparison cards no longer emit an invalid nested-`<button>` HTML warning under React 19 (the card header is now a `role="button"` element, keeping click and keyboard toggling).
 - SQLite driver is now loaded lazily so the server can start and serve JSON-backed data even when the native binding is unavailable.

docs/cli.md

@@ -92,10 +92,16 @@ Remove person files that are not part of SQLite database. Useful for cleaning up
 ```bash
 npx tsx scripts/rebuild.ts DB_ID     # Rebuild specific database
 npx tsx scripts/rebuild.ts --all     # Rebuild all databases
+npx tsx scripts/rebuild.ts DB_ID --max=10
 ```
 
 Re-extract person data from cached JSON files using the latest schema. Useful after code updates that add new fields.
 
+With `DATABASE_URL` set, the command also rebuilds PostgreSQL transactionally by
+walking `data/person/*.json` from `DB_ID`. A specific root can populate a clean
+PostgreSQL store even when no legacy `db-DB_ID.json` exists. Without `DATABASE_URL`,
+the existing JSON/SQLite workflow is unchanged.
+
 ## Data Migration
 
 ### Run Migrations

docs/development.md

@@ -66,17 +66,33 @@ export DATABASE_URL='postgresql://sparsetree:password@localhost:5432/sparsetree'
 pm2 restart ecosystem.config.cjs --update-env
 ```
 
-Credentials are not stored in `ecosystem.config.cjs`. When `DATABASE_URL` is absent
-or unreachable, the PostgreSQL service reports itself unavailable without changing
-the current SQLite/JSON startup behavior. This foundation is not selected by the
-application yet; later migration slices will move writers and readers onto it.
+Credentials are not stored in `ecosystem.config.cjs`. When `DATABASE_URL` is absent,
+indexing and rebuild commands keep their current SQLite/JSON behavior. When it is
+present, the completed JSON graph is also synchronized into PostgreSQL in one
+transaction; application reads still use SQLite/JSON until the later cutover slices.
+An unreachable configured database fails that explicit PostgreSQL write instead of
+silently leaving a partially refreshed query store.
+
+To rebuild a clean PostgreSQL query store directly from the read-only person cache:
+
+```bash
+DATABASE_URL='postgresql://sparsetree:password@localhost:5432/sparsetree' \
+  npx tsx scripts/rebuild.ts FAMILYSEARCH_ROOT_ID
+
+# Limit traversal to the same ancestor depth as an index run
+DATABASE_URL="$DATABASE_URL" npx tsx scripts/rebuild.ts FAMILYSEARCH_ROOT_ID --max=10
+```
+
+The root and its parents are loaded from `data/person/*.json`; no rows are copied
+from `data/sparsetree.db`. Re-running the command updates provider-derived rows in
+place while preserving canonical ULIDs and local rows that reference them.
 
 The PostgreSQL integration test creates and removes a unique schema inside the
 database named by `SPARSETREE_TEST_DATABASE_URL`:
 
 ```bash
 SPARSETREE_TEST_DATABASE_URL="$DATABASE_URL" \
-  npm test -- --run tests/integration/db/postgresSchema.spec.ts
+  npm test -- --run tests/integration/db/postgresWriter.spec.ts
 ```
 
 ## Build

scripts/index.ts

@@ -24,6 +24,7 @@ import { config } from '../server/src/lib/config.js';
 import { sleep } from '../server/src/utils/sleep.js';
 import { randInt } from '../server/src/utils/randInt.js';
 import { sqliteWriter } from '../server/src/lib/sqlite-writer.js';
+import { postgresWriter } from '../server/src/lib/postgres-writer.js';
 import { logPerson } from './utils/logPerson.js';
 import type { Person, Database } from '@fsf/shared';
 
@@ -262,6 +263,22 @@ const saveDB = async (): Promise<void> => {
   const dbId = sqliteWriter.getOrCreatePersonId(selfID, db[selfID]?.name || 'Unknown');
   sqliteWriter.finalizeDatabase(dbId, selfID, db, maxGenerations);
 
+  // PostgreSQL is an explicit staged opt-in. Mirror the complete graph in one
+  // transaction and reuse SQLite's canonical IDs while both stores coexist.
+  if (postgresWriter.isConfigured()) {
+    const canonicalIds = new Map<string, string>();
+    for (const externalId of Object.keys(db)) {
+      const canonicalId = sqliteWriter.getPersonId(externalId);
+      if (canonicalId) canonicalIds.set(externalId, canonicalId);
+    }
+    await postgresWriter.rebuildDatabase({
+      rootExternalId: selfID,
+      database: db,
+      databaseId: dbId,
+      canonicalIds,
+    });
+  }
+
   console.log(
     `finished building ${fileName} with ${
       Object.keys(db).length
@@ -274,13 +291,17 @@ const saveDB = async (): Promise<void> => {
 };
 
 process.on('SIGINT', async () => {
-  await saveDB();
-  sqliteWriter.close();
+  await saveDB().finally(async () => {
+    sqliteWriter.close();
+    await postgresWriter.close();
+  });
   process.exit();
 });
 
-(async () => {
+void (async () => {
   await getPerson(selfID, 0);
   await saveDB();
+})().finally(async () => {
   sqliteWriter.close();
-})();
+  await postgresWriter.close();
+});

scripts/rebuild.ts

@@ -14,15 +14,19 @@ import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
 import { json2person } from '../server/src/lib/familysearch/transformer.js';
+import { loadFamilySearchTreeFromJson } from '../server/src/lib/json-tree-loader.js';
+import { postgresWriter } from '../server/src/lib/postgres-writer.js';
 import type { Person, Database } from '@fsf/shared';
 
 const argv = yargs(hideBin(process.argv)).argv as {
   _: (string | number)[];
   all?: boolean;
+  max?: number;
 };
 
 const [dbId] = argv._ as string[];
 const rebuildAll = argv.all;
+const requestedMaxGenerations = argv.max === undefined ? undefined : Number(argv.max);
 
 const DATA_DIR = './data';
 const PERSON_DIR = `${DATA_DIR}/person`;
@@ -135,39 +139,87 @@ const rebuildDatabase = (dbPath: string): Database => {
   return db;
 };
 
+const maxGenerationsFromFilename = (filename: string): number => {
+  const match = filename.match(/-(\d+)\.json$/);
+  return match ? Number(match[1]) : Infinity;
+};
+
+/**
+ * Populate PostgreSQL directly from the read-only raw person cache. This path
+ * deliberately does not consult the legacy SQLite database.
+ */
+const rebuildPostgresDatabase = async (
+  rootExternalId: string,
+  maxGenerations: number
+): Promise<void> => {
+  const { database, missingPersonIds } = await loadFamilySearchTreeFromJson({
+    rootExternalId,
+    personDir: PERSON_DIR,
+    maxGenerations,
+  });
+  if (!database[rootExternalId]) {
+    throw new Error(`Root source file not found: ${path.join(PERSON_DIR, `${rootExternalId}.json`)}`);
+  }
+  if (missingPersonIds.length > 0) {
+    console.warn(`  Warning: ${missingPersonIds.length} referenced person file(s) were missing`);
+  }
+
+  const result = await postgresWriter.rebuildDatabase({
+    rootExternalId,
+    database,
+  });
+  console.log(
+    `  PostgreSQL query store rebuilt as ${result.databaseId}: ` +
+    `${result.personCount} persons, ${result.parentEdgeCount} parent edges, ` +
+    `${result.spouseEdgeCount} spouse edges`
+  );
+};
+
 /**
  * Main entry point
  */
-const main = (): void => {
+const main = async (): Promise<void> => {
   if (!rebuildAll && !dbId) {
-    console.error('Usage: npx tsx scripts/rebuild.ts DB_ID  or  npx tsx scripts/rebuild.ts --all');
+    console.error('Usage: npx tsx scripts/rebuild.ts DB_ID [--max=N]  or  npx tsx scripts/rebuild.ts --all');
     process.exit(1);
   }
 
   const databases = getDatabaseFiles();
 
   if (rebuildAll) {
     console.log(`Found ${databases.length} databases to rebuild`);
-    for (const { filename } of databases) {
+    for (const { filename, rootId } of databases) {
       rebuildDatabase(path.join(DATA_DIR, filename));
+      if (postgresWriter.isConfigured()) {
+        await rebuildPostgresDatabase(
+          rootId,
+          requestedMaxGenerations ?? maxGenerationsFromFilename(filename)
+        );
+      }
     }
   } else {
     // Find matching database
     const match = databases.find(
       (d) => d.rootId === dbId || d.filename === `db-${dbId}.json`
     );
 
-    if (!match) {
+    if (!match && !postgresWriter.isConfigured()) {
       console.error(`Database not found for ID: ${dbId}`);
       console.log('Available databases:');
       databases.forEach((d) => console.log(`  - ${d.rootId} (${d.filename})`));
       process.exit(1);
     }
 
-    rebuildDatabase(path.join(DATA_DIR, match.filename));
+    if (match) rebuildDatabase(path.join(DATA_DIR, match.filename));
+    if (postgresWriter.isConfigured()) {
+      await rebuildPostgresDatabase(
+        match?.rootId ?? dbId,
+        requestedMaxGenerations ?? (match ? maxGenerationsFromFilename(match.filename) : Infinity)
+      );
+    }
   }
 
   console.log('\nRebuild complete!');
 };
 
-main();
+void main().finally(() => postgresWriter.close());

server/src/lib/json-tree-loader.ts

@@ -0,0 +1,70 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import type { Database, Person } from '@fsf/shared';
+// @ts-ignore - the legacy transformer is JavaScript and has no declarations
+import { json2person } from './familysearch/transformer.js';
+
+export interface JsonTreeLoadOptions {
+  rootExternalId: string;
+  personDir?: string;
+  maxGenerations?: number;
+}
+
+export interface JsonTreeLoadResult {
+  database: Database;
+  missingPersonIds: string[];
+}
+
+/**
+ * Rebuild an ancestor graph directly from the raw FamilySearch JSON cache.
+ * Source files are read-only: children are derived on the returned in-memory
+ * graph and are never written back into data/person.
+ */
+export async function loadFamilySearchTreeFromJson({
+  rootExternalId,
+  personDir = path.resolve('data/person'),
+  maxGenerations = Infinity,
+}: JsonTreeLoadOptions): Promise<JsonTreeLoadResult> {
+  const database: Database = {};
+  const missingPersonIds: string[] = [];
+  const visited = new Set<string>();
+  const queue: Array<{ externalId: string; generation: number }> = [
+    { externalId: rootExternalId, generation: 0 },
+  ];
+
+  for (let cursor = 0; cursor < queue.length; cursor++) {
+    const { externalId, generation } = queue[cursor];
+    if (visited.has(externalId) || generation > maxGenerations) continue;
+    visited.add(externalId);
+
+    const sourcePath = path.join(personDir, `${externalId}.json`);
+    const source = await readFile(sourcePath, 'utf8').catch((error: NodeJS.ErrnoException) => {
+      if (error.code === 'ENOENT') return undefined;
+      throw error;
+    });
+    if (source === undefined) {
+      missingPersonIds.push(externalId);
+      continue;
+    }
+
+    const person = json2person(JSON.parse(source)) as Person | undefined;
+    if (!person) continue;
+    database[externalId] = person;
+
+    if (generation === maxGenerations) continue;
+    for (const parentId of person.parents) {
+      if (parentId && !visited.has(parentId)) {
+        queue.push({ externalId: parentId, generation: generation + 1 });
+      }
+    }
+  }
+
+  for (const [childId, person] of Object.entries(database)) {
+    for (const parentId of person.parents) {
+      const parent = parentId ? database[parentId] : undefined;
+      if (parent && !parent.children.includes(childId)) parent.children.push(childId);
+    }
+  }
+
+  return { database, missingPersonIds };
+}