diff --git a/packages/host/config/schema/1788216332849_schema.sql b/packages/host/config/schema/1788540959469_schema.sql similarity index 100% rename from packages/host/config/schema/1788216332849_schema.sql rename to packages/host/config/schema/1788540959469_schema.sql diff --git a/packages/postgres/migrations/1788540959469_add-host-shell-generation.js b/packages/postgres/migrations/1788540959469_add-host-shell-generation.js new file mode 100644 index 00000000000..c88a1c4d31e --- /dev/null +++ b/packages/postgres/migrations/1788540959469_add-host-shell-generation.js @@ -0,0 +1,49 @@ +exports.shorthands = undefined; + +// An ordering over host shells, so a row can be asked whether it was rendered +// before or after the shell currently being served. +// +// The shell itself is identified by a hash of the host app's index HTML, which +// is the only thing the realm server can observe about a bundle it fetches +// over HTTP. A hash cannot be ordered, so it cannot answer "is this row's +// shell older than the current one" — the question every repair of +// deploy-skewed rows has to ask. This table supplies that ordering. +// +// One row, ever. `generation` advances by one each time a realm server +// observes a shell hash different from the one recorded here, and every +// server that observes the same shell reads back the same number. Advancing on +// *transition* rather than per distinct hash is what makes a rollback behave: +// redeploying a bundle that ran before is a new generation, higher than the +// one it is replacing, because the question is when a render happened and not +// which artifact is semantically newer. +// +// Seeded rather than left empty so the claim is always an UPDATE against an +// existing row — which is what makes concurrent claims safe without a lock. +// Generation 0 with an empty hash means "no shell observed yet", which no real +// hash can collide with. +exports.up = (pgm) => { + pgm.createTable('host_shell_generation', { + // Pinned to 1 by the constraint below: this table holds the current shell, + // not a history of them. + id: { type: 'integer', primaryKey: true }, + shell_hash: { type: 'varchar', notNull: true }, + generation: { type: 'integer', notNull: true }, + // Unix ms, as a bigint, matching `prerendered_html.rendered_at` and the + // media-cache ledger (pg returns these as JS strings). + observed_at: { type: 'bigint', notNull: true }, + }); + pgm.addConstraint( + 'host_shell_generation', + 'host_shell_generation_singleton', + { check: 'id = 1' }, + ); + pgm.sql(` + INSERT INTO host_shell_generation (id, shell_hash, generation, observed_at) + VALUES (1, '', 0, 0) + ON CONFLICT (id) DO NOTHING + `); +}; + +exports.down = (pgm) => { + pgm.dropTable('host_shell_generation'); +}; diff --git a/packages/postgres/scripts/schema-dump.sh b/packages/postgres/scripts/schema-dump.sh index 1f98c469b49..7363a1e0c07 100755 --- a/packages/postgres/scripts/schema-dump.sh +++ b/packages/postgres/scripts/schema-dump.sh @@ -33,6 +33,7 @@ docker exec boxel-pg pg_dump \ --exclude-table-and-children=proxy_endpoints \ --exclude-table-and-children=claimed_domains_for_sites \ --exclude-table-and-children=session_rooms \ + --exclude-table-and-children=host_shell_generation \ --no-tablespaces \ --no-table-access-method \ --no-owner \ diff --git a/packages/realm-server/tests/host-shell-generation-test.ts b/packages/realm-server/tests/host-shell-generation-test.ts new file mode 100644 index 00000000000..4fb79ba4e3b --- /dev/null +++ b/packages/realm-server/tests/host-shell-generation-test.ts @@ -0,0 +1,280 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; +import { basename } from 'path'; +import type { PgAdapter } from '@cardstack/postgres'; +import { + claimHostShellGeneration, + currentHostShellGeneration, + dbAdapterQuerier, + NO_HOST_SHELL_OBSERVED, + param, + type HostShellGeneration, + type Querier, +} from '@cardstack/runtime-common'; +import { setupDB } from './helpers/index.ts'; + +// The ordering that lets a row be asked whether it was rendered before or +// after the shell currently being served. A hash can only answer "same or +// different", which is why this exists; see `host-shell-generation.ts`. +module(basename(import.meta.filename), function (hooks) { + let dbAdapter: PgAdapter; + + setupDB(hooks, { + beforeEach: async (adapter) => { + dbAdapter = adapter; + }, + }); + + test('a fresh database has observed no shell', async function (assert) { + let current = await currentHostShellGeneration(dbAdapter); + assert.deepEqual( + current, + { generation: NO_HOST_SHELL_OBSERVED, shellHash: '' }, + 'the seeded row reads as "nothing observed" rather than as a shell', + ); + }); + + test('the first shell observed takes the first generation', async function (assert) { + let claimed = await claimHostShellGeneration(dbAdapter, 'aaaaaaaa', 1000); + assert.deepEqual(claimed, { generation: 1, shellHash: 'aaaaaaaa' }); + assert.deepEqual( + await currentHostShellGeneration(dbAdapter), + { generation: 1, shellHash: 'aaaaaaaa' }, + 'and is what a reader sees as current', + ); + }); + + test('re-reporting the current shell does not advance it', async function (assert) { + await claimHostShellGeneration(dbAdapter, 'aaaaaaaa', 1000); + // Every realm server reports on boot and again from the post-deployment + // hook, so repeat claims for one shell are the normal case, not a corner. + for (let attempt of [1, 2, 3]) { + let claimed = await claimHostShellGeneration(dbAdapter, 'aaaaaaaa', 2000); + assert.strictEqual( + claimed.generation, + 1, + `report ${attempt} reads back the same generation`, + ); + } + }); + + test('each new shell advances the generation', async function (assert) { + assert.strictEqual( + (await claimHostShellGeneration(dbAdapter, 'aaaaaaaa', 1000)).generation, + 1, + ); + assert.strictEqual( + (await claimHostShellGeneration(dbAdapter, 'bbbbbbbb', 2000)).generation, + 2, + ); + assert.strictEqual( + (await claimHostShellGeneration(dbAdapter, 'cccccccc', 3000)).generation, + 3, + ); + }); + + // The case that rules out deriving the ordering from the artifact itself: a + // rollback deploys a bundle that ran before, so anything read off the + // artifact would go backwards. A generation records *when* a render + // happened, so returning to an earlier bundle is a later generation. + test('rolling back to an earlier shell moves the generation forward', async function (assert) { + await claimHostShellGeneration(dbAdapter, 'aaaaaaaa', 1000); + await claimHostShellGeneration(dbAdapter, 'bbbbbbbb', 2000); + + let rolledBack = await claimHostShellGeneration( + dbAdapter, + 'aaaaaaaa', + 3000, + ); + assert.deepEqual( + rolledBack, + { generation: 3, shellHash: 'aaaaaaaa' }, + 'the old bundle is a new generation, not the one it had before', + ); + // What this buys: rows rendered during the bundle just rolled away from + // are below the current generation, so a repair can find them. Had the + // rollback reused generation 1, those rows would have been *above* the + // current generation and invisible to it. + assert.true( + 2 < rolledBack.generation, + 'rows from the rolled-back bundle are selectable as stale', + ); + }); + + // A rolling deploy has several realm-server tasks computing the same hash and + // reporting it at once, so this is the ordinary case rather than a corner. + // Note that it does not discriminate between this implementation and a + // read-then-write — claimants of one shell all compute the same successor + // either way. The test below it is the one that does. + test('concurrent first claims of one shell agree on its generation', async function (assert) { + let claims = await Promise.all( + Array.from({ length: 8 }, () => + claimHostShellGeneration(dbAdapter, 'bbbbbbbb', 5000), + ), + ); + + assert.deepEqual( + [...new Set(claims.map((c) => c.generation))], + [1], + 'every concurrent claimant reports the same generation', + ); + assert.deepEqual( + await currentHostShellGeneration(dbAdapter), + { generation: 1, shellHash: 'bbbbbbbb' }, + 'and the shell advanced exactly once', + ); + }); + + // The property that makes a single UPDATE the right shape, tested against a + // real lock wait rather than a hopeful `Promise.all`. A claim for a + // different shell has to arrive *while* another transition is uncommitted; + // that is when a read-then-write reads a generation that is about to be + // superseded, and writes its own shell over the transition it never saw. + // + // Deterministic because the commit waits for the second claim to actually be + // blocked on the row — asked of `pg_stat_activity`, not of a clock. + test('a claim arriving mid-transition counts the transition it missed', async function (assert) { + let blocked: Promise | undefined; + + await dbAdapter.withConnection(async (query) => { + await query(['BEGIN']); + await query([ + `UPDATE host_shell_generation SET shell_hash = `, + param('aaaaaaaa'), + `, generation = generation + 1, observed_at = 1000 + WHERE id = 1 AND shell_hash <> `, + param('aaaaaaaa'), + ]); + // Uncommitted, so the row is locked and a competing claim must wait. + blocked = claimHostShellGeneration(dbAdapter, 'bbbbbbbb', 2000); + await waitForLockWait(); + await query(['COMMIT']); + }); + + assert.deepEqual( + await blocked!, + { generation: 2, shellHash: 'bbbbbbbb' }, + 'both transitions are counted, so the two shells keep distinct generations', + ); + }); + + // Wait until some backend on this database is blocked on a lock. Polls the + // server's own view of who is waiting, so it reports the state the test + // needs rather than guessing at how long to sleep for it. + async function waitForLockWait(): Promise { + for (let attempt = 0; attempt < 200; attempt++) { + let waiting = await dbAdapter.execute( + `SELECT count(*)::int AS waiting FROM pg_stat_activity + WHERE datname = current_database() + AND wait_event_type = 'Lock' + AND pid <> pg_backend_pid()`, + ); + if (Number(waiting[0]?.waiting ?? 0) > 0) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + throw new Error( + 'no backend ever blocked on the row lock, so this test never created the overlap it asserts about', + ); + } + + // The contract a caller depends on: the answer describes the shell that was + // asked about. Anything else lets a render be stamped with another shell's + // number, which is the ordering failing in the one situation it exists for. + test('the answer always describes the shell that was claimed', async function (assert) { + for (let hash of ['aaaaaaaa', 'bbbbbbbb', 'aaaaaaaa', 'cccccccc']) { + let claimed = await claimHostShellGeneration(dbAdapter, hash, 1000); + assert.strictEqual( + claimed.shellHash, + hash, + `claiming ${hash} reported ${claimed.shellHash}`, + ); + } + }); + + // Drives the gap directly rather than hoping to hit it. A querier that lets + // another shell be claimed *after* the claim's own statement runs is exactly + // the interleaving a second read-back query would have been exposed to: the + // lock is released by then, so the row has moved on. One statement has + // nothing to interpose on, so the claim still answers about its own shell. + test('a competing claim landing after the statement cannot change the answer', async function (assert) { + await claimHostShellGeneration(dbAdapter, 'aaaaaaaa', 1000); + + let interposed = false; + let interposing: Querier = async (expression) => { + let rows = await dbAdapterQuerier(dbAdapter)(expression); + if (!interposed) { + interposed = true; + // A different shell wins the row the instant this claim lets go of it. + await claimHostShellGeneration(dbAdapter, 'bbbbbbbb', 2000); + } + return rows; + }; + + let claimed = await claimHostShellGeneration( + dbAdapter, + 'aaaaaaaa', + 3000, + interposing, + ); + assert.true(interposed, 'the competing claim really did run'); + assert.strictEqual( + claimed.shellHash, + 'aaaaaaaa', + 'answers about the shell it was asked about, not the one that overtook it', + ); + assert.deepEqual( + await currentHostShellGeneration(dbAdapter), + { generation: 2, shellHash: 'bbbbbbbb' }, + 'and the competing claim is what the row now holds', + ); + }); + + // The case the no-throw comment names, and the one the zero-row guard cannot + // reach: a database with no such table at all. Reachable two ways — a + // database predating the migration, and the browser's SQLite, which the + // schema dump deliberately excludes this table from. Both must yield the + // sentinel rather than propagate, because the intent stated at these + // functions is that a boot sequence never fails for a diagnostic. + test('a database without the table reports nothing observed', async function (assert) { + await dbAdapter.execute('DROP TABLE host_shell_generation'); + + assert.deepEqual( + await currentHostShellGeneration(dbAdapter), + { generation: NO_HOST_SHELL_OBSERVED, shellHash: '' }, + 'the read reports no ordering rather than throwing', + ); + assert.deepEqual( + await claimHostShellGeneration(dbAdapter, 'aaaaaaaa', 1000), + { generation: NO_HOST_SHELL_OBSERVED, shellHash: '' }, + 'and so does the claim', + ); + }); + + // Narrow on purpose: only the missing table is absorbed. A fault a caller + // should hear about still propagates. + test('any other failure still propagates', async function (assert) { + let broken: Querier = async () => { + throw Object.assign(new Error('permission denied for table'), { + code: '42501', + }); + }; + await assert.rejects( + claimHostShellGeneration(dbAdapter, 'aaaaaaaa', 1000, broken), + /permission denied/, + 'a permissions failure is not mistaken for an absent table', + ); + }); + + test('the singleton constraint keeps a second row out', async function (assert) { + await assert.rejects( + dbAdapter.execute( + `INSERT INTO host_shell_generation (id, shell_hash, generation, observed_at) + VALUES (2, 'cccccccc', 99, 0)`, + ), + /host_shell_generation_singleton/, + 'the table holds the current shell, so a second row is a bug not a record', + ); + }); +}); diff --git a/packages/runtime-common/host-shell-generation.ts b/packages/runtime-common/host-shell-generation.ts new file mode 100644 index 00000000000..b949bed153d --- /dev/null +++ b/packages/runtime-common/host-shell-generation.ts @@ -0,0 +1,152 @@ +import type { DBAdapter } from './db.ts'; +import { dbAdapterQuerier, param, type Querier } from './expression.ts'; + +// An ordering over host shells. +// +// A realm server can only identify the host bundle it serves by hashing the +// index HTML it fetches, and a hash answers "is this the same shell?" but never +// "is this shell older?". Repairing rows a deploy left behind needs the second +// question, so `host_shell_generation` carries a number that advances every +// time the served shell changes. A row stamped with a generation below the +// current one was rendered against a bundle that is no longer being served. +// +// The number is assigned here rather than by the deploy pipeline, because the +// host is not a container: it is static files in S3, with no environment to +// inject and no task definition to stamp. Assigning on first observation also +// makes a rollback behave — see `claimHostShellGeneration`. + +// Generation 0 with an empty hash is what the migration seeds, and means no +// shell has been observed yet. No real hash can collide with it. +export const NO_HOST_SHELL_OBSERVED = 0; + +const NOT_OBSERVED: HostShellGeneration = { + generation: NO_HOST_SHELL_OBSERVED, + shellHash: '', +}; + +// Whether a query failed because the table is not there. +// +// Two ways to reach it, and neither is a fault worth failing a boot over. A +// database that predates the migration has no such table — the zero-row guard +// below cannot cover that case, because the statement throws before returning +// rows. And the table is deliberately absent from the browser's SQLite schema +// (`schema-dump.sh` excludes it, being realm-server operational state), so any +// host-side caller meets this by design rather than by accident. +// +// Narrow on purpose: only the missing table yields the sentinel. A syntax +// error, a permissions failure or a dead connection still propagates, because +// those are faults a caller should hear about. +function isMissingTableError(err: unknown): boolean { + // Postgres raises SQLSTATE 42P01 (undefined_table); SQLite has no codes, so + // its message is the only signal. + if ((err as { code?: unknown })?.code === '42P01') { + return true; + } + let message = (err as Error)?.message ?? ''; + return /no such table/i.test(message); +} + +export interface HostShellGeneration { + generation: number; + shellHash: string; +} + +// Record `shellHash` as the shell now being served, and return the generation +// it belongs to. Idempotent: a server re-reporting the shell already recorded +// gets that shell's generation back without advancing anything. +// +// Concurrency is the whole reason this is one statement. A rolling deploy +// overlaps a task booting against the outgoing bundle with its neighbour on the +// new one, so two *different* shells are claimed at once. Two readers see the +// same starting generation, compute the same successor, and two distinct shells +// end up sharing one number — which destroys the ordering, because rows from +// either then carry the same generation and nothing can tell them apart. +// +// The `WHERE id = 1` is unconditional so the statement always takes the row +// lock and always returns the row it observed. A second claimant blocks; once +// the first commits, READ COMMITTED re-evaluates against the committed row, and +// the `CASE` decides from that value whether this claim is a transition to +// count or the shell already recorded. Advancing on a *transition* rather than +// per distinct hash is what makes a rollback correct: redeploying a bundle that +// ran before takes a new, higher generation than the one it replaces, because a +// row's generation records when it was rendered, not which artifact is +// semantically newer. +// +// Reading the idempotent answer back in a second query would reopen the gap +// this closes. That statement commits and releases the lock before the read +// runs, so a different-shell claim landing in between would make this return +// *that* shell's hash and generation — and a caller stamping its own render +// with the number would defeat the ordering during exactly the concurrency +// this exists to survive. +export async function claimHostShellGeneration( + dbAdapter: DBAdapter, + shellHash: string, + observedAt: number, + querier?: Querier, +): Promise { + let q = querier ?? dbAdapterQuerier(dbAdapter); + let claimed: Awaited>; + try { + claimed = await q([ + `UPDATE host_shell_generation SET shell_hash = `, + param(shellHash), + `, generation = CASE WHEN shell_hash <> `, + param(shellHash), + ` THEN generation + 1 ELSE generation END`, + `, observed_at = CASE WHEN shell_hash <> `, + param(shellHash), + ` THEN `, + param(observedAt), + ` ELSE observed_at END`, + ` WHERE id = 1 RETURNING generation, shell_hash`, + ]); + } catch (err: unknown) { + if (isMissingTableError(err)) { + return NOT_OBSERVED; + } + throw err; + } + if (claimed.length === 0) { + // The migration seeds the row, so its absence here means something removed + // it from a table that does exist. Reporting "nothing observed" keeps + // callers on their no-ordering-available path instead of throwing into a + // boot sequence that must not fail for a diagnostic. A database with no + // such table at all is handled above, where the statement itself throws. + return NOT_OBSERVED; + } + return rowToGeneration(claimed[0]); +} + +// The generation of the shell currently being served, for comparing against +// the generation stamped on a row. +export async function currentHostShellGeneration( + dbAdapter: DBAdapter, + querier?: Querier, +): Promise { + let q = querier ?? dbAdapterQuerier(dbAdapter); + let rows: Awaited>; + try { + rows = await q([ + `SELECT generation, shell_hash FROM host_shell_generation WHERE id = 1`, + ]); + } catch (err: unknown) { + if (isMissingTableError(err)) { + return NOT_OBSERVED; + } + throw err; + } + if (rows.length === 0) { + // As in `claimHostShellGeneration`: a seeded row that has gone missing from + // a table that does exist. The absent-table case throws and is caught + // above. + return NOT_OBSERVED; + } + return rowToGeneration(rows[0]); +} + +function rowToGeneration(row: Record): HostShellGeneration { + return { + generation: Number(row.generation), + shellHash: String(row.shell_hash ?? ''), + }; +} diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index edb7dafe056..0aa63a88f10 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -1315,6 +1315,7 @@ export * from './matrix-constants.ts'; export * from './session-token.ts'; export * from './matrix-client.ts'; export * from './queue.ts'; +export * from './host-shell-generation.ts'; export * from './job-utils.ts'; export * from './prerender-html-reconcile.ts'; export * from './media-cache.ts';