diff --git a/tests/integration/static-egress-receipt-capacity-cases.ts b/tests/integration/static-egress-receipt-capacity-cases.ts new file mode 100644 index 0000000000..22a0415fc3 --- /dev/null +++ b/tests/integration/static-egress-receipt-capacity-cases.ts @@ -0,0 +1,188 @@ +import assert from "node:assert/strict"; +import type { TestContext } from "node:test"; +import type { EgressRequest } from "../../packages/contracts/src/index.ts"; +import { + CredentialProxy, + OPERATION_SANDBOX_SHARE, +} from "../../packages/occ/src/credential-proxy.ts"; +import { EgressSendReceipts } from "../../packages/occ/src/egress-send-receipts.ts"; +import { InMemoryPlatformState } from "../../packages/occ/src/state/platform-state.ts"; +import { syntheticDispatchFixture } from "../helpers/static-egress-dispatch.ts"; +import { staticEgressFixture } from "../helpers/static-egress.mjs"; + +const saturated = { kind: "refused", error: { kind: "unavailable", code: "resolver" } }; + +export async function runStaticEgressReceiptCapacityCase(t: TestContext): Promise { + for (const phase of ["begin", "authorize", "deny", "observe", "expiry"] as const) { + await t.test(phase, async (t) => { + const backingReceipts = new EgressSendReceipts(new InMemoryPlatformState()); + const f = await syntheticDispatchFixture(t, backingReceipts); + const waiting = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const fillerGate = Promise.withResolvers(); + const cancellation = new AbortController(); + const fillerCancellation = new AbortController(); + let now = f.ports.now(); + let filling = 0; + const filled = Promise.withResolvers(); + const pause = async (name: "begin" | "authorize" | "deny" | "observe") => { + if (phase === name || (phase === "expiry" && name === "authorize")) { + entered.resolve(); + await waiting.promise; + } + }; + const receipts: Pick = { + async begin(...args) { + await pause("begin"); + return backingReceipts.begin(...args); + }, + async authorize(...args) { + await pause("authorize"); + return backingReceipts.authorize(...args); + }, + async record(...args) { + await pause("deny"); + return backingReceipts.record(...args); + }, + async observe(...args) { + await pause("observe"); + return backingReceipts.observe(...args); + }, + }; + const proxy = new CredentialProxy({ + ...f.ports, + now: () => now, + exchangeOwner: { + ...f.ports.exchangeOwner, + // Fillers and the probe authenticate as the target's Sandbox, so they share its + // slots, with a peer that outlives the target's deadline. + async verify(authorization) { + const peer = await f.ports.exchangeOwner.verify(authorization); + assert.ok(peer); + return ["filler", "probe"].includes(authorization) + ? { ...peer, expiresAt: new Date(now + 3_600_000).toISOString() } + : peer; + }, + }, + // The first requests hold the share's slots in their installation lookup. + async installation(peer) { + if (filling < OPERATION_SANDBOX_SHARE - 1) { + if (++filling === OPERATION_SANDBOX_SHARE - 1) { + filled.resolve(); + } + await fillerGate.promise; + return undefined; + } + return f.ports.installation(peer); + }, + check: async (use, evidence) => + phase === "deny" && evidence + ? { kind: "denied", code: "withdrawn" } + : f.ports.check(use, evidence), + receipts, + }); + const sandboxId = f.association.installation.sandboxId; + const resolve = ( + authorization: string, + signal: AbortSignal, + request: EgressRequest = f.request, + ) => proxy.resolve(authorization, sandboxId, request, signal); + // The probe is refused at its deadline once it gets a slot. + const probe = () => + resolve("probe", fillerCancellation.signal, { + ...f.request, + deadlineAt: new Date(0).toISOString(), + }); + // Leave the Sandbox's share one slot. + const fillers = Array.from({ length: OPERATION_SANDBOX_SHARE - 1 }, () => + resolve("filler", fillerCancellation.signal), + ); + await filled.promise; + const target = (async () => { + const ready = await resolve("synthetic-peer", cancellation.signal); + assert.equal(ready.kind, "ready"); + assert.deepEqual(await probe(), saturated); + const decision = await proxy.authorizeSend( + "synthetic-peer", + sandboxId, + f.request.id, + ready.handle, + ready.evidence, + ready.injectionDigest, + f.request.deadlineAt, + cancellation.signal, + ); + if (phase !== "observe") { + assert.equal(decision.kind, "refused"); + return; + } + assert.equal(decision.kind, "allow"); + const handle = await proxy.beginResponse("synthetic-peer", sandboxId, f.request.id); + assert.ok(handle); + assert.equal( + await proxy.protectResponse( + handle, + { status: 200, headers: [], body: new Uint8Array(), trailers: [] }, + cancellation.signal, + false, + ), + undefined, + ); + })(); + try { + await Promise.race([ + entered.promise, + target.then(() => { + throw new Error("receipt work did not reach the expected phase"); + }), + ]); + if (phase === "expiry") { + now = Date.parse(f.request.deadlineAt); + } else if (phase !== "deny") { + cancellation.abort(); + } + assert.deepEqual( + await probe(), + saturated, + "revoking the handle must not release pending receipt work", + ); + waiting.resolve(); + await target; + assert.deepEqual( + await probe(), + { kind: "refused", error: { kind: "deadline_exceeded" } }, + "capacity returns when the final outstanding work settles", + ); + assert.equal(f.arrivals, 0); + } finally { + cancellation.abort(); + fillerCancellation.abort(); + waiting.resolve(); + fillerGate.resolve(); + await Promise.allSettled([target, ...fillers]); + } + }); + } +} + +export async function runStaticEgressElapsedDeadlineCase(t: TestContext): Promise { + const f = await staticEgressFixture(t); + const captured = f.capture(); + const before = f.secretReads; + const placement: { reached: Promise; release(): void } = f.pauseNamespaceVerification(); + t.after(placement.release); + const pending = f.resolve(captured); + await placement.reached; + + // Advance only wall time. This separates the absolute deadline from a timeout callback + // that may not have run yet, for example when an event-loop turn takes too long. + const { seconds, nanos } = captured.deadline; + const deadline = Number(seconds) * 1000 + nanos / 1e6; + const clock = t.mock.method(Date, "now", () => deadline); + placement.release(); + const result = await pending; + clock.mock.restore(); + assert.ok(result.refused); + assert.equal(f.secretReads, before, "no Secret GET reached Kubernetes after the deadline"); + assert.equal(f.providerRequests.length, 0); +} diff --git a/tests/integration/static-egress-receipt-persistence-cases.ts b/tests/integration/static-egress-receipt-persistence-cases.ts new file mode 100644 index 0000000000..f62de30d8d --- /dev/null +++ b/tests/integration/static-egress-receipt-persistence-cases.ts @@ -0,0 +1,313 @@ +import assert from "node:assert/strict"; +import type { TestContext } from "node:test"; +import { staticEgressFixture } from "../helpers/static-egress.mjs"; +import { + createReceiptPersistenceFaults, + readReceipt, + stageReceiptExchange, +} from "./static-egress-receipt-support.ts"; + +export async function runStaticEgressReceiptRestartCase(t: TestContext): Promise { + const f = await staticEgressFixture(t); + const { captured, ready, input, permit } = await stageReceiptExchange(f); + assert.ok((await f.authorize(permit)).allowed); + const possible = await readReceipt(f, input); + assert.ok(possible); + assert.deepEqual(possible.outcome, { kind: "uncertain", dispatch: "possible" }); + assert.equal(possible.observation, null); + const response = await f.send(captured, ready); + const foreign = await f.beginExchange( + f.metadata({ instance_id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }), + ); + foreign.stream.write({ response }); + await assert.rejects( + foreign.next(), + { code: 7 }, + "another response stream cannot claim the accepted reply", + ); + const protectedResponse = await f.protect(captured, response); + assert.ok("body" in protectedResponse && protectedResponse.body); + const observed = await readReceipt(f, input); + assert.ok(observed); + assert.deepEqual(observed.observation, { kind: "response", dispatch: "confirmed", status: 200 }); + assert.deepEqual(observed.outcome, possible.outcome); + assert.deepEqual(observed.authorization, possible.authorization); + assert.equal( + observed.settled, + false, + "body decision bytes are not final supplier application proof", + ); + assert.equal(observed.feedback, null); + assert.deepEqual( + Object.keys(observed).sort(), + [ + "sender", + "instanceId", + "useId", + "attemptId", + "material", + "deadlineAt", + "authorization", + "observation", + "outcome", + "settled", + "feedback", + ].sort(), + ); + assert.ok(!JSON.stringify(observed).includes("synthetic-key-initial")); + await f.restartAuthority(); + const replay = (await f.resolve(captured)).ready; + assert.ok(replay, "restarted authority has no transient operation registry"); + assert.ok((await f.authorize(f.permitRequest(captured, replay))).refused); + assert.deepEqual(await readReceipt(f, input), observed); + assert.equal(f.providerRequests.length, 1, "recovery never dispatches another request"); +} + +export async function runStaticEgressCredentialRejectionCase(t: TestContext): Promise { + const f = await staticEgressFixture(t); + type ProviderBody = NonNullable[0]["body"]>; + async function exchange(status: number, body: ProviderBody) { + f.setResponse({ status, body }); + const { captured, ready, input, permit } = await stageReceiptExchange(f); + assert.ok((await f.authorize(permit)).allowed); + const result = await f.protect(captured, await f.send(captured, ready)); + const receipt = await readReceipt(f, input); + assert.ok(receipt); + return { result, receipt }; + } + const nonRejections: [number, ProviderBody][] = [ + [401, { error: "invalid credential" }], + [401, { error: { code: "insufficient_quota", type: "invalid_request_error" } }], + [403, { error: { code: "invalid_api_key", type: "invalid_request_error" } }], + ]; + for (const [status, body] of nonRejections) { + const value = await exchange(status, body); + assert.ok("body" in value.result && value.result.body); + assert.deepEqual(value.receipt.observation, { + kind: "response", + dispatch: "confirmed", + status, + }); + assert.equal(value.receipt.feedback, null, "HTTP failure alone is not credential rejection"); + } + // Even recognizable bytes in an unfinished supplier unit do not establish a + // complete provider rejection. Block the response without fencing the credential. + f.setResponse({ + status: 401, + body: { error: { code: "invalid_api_key", type: "invalid_request_error" } }, + }); + const partial = await stageReceiptExchange(f); + assert.ok((await f.authorize(partial.permit)).allowed); + const partialResponse = await f.send(partial.captured, partial.ready); + const incomplete = await f.protect(partial.captured, { + ...partialResponse, + response_complete: false, + }); + assert.ok("blocked" in incomplete); + assert.equal(incomplete.blocked, true); + const partialReceipt = await readReceipt(f, partial.input); + assert.ok(partialReceipt?.observation); + assert.equal(partialReceipt.observation.kind, "response_blocked"); + assert.equal(partialReceipt.feedback, null); + const acceptedRejection = await exchange(401, { + error: { code: "invalid_api_key", type: "invalid_request_error" }, + }); + assert.ok("body" in acceptedRejection.result && acceptedRejection.result.body); + assert.deepEqual(acceptedRejection.receipt.observation, { + kind: "provider_rejected", + dispatch: "confirmed", + status: 401, + }); + assert.equal(acceptedRejection.receipt.feedback, "fenced"); + await f.rotateSecret("synthetic-key-next-version"); + const rejection = { + error: { + code: "invalid_api_key", + type: "invalid_request_error", + message: "synthetic-key-next-version", + }, + }; + const blocked = await exchange(401, rejection); + assert.ok("blocked" in blocked.result && blocked.result.blocked); + assert.deepEqual(blocked.receipt.observation, { + kind: "response_blocked", + dispatch: "confirmed", + code: "response_policy", + }); + assert.equal( + blocked.receipt.feedback, + "fenced", + "actual rejection fences even when response bytes cannot be returned", + ); + assert.equal(blocked.receipt.settled, false); + assert.ok(!JSON.stringify(blocked.receipt).includes("synthetic-key-next-version")); + const next = f.capture(); + const ready = (await f.resolve(next)).ready; + assert.ok(ready); + assert.ok((await f.authorize(f.permitRequest(next, ready))).refused); + assert.equal(f.providerRequests.length, 6); +} + +export async function runStaticEgressReceiptAuthorizationFailureCase( + t: TestContext, +): Promise { + for (const scenario of [ + "reject", + "lost_ack", + "revoke", + "deny_lost_ack", + "deadline", + "check_deadline", + "cancel", + ] as const) { + await t.test(scenario, async (t) => { + let elapsed = 0; + const fault = createReceiptPersistenceFaults(); + const f = await staticEgressFixture(t, { + now: () => new Date(Date.now() + elapsed), + appFactory: fault.appFactory, + }); + const { captured, input, permit } = await stageReceiptExchange(f); + let method: Parameters[0] = "insert"; + if (scenario === "check_deadline") { + method = "isEgressMaterialRejected"; + } else if (scenario === "lost_ack") { + method = "authorize"; + } + const kind = scenario === "reject" || scenario === "lost_ack" ? scenario : "pause"; + const gate = fault.arm(method, kind); + t.after(gate.release); + const result = f.authorize(permit, scenario === "cancel" ? { cancelAfterMs: 50 } : {}); + if (scenario !== "reject") { + const cancellation = + scenario === "cancel" ? assert.rejects(result, { code: 1 }) : undefined; + await gate.reached; + if (scenario === "revoke" || scenario === "deny_lost_ack") { + await f.revoke(); + } + // The refused settlement commits but its acknowledgment is lost: the sender sees + // unavailable, while the receipt records the committed refusal. + const settlement = scenario === "deny_lost_ack" ? fault.arm("save", "lost_ack") : undefined; + if (settlement) { + t.after(settlement.release); + } + if (scenario === "deadline" || scenario === "check_deadline") { + elapsed = 31_000; + } + if (scenario === "check_deadline") { + // The final policy-and-receipt transaction has committed. Prune the + // expired handle before its acknowledgment returns; no permit may escape. + assert.ok((await f.resolve(f.capture())).ready); + } + if (cancellation) { + await cancellation; + } + gate.release(); + if (settlement) { + await settlement.reached; + settlement.release(); + } + } + if (scenario !== "cancel") { + const { refused } = await result; + assert.ok(refused, scenario); + if (scenario === "deny_lost_ack") { + assert.equal(refused.code, f.refusalCode("UNAVAILABLE")); + } + } + const stored = await readReceipt(f, input); + if (scenario === "reject") { + assert.equal(stored, undefined); + } else { + assert.ok(stored); + if (scenario === "lost_ack") { + assert.ok( + stored.authorization?.association.custodyId, + "lost the exact authorized dispatch acknowledgment, not an earlier begin", + ); + assert.equal(stored.settled, false); + assert.deepEqual(stored.outcome, { kind: "uncertain", dispatch: "possible" }); + await f.restartAuthority(); + const replay = (await f.resolve(captured)).ready; + assert.ok(replay); + assert.ok((await f.authorize(f.permitRequest(captured, replay))).refused); + } + if (["revoke", "deadline"].includes(scenario)) { + assert.equal(stored.outcome.kind, "refused"); + assert.equal(stored.outcome.dispatched, false); + } + if (scenario === "check_deadline") { + assert.ok(stored.authorization?.association.custodyId); + assert.deepEqual(stored.outcome, { kind: "uncertain", dispatch: "possible" }); + assert.equal(stored.settled, false); + } + if (scenario === "deny_lost_ack") { + assert.equal( + stored.settled, + true, + "refusal committed before its acknowledgment was lost", + ); + assert.equal(stored.authorization, null); + assert.deepEqual(stored.outcome, { + kind: "refused", + dispatched: false, + error: { kind: "denied", code: "unauthorized" }, + }); + } + } + assert.equal(f.providerRequests.length, 0); + }); + } +} + +export async function runStaticEgressResponseCommitCase(t: TestContext): Promise { + for (const scenario of ["reject", "lost_ack", "deadline"] as const) { + await t.test(scenario, async (t) => { + let elapsed = 0; + const fault = createReceiptPersistenceFaults(); + const f = await staticEgressFixture(t, { + now: () => new Date(Date.now() + elapsed), + appFactory: fault.appFactory, + }); + f.setResponse({ + status: 401, + body: { error: { code: "invalid_api_key", type: "invalid_request_error" } }, + }); + const { captured, ready, input, permit } = await stageReceiptExchange(f); + assert.ok((await f.authorize(permit)).allowed); + const response = await f.send(captured, ready); + const gate = fault.arm("save", scenario === "deadline" ? "pause" : scenario); + t.after(gate.release); + const pending = f.protect(captured, response); + const failed = scenario === "deadline" ? undefined : assert.rejects(pending, { code: 14 }); + if (scenario !== "reject") { + await gate.reached; + if (scenario === "deadline") { + elapsed = 31_000; + } + gate.release(); + } + if (failed) { + await failed; + } else { + const result = await pending; + assert.ok("blocked" in result && result.blocked); + } + const receipt = await readReceipt(f, input); + assert.ok(receipt); + assert.deepEqual(receipt.outcome, { kind: "uncertain", dispatch: "possible" }); + assert.equal(receipt.settled, false); + const fenced = await f.controller.transact((unit) => + unit.credentialSources.isEgressMaterialRejected(input.material), + ); + assert.equal( + fenced, + scenario !== "reject", + "observation and exact-version fence share one real state transaction", + ); + assert.equal(receipt.feedback, fenced ? "fenced" : null); + assert.equal(receipt.observation?.kind, fenced ? "provider_rejected" : undefined); + assert.equal(f.providerRequests.length, 1); + }); + } +} diff --git a/tests/integration/static-egress-receipt-support.ts b/tests/integration/static-egress-receipt-support.ts new file mode 100644 index 0000000000..0d6dbce07b --- /dev/null +++ b/tests/integration/static-egress-receipt-support.ts @@ -0,0 +1,164 @@ +import assert from "node:assert/strict"; +import type { TestContext } from "node:test"; +import type { + EgressRef, + EgressSourceKey, + StaticMaterialEvidence, +} from "../../packages/contracts/src/index.ts"; +import type { SendAttempt } from "../../packages/occ/src/egress-send-receipts.ts"; +import type { PlatformStateStore } from "../../packages/occ/src/state/platform-state.ts"; +import { createConsoleAppFixture } from "../helpers/console-app.mjs"; +import { isRecord, parseJson } from "../helpers/egress-observation-values.ts"; +import { staticEgressFixture } from "../helpers/static-egress.mjs"; + +type DeployedFixture = Extract< + Awaited>, + { fixtureKind: "deployed" } +>; +type StaticSendAttempt = SendAttempt & { readonly material: StaticMaterialEvidence }; +type FaultMethod = + "findCredentialWithdrawal" | "isEgressMaterialRejected" | "insert" | "authorize" | "save"; +type FaultKind = "pause" | "reject" | "lost_ack"; +type ArmedFault = { + method: FaultMethod; + kind: FaultKind; + reached: PromiseWithResolvers; + release: PromiseWithResolvers; +}; + +function isRef(value: unknown): value is EgressRef { + return ( + isRecord(value) && + typeof value.id === "string" && + typeof value.generation === "number" && + Number.isSafeInteger(value.generation) && + value.generation > 0 + ); +} + +function isSourceKey(value: unknown): value is EgressSourceKey { + return ( + isRecord(value) && + typeof value.installationId === "string" && + typeof value.namespaceId === "string" && + isRef(value.source) && + isRef(value.backend) && + isRef(value.provider) + ); +} + +function isStaticMaterial(value: unknown): value is StaticMaterialEvidence { + return ( + isRecord(value) && + value.kind === "static" && + isSourceKey(value.source) && + isRef(value.binding) && + typeof value.grantHash === "string" && + typeof value.isolationId === "string" && + value.adoption === "live" && + typeof value.materialRefId === "string" && + isRecord(value.secret) && + value.secret.kind === "secret" && + typeof value.secret.namespaceId === "string" && + typeof value.secret.id === "string" && + isRef(value.secretDriver) && + typeof value.materialId === "string" && + typeof value.materialVersion === "string" && + typeof value.authoritativeObservedAt === "string" && + typeof value.usableUntil === "string" && + value.retention === "none" + ); +} + +export async function stageReceiptExchange(f: DeployedFixture) { + const captured = f.capture(); + const { ready } = await f.resolve(captured); + assert.ok(ready, "authority resolved the staged operation"); + const useId = captured.request.context.request_id; + const material = parseJson(ready.evidence.authority_evidence.toString()); + assert.ok(isStaticMaterial(material), "the authority returned static material evidence"); + const input: StaticSendAttempt = { + sender: f.egress.sourceControl.sender, + instanceId: f.installation.enforcement.instanceId, + useId, + attemptId: useId, + material, + deadlineAt: new Date( + Number(captured.deadline.seconds) * 1000 + captured.deadline.nanos / 1e6, + ).toISOString(), + }; + return { captured, ready, permit: f.permitRequest(captured, ready), input }; +} + +export function readReceipt(f: DeployedFixture, input: SendAttempt) { + return f.controller.transact((unit) => unit.egressSendReceipts.find(input)); +} + +// Faults wrap the real state transaction owner: operations and rollback still run in the +// actual memory adapter. Lost acknowledgments here are scheduling proof, not real PG proof. +export function createReceiptPersistenceFaults() { + let armed: ArmedFault | undefined; + + function wrapState(state: PlatformStateStore): PlatformStateStore { + return { + read: (work) => state.read(work), + async transact(work) { + const effect: { current?: ArmedFault } = {}; + async function intercept(method: FaultMethod, run: () => Promise) { + if (armed?.method === method) { + effect.current = armed; + armed = undefined; + if (effect.current.kind === "reject") { + throw new Error("receipt persistence unavailable"); + } + } + return run(); + } + const result = await state.transact((unit) => + work({ + ...unit, + egressSendReceipts: { + ...unit.egressSendReceipts, + insert: (...args) => + intercept("insert", () => unit.egressSendReceipts.insert(...args)), + authorize: (...args) => + intercept("authorize", () => unit.egressSendReceipts.authorize(...args)), + save: (...args) => intercept("save", () => unit.egressSendReceipts.save(...args)), + }, + credentialSources: { + ...unit.credentialSources, + findCredentialWithdrawal: (...args) => + intercept("findCredentialWithdrawal", () => + unit.credentialSources.findCredentialWithdrawal(...args), + ), + isEgressMaterialRejected: (...args) => + intercept("isEgressMaterialRejected", () => + unit.credentialSources.isEgressMaterialRejected(...args), + ), + }, + }), + ); + if (effect.current) { + effect.current.reached.resolve(); + await effect.current.release.promise; + if (effect.current.kind === "lost_ack") { + throw new Error("receipt commit acknowledgment unavailable"); + } + } + return result; + }, + }; + } + + return { + arm(method: FaultMethod, kind: FaultKind = "pause") { + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + armed = { method, kind, reached, release }; + return { reached: reached.promise, release: () => release.resolve() }; + }, + appFactory(t: TestContext, options?: Parameters[1]) { + return createConsoleAppFixture(t, { ...options, wrapState }); + }, + }; +} diff --git a/tests/integration/static-egress-request-capacity-cases.ts b/tests/integration/static-egress-request-capacity-cases.ts index 255e56b387..706342ddba 100644 --- a/tests/integration/static-egress-request-capacity-cases.ts +++ b/tests/integration/static-egress-request-capacity-cases.ts @@ -854,3 +854,15 @@ export function registerStaticEgressResponsesAndCapacityCases( registerStaticEgressResponsesWireCases(unicodeEscapes); registerStaticEgressRequestCapacityCases(); } + +export { + runStaticEgressElapsedDeadlineCase, + runStaticEgressReceiptCapacityCase, +} from "./static-egress-receipt-capacity-cases.ts"; +export { + runStaticEgressCredentialRejectionCase, + runStaticEgressReceiptAuthorizationFailureCase, + runStaticEgressReceiptRestartCase, + runStaticEgressResponseCommitCase, +} from "./static-egress-receipt-persistence-cases.ts"; +export { createReceiptPersistenceFaults } from "./static-egress-receipt-support.ts"; diff --git a/tests/integration/static-egress.test.mjs b/tests/integration/static-egress.test.mjs index 127ae4a3c1..213ed1a8d2 100644 --- a/tests/integration/static-egress.test.mjs +++ b/tests/integration/static-egress.test.mjs @@ -9,11 +9,7 @@ import { EgressSendReceipts } from "../../packages/occ/src/egress-send-receipts. import { InMemoryPlatformState } from "../../packages/occ/src/state/platform-state.ts"; import { PROXY_REGISTRATION_UNAVAILABLE } from "../helpers/refusal-text.mjs"; import { OpenShellEgressIdentity } from "../../apps/controller/src/drivers/egress-proxy/openshell-identity.ts"; -import { - CredentialProxy, - OPERATION_SANDBOX_SHARE, - egressInjectionDigest, -} from "../../packages/occ/src/credential-proxy.ts"; +import { CredentialProxy, egressInjectionDigest } from "../../packages/occ/src/credential-proxy.ts"; import { deploymentErrorForWork } from "../../packages/occ/src/state/controller-work.ts"; import { reconcileSandboxPolicy } from "../../apps/controller/src/worker/sandbox-policy.ts"; import { @@ -30,7 +26,16 @@ import { } from "../helpers/static-egress.mjs"; import { syntheticDispatchFixture } from "../helpers/static-egress-dispatch.ts"; -import { registerStaticEgressResponsesAndCapacityCases } from "./static-egress-request-capacity-cases.ts"; +import { + createReceiptPersistenceFaults, + registerStaticEgressResponsesAndCapacityCases, + runStaticEgressCredentialRejectionCase, + runStaticEgressElapsedDeadlineCase, + runStaticEgressReceiptAuthorizationFailureCase, + runStaticEgressReceiptCapacityCase, + runStaticEgressReceiptRestartCase, + runStaticEgressResponseCommitCase, +} from "./static-egress-request-capacity-cases.ts"; import { registerStaticEgressResponsesPrefixCases, runStaticEgressNativeAgentMessageCase, @@ -1095,236 +1100,13 @@ test("Responses admits bounded native agent messages", async (t) => { registerStaticEgressResponsesAndCapacityCases(unicodeEscapes); -const saturated = { kind: "refused", error: { kind: "unavailable", code: "resolver" } }; - test("cancelled and denied exchanges retain capacity until receipt work settles", async (t) => { - for (const phase of ["begin", "authorize", "deny", "observe", "expiry"]) { - await t.test(phase, async (t) => { - const f = await syntheticDispatchFixture(t); - const waiting = Promise.withResolvers(); - const entered = Promise.withResolvers(); - const fillerGate = Promise.withResolvers(); - const cancellation = new AbortController(); - const fillerCancellation = new AbortController(); - let now = f.ports.now(); - let filling = 0; - const filled = Promise.withResolvers(); - const pause = async (name) => { - if (phase === name || (phase === "expiry" && name === "authorize")) { - entered.resolve(); - await waiting.promise; - } - }; - const proxy = new CredentialProxy({ - ...f.ports, - now: () => now, - exchangeOwner: { - ...f.ports.exchangeOwner, - // Fillers and the probe authenticate as the target's Sandbox, so they share its - // slots, with a peer that outlives the target's deadline. - async verify(authorization) { - const peer = await f.ports.exchangeOwner.verify(authorization); - return ["filler", "probe"].includes(authorization) - ? { ...peer, expiresAt: new Date(now + 3_600_000).toISOString() } - : peer; - }, - }, - // The first requests hold the share's slots in their installation lookup. - async installation(peer) { - if (filling < OPERATION_SANDBOX_SHARE - 1) { - if (++filling === OPERATION_SANDBOX_SHARE - 1) { - filled.resolve(); - } - await fillerGate.promise; - return undefined; - } - return f.ports.installation(peer); - }, - check: async (use, evidence) => - phase === "deny" && evidence - ? { kind: "denied", code: "withdrawn" } - : f.ports.check(use, evidence), - receipts: { - async begin() { - await pause("begin"); - return { created: true }; - }, - async authorize(_sender, _attempt, authorization) { - await pause("authorize"); - return { authorization }; - }, - async record() { - await pause("deny"); - }, - async observe() { - await pause("observe"); - }, - }, - }); - const sandboxId = f.association.installation.sandboxId; - const resolve = (authorization, signal, request = f.request) => - proxy.resolve(authorization, sandboxId, request, signal); - // The probe is refused at its deadline once it gets a slot. - const probe = () => - resolve("probe", fillerCancellation.signal, { - ...f.request, - deadlineAt: new Date(0).toISOString(), - }); - // Leave the Sandbox's share one slot. - const fillers = Array.from({ length: OPERATION_SANDBOX_SHARE - 1 }, () => - resolve("filler", fillerCancellation.signal), - ); - await filled.promise; - const target = (async () => { - const ready = await resolve("synthetic-peer", cancellation.signal); - assert.equal(ready.kind, "ready"); - assert.deepEqual(await probe(), saturated); - const decision = await proxy.authorizeSend( - "synthetic-peer", - sandboxId, - f.request.id, - ready.handle, - ready.evidence, - ready.injectionDigest, - f.request.deadlineAt, - cancellation.signal, - ); - if (phase !== "observe") { - assert.equal(decision.kind, "refused"); - return; - } - assert.equal(decision.kind, "allow"); - const handle = await proxy.beginResponse("synthetic-peer", sandboxId, f.request.id); - assert.ok(handle); - assert.equal( - await proxy.protectResponse( - handle, - { status: 200, headers: [], body: new Uint8Array(), trailers: [] }, - cancellation.signal, - false, - ), - undefined, - ); - })(); - try { - await Promise.race([ - entered.promise, - target.then(() => { - throw new Error("receipt work did not reach the expected phase"); - }), - ]); - if (phase === "expiry") { - now = Date.parse(f.request.deadlineAt); - } else if (phase !== "deny") { - cancellation.abort(); - } - assert.deepEqual( - await probe(), - saturated, - "revoking the handle must not release pending receipt work", - ); - waiting.resolve(); - await target; - assert.deepEqual( - await probe(), - { kind: "refused", error: { kind: "deadline_exceeded" } }, - "capacity returns when the final outstanding work settles", - ); - assert.equal(f.arrivals, 0); - } finally { - cancellation.abort(); - fillerCancellation.abort(); - waiting.resolve(); - fillerGate.resolve(); - await Promise.allSettled([target, ...fillers]); - } - }); - } + await runStaticEgressReceiptCapacityCase(t); }); -async function stageReceiptExchange(f) { - const captured = f.capture(); - const { ready } = await f.resolve(captured); - assert.ok(ready, "authority resolved the staged operation"); - const useId = captured.request.context.request_id; - return { - captured, - ready, - permit: f.permitRequest(captured, ready), - input: { - sender: f.egress.sourceControl.sender, - instanceId: f.installation.enforcement.instanceId, - useId, - attemptId: useId, - material: JSON.parse(ready.evidence.authority_evidence.toString()), - deadlineAt: new Date( - Number(captured.deadline.seconds) * 1000 + captured.deadline.nanos / 1e6, - ).toISOString(), - }, - }; -} - -const readReceipt = (f, input) => - f.controller.transact((unit) => unit.egressSendReceipts.find(input)); - -// Faults wrap the real state transaction owner: operations and rollback still run in the -// actual memory adapter. Lost acknowledgments here are scheduling proof, not real PG proof. -function receiptPersistenceFaults() { - let armed; - return { - arm(method, kind = "pause") { - const reached = Promise.withResolvers(); - const release = Promise.withResolvers(); - armed = { method, kind, reached, release }; - return { reached: reached.promise, release: () => release.resolve() }; - }, - appFactory: (t, options) => createConsoleAppFixture(t, { ...options, wrapState }), - }; - - function wrapState(state) { - return { - read: (work) => state.read(work), - async transact(work) { - let effect; - const intercept = (repository) => - Object.fromEntries( - Object.entries(repository).map(([name, method]) => [ - name, - async (...args) => { - if (armed?.method === name) { - effect = armed; - armed = undefined; - if (effect.kind === "reject") { - throw new Error("receipt persistence unavailable"); - } - } - return method(...args); - }, - ]), - ); - const result = await state.transact((unit) => - work({ - ...unit, - egressSendReceipts: intercept(unit.egressSendReceipts), - credentialSources: intercept(unit.credentialSources), - }), - ); - if (effect) { - effect.reached.resolve(); - await effect.release.promise; - if (effect.kind === "lost_ack") { - throw new Error("receipt commit acknowledgment unavailable"); - } - } - return result; - }, - }; - } -} - for (const stage of ["policy check", "Kubernetes namespace verification"]) { test(`closing the accepted duplex during ${stage} prevents the Secret GET`, async (t) => { - const fault = receiptPersistenceFaults(); + const fault = createReceiptPersistenceFaults(); const finished = Promise.withResolvers(); const f = await staticEgressFixture(t, { resolutionFinished: finished.resolve, @@ -1360,25 +1142,7 @@ for (const stage of ["policy check", "Kubernetes namespace verification"]) { } test("an elapsed deadline prevents the Kubernetes Secret GET even before the timeout callback runs", async (t) => { - const f = await staticEgressFixture(t); - const captured = f.capture(); - const before = f.secretReads; - const placement = f.pauseNamespaceVerification(); - t.after(placement.release); - const pending = f.resolve(captured); - await placement.reached; - - // Advance only wall time. This separates the absolute deadline from a timeout callback - // that may not have run yet, for example when an event-loop turn takes too long. - const { seconds, nanos } = captured.deadline; - const deadline = Number(seconds) * 1000 + nanos / 1e6; - const clock = t.mock.method(Date, "now", () => deadline); - placement.release(); - const result = await pending; - clock.mock.restore(); - assert.ok(result.refused); - assert.equal(f.secretReads, before, "no Secret GET reached Kubernetes after the deadline"); - assert.equal(f.providerRequests.length, 0); + await runStaticEgressElapsedDeadlineCase(t); }); for (const stage of ["check", "read"]) { @@ -1411,289 +1175,19 @@ for (const stage of ["check", "read"]) { registerStreamReceiptPersistenceCases(() => new EgressSendReceipts(new InMemoryPlatformState())); test("authenticated sender receipts survive authority restart without replay permission and distinguish observations", async (t) => { - const f = await staticEgressFixture(t); - const { captured, ready, input, permit } = await stageReceiptExchange(f); - assert.ok((await f.authorize(permit)).allowed); - const possible = await readReceipt(f, input); - assert.deepEqual(possible.outcome, { kind: "uncertain", dispatch: "possible" }); - assert.equal(possible.observation, null); - const response = await f.send(captured, ready); - const foreign = await f.beginExchange( - f.metadata({ instance_id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }), - ); - foreign.stream.write({ response }); - await assert.rejects( - foreign.next(), - { code: 7 }, - "another response stream cannot claim the accepted reply", - ); - assert.ok((await f.protect(captured, response)).body); - const observed = await readReceipt(f, input); - assert.deepEqual(observed.observation, { kind: "response", dispatch: "confirmed", status: 200 }); - assert.deepEqual(observed.outcome, possible.outcome); - assert.deepEqual(observed.authorization, possible.authorization); - assert.equal( - observed.settled, - false, - "body decision bytes are not final supplier application proof", - ); - assert.equal(observed.feedback, null); - assert.deepEqual( - Object.keys(observed).sort(), - [ - "sender", - "instanceId", - "useId", - "attemptId", - "material", - "deadlineAt", - "authorization", - "observation", - "outcome", - "settled", - "feedback", - ].sort(), - ); - assert.ok(!JSON.stringify(observed).includes("synthetic-key-initial")); - await f.restartAuthority(); - const replay = (await f.resolve(captured)).ready; - assert.ok(replay, "restarted authority has no transient operation registry"); - assert.ok((await f.authorize(f.permitRequest(captured, replay))).refused); - assert.deepEqual(await readReceipt(f, input), observed); - assert.equal(f.providerRequests.length, 1, "recovery never dispatches another request"); + await runStaticEgressReceiptRestartCase(t); }); test("receipt observation fences only positive credential rejection, including blocked provider responses", async (t) => { - const f = await staticEgressFixture(t); - async function exchange(status, body) { - f.setResponse({ status, body }); - const { captured, ready, input, permit } = await stageReceiptExchange(f); - assert.ok((await f.authorize(permit)).allowed); - const result = await f.protect(captured, await f.send(captured, ready)); - return { result, receipt: await readReceipt(f, input) }; - } - for (const [status, body] of [ - [401, { error: "invalid credential" }], - [401, { error: { code: "insufficient_quota", type: "invalid_request_error" } }], - [403, { error: { code: "invalid_api_key", type: "invalid_request_error" } }], - ]) { - const value = await exchange(status, body); - assert.ok(value.result.body); - assert.deepEqual(value.receipt.observation, { - kind: "response", - dispatch: "confirmed", - status, - }); - assert.equal(value.receipt.feedback, null, "HTTP failure alone is not credential rejection"); - } - // Even recognizable bytes in an unfinished supplier unit do not establish a - // complete provider rejection. Block the response without fencing the credential. - f.setResponse({ - status: 401, - body: { error: { code: "invalid_api_key", type: "invalid_request_error" } }, - }); - const partial = await stageReceiptExchange(f); - assert.ok((await f.authorize(partial.permit)).allowed); - const partialResponse = await f.send(partial.captured, partial.ready); - const incomplete = await f.protect(partial.captured, { - ...partialResponse, - response_complete: false, - }); - assert.equal(incomplete.blocked, true); - const partialReceipt = await readReceipt(f, partial.input); - assert.equal(partialReceipt.observation.kind, "response_blocked"); - assert.equal(partialReceipt.feedback, null); - const acceptedRejection = await exchange(401, { - error: { code: "invalid_api_key", type: "invalid_request_error" }, - }); - assert.ok(acceptedRejection.result.body); - assert.deepEqual(acceptedRejection.receipt.observation, { - kind: "provider_rejected", - dispatch: "confirmed", - status: 401, - }); - assert.equal(acceptedRejection.receipt.feedback, "fenced"); - await f.rotateSecret("synthetic-key-next-version"); - const rejection = { - error: { - code: "invalid_api_key", - type: "invalid_request_error", - message: "synthetic-key-next-version", - }, - }; - const blocked = await exchange(401, rejection); - assert.ok(blocked.result.blocked); - assert.deepEqual(blocked.receipt.observation, { - kind: "response_blocked", - dispatch: "confirmed", - code: "response_policy", - }); - assert.equal( - blocked.receipt.feedback, - "fenced", - "actual rejection fences even when response bytes cannot be returned", - ); - assert.equal(blocked.receipt.settled, false); - assert.ok(!JSON.stringify(blocked.receipt).includes("synthetic-key-next-version")); - const next = f.capture(); - const ready = (await f.resolve(next)).ready; - assert.ok((await f.authorize(f.permitRequest(next, ready))).refused); - assert.equal(f.providerRequests.length, 6); + await runStaticEgressCredentialRejectionCase(t); }); test("receipt persistence failure, lost acknowledgment and late authorization never grant send permission", async (t) => { - for (const scenario of [ - "reject", - "lost_ack", - "revoke", - "deny_lost_ack", - "deadline", - "check_deadline", - "cancel", - ]) { - await t.test(scenario, async (t) => { - let elapsed = 0; - const fault = receiptPersistenceFaults(); - const f = await staticEgressFixture(t, { - now: () => new Date(Date.now() + elapsed), - appFactory: fault.appFactory, - }); - const { captured, input, permit } = await stageReceiptExchange(f); - const gate = fault.arm( - scenario === "check_deadline" - ? "isEgressMaterialRejected" - : scenario === "lost_ack" - ? "authorize" - : "insert", - ["reject", "lost_ack"].includes(scenario) ? scenario : "pause", - ); - t.after(gate.release); - const result = f.authorize(permit, { - cancelAfterMs: scenario === "cancel" ? 50 : undefined, - }); - if (scenario !== "reject") { - const cancellation = - scenario === "cancel" ? assert.rejects(result, { code: 1 }) : undefined; - await gate.reached; - if (scenario === "revoke" || scenario === "deny_lost_ack") { - await f.revoke(); - } - // The refused settlement commits but its acknowledgment is lost: the sender sees - // unavailable, while the receipt records the committed refusal. - const settlement = scenario === "deny_lost_ack" ? fault.arm("save", "lost_ack") : undefined; - if (settlement) { - t.after(settlement.release); - } - if (scenario === "deadline" || scenario === "check_deadline") { - elapsed = 31_000; - } - if (scenario === "check_deadline") { - // The final policy-and-receipt transaction has committed. Prune the - // expired handle before its acknowledgment returns; no permit may escape. - assert.ok((await f.resolve(f.capture())).ready); - } - if (cancellation) { - await cancellation; - } - gate.release(); - if (settlement) { - await settlement.reached; - settlement.release(); - } - } - if (scenario !== "cancel") { - const { refused } = await result; - assert.ok(refused, scenario); - if (scenario === "deny_lost_ack") { - assert.equal(refused.code, f.refusalCode("UNAVAILABLE")); - } - } - const stored = await readReceipt(f, input); - if (scenario === "reject") { - assert.equal(stored, undefined); - } - if (scenario === "lost_ack") { - assert.ok( - stored.authorization?.association.custodyId, - "lost the exact authorized dispatch acknowledgment, not an earlier begin", - ); - assert.equal(stored.settled, false); - assert.deepEqual(stored.outcome, { kind: "uncertain", dispatch: "possible" }); - await f.restartAuthority(); - const replay = (await f.resolve(captured)).ready; - assert.ok((await f.authorize(f.permitRequest(captured, replay))).refused); - } - if (["revoke", "deadline"].includes(scenario)) { - assert.equal(stored.outcome.kind, "refused"); - assert.equal(stored.outcome.dispatched, false); - } - if (scenario === "check_deadline") { - assert.ok(stored.authorization?.association.custodyId); - assert.deepEqual(stored.outcome, { kind: "uncertain", dispatch: "possible" }); - assert.equal(stored.settled, false); - } - if (scenario === "deny_lost_ack") { - assert.equal(stored.settled, true, "refusal committed before its acknowledgment was lost"); - assert.equal(stored.authorization, null); - assert.deepEqual(stored.outcome, { - kind: "refused", - dispatched: false, - error: { kind: "denied", code: "unauthorized" }, - }); - } - assert.equal(f.providerRequests.length, 0); - }); - } + await runStaticEgressReceiptAuthorizationFailureCase(t); }); test("response persistence must commit its observation and fence before policy bytes are acknowledged", async (t) => { - for (const scenario of ["reject", "lost_ack", "deadline"]) { - await t.test(scenario, async (t) => { - let elapsed = 0; - const fault = receiptPersistenceFaults(); - const f = await staticEgressFixture(t, { - now: () => new Date(Date.now() + elapsed), - appFactory: fault.appFactory, - }); - f.setResponse({ - status: 401, - body: { error: { code: "invalid_api_key", type: "invalid_request_error" } }, - }); - const { captured, ready, input, permit } = await stageReceiptExchange(f); - assert.ok((await f.authorize(permit)).allowed); - const response = await f.send(captured, ready); - const gate = fault.arm("save", scenario === "deadline" ? "pause" : scenario); - t.after(gate.release); - const pending = f.protect(captured, response); - const failed = scenario === "deadline" ? undefined : assert.rejects(pending, { code: 14 }); - if (scenario !== "reject") { - await gate.reached; - if (scenario === "deadline") { - elapsed = 31_000; - } - gate.release(); - } - if (failed) { - await failed; - } else { - assert.ok((await pending).blocked); - } - const receipt = await readReceipt(f, input); - assert.deepEqual(receipt.outcome, { kind: "uncertain", dispatch: "possible" }); - assert.equal(receipt.settled, false); - const fenced = await f.controller.transact((unit) => - unit.credentialSources.isEgressMaterialRejected(input.material), - ); - assert.equal( - fenced, - scenario !== "reject", - "observation and exact-version fence share one real state transaction", - ); - assert.equal(receipt.feedback, fenced ? "fenced" : null); - assert.equal(receipt.observation?.kind, fenced ? "provider_rejected" : undefined); - assert.equal(f.providerRequests.length, 1); - }); - } + await runStaticEgressResponseCommitCase(t); }); function admissionCases(revision) {