Cache api key validations per isolate (#1864)

* Cache successful api key validations per isolate

* Add cloud e2e coverage for the api key validation cache

* Build the MCP auth layer once per isolate

* Key the memoized MCP auth runtime on its WorkOS bindings

* Encode the auth-runtime binding fingerprint as JSON

Author: RhysSullivan

.changeset/apikey-validate-cache.md

@@ -0,0 +1,9 @@
+---
+"@executor-js/cloud": patch
+---
+
+**API key validation is cached per isolate**
+
+Every MCP request and every API-key-authenticated `/api/*` request used to pay a live WorkOS round trip (~100-150ms) to validate the presented key, on every single request. The JWT bearer path beside it already verified locally against a JWKS cached for an hour; API keys had no cache at all.
+
+Successful validations are now cached in a bounded per-isolate map for 60 seconds, keyed by the SHA-256 digest of the key value (never the raw credential). The MCP handler used to rebuild its whole auth layer (and with it the cache) on every request; it now builds the layer once per isolate, so the cache holds on both the `/api/*` and `/mcp` planes. Invalid keys and upstream failures are never cached, so probing bad keys cannot pollute the map and a freshly created key works immediately. The tradeoff: a revoked key remains usable for up to 60 seconds within an isolate that validated it before revocation — far tighter than the one-hour rotation window the JWT path already accepts.

apps/cloud/src/auth/api-key-validation-cache.node.test.ts

@@ -0,0 +1,210 @@
+import { describe, expect, it } from "@effect/vitest";
+import { Effect, Exit, Layer } from "effect";
+
+import {
+  ApiKeyService,
+  ApiKeyValidationError,
+  makeCachedApiKeyValidate,
+  type ApiKeyOwner,
+} from "./api-keys";
+import { WorkOSClient, type WorkOSClientService } from "./workos";
+
+const stubWorkOS = (overrides: Partial<WorkOSClientService>) =>
+  Layer.succeed(
+    WorkOSClient,
+    new Proxy({} as WorkOSClientService, {
+      get: (_target, prop) => {
+        if (prop in overrides) return overrides[prop as keyof WorkOSClientService];
+        return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`);
+      },
+    }),
+  );
+
+const userOwner = (keyId: string): ApiKeyOwner => ({
+  scope: "user",
+  accountId: "user_123",
+  organizationId: "org_123",
+  keyId,
+});
+
+describe("makeCachedApiKeyValidate", () => {
+  it.effect("serves a repeat validation from cache without a second upstream call", () =>
+    Effect.gen(function* () {
+      let calls = 0;
+      const cached = makeCachedApiKeyValidate(() =>
+        Effect.sync(() => {
+          calls += 1;
+          return userOwner("api_key_1");
+        }),
+      );
+
+      const first = yield* cached.validate("sk_test_abc");
+      const second = yield* cached.validate("sk_test_abc");
+
+      expect(calls).toBe(1);
+      expect(first).toEqual(userOwner("api_key_1"));
+      expect(second).toEqual(first);
+    }),
+  );
+
+  it.effect("re-validates once the TTL has expired", () =>
+    Effect.gen(function* () {
+      let calls = 0;
+      let nowMs = 0;
+      const cached = makeCachedApiKeyValidate(
+        () =>
+          Effect.sync(() => {
+            calls += 1;
+            return userOwner("api_key_1");
+          }),
+        { ttlMs: 1_000, now: () => nowMs },
+      );
+
+      yield* cached.validate("sk_test_abc");
+      nowMs = 999;
+      yield* cached.validate("sk_test_abc");
+      expect(calls).toBe(1);
+
+      nowMs = 1_000;
+      yield* cached.validate("sk_test_abc");
+      expect(calls).toBe(2);
+    }),
+  );
+
+  it.effect("never caches an invalid key", () =>
+    Effect.gen(function* () {
+      let calls = 0;
+      const cached = makeCachedApiKeyValidate(() =>
+        Effect.sync(() => {
+          calls += 1;
+          return null;
+        }),
+      );
+
+      const first = yield* cached.validate("sk_test_bad");
+      const second = yield* cached.validate("sk_test_bad");
+
+      expect(first).toBeNull();
+      expect(second).toBeNull();
+      expect(calls).toBe(2);
+      expect(cached.cacheKeys()).toHaveLength(0);
+    }),
+  );
+
+  it.effect("never caches an upstream failure", () =>
+    Effect.gen(function* () {
+      let calls = 0;
+      const cached = makeCachedApiKeyValidate(() =>
+        Effect.suspend(() => {
+          calls += 1;
+          return Effect.fail(new ApiKeyValidationError({ cause: "workos_down" }));
+        }),
+      );
+
+      const first = yield* Effect.exit(cached.validate("sk_test_abc"));
+      const second = yield* Effect.exit(cached.validate("sk_test_abc"));
+
+      expect(Exit.isFailure(first)).toBe(true);
+      expect(Exit.isFailure(second)).toBe(true);
+      expect(calls).toBe(2);
+      expect(cached.cacheKeys()).toHaveLength(0);
+    }),
+  );
+
+  it.effect("resets a saturated map of live entries rather than growing past the bound", () =>
+    Effect.gen(function* () {
+      const cached = makeCachedApiKeyValidate((value) => Effect.succeed(userOwner(value)), {
+        maxEntries: 3,
+      });
+
+      for (const value of ["sk_1", "sk_2", "sk_3"]) {
+        yield* cached.validate(value);
+      }
+      expect(cached.cacheKeys()).toHaveLength(3);
+
+      // Nothing is expired, so a fourth live entry resets the map instead of
+      // growing it (mirrors execution-gate's writeCache).
+      yield* cached.validate("sk_4");
+      expect(cached.cacheKeys()).toHaveLength(1);
+
+      yield* cached.validate("sk_5");
+      expect(cached.cacheKeys()).toHaveLength(2);
+    }),
+  );
+
+  it.effect("evicts expired entries before resetting a saturated map", () =>
+    Effect.gen(function* () {
+      let nowMs = 0;
+      const cached = makeCachedApiKeyValidate((value) => Effect.succeed(userOwner(value)), {
+        ttlMs: 1_000,
+        maxEntries: 3,
+        now: () => nowMs,
+      });
+
+      for (const value of ["sk_1", "sk_2", "sk_3"]) {
+        yield* cached.validate(value);
+      }
+      expect(cached.cacheKeys()).toHaveLength(3);
+
+      // Everything is expired: the sweep drops the stale entries and the new
+      // one fits without a full reset.
+      nowMs = 2_000;
+      yield* cached.validate("sk_4");
+      expect(cached.cacheKeys()).toHaveLength(1);
+    }),
+  );
+
+  it.effect("keys the cache by the SHA-256 digest, never the raw key value", () =>
+    Effect.gen(function* () {
+      const rawKey = "sk_live_super_secret_value";
+      const cached = makeCachedApiKeyValidate(() => Effect.succeed(userOwner("api_key_1")));
+
+      yield* cached.validate(rawKey);
+
+      const expectedDigest = yield* Effect.promise(async () => {
+        const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(rawKey));
+        return Array.from(new Uint8Array(digest), (byte) =>
+          byte.toString(16).padStart(2, "0"),
+        ).join("");
+      });
+      expect(cached.cacheKeys()).toEqual([expectedDigest]);
+      expect(cached.cacheKeys()).not.toContain(rawKey);
+    }),
+  );
+});
+
+describe("ApiKeyService.WorkOS validation cache", () => {
+  it.effect("validates a repeated key through one upstream call per layer build", () =>
+    Effect.gen(function* () {
+      let calls = 0;
+      const layer = ApiKeyService.WorkOS.pipe(
+        Layer.provide(
+          stubWorkOS({
+            validateApiKey: () =>
+              Effect.sync(() => {
+                calls += 1;
+                return {
+                  apiKey: {
+                    id: "api_key_123",
+                    owner: { type: "user", id: "user_123", organizationId: "org_123" },
+                  },
+                };
+              }),
+          }),
+        ),
+      );
+
+      const { first, second } = yield* Effect.gen(function* () {
+        const apiKeys = yield* ApiKeyService;
+        return {
+          first: yield* apiKeys.validate("sk_test_abc"),
+          second: yield* apiKeys.validate("sk_test_abc"),
+        };
+      }).pipe(Effect.provide(layer));
+
+      expect(calls).toBe(1);
+      expect(first?.keyId).toBe("api_key_123");
+      expect(second).toEqual(first);
+    }),
+  );
+});

apps/cloud/src/auth/api-keys.ts

@@ -1,5 +1,7 @@
 import { Context, Data, Effect, Layer, Option, Schema } from "effect";
 
+import { sha256Hex } from "@executor-js/sdk";
+
 import { ApiKeyManagementError } from "./errors";
 import { WorkOSClient } from "./workos";
 
@@ -277,6 +279,98 @@ const createdFromResponse = (value: unknown): CreatedApiKey | null =>
     },
   });
 
+// ---------------------------------------------------------------------------
+// Per-isolate validation cache.
+//
+// Every /mcp request and every api-key-authenticated /api/* request funnels
+// through `validate`, and each call is a live WorkOS round trip (~100-150ms).
+// The JWT bearer path beside it verifies locally against a JWKS cached for an
+// hour; api keys had no cache at all. This is the same bounded TTL map the
+// engine already uses for billing outcomes (engine/execution-gate.ts).
+//
+// Security shape, stated explicitly because this is auth code:
+//   - The map key is the SHA-256 digest of the presented key value, never the
+//     raw credential, so the map's keys cannot be reversed into a usable key.
+//     The digest reaches no log, span, or error message.
+//   - Only SUCCESSFUL validations are cached. An invalid key and an upstream
+//     failure always miss: an attacker probing bad keys cannot poison the map
+//     (the size bound alone handles memory), and a just-created key works
+//     immediately instead of waiting out a stale negative entry.
+//   - REVOCATION WINDOW: a revoked key keeps working for up to the TTL (60s)
+//     in any isolate that validated it before revocation. That is the price
+//     of the cache, and it is far tighter than the 1h JWKS rotation window
+//     the JWT path already accepts.
+//   - Concurrent misses for the same key each call WorkOS; there is no
+//     in-flight dedupe. The duplicate window is one round trip per key per
+//     TTL per isolate — exactly the cost EVERY request paid before this
+//     cache — and dedupe would add interruption-safety machinery (a
+//     cancelled leader must not strand its waiters) to auth-critical code.
+// ---------------------------------------------------------------------------
+
+const API_KEY_VALIDATION_CACHE_TTL_MS = 60_000;
+// Sweep guard so a long-lived isolate serving many keys can't grow the cache
+// map unbounded (mirrors BALANCE_CACHE_MAX_ENTRIES in execution-gate.ts).
+const API_KEY_VALIDATION_CACHE_MAX_ENTRIES = 10_000;
+
+export type ApiKeyValidate = (
+  value: string,
+) => Effect.Effect<ApiKeyOwner | null, ApiKeyValidationError>;
+
+/**
+ * Wrap a `validate` function with the per-isolate success cache described
+ * above. Exported for tests only — production use is the single instance the
+ * `ApiKeyService.WorkOS` layer builds, which lives in `boot` and is therefore
+ * constructed once per isolate.
+ *
+ * The `ttlMs` / `maxEntries` / `now` knobs exist so tests can exercise expiry
+ * and the size bound without waiting on wall time; production passes nothing.
+ */
+export const makeCachedApiKeyValidate = (
+  validate: ApiKeyValidate,
+  options?: {
+    readonly ttlMs?: number;
+    readonly maxEntries?: number;
+    readonly now?: () => number;
+  },
+): {
+  readonly validate: ApiKeyValidate;
+  /** Test seam: the digests currently cached, so tests can assert the raw
+   *  credential never appears as a map key. */
+  readonly cacheKeys: () => ReadonlyArray<string>;
+} => {
+  const ttlMs = options?.ttlMs ?? API_KEY_VALIDATION_CACHE_TTL_MS;
+  const maxEntries = options?.maxEntries ?? API_KEY_VALIDATION_CACHE_MAX_ENTRIES;
+  const now = options?.now ?? Date.now;
+  const cache = new Map<string, { readonly owner: ApiKeyOwner; readonly expiresAtMs: number }>();
+
+  const writeCache = (digest: string, owner: ApiKeyOwner, nowMs: number): void => {
+    if (cache.size >= maxEntries) {
+      for (const [key, entry] of cache) {
+        if (entry.expiresAtMs <= nowMs) cache.delete(key);
+      }
+      // Still saturated after dropping expired entries: reset rather than grow.
+      if (cache.size >= maxEntries) cache.clear();
+    }
+    cache.set(digest, { owner, expiresAtMs: nowMs + ttlMs });
+  };
+
+  return {
+    cacheKeys: () => [...cache.keys()],
+    validate: (value) =>
+      Effect.gen(function* () {
+        const digest = yield* sha256Hex(value);
+        const nowMs = now();
+        const cached = cache.get(digest);
+        if (cached && cached.expiresAtMs > nowMs) return cached.owner;
+        const owner = yield* validate(value);
+        // Cache only a resolved owner — see the block comment above for why
+        // null results and failures always stay misses.
+        if (owner) writeCache(digest, owner, nowMs);
+        return owner;
+      }),
+  };
+};
+
 export class ApiKeyService extends Context.Service<
   ApiKeyService,
   {
@@ -320,12 +414,15 @@ export class ApiKeyService extends Context.Service<
   static WorkOS = Layer.effect(this)(
     Effect.gen(function* () {
       const workos = yield* WorkOSClient;
+      // Built once with the boot layer, so the success cache is per-isolate.
+      const cachedValidate = makeCachedApiKeyValidate((value) =>
+        workos.validateApiKey(value).pipe(
+          Effect.map(ownerFromResponse),
+          Effect.mapError((cause) => new ApiKeyValidationError({ cause })),
+        ),
+      );
       return {
-        validate: (value: string) =>
-          workos.validateApiKey(value).pipe(
-            Effect.map(ownerFromResponse),
-            Effect.mapError((cause) => new ApiKeyValidationError({ cause })),
-          ),
+        validate: cachedValidate.validate,
         listUserKeys: ({ accountId, organizationId }) =>
           workos.listUserApiKeys(accountId, organizationId).pipe(
             Effect.map(listFromResponse),