From e8748aee7a8827c7ae90124b6d0f1b2891c25d49 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:12:24 -0700 Subject: [PATCH 01/22] fix(pi): restore model catalog after settings sync failure Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- agents/pi/generate-config.ts | 29 ++++++++++++-- .../pi-candidate-runtime-artifacts.test.ts | 39 +++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/agents/pi/generate-config.ts b/agents/pi/generate-config.ts index 30e5a98a821..2a3b9a1b927 100644 --- a/agents/pi/generate-config.ts +++ b/agents/pi/generate-config.ts @@ -17,7 +17,7 @@ import { NVIDIA_HOSTED_NATIVE_ENDPOINT, } from "../../src/lib/inference/native-nvidia/contract.ts"; -import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; @@ -211,9 +211,30 @@ function main(): void { const configPath = join(configDir, "models.json"); const config = buildConfig(settings); - writeFileSync(configPath, config.text, { mode: 0o600 }); - chmodSync(configPath, 0o600); - syncManagedDefaults(configDir, config.model, config.nativeLocal); + let previousCatalog: Buffer | null; + try { + previousCatalog = readFileSync(configPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + previousCatalog = null; + } + try { + writeFileSync(configPath, config.text, { mode: 0o600 }); + chmodSync(configPath, 0o600); + syncManagedDefaults(configDir, config.model, config.nativeLocal); + } catch (error) { + try { + if (previousCatalog === null) { + rmSync(configPath, { force: true }); + } else { + writeFileSync(configPath, previousCatalog, { mode: 0o600 }); + chmodSync(configPath, 0o600); + } + } catch (rollbackError) { + throw new AggregateError([error, rollbackError], "Pi catalog rollback failed."); + } + throw error; + } console.log(`[config] Wrote ${configPath} (model=${config.model}, base_url=${config.baseUrl})`); } diff --git a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts index e1a7c410147..fccbb4b45ad 100644 --- a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts +++ b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts @@ -254,12 +254,51 @@ describe("Pi managed model catalog generation", () => { try { expect(status).not.toBe(0); expect(fs.readFileSync(path.join(home, ".pi/agent/settings.json"), "utf8")).toBe(settings); + expect(fs.existsSync(path.join(home, ".pi/agent/models.json"))).toBe(false); } finally { fs.rmSync(home, { recursive: true, force: true }); } }, ); + it("preserves the previous Pi catalog when settings sync fails (#13362)", () => { + const selected = { + NEMOCLAW_MODEL: "previous-model", + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", + NEMOCLAW_INFERENCE_BASE_URL: "http://host.openshell.internal:11434/v1", + }; + const { home, status, stderr } = generate(selected); + try { + expect(status, stderr).toBe(0); + const configDir = path.join(home, ".pi/agent"); + const modelsPath = path.join(configDir, "models.json"); + const settingsPath = path.join(configDir, "settings.json"); + const previousCatalog = fs.readFileSync(modelsPath); + fs.writeFileSync(settingsPath, "{invalid json"); + + const failed = spawnSync( + process.execPath, + [path.join(root, "agents/pi/generate-config.ts")], + { + cwd: root, + encoding: "utf8", + env: { + PATH: process.env.PATH ?? "", + HOME: home, + ...selected, + NEMOCLAW_MODEL: "next-model", + }, + }, + ); + expect(failed.status).not.toBe(0); + expect(fs.readFileSync(modelsPath)).toEqual(previousCatalog); + expect(fs.statSync(modelsPath).mode & 0o777).toBe(0o600); + expect(fs.readFileSync(settingsPath, "utf8")).toBe("{invalid json"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("updates Pi's managed defaults when the native-local model changes and preserves other settings", () => { const selected = { NEMOCLAW_MODEL: "selected-local-model", From 75016202a34b4d9a6bd77f78487d13203e0ff846 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:47:37 -0700 Subject: [PATCH 02/22] chore(pi): refresh candidate qualification receipts Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- ci/pi-agent-qualification-v1-linux-amd64.json | 8 ++++---- ci/pi-agent-qualification-v1-linux-arm64.json | 8 ++++---- src/lib/agent/candidate-authority.ts | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/pi-agent-qualification-v1-linux-amd64.json b/ci/pi-agent-qualification-v1-linux-amd64.json index 62f5f58660f..ceea1cf510f 100644 --- a/ci/pi-agent-qualification-v1-linux-amd64.json +++ b/ci/pi-agent-qualification-v1-linux-amd64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/amd64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:303602a81eb87456a3f653654737b62aea8797128ce31bd38a0a759053c42c34", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:303602a81eb87456a3f653654737b62aea8797128ce31bd38a0a759053c42c34", + "digest": "sha256:756190ee5006cebf9d3029403fc80e8cabe4fe3d45ce00be1cd1acd57d4dcca6", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:756190ee5006cebf9d3029403fc80e8cabe4fe3d45ce00be1cd1acd57d4dcca6", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "46762b985f8527db2ae7cedfcb6c7fc07a7a44bf", + "revision": "e8748aee7a8827c7ae90124b6d0f1b2891c25d49", "release": "v0.1.0", - "cohort": "ghrun-38873268267-1" + "cohort": "ghrun-38955028077-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/ci/pi-agent-qualification-v1-linux-arm64.json b/ci/pi-agent-qualification-v1-linux-arm64.json index ae5ef88027f..f9bcba16d80 100644 --- a/ci/pi-agent-qualification-v1-linux-arm64.json +++ b/ci/pi-agent-qualification-v1-linux-arm64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/arm64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:828075315daa5dd4e555be3b0007a28df5012664637ca805e83e5c9c80823490", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:828075315daa5dd4e555be3b0007a28df5012664637ca805e83e5c9c80823490", + "digest": "sha256:4ac7f16340a836c613a02b3b85b0c0e211805aa8217f9bb23ac118acd59359f3", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:4ac7f16340a836c613a02b3b85b0c0e211805aa8217f9bb23ac118acd59359f3", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "46762b985f8527db2ae7cedfcb6c7fc07a7a44bf", + "revision": "e8748aee7a8827c7ae90124b6d0f1b2891c25d49", "release": "v0.1.0", - "cohort": "ghrun-38873268267-1" + "cohort": "ghrun-38955028077-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/src/lib/agent/candidate-authority.ts b/src/lib/agent/candidate-authority.ts index e5f1a4a8e1d..3c81f0c3079 100644 --- a/src/lib/agent/candidate-authority.ts +++ b/src/lib/agent/candidate-authority.ts @@ -13,8 +13,8 @@ export const CANDIDATE_QUALIFICATION_RECEIPT_DIGESTS: Readonly< Record > = Object.freeze({ pi: Object.freeze([ - "0f9f1f3521e54383e29ee3c6bb700b207bc83b5f893e2ae10737eb311a4352a0", - "4deadd4cb0f3f5d55d0ffd30497b59334853c754aa94fbab8f11145aca8cb91e", + "960e2c7e62210547cb1bf2063b8fabbdc7baa536080943f98976e0ebb0996d96", + "633c42e6fd9b295714125387a25f44f1a651a0102f956c49523c72870b897504", ]), }); From 573eff38da88a43eff993abd8fbea7591f1c7228 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:21:10 -0700 Subject: [PATCH 03/22] chore(pi): refresh combined-source qualification receipts --- ci/pi-agent-qualification-v1-linux-amd64.json | 8 ++++---- ci/pi-agent-qualification-v1-linux-arm64.json | 8 ++++---- src/lib/agent/candidate-authority.ts | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/pi-agent-qualification-v1-linux-amd64.json b/ci/pi-agent-qualification-v1-linux-amd64.json index ceea1cf510f..ab071ab179e 100644 --- a/ci/pi-agent-qualification-v1-linux-amd64.json +++ b/ci/pi-agent-qualification-v1-linux-amd64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/amd64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:756190ee5006cebf9d3029403fc80e8cabe4fe3d45ce00be1cd1acd57d4dcca6", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:756190ee5006cebf9d3029403fc80e8cabe4fe3d45ce00be1cd1acd57d4dcca6", + "digest": "sha256:35683a18dff9233b4465f6de83f7749f6bcd02a284765fc77a822d7188b51a38", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:35683a18dff9233b4465f6de83f7749f6bcd02a284765fc77a822d7188b51a38", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "e8748aee7a8827c7ae90124b6d0f1b2891c25d49", + "revision": "f4bc4efd9e29961624fc82bc9866862af98bb5d2", "release": "v0.1.0", - "cohort": "ghrun-38955028077-1" + "cohort": "ghrun-38991416051-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/ci/pi-agent-qualification-v1-linux-arm64.json b/ci/pi-agent-qualification-v1-linux-arm64.json index f9bcba16d80..4308e7ef690 100644 --- a/ci/pi-agent-qualification-v1-linux-arm64.json +++ b/ci/pi-agent-qualification-v1-linux-arm64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/arm64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:4ac7f16340a836c613a02b3b85b0c0e211805aa8217f9bb23ac118acd59359f3", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:4ac7f16340a836c613a02b3b85b0c0e211805aa8217f9bb23ac118acd59359f3", + "digest": "sha256:46108cfed56bad0ca341147cd7b7f6f1ae410fe7e50be9db7c19c2883bea8bfb", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:46108cfed56bad0ca341147cd7b7f6f1ae410fe7e50be9db7c19c2883bea8bfb", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "e8748aee7a8827c7ae90124b6d0f1b2891c25d49", + "revision": "f4bc4efd9e29961624fc82bc9866862af98bb5d2", "release": "v0.1.0", - "cohort": "ghrun-38955028077-1" + "cohort": "ghrun-38991416051-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/src/lib/agent/candidate-authority.ts b/src/lib/agent/candidate-authority.ts index 3c81f0c3079..58185cbd898 100644 --- a/src/lib/agent/candidate-authority.ts +++ b/src/lib/agent/candidate-authority.ts @@ -13,8 +13,8 @@ export const CANDIDATE_QUALIFICATION_RECEIPT_DIGESTS: Readonly< Record > = Object.freeze({ pi: Object.freeze([ - "960e2c7e62210547cb1bf2063b8fabbdc7baa536080943f98976e0ebb0996d96", - "633c42e6fd9b295714125387a25f44f1a651a0102f956c49523c72870b897504", + "8ff0bc51fa60d58057cc5505d95261c9833b934fcd55d1f900967a66c028090c", + "0a346869512ea536379c2c2ea58209ddffd7b37654a690ba95085544c14d8ecc", ]), }); From 8da11a7952539e8a990f576c7b353ad8c86cacd0 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Sun, 18 Oct 2026 04:55:10 +0100 Subject: [PATCH 04/22] refactor(hermes): share tool gateway profile contract (#13377) ## Outcome Broker registration and direct sandbox creation now share one Hermes tool gateway profile contract. ## Reason The two paths chose the provider type, profile path, and profile import separately, so they could drift. ### Related issues Fixes #13196 ## Changes - Add `hermes-tool-gateway-provider-profile.ts` with the profile type, legacy type, `matchHermesToolGatewayProviderType`, and `importHermesToolGatewayProfile`. - Use it in the broker and in `provider-publication.ts`. Each caller keeps its own cleanup and errors. - Drop the profile type and credential env from the direct-create input. The caller now needs only the provider name. - Tests cover the owned profile, the exact legacy `generic` binding, and each mismatch. ## Verification - `npm run typecheck:cli`: passed - `npx vitest run --project cli src/lib/hermes-tool-gateway src/lib/onboard/sandbox-create`: 307 passed - Broker, connect-flow, and provider-auth tests: 103 passed - Commit and pre-push hooks: passed - No secrets, API keys, or credentials in the diff. --- Signed-off-by: Eric Curtin ## Summary by CodeRabbit * **Bug Fixes** * Improved recognition of existing Hermes tool gateway providers, including supported legacy configurations. * Provider validation now checks that the provider type, name, and credential and configuration bindings match before proceeding. Mismatched configurations are rejected. * **Compatibility** * Existing providers using the supported legacy `generic` type remain recognized when their bindings match. Signed-off-by: Eric Curtin --- src/lib/hermes-tool-gateway-broker.ts | 45 ++---- ...rmes-tool-gateway-provider-profile.test.ts | 37 +++++ .../hermes-tool-gateway-provider-profile.ts | 55 ++++++++ .../onboard/sandbox-create/orchestration.ts | 14 +- .../provider-publication.test.ts | 128 ++++++++++-------- .../sandbox-create/provider-publication.ts | 43 ++---- 6 files changed, 187 insertions(+), 135 deletions(-) create mode 100644 src/lib/hermes-tool-gateway-provider-profile.ts diff --git a/src/lib/hermes-tool-gateway-broker.ts b/src/lib/hermes-tool-gateway-broker.ts index 52dc1d94134..0b2db712e48 100644 --- a/src/lib/hermes-tool-gateway-broker.ts +++ b/src/lib/hermes-tool-gateway-broker.ts @@ -16,8 +16,12 @@ const { getCredsDir } = require("./credentials/store"); const oauth = require("./oauth-device-code"); const onboardProviders = require("./onboard/providers"); const { - matchesGatewayCredentialOnlyProviderBinding, -} = require("./onboard/gateway-provider-metadata"); + HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE, + HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, + LEGACY_HERMES_TOOL_GATEWAY_PROVIDER_TYPE, + importHermesToolGatewayProfile, + matchHermesToolGatewayProviderType, +} = require("./hermes-tool-gateway-provider-profile"); const { createCliOpenShellProviderAdapter } = require("./adapters/openshell/provider-adapter-cli"); const { HERMES_CLONE_CONTROL_CLIENT_TIMEOUT_MS, @@ -28,8 +32,6 @@ const { newControlDeadline, } = require(path.join(ROOT, "agents", "hermes", "host", "tool-gateway-control-contract.ts")); -const HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV = "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN"; -const HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE = "nemoclaw-hermes-tool-gateway-v1"; const HERMES_TOOL_GATEWAY_PORT = 11436; const HERMES_TOOL_GATEWAY_STATE_DIR = path.join(getCredsDir(), "hermes-tool-gateway"); const HERMES_TOOL_GATEWAY_PID_PATH = path.join(getCredsDir(), "hermes-tool-gateway-broker.pid"); @@ -342,16 +344,7 @@ function brokerRuntimeFileHash(file) { async function inspectHermesToolGatewayRefreshProvider(sandboxName, runOpenshell) { const providerName = getHermesToolGatewayProviderName(sandboxName); const adapter = createCliOpenShellProviderAdapter({ run: runOpenshell }); - const profile = await adapter.importProviderProfile({ - target: { kind: "selected" }, - profilePath: path.join( - ROOT, - "agents", - "hermes", - "host", - `${HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE}.yaml`, - ), - }); + const profile = await importHermesToolGatewayProfile(adapter, { kind: "selected" }, ROOT); if (!profile.ok) { if (profile.error.kind === "command" && profile.error.reason === "profile_incompatible") { throw new Error( @@ -366,23 +359,10 @@ async function inspectHermesToolGatewayRefreshProvider(sandboxName, runOpenshell target: { kind: "selected" }, providerName, }); - const providerType = - observed.ok && - matchesGatewayCredentialOnlyProviderBinding(observed.value, { - name: providerName, - type: "generic", - credentialKey: HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, - }) - ? "generic" - : HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE; - if ( - observed.ok && - !matchesGatewayCredentialOnlyProviderBinding(observed.value, { - name: providerName, - type: providerType, - credentialKey: HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, - }) - ) { + const matchedType = observed.ok + ? matchHermesToolGatewayProviderType(observed.value, providerName) + : null; + if (observed.ok && !matchedType) { throw new Error( `Existing provider '${providerName}' does not match the Hermes tool gateway credential binding. Ask the gateway operator to inspect its attachments before retrying.`, ); @@ -393,6 +373,7 @@ async function inspectHermesToolGatewayRefreshProvider(sandboxName, runOpenshell ) { throw new Error(`Could not inspect provider '${providerName}' before Hermes tool setup.`); } + const providerType = matchedType ?? HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE; return { adapter, exists: observed.ok, providerName, providerType }; } @@ -420,7 +401,7 @@ async function registerHermesToolGatewayRefreshProvider(sandboxName, refreshToke { requireExactBinding: true, providerAdapter: adapter, - ...(providerType === "generic" ? { knownExists: true } : {}), + ...(providerType === LEGACY_HERMES_TOOL_GATEWAY_PROVIDER_TYPE ? { knownExists: true } : {}), }, ); if (!result.ok) { diff --git a/src/lib/hermes-tool-gateway-provider-profile.test.ts b/src/lib/hermes-tool-gateway-provider-profile.test.ts index ee0717788d8..bfd76be7cdc 100644 --- a/src/lib/hermes-tool-gateway-provider-profile.test.ts +++ b/src/lib/hermes-tool-gateway-provider-profile.test.ts @@ -9,6 +9,10 @@ import { describe, expect, it, vi } from "vitest"; import YAML from "yaml"; import { REPOSITORY_ROOT } from "./core/repository-root"; +import { + importHermesToolGatewayProfile, + matchHermesToolGatewayProviderType, +} from "./hermes-tool-gateway-provider-profile"; const PROFILE_ID = "nemoclaw-hermes-tool-gateway-v1"; const PROFILE_PATH = path.join(REPOSITORY_ROOT, "agents", "hermes", "host", `${PROFILE_ID}.yaml`); @@ -289,3 +293,36 @@ describe("Hermes tool gateway provider profile", () => { }); }); }); + +describe("Hermes tool gateway provider contract", () => { + const exact = { + name: "alpha-hermes-tool-gateway", + type: PROFILE_ID, + credentialKeys: [CREDENTIAL_ENV], + configKeys: [], + }; + + it.each([ + ["the owned profile", exact, PROFILE_ID], + ["an exact legacy generic provider", { ...exact, type: "generic" }, "generic"], + ["an unrelated type", { ...exact, type: "unrelated" }, null], + ["another provider name", { ...exact, name: "other" }, null], + ["an extra credential key", { ...exact, credentialKeys: [CREDENTIAL_ENV, "OTHER_KEY"] }, null], + ["another credential key", { ...exact, credentialKeys: ["OTHER_KEY"] }, null], + ["a config key", { ...exact, configKeys: ["OTHER_KEY"] }, null], + ])("matches %s", (_case, observed, expected) => { + expect(matchHermesToolGatewayProviderType(observed, "alpha-hermes-tool-gateway")).toBe( + expected, + ); + }); + + it("imports the checked-in profile on the requested gateway", () => { + const importProviderProfile = vi.fn(() => ({ ok: true as const })); + const target = { kind: "named", gatewayName: "nemoclaw" } as const; + + expect( + importHermesToolGatewayProfile({ importProviderProfile }, target, REPOSITORY_ROOT), + ).toEqual({ ok: true }); + expect(importProviderProfile).toHaveBeenCalledWith({ target, profilePath: PROFILE_PATH }); + }); +}); diff --git a/src/lib/hermes-tool-gateway-provider-profile.ts b/src/lib/hermes-tool-gateway-provider-profile.ts new file mode 100644 index 00000000000..ac2bada308e --- /dev/null +++ b/src/lib/hermes-tool-gateway-provider-profile.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import type { OpenShellProviderAdapter } from "./adapters/openshell/provider-adapter"; +import type { OpenShellGatewayTarget } from "./adapters/openshell/sandbox-observer"; +import { + matchesGatewayCredentialOnlyProviderBinding, + type GatewayProviderMetadata, +} from "./onboard/gateway-provider-metadata"; + +export const HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV = + "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN"; +export const HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE = "nemoclaw-hermes-tool-gateway-v1"; +export const LEGACY_HERMES_TOOL_GATEWAY_PROVIDER_TYPE = "generic"; + +/** Import or verify the NemoClaw-owned profile on the given gateway. */ +export function importHermesToolGatewayProfile( + adapter: Pick, + target: OpenShellGatewayTarget, + root: string, +) { + return adapter.importProviderProfile({ + target, + profilePath: path.join( + root, + "agents", + "hermes", + "host", + `${HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE}.yaml`, + ), + }); +} + +/** Return the provider type that holds the exact credential binding, or null if none does. */ +export function matchHermesToolGatewayProviderType( + observed: GatewayProviderMetadata, + providerName: string, +): + | typeof HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE + | typeof LEGACY_HERMES_TOOL_GATEWAY_PROVIDER_TYPE + | null { + return ( + ( + [HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE, LEGACY_HERMES_TOOL_GATEWAY_PROVIDER_TYPE] as const + ).find((type) => + matchesGatewayCredentialOnlyProviderBinding(observed, { + name: providerName, + type, + credentialKey: HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, + }), + ) ?? null + ); +} diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index f07a6afa331..80b75146f1f 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1784,21 +1784,11 @@ function directCreateHermesToolGatewayProfile(input: { readonly providerNames: readonly string[]; readonly getBroker: () => { getHermesToolGatewayProviderName(sandboxName: string): string; - HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE: string; - HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV: string; }; }): { readonly hermesToolGateway?: ProviderPreparationInput["hermesToolGateway"] } { if (!input.enabled) return {}; - const broker = input.getBroker(); - const providerName = broker.getHermesToolGatewayProviderName(input.sandboxName); - if (!input.providerNames.includes(providerName)) return {}; - return { - hermesToolGateway: { - providerName, - profileType: broker.HERMES_TOOL_GATEWAY_PROVIDER_PROFILE_TYPE, - credentialEnv: broker.HERMES_TOOL_GATEWAY_REFRESH_CREDENTIAL_ENV, - }, - }; + const providerName = input.getBroker().getHermesToolGatewayProviderName(input.sandboxName); + return input.providerNames.includes(providerName) ? { hermesToolGateway: { providerName } } : {}; } /** diff --git a/src/lib/onboard/sandbox-create/provider-publication.test.ts b/src/lib/onboard/sandbox-create/provider-publication.test.ts index 5b3612e0fc8..3775199e02f 100644 --- a/src/lib/onboard/sandbox-create/provider-publication.test.ts +++ b/src/lib/onboard/sandbox-create/provider-publication.test.ts @@ -107,6 +107,47 @@ async function prepareProviders( await publishAttachedProvidersBeforeDockerSandboxCreation(input, deps); } +const HERMES_PROFILE_TYPE = "nemoclaw-hermes-tool-gateway-v1"; +const HERMES_CREDENTIAL_ENV = "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN"; +const hermesProviderName = "alpha-hermes-tool-gateway"; + +function hermesProvider( + overrides: { + type?: string; + name?: string; + credentialKeys?: string[]; + configKeys?: string[]; + } = {}, +) { + return vi.fn(async () => ({ + ok: true as const, + value: { + name: hermesProviderName, + type: HERMES_PROFILE_TYPE, + credentialKeys: [HERMES_CREDENTIAL_ENV], + configKeys: [], + ...overrides, + }, + })); +} + +function hermesBoundary(harness: ReturnType) { + return createProviderEffectBoundary({ + deferred: false, + sandboxName: "alpha", + gatewayName: "nemoclaw", + preparationInput: publicationInput({ + messagingProviders: [], + messagingProviderRequests: [], + hermesToolGateway: { providerName: hermesProviderName }, + }), + preparationDeps: harness.deps, + runVerifiedSandboxCreateEffects: null, + activateDeferredProviderEffects: null, + revalidateSandboxIdentityBeforeCreate: vi.fn(), + }); +} + describe("sandbox provider preparation", () => { it.each([ ["brave", true, expect.anything()], @@ -382,44 +423,16 @@ describe("sandbox provider preparation", () => { expect(harness.runOpenshell).not.toHaveBeenCalled(); }); - it.each(["nemoclaw-hermes-tool-gateway-v1", "generic"])( + it.each([HERMES_PROFILE_TYPE, "generic"])( "preserves the %s Hermes tool gateway binding on direct create", async (type) => { const warningCheck = vi.fn(); - const providerName = "alpha-hermes-tool-gateway"; const adapter = Object.assign( - typedProviderAdapter({ - getProvider: vi.fn(async () => ({ - ok: true as const, - value: { - name: providerName, - type, - credentialKeys: ["NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN"], - configKeys: [], - }, - })), - }), + typedProviderAdapter({ getProvider: hermesProvider({ type }) }), { warnIfVerifiedProfilesChangedAfterSandboxCreate: warningCheck }, ); const harness = createHarness(adapter); - const boundary = createProviderEffectBoundary({ - deferred: false, - sandboxName: "alpha", - gatewayName: "nemoclaw", - preparationInput: publicationInput({ - messagingProviders: [], - messagingProviderRequests: [], - hermesToolGateway: { - providerName, - profileType: "nemoclaw-hermes-tool-gateway-v1", - credentialEnv: "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN", - }, - }), - preparationDeps: harness.deps, - runVerifiedSandboxCreateEffects: null, - activateDeferredProviderEffects: null, - revalidateSandboxIdentityBeforeCreate: vi.fn(), - }); + const boundary = hermesBoundary(harness); await boundary.validateBeforeCreate(); expect(adapter.importProviderProfile).toHaveBeenCalledTimes(type === "generic" ? 0 : 1); @@ -440,17 +453,8 @@ describe("sandbox provider preparation", () => { ); it("blocks a changed Hermes tool gateway profile before sandbox creation", async () => { - const providerName = "alpha-hermes-tool-gateway"; const adapter = typedProviderAdapter({ - getProvider: vi.fn(async () => ({ - ok: true as const, - value: { - name: providerName, - type: "nemoclaw-hermes-tool-gateway-v1", - credentialKeys: ["NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN"], - configKeys: [], - }, - })), + getProvider: hermesProvider(), importProviderProfile: vi.fn(async () => ({ ok: false as const, error: { @@ -461,32 +465,36 @@ describe("sandbox provider preparation", () => { })), }); const harness = createHarness(adapter); - const boundary = createProviderEffectBoundary({ - deferred: false, - sandboxName: "alpha", - gatewayName: "nemoclaw", - preparationInput: publicationInput({ - messagingProviders: [], - messagingProviderRequests: [], - hermesToolGateway: { - providerName, - profileType: "nemoclaw-hermes-tool-gateway-v1", - credentialEnv: "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN", - }, - }), - preparationDeps: harness.deps, - runVerifiedSandboxCreateEffects: null, - activateDeferredProviderEffects: null, - revalidateSandboxIdentityBeforeCreate: vi.fn(), - }); - await expect(boundary.validateBeforeCreate()).rejects.toThrow( + await expect(hermesBoundary(harness).validateBeforeCreate()).rejects.toThrow( "Could not verify Hermes tool gateway profile before sandbox creation", ); expect(harness.cleanupCreateSources).toHaveBeenCalledOnce(); expect(adapter.updateProvider).not.toHaveBeenCalled(); }); + it.each([ + ["unrelated provider type", { type: "unrelated" }], + ["legacy provider name", { type: "generic", name: "other" }], + [ + "legacy credential keys", + { type: "generic", credentialKeys: [HERMES_CREDENTIAL_ENV, "OTHER_KEY"] }, + ], + ["legacy config keys", { type: "generic", configKeys: ["OTHER_KEY"] }], + ["owned credential key", { credentialKeys: ["OTHER_KEY"] }], + ])("stops on %s before sandbox or provider mutation", async (_case, overrides) => { + const adapter = typedProviderAdapter({ getProvider: hermesProvider(overrides) }); + const harness = createHarness(adapter); + + await expect(hermesBoundary(harness).validateBeforeCreate()).rejects.toThrow( + "Hermes tool gateway provider binding changed before sandbox creation", + ); + expect(harness.cleanupCreateSources).toHaveBeenCalledOnce(); + expect(adapter.importProviderProfile).not.toHaveBeenCalled(); + expect(adapter.createProvider).not.toHaveBeenCalled(); + expect(adapter.updateProvider).not.toHaveBeenCalled(); + }); + it("checks one verified messaging profile after direct sandbox creation", async () => { const warningCheck = vi.fn(); const adapter = Object.assign(typedProviderAdapter(), { diff --git a/src/lib/onboard/sandbox-create/provider-publication.ts b/src/lib/onboard/sandbox-create/provider-publication.ts index 5487f81848f..1aa166c9296 100644 --- a/src/lib/onboard/sandbox-create/provider-publication.ts +++ b/src/lib/onboard/sandbox-create/provider-publication.ts @@ -1,8 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import path from "node:path"; - +import { + importHermesToolGatewayProfile, + LEGACY_HERMES_TOOL_GATEWAY_PROVIDER_TYPE, + matchHermesToolGatewayProviderType, +} from "../../hermes-tool-gateway-provider-profile"; import { ensureNativeLocalProviderAttached } from "../../inference/native-local/profile"; import { isNativeHostedProviderName } from "../../inference/native-provider/hosted-attachment"; import { hostedNativeProviderForAttachment } from "../../inference/native-provider/hosted-attachment"; @@ -36,10 +39,7 @@ import { } from "../../messaging/provider-profile"; import { listMessagingBridgeProfiles } from "../messaging-bridge-provider"; import type { SandboxEntry } from "../../state/registry"; -import { - matchesGatewayCredentialFamilyProviderBinding, - matchesGatewayCredentialOnlyProviderBinding, -} from "../gateway-provider-metadata"; +import { matchesGatewayCredentialFamilyProviderBinding } from "../gateway-provider-metadata"; import { resolveRegisteredRuntimeProvider } from "../runtime-provider/selection"; import type { SandboxCreateIntent } from "../sandbox-create-intent-types"; @@ -52,11 +52,7 @@ type ProviderPreparationInput = { readonly extraProviders: readonly string[]; readonly gatewayName: string; readonly directCreateProviderNames?: readonly string[]; - readonly hermesToolGateway?: Readonly<{ - providerName: string; - profileType: string; - credentialEnv: string; - }>; + readonly hermesToolGateway?: Readonly<{ providerName: string }>; }; type ProviderPreparationDeps = Pick & { @@ -208,37 +204,22 @@ export async function validateAttachedHermesToolGatewayProfileBeforeSandboxCreat `Could not inspect Hermes tool gateway provider before sandbox creation.`, ); } - if ( - ![expected.profileType, "generic"].includes(observed.value.type) || - !matchesGatewayCredentialOnlyProviderBinding(observed.value, { - name: expected.providerName, - type: observed.value.type, - credentialKey: expected.credentialEnv, - }) - ) { + const providerType = matchHermesToolGatewayProviderType(observed.value, expected.providerName); + if (!providerType) { throwAfterCleanup( deps, `Hermes tool gateway provider binding changed before sandbox creation.`, ); } - if (observed.value.type === "generic") return null; - const profile = await adapter.importProviderProfile({ - target, - profilePath: path.join( - REPOSITORY_ROOT, - "agents", - "hermes", - "host", - `${expected.profileType}.yaml`, - ), - }); + if (providerType === LEGACY_HERMES_TOOL_GATEWAY_PROVIDER_TYPE) return null; + const profile = await importHermesToolGatewayProfile(adapter, target, REPOSITORY_ROOT); if (!profile.ok) { throwAfterCleanup( deps, `Could not verify Hermes tool gateway profile before sandbox creation.`, ); } - return expected.profileType; + return providerType; } export function resolveProviderAdapter( From 762aef4c82dec358e307a771d2b66da352a904e9 Mon Sep 17 00:00:00 2001 From: "Deepak Jain (DJ)" Date: Sun, 18 Oct 2026 06:26:30 -0700 Subject: [PATCH 05/22] test(e2e): wait for approved cron completion (#13355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Outcome The OpenClaw approval fixture requires successful completion of its cron run before the next agent request. Failures retain bounded backend diagnostics and allowlisted cron reason/stage categories and fixed error-text signatures without exposing private output. Diagnostic failures preserve the original exit code. GPU qualification selection now fails when changed-file discovery is unsuccessful, malformed or incomplete. ## Reason The failures in #13342 accepted enqueue-only cron output before another agent request returned HTTP 503. Those logs do not establish causality. Completion checks close the sequencing gap; structured diagnostics preserve useful evidence when a later cron run fails. ### Related issues Refs #13342. Part of #13338. ## Changes - Use pinned OpenClaw 2026.9.2's `--wait` and `--no-deliver`. Require agreeing successful envelope and nested fields bound to the created job and returned run. The approval helper tests reject enqueue-only, contradictory, malformed and mismatched responses. - Capture llama.cpp setup, post-agent and failure logs through the existing engine and redacting artifact writer. Preserve the original exception when capture or storage fails. The diagnostics tests cover these boundaries; the workflow selector test covers the fixture's path. - After verifying terminal status and job/run identity, report fixed cron categories, 16 allowlisted error reasons, and deduplicated source/severity pairs from at most 10 diagnostic entries. Unknown fields produce `unavailable`; raw messages and identifiers are excluded. The existing emitter owns this projection so private output does not need a second capture path. - Observe six exact upstream error messages and anchored HTTP 400–599 error-text forms from the bound run error and at most ten validated error diagnostics. Output only finite fixed labels; exclude raw text, identifiers and tool names. These labels do not establish provider or tool causality. - Preserve exit 29 when diagnostic projection itself fails. Actual-shell tests cover this failure, privacy sentinels, the 64 KiB capture bound, producer draining, private-file removal and unchanged cleanup failure behavior. - Fail GPU qualification selection closed on GitHub API failure, malformed page/file data or a changed-file count mismatch. Apply the existing path predicate only after complete validated discovery. Actual-shell regressions require nonzero exit and no job outputs for rejected discovery. Approval limits, deadlines, inference retries, provider ownership and security assertions are unchanged. Mock/live ownership maps the fixtures to their executing tests. ## Verification Commit under evaluation: `5873288f4ea8a35ab9bdcfcb47e88468b0c7c7f0` (repair 7). - GPU selector: 77 passing local cases, including 22 new cases, 51 existing cases strengthened through shared process assertions, and four unchanged structural cases. Nineteen new rejection cases fail against predecessor `6849399` and pass with the discovery repair. They cover API failures, malformed page/file data, missing or invalid authoritative counts, partial results and empty results conflicting with a positive count. Valid empty and multipage selection also pass. - These tests execute real Bash and jq with simulated GitHub responses. They do not qualify the live GPU cron path. - Targeted lint, formatting and source-shape checks passed. Two source-shape guard failures and the first commit hook rejection for conditional growth are retained. The final harness asserts process status, signal and output directly, without adding a conditional branch or policy exception. An intermediate malformed-JSON status expectation failed; the corrected case accepts jq parse-error codes 4 or 5, both nonzero, and all 77 cases pass. No assertion, test budget or runtime bound was weakened. - Canonical TypeScript and normal signed commit/publication validation: passed; normal signed commit and pre-push hooks completed without bypass. The committed patch matches the independently reviewed diff. - Current-head [hosted CI](https://github.com/NVIDIA/NemoClaw/actions/runs/39065837343) passed, including all 12 shards and aggregate. All nine [Advisor reports](https://github.com/NVIDIA/NemoClaw/actions/runs/39067424690) were authenticated and read; all are clear and confirm the selector finding is resolved. - [Automatic runtime qualification](https://github.com/NVIDIA/NemoClaw/actions/runs/39065838891) passed GPU, Hermes, image security/glibc, ports and nonroot, with all artifacts audited. GPU strict approval/cron completed in 29.459 seconds, followed by a real agent turn in 7.424 seconds, full GPU offload and owned cleanup/state removal/cache preservation. The GPU lane used verified base image revision `64ca9329`, separate from the current activation cohort. - [Selected CPU qualification](https://github.com/NVIDIA/NemoClaw/actions/runs/39068653459) passed cloud-onboard, full-e2e, security-openclaw, security-hermes and scope-upgrade; all five artifacts and the aggregate receipt were authenticated and audited. Scope strict approval/cron completed in 39.915 seconds. Full E2E produced its real sentinel turn in 5.454 seconds; Hermes passed seven ACP cases. CPU images used authenticated cohort `ghrun-38985251532-1`, revision `1e82499`. Full E2E and OpenClaw security use `verifyCron=false`; their passes are not credited as cron completion. Scope has no subsequent-agent assertion; the separate GPU lane supplies that boundary. - [Managed image activation](https://github.com/NVIDIA/NemoClaw/actions/runs/39065837403) passed AMD64, ARM64 and rootless Podman, with exact `5873288` contracts and completed artifact/assertion/cleanup audits. Each lane passed 13 phases and 28 authenticated inference requests. Agents and containers are real; inference responses are synthetic. - Limits remain explicit: full E2E passed its performance policy with a nonblocking first-turn tail of 15.908 seconds versus 14 seconds; Podman runner orphan termination and best-effort service shutdown remain recorded. Cleanup claims cover owned resources, not universal host process absence. Private raw cron/ACP receipts are intentionally unretained. Current passes do not establish the causes of historical failures or resolve the outstanding human-review disposition. No unchanged retry or retry-budget reset was performed. ### Predecessor 6849399 evidence Repair 6 qualification at `6849399`: [CI](https://github.com/NVIDIA/NemoClaw/actions/runs/39062023197) passed. All nine [Advisor reports](https://github.com/NVIDIA/NemoClaw/actions/runs/39063532948) were authenticated and read; eight are clear and the security specialist identified incomplete changed-file discovery in the GPU selector. Repair 7 addresses that finding with actual-shell regression tests. CodeRabbit completed without an actionable finding. [Automatic runtime qualification](https://github.com/NVIDIA/NemoClaw/actions/runs/39062026295) failed at terminal cron completion. The approval command exited 29 after 35.597 seconds without an outer timeout or signal. Its bound diagnostic reason and error-text signature are unavailable, with tool:error and agent-run:error stages. This does not establish an underlying cause. The first agent request and full GPU offload passed; the subsequent agent request and final success-path absence checks were unreached. The failure artifact was authenticated and audited, including the complete job log, current embedded emitter and cron predicate, and registered cleanup within its existing limits. Hermes, image security/glibc, ports and nonroot lanes passed with completed artifact audits. [Managed image qualification](https://github.com/NVIDIA/NemoClaw/actions/runs/39062023067) passed AMD64 and ARM64 activation with completed artifact/assertion/cleanup audits; Podman also passed with its artifact/assertion/cleanup audit complete. These activations use real agents/runtime with synthetic inference responses. Manual CPU qualification of repair 6 was not dispatched because its validated selector finding required repair. Repair 7 qualification is recorded above. Earlier passing evidence does not qualify a later commit. ### Predecessor dd90af4 evidence Predecessor [CI](https://github.com/NVIDIA/NemoClaw/actions/runs/38999646062) and all nine [Advisor reports](https://github.com/NVIDIA/NemoClaw/actions/runs/39000943212) passed; the reports were authenticated and audited. [Managed image qualification](https://github.com/NVIDIA/NemoClaw/actions/runs/38999646076) passed AMD64, ARM64 and Podman activation with completed artifact/assertion/cleanup audits. These activations use real agents/runtime with synthetic inference responses. [Automatic runtime qualification](https://github.com/NVIDIA/NemoClaw/actions/runs/38999645912) failed at the GPU terminal cron check: exit 29 after a 26.027-second approval command, reason unavailable, tool:error and agent-run:error. Hermes, image security/glibc, ports and nonroot lanes passed and were audited. [Manual CPU qualification](https://github.com/NVIDIA/NemoClaw/actions/runs/39003096803) passed cloud-onboard, full-e2e, security-openclaw and security-hermes, and failed scope-upgrade with a bound terminal cron error classified timeout after a 47.394-second approval command. All five artifacts and the failed aggregate were authenticated and audited. CPU jobs used base-cohort images at revision 1e82499; full-e2e and security-openclaw had verifyCron=false and supply no cron-completion evidence. The timeout category includes some HTTP, network and provider/internal errors; it does not establish timer expiration. Neither failure has an attributed underlying cause, and their relationship is unknown. Registered cleanup passed within its documented limits; unreached success-path assertions are not passes. Existing responses preserve the [GPU evidence](https://github.com/NVIDIA/NemoClaw/pull/13355#issuecomment-6229299445) and [CPU evidence](https://github.com/NVIDIA/NemoClaw/pull/13355#issuecomment-6229860089). ## Review notes Repair 7 changes only `.github/workflows/pr-self-hosted.yaml` and its selector test. Independent parent review read the complete final patch, verified the selection predicate is unchanged, and checked the regression evidence. Reviewed patch SHA-256: `43f18f926cd3fead9325784f1adbf0d62da61646f3f1a098d4e31a1696b3d826`. The workflow is a sensitive path. This review provides pre-publication context; it does not grant maintainer approval. Repair 6 added bounded error-text signatures in the approval fixture and tests, with 124 distinct local passes (34 new, 36 modified existing) and 33 selected red controls against `dd90af4`. Its original 25 expectation failures, one unchanged 15-second timeout and raw default-heap TypeScript failure remain recorded; corrected expectations, affected tests and the canonical compiler wrapper passed. That diagnostic change did not establish a live root cause. Earlier evidence remains separate from qualification of repair 7: - At `890fff6`, [manual qualification](https://github.com/NVIDIA/NemoClaw/actions/runs/38965310220) passed six runtime jobs; CI, nine Advisor specialists and three image activations also passed and were audited. - The [automatic GPU first attempt](https://github.com/NVIDIA/NemoClaw/actions/runs/38970681640/job/117208564327) then failed with a bound terminal cron error after 28.781 seconds. Registered cleanup passed, but final success-path absence checks were unreached. Its private detailed error was not retained; the cause remains unknown. - The single permitted [failed-job retry](https://github.com/NVIDIA/NemoClaw/actions/runs/38970681640/job/117215147301) passed on unchanged `890fff6`. Approval completed in 99.046 seconds, then a real agent request in 4.142 seconds. Authenticated artifacts support strict completion, full GPU offload, owned resource removal and cache preservation. This pass does not explain the failure; that retry allowance remains consumed. - Earlier `bac2941` cron failure and historical HTTP 503 causes also remain unknown. The [review response](https://github.com/NVIDIA/NemoClaw/pull/13355#issuecomment-6226498140) corrects the cited baseline's identity: run 38953387248 executed earlier candidate `5cbc857`, not an independent base execution. Repair 6 addresses a demonstrated observation gap: different upstream error texts previously collapsed to identical safe diagnostic output. It cannot retroactively attribute a deleted response. Rebecca's evidence concern remains unresolved; no waiver, approval or merge readiness is claimed. The PR remains non-draft as explicitly directed by the author. Existing failed attempts and the consumed retry allowance remain recorded; a new diagnostic observation is not an unchanged retry. --- Signed-off-by: Deepak Jain ## Summary by CodeRabbit * **Bug Fixes** * Administrative cron approvals now require a completed successful run with matching job and run details; incomplete or inconsistent results are rejected. * Cron failures now provide clearer diagnostics, including wait timeouts, while keeping sensitive values out of reported output. * **Tests** * Added coverage for bounded, redacted llama.cpp runtime logs during setup and agent failures, including cases where diagnostic capture fails. * Updated automated checks for approval failures and GPU test-job selection. --------- Signed-off-by: Deepak Jain --- .github/workflows/pr-self-hosted.yaml | 19 +- test/e2e/fixtures/admin-approval-connect.sh | 109 +++- .../fixtures/llama-cpp-runtime-diagnostics.ts | 43 ++ test/e2e/live/llama-cpp-generic-gpu.test.ts | 76 ++- test/e2e/mock-parity.json | 41 +- .../llama-cpp-generic-gpu-diagnostics.test.ts | 124 ++++ .../pr-self-hosted-llama-selector.test.ts | 191 +++++- test/security/admin-approval-helper.test.ts | 584 +++++++++++++++++- 8 files changed, 1107 insertions(+), 80 deletions(-) create mode 100644 test/e2e/fixtures/llama-cpp-runtime-diagnostics.ts create mode 100644 test/e2e/support/llama-cpp-generic-gpu-diagnostics.test.ts diff --git a/.github/workflows/pr-self-hosted.yaml b/.github/workflows/pr-self-hosted.yaml index d0c3d846083..5486bd8c549 100644 --- a/.github/workflows/pr-self-hosted.yaml +++ b/.github/workflows/pr-self-hosted.yaml @@ -141,9 +141,21 @@ jobs: echo "::error::Copied PR branch SHA does not match the current PR head" >&2 exit 1 } - if gh api --paginate --slurp \ + changed_files="$(gh api --paginate --slurp \ "repos/$GITHUB_REPOSITORY/pulls/$pr_number/files?per_page=100" \ - | jq -e ' + )" + jq -e ' + type == "array" + and all(.[]; type == "array") + and all(.[][]; type == "object" and (.filename | type == "string")) + ' <<<"$changed_files" >/dev/null + expected_file_count="$(jq -er '.changed_files | select(type == "number" and floor == . and . >= 0)' <<<"$pr_json")" + actual_file_count="$(jq -er 'flatten | length' <<<"$changed_files")" + [[ "$actual_file_count" -eq "$expected_file_count" ]] || { + echo "::error::Changed-file response is incomplete ($actual_file_count of $expected_file_count files)" >&2 + exit 1 + } + if jq -e ' flatten | any( .filename == ".github/workflows/e2e.yaml" @@ -161,6 +173,7 @@ jobs: or .filename == "test/e2e/lib/issue-4462-admin-request-selector.py" or .filename == "test/e2e/live/llama-cpp-generic-gpu.test.ts" or .filename == "test/e2e/live/gpu-e2e-helpers.ts" + or .filename == "test/e2e/fixtures/llama-cpp-runtime-diagnostics.ts" or .filename == "test/e2e/mock-parity.json" or .filename == "tools/e2e/cli-artifact-workflow-boundary.mts" or .filename == "tools/e2e/workflow-boundary.mts" @@ -173,7 +186,7 @@ jobs: or .filename == "src/lib/onboard/preflight.ts" or (.filename | startswith("src/lib/onboard/runtime-provider/docker-llama-cpp")) ) - ' >/dev/null; then + ' <<<"$changed_files" >/dev/null; then selected=true else selected=false diff --git a/test/e2e/fixtures/admin-approval-connect.sh b/test/e2e/fixtures/admin-approval-connect.sh index fb93dc06eb7..b685e237f65 100644 --- a/test/e2e/fixtures/admin-approval-connect.sh +++ b/test/e2e/fixtures/admin-approval-connect.sh @@ -27,14 +27,93 @@ cron_name=__NEMOCLAW_ADMIN_CRON_NAME__ expected_request_id=__NEMOCLAW_ADMIN_EXPECTED_REQUEST_ID__ verify_cron=__NEMOCLAW_ADMIN_VERIFY_CRON__ emit_admin_diagnostic() { - python3 - "$1" "${2:-command}" <<'PY_ADMIN_DIAGNOSTIC' -import re, sys + python3 - "$1" "${2:-command}" "${3:-}" <<'PY_ADMIN_DIAGNOSTIC' +import json, re, sys from pathlib import Path try: with Path(sys.argv[1]).open('rb') as stream: raw=stream.read(65536).decode('utf-8', errors='replace') except FileNotFoundError: raw='' # Emit only fixed categories, never device records, identities, tokens or traceback text. +if sys.argv[2] == 'cron-run': + terminal_labels={ + ('error', 'failed'): 'cron-run-error', + ('skipped', 'failed'): 'cron-run-skipped', + ('ok', 'failed'): 'cron-completion-failed', + ('ok', 'unknown'): 'cron-completion-unknown', + } + decoder=json.JSONDecoder() + for index, char in enumerate(raw): + if char != '{': continue + try: value,_=decoder.raw_decode(raw[index:]) + except (ValueError, RecursionError): continue + if not isinstance(value, dict) or value.get('ok') is not True or value.get('completed') is not True: continue + run_id=value.get('runId') + run=value.get('run') + status=value.get('status') + completion=value.get('completionStatus') + if not isinstance(run_id, str) or not run_id.strip() or not isinstance(run, dict): continue + if not isinstance(status, str) or not isinstance(completion, str): continue + if run.get('jobId') != sys.argv[3] or run.get('runId') != run_id: continue + if run.get('status') != status or run.get('completionStatus') != completion: continue + label=terminal_labels.get((status, completion)) + if label: + print(f'ADMIN_DIAGNOSTIC={label}', file=sys.stderr) + # Pinned cron metadata has closed vocabularies; never export free-text errors. + reasons=( + 'auth', 'auth_permanent', 'format', 'rate_limit', 'overloaded', 'billing', + 'server_error', 'timeout', 'tls_certificate', 'context_overflow', + 'model_not_found', 'session_expired', 'empty_response', 'no_error_details', + 'unclassified', 'unknown', + ) + reason=run.get('errorReason') + reason=reason if isinstance(reason, str) and reason in reasons else 'unavailable' + print(f'ADMIN_CRON_ERROR_REASON={reason}', file=sys.stderr) + diagnostics=run.get('diagnostics') + entries=diagnostics.get('entries') if isinstance(diagnostics, dict) else None + sources=('cron-preflight', 'cron-setup', 'model-preflight', 'agent-run', 'tool', 'exec', 'delivery') + # Observe fixed error-text signatures, not provider or tool causality. + messages={ + 'LLM request timed out.': 'message-llm-timeout', + 'LLM request failed.': 'message-llm-failed', + 'LLM request failed with an unknown error.': 'message-llm-unknown', + 'LLM request failed: provider returned an internal error.': 'message-llm-provider-internal', + 'The selected model was not found by the provider. Check the model id or choose a different model.': 'message-model-not-found', + 'Code Mode could not resolve a configured MCP tool.': 'message-code-mode-catalog-miss', + } + http_signatures={str(code): f'error-text-http-{code}' for code in range(400, 600)} + # Anchored subset of the pinned assistant-error-format HTTP grammar. + http_patterns=( + r'(?:http\s*)?([45][0-9]{2})(?:(?:\s*:\s*|\s+)[\s\S]+)?', + r'api\s*error\s*\(([45][0-9]{2})\)(?:\s*:\s*[\s\S]*)?', + r'(?:status code|unexpected status|http status)\s*[:=]?\s*([45][0-9]{2})\b(?:\s*[:,]?\s*(?:message\s*:\s*)?[\s\S]*)?', + ) + def error_signature(message): + if not isinstance(message, str): return None + if message in messages: return messages[message] + text=re.sub(r'^error\s*[:,]\s*', '', message.strip(), count=1, flags=re.IGNORECASE | re.ASCII) + for pattern in http_patterns: + match=re.fullmatch(pattern, text, flags=re.IGNORECASE | re.ASCII) + if match: return http_signatures.get(match[1]) + return None + signature=error_signature(run.get('error')) + signatures=[f'run:{signature}'] if signature else [] + pairs=[] + for entry in entries[:10] if isinstance(entries, list) else []: + if not isinstance(entry, dict): continue + source=entry.get('source') + severity=entry.get('severity') + if not isinstance(source, str) or source not in sources: continue + if not isinstance(severity, str) or severity not in ('info', 'warn', 'error'): continue + pair=f'{source}:{severity}' + if pair not in pairs: pairs.append(pair) + signature=error_signature(entry.get('message')) if severity == 'error' else None + if signature: + observation=f'{source}:{signature}' + if observation not in signatures: signatures.append(observation) + print('ADMIN_CRON_DIAGNOSTIC_SOURCES=' + (','.join(pairs) or 'unavailable'), file=sys.stderr) + print('ADMIN_CRON_ERROR_SIGNATURES=' + (','.join(signatures) or 'unavailable'), file=sys.stderr) + raise SystemExit(0) selection_checks=( ('selection-state-read', r'canonical pairing-state|canonical device state|canonical primary device identity|FileNotFoundError|PermissionError|sqlite3\.'), ('selection-local-identity', r'local CLI identity'), @@ -51,6 +130,7 @@ checks=( ('approval-not-pending', r'No pending device request matches'), ('self-approval-rejected', r"This device can't approve its own scope upgrade"), ('approval-token-handoff-failed', r'Admin approval completed, but its token handoff'), + ('cron-wait-timeout', r'timed out waiting for cron run '), ('timeout', r'timed?\s*out|timeout'), ('pairing-required', r'device pairing|required.*pairing|pairing required'), ('authorization-rejected', r'denied|forbidden|unauthorized|approval.*(?:failed|rejected)'), @@ -126,7 +206,7 @@ if [ "$verify_cron" = 0 ]; then echo "ISSUE_5324_ADMIN_APPROVAL_OK" exit fi -if ! run_with_bounded_output "$cron_output" openclaw cron add --name "$cron_name" --every 2h --agent main --session isolated --message "hello"; then +if ! run_with_bounded_output "$cron_output" openclaw cron add --name "$cron_name" --every 2h --agent main --session isolated --message "hello" --no-deliver; then echo "ADMIN_CRON_RETRY_FAILED" >&2 emit_admin_diagnostic "$cron_output" exit 28 @@ -157,12 +237,13 @@ cron_id="$(cat "$cron_id_file")" exit 28 } echo "ISSUE_5324_STAGE=cron-run" -if ! run_with_bounded_output "$cron_run_output" openclaw cron run "$cron_id"; then +# Keep the completion wait inside the caller's four-minute approval budget. +if ! run_with_bounded_output "$cron_run_output" openclaw cron run "$cron_id" --wait --wait-timeout 2m --timeout 10000; then echo "ADMIN_CRON_RUN_FAILED" >&2 - emit_admin_diagnostic "$cron_run_output" + emit_admin_diagnostic "$cron_run_output" cron-run "$cron_id" || echo "ADMIN_DIAGNOSTIC=unavailable" >&2 exit 29 fi -if ! python3 - "$cron_run_output" <<'PY_CRON_RUN'; then +if ! python3 - "$cron_run_output" "$cron_id" <<'PY_CRON_RUN'; then import json, sys from pathlib import Path raw=Path(sys.argv[1]).read_text(encoding='utf-8') @@ -172,9 +253,19 @@ for index, char in enumerate(raw): try: value,_=decoder.raw_decode(raw[index:]) except Exception: continue if not isinstance(value, dict) or value.get('ok') is not True: continue - if value.get('ran') is True: raise SystemExit(0) - if value.get('enqueued') is True and str(value.get('runId') or '').strip(): raise SystemExit(0) -raise SystemExit('cron run did not report a successful run or enqueue') + run_id=value.get('runId') + run=value.get('run') + if not isinstance(run_id, str) or not run_id.strip() or not isinstance(run, dict): continue + if ( + value.get('completed') is True + and value.get('status') == 'ok' + and value.get('completionStatus') == 'succeeded' + and run.get('status') == 'ok' + and run.get('completionStatus') == 'succeeded' + and run.get('jobId') == sys.argv[2] + and run.get('runId') == run_id + ): raise SystemExit(0) +raise SystemExit('cron run did not report successful completion for its job and run') PY_CRON_RUN echo "ADMIN_CRON_RUN_RESULT_INVALID" >&2 exit 30 diff --git a/test/e2e/fixtures/llama-cpp-runtime-diagnostics.ts b/test/e2e/fixtures/llama-cpp-runtime-diagnostics.ts new file mode 100644 index 00000000000..2e2bfa24547 --- /dev/null +++ b/test/e2e/fixtures/llama-cpp-runtime-diagnostics.ts @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ContainerEngine } from "../../../src/lib/adapters/container-engine.ts"; +import type { ArtifactSink } from "./artifacts.ts"; + +export async function captureManagedLlamaCppRuntimeLogs( + engine: Pick, + artifacts: Pick, + runtime: { providerId: string; runtimeId: string }, + phase: "setup" | "post-agent" | "failure", +): Promise { + const logs = engine.capture(["container", "logs", "--tail", "20000", runtime.runtimeId], 30_000); + await artifacts.writeJson(`managed-runtime-logs-${phase}.json`, { + ...runtime, + status: logs.status, + error: logs.error?.message ?? null, + stdout: logs.stdout, + stderr: logs.stderr, + }); +} + +/** Keep the owned backend's failure evidence before registered teardown removes it. */ +export async function withManagedLlamaCppFailureDiagnostics( + run: () => Promise, + engine: Pick, + artifacts: Pick, + runtime: { providerId: string; runtimeId: string }, +): Promise { + let completed = false; + try { + await run(); + completed = true; + } finally { + if (!completed) { + try { + await captureManagedLlamaCppRuntimeLogs(engine, artifacts, runtime, "failure"); + } catch { + // Diagnostic transport or storage failure must preserve the original assertion. + } + } + } +} diff --git a/test/e2e/live/llama-cpp-generic-gpu.test.ts b/test/e2e/live/llama-cpp-generic-gpu.test.ts index 51980d766e1..f794f28916b 100644 --- a/test/e2e/live/llama-cpp-generic-gpu.test.ts +++ b/test/e2e/live/llama-cpp-generic-gpu.test.ts @@ -23,6 +23,10 @@ import { trustedSandboxShellScript, validateSandboxName } from "../fixtures/clie import { expect, test } from "../fixtures/e2e-test.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import { assertAgentExecutionSucceeded, hasExactReadyPhase } from "./gpu-e2e-helpers.ts"; +import { + captureManagedLlamaCppRuntimeLogs, + withManagedLlamaCppFailureDiagnostics, +} from "../fixtures/llama-cpp-runtime-diagnostics.ts"; import { approveOpenClawAdminScope } from "./openclaw-admin-scope.ts"; const TIMEOUT_MS = 110 * 60_000; @@ -172,21 +176,13 @@ test( const runtimeId = receipt.runtime.runtimeId; const runtimeOperation = runtimeProvider.hostLocalInference.createOperation({ env: env() }); runtimeOperation.assertAuthority(); - const captureManagedRuntimeLogs = async (phase: "setup" | "post-agent") => { - const runtimeLogs = runtimeOperation.engine.capture( - ["container", "logs", "--tail", "20000", runtimeId], - 30_000, - ); - await artifacts.writeJson(`managed-runtime-logs-${phase}.json`, { - providerId: receipt.providerId, - runtimeId, - status: runtimeLogs.status, - error: runtimeLogs.error?.message ?? null, - stdout: runtimeLogs.stdout, - stderr: runtimeLogs.stderr, - }); - }; - await captureManagedRuntimeLogs("setup"); + const runtimeIdentity = { providerId: receipt.providerId, runtimeId }; + await captureManagedLlamaCppRuntimeLogs( + runtimeOperation.engine, + artifacts, + runtimeIdentity, + "setup", + ); const runtimeInspection = runtimeOperation.engine.capture( ["container", "inspect", receipt.runtime.runtimeId], 30_000, @@ -340,26 +336,38 @@ NODE`), "-m", "Respond with a short greeting.", ]; - const preApprovalAgent = await host.nemoclaw(agentArgs, { - artifactName: "openclaw-agent-before-admin-approval", - env: env(), - timeoutMs: 12 * 60_000, - }); - expect(preApprovalAgent.exitCode, resultText(preApprovalAgent)).toBe(0); - assertAgentExecutionSucceeded( - preApprovalAgent.stdout, - "inference", - recipe.spec.model.servedName, + await withManagedLlamaCppFailureDiagnostics( + async () => { + const preApprovalAgent = await host.nemoclaw(agentArgs, { + artifactName: "openclaw-agent-before-admin-approval", + env: env(), + timeoutMs: 12 * 60_000, + }); + expect(preApprovalAgent.exitCode, resultText(preApprovalAgent)).toBe(0); + assertAgentExecutionSucceeded( + preApprovalAgent.stdout, + "inference", + recipe.spec.model.servedName, + ); + await approveOpenClawAdminScope(host, sandbox, SANDBOX_NAME, env(), [apiKey]); + const agent = await host.nemoclaw(agentArgs, { + artifactName: "openclaw-agent-through-managed-llama-cpp", + env: env(), + timeoutMs: 12 * 60_000, + }); + await captureManagedLlamaCppRuntimeLogs( + runtimeOperation.engine, + artifacts, + runtimeIdentity, + "post-agent", + ); + expect(agent.exitCode, resultText(agent)).toBe(0); + assertAgentExecutionSucceeded(agent.stdout, "inference", recipe.spec.model.servedName); + }, + runtimeOperation.engine, + artifacts, + runtimeIdentity, ); - await approveOpenClawAdminScope(host, sandbox, SANDBOX_NAME, env(), [apiKey]); - const agent = await host.nemoclaw(agentArgs, { - artifactName: "openclaw-agent-through-managed-llama-cpp", - env: env(), - timeoutMs: 12 * 60_000, - }); - await captureManagedRuntimeLogs("post-agent"); - expect(agent.exitCode, resultText(agent)).toBe(0); - assertAgentExecutionSucceeded(agent.stdout, "inference", recipe.spec.model.servedName); const readySandbox = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { artifactName: "openshell-sandbox-ready-after-agent", diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 74bd78f795b..944c8ddccf2 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -52,7 +52,9 @@ "test/e2e/fixtures/admin-request-selector.ts", "test/e2e/fixtures/issue-4462-admin-approval-evidence.ts", "test/e2e/lib/issue-4462-admin-request-selector.py", - "test/e2e/live/openclaw-admin-scope.ts" + "test/e2e/live/openclaw-admin-scope.ts", + "test/e2e/live/gpu-e2e-helpers.ts", + "test/e2e/fixtures/llama-cpp-runtime-diagnostics.ts" ], "fast": [ "src/lib/inference/llama-cpp/index.test.ts", @@ -64,7 +66,8 @@ "test/security/admin-approval-helper.test.ts", "test/e2e/support/pr-self-hosted-llama-selector.test.ts", "test/e2e/support/onboard-timeout-contract.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/e2e/support/llama-cpp-generic-gpu-diagnostics.test.ts" ] }, { @@ -107,7 +110,8 @@ "src/lib/onboard/command.test.ts", "src/lib/onboard/sandbox-workload-preparation.test.ts", "test/e2e/support/managed-image-activation-diagnostics.test.ts", - "test/inference/managed/managed-image-publication-workflow.test.ts" + "test/inference/managed/managed-image-publication-workflow.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { @@ -202,14 +206,13 @@ "test/e2e/support/onboard-timeout-contract.test.ts", "test/onboarding/onboard-inference-gateway-scope.test.ts", "test/e2e/support/managed-image-activation-diagnostics.test.ts", - "test/e2e/support/hosted-provider-smoke-selection.test.ts" + "test/e2e/support/hosted-provider-smoke-selection.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { "live": "test/e2e/live/inference-routing-provider-smoke.test.ts", - "liveSources": [ - "test/e2e/live/inference-routing-credential-scan.ts" - ], + "liveSources": ["test/e2e/live/inference-routing-credential-scan.ts"], "fast": [ "test/agents/openclaw/runtime/provider-placeholder-refresh.test.ts", "test/generation/generate-native-hosted-config.test.ts", @@ -253,7 +256,8 @@ "test/e2e/support/e2e-clients.test.ts", "test/e2e/support/managed-image-receipt.test.ts", "test/helpers/e2e-answer-assertions.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { @@ -364,7 +368,8 @@ "test/mcp/mcp-agent-matrix-artifact-proof.test.ts", "test/e2e/support/mcp-workflow-boundary.test.ts", "test/e2e/support/runtime-provider-fixture.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { @@ -498,7 +503,8 @@ "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", "test/e2e/support/openclaw-agent-output.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { @@ -559,7 +565,8 @@ "test/e2e/support/retry-policy.test.ts", "test/e2e/support/e2e-clients.test.ts", "test/e2e/support/openclaw-agent-output.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { @@ -708,7 +715,8 @@ "test/e2e/support/issue-4462-gateway-snapshot.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { @@ -778,7 +786,8 @@ "test/e2e/support/mcp-bridge-https-diagnostics.test.ts", "src/lib/onboard/runtime-provider/docker-stopped-state-capture.test.ts", "test/state/snapshot.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { @@ -997,7 +1006,8 @@ "test/e2e/support/json-envelope.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/security/admin-approval-helper.test.ts" ] }, { @@ -1007,7 +1017,8 @@ "test/e2e/support/skill-agent-classifiers.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", - "test/e2e/support/managed-image-activation-diagnostics.test.ts" + "test/e2e/support/managed-image-activation-diagnostics.test.ts", + "test/security/admin-approval-helper.test.ts" ], "liveSources": [ "test/e2e/live/openclaw-admin-scope.ts", diff --git a/test/e2e/support/llama-cpp-generic-gpu-diagnostics.test.ts b/test/e2e/support/llama-cpp-generic-gpu-diagnostics.test.ts new file mode 100644 index 00000000000..50a1a3ca8fc --- /dev/null +++ b/test/e2e/support/llama-cpp-generic-gpu-diagnostics.test.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { expect, it, vi } from "vitest"; +import { ArtifactSink } from "../fixtures/artifacts.ts"; +import { + captureManagedLlamaCppRuntimeLogs, + withManagedLlamaCppFailureDiagnostics, +} from "../fixtures/llama-cpp-runtime-diagnostics.ts"; + +const runtime = { providerId: "docker", runtimeId: "a".repeat(64) }; + +it("captures bounded owned runtime logs when approval throws and redacts their credentials", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "llama-failure-")); + const secret = 'fixture-secret-"\nvalue'; + const original = new Error("approval failed"); + const capture = vi.fn().mockReturnValue({ status: 0, stdout: secret, stderr: secret }); + const nextAgent = vi.fn(); + try { + await expect( + withManagedLlamaCppFailureDiagnostics( + async () => { + await Promise.reject(original); + nextAgent(); + }, + { capture }, + new ArtifactSink(root, [secret]), + runtime, + ), + ).rejects.toBe(original); + expect(nextAgent).not.toHaveBeenCalled(); + expect(capture).toHaveBeenCalledExactlyOnceWith( + ["container", "logs", "--tail", "20000", runtime.runtimeId], + 30_000, + ); + const raw = await readFile(path.join(root, "managed-runtime-logs-failure.json"), "utf8"); + expect(raw).not.toContain(secret); + expect(raw).not.toContain(JSON.stringify(secret).slice(1, -1)); + expect(JSON.parse(raw)).toMatchObject({ + ...runtime, + status: 0, + stdout: "[REDACTED]", + stderr: "[REDACTED]", + }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +it("preserves the original failure when diagnostics capture fails", async () => { + const original = new Error("agent failed"); + const capture = vi.fn(() => { + throw new Error("engine unavailable"); + }); + const writeJson = vi.fn().mockRejectedValue(new Error("artifact unavailable")); + await expect( + withManagedLlamaCppFailureDiagnostics( + async () => { + throw original; + }, + { capture }, + { writeJson }, + runtime, + ), + ).rejects.toBe(original); + expect(capture).toHaveBeenCalledTimes(1); + expect(writeJson).toHaveBeenCalledTimes(0); +}); + +it("preserves the original failure when diagnostics persist fails", async () => { + const original = new Error("agent failed"); + const capture = vi.fn().mockReturnValue({ status: 1, stdout: "", stderr: "container absent" }); + const writeJson = vi.fn().mockRejectedValue(new Error("artifact unavailable")); + await expect( + withManagedLlamaCppFailureDiagnostics( + async () => { + throw original; + }, + { capture }, + { writeJson }, + runtime, + ), + ).rejects.toBe(original); + expect(capture).toHaveBeenCalledTimes(1); + expect(writeJson).toHaveBeenCalledTimes(1); +}); + +it("does not capture extra logs on ordinary success", async () => { + const run = vi.fn().mockResolvedValue(undefined); + const capture = vi.fn(); + const writeJson = vi.fn(); + await withManagedLlamaCppFailureDiagnostics(run, { capture }, { writeJson }, runtime); + expect(run).toHaveBeenCalledTimes(1); + expect(capture).not.toHaveBeenCalled(); + expect(writeJson).not.toHaveBeenCalled(); +}); + +it.each(["setup", "post-agent"] as const)( + "captures %s through the same bounded runtime owner", + async (phase) => { + const capture = vi.fn().mockReturnValue({ + status: 1, + error: new Error("runtime unavailable"), + stdout: "", + stderr: "container absent", + }); + const writeJson = vi.fn().mockResolvedValue(undefined); + await captureManagedLlamaCppRuntimeLogs({ capture }, { writeJson }, runtime, phase); + expect(capture).toHaveBeenCalledExactlyOnceWith( + ["container", "logs", "--tail", "20000", runtime.runtimeId], + 30_000, + ); + expect(writeJson).toHaveBeenCalledExactlyOnceWith(`managed-runtime-logs-${phase}.json`, { + ...runtime, + status: 1, + error: "runtime unavailable", + stdout: "", + stderr: "container absent", + }); + }, +); diff --git a/test/e2e/support/pr-self-hosted-llama-selector.test.ts b/test/e2e/support/pr-self-hosted-llama-selector.test.ts index 0c98ac4a276..816e695903d 100644 --- a/test/e2e/support/pr-self-hosted-llama-selector.test.ts +++ b/test/e2e/support/pr-self-hosted-llama-selector.test.ts @@ -133,8 +133,15 @@ async function executeSelector( copiedSha = CANDIDATE_SHA, baseSha = BASE_SHA, filesRequestStatus = 0, - reportedChangedFileCount = changedFiles.length, - expectedStatus = filesRequestStatus, + reportedChangedFileCount: unknown = changedFiles.length, + expectedStatus: number | readonly number[] = filesRequestStatus, + filesResponse = JSON.stringify([changedFiles.map((filename) => ({ filename }))]), + prResponse = JSON.stringify({ + number: 8748, + base: { sha: baseSha }, + changed_files: reportedChangedFileCount, + head: { sha: CANDIDATE_SHA }, + }), ) { const directory = mkdtempSync(join(tmpdir(), `nemoclaw-${fixtureName}-selector-`)); const binDirectory = join(directory, "bin"); @@ -168,20 +175,16 @@ fi GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_SHA: copiedSha, PATH: `${binDirectory}:${process.env.PATH ?? ""}`, - PR_FILES_JSON: JSON.stringify([changedFiles.map((filename) => ({ filename }))]), + PR_FILES_JSON: filesResponse, PR_FILES_EXIT: String(filesRequestStatus), - PR_JSON: JSON.stringify({ - number: 8748, - base: { sha: baseSha }, - changed_files: reportedChangedFileCount, - head: { sha: CANDIDATE_SHA }, - }), + PR_JSON: prResponse, }, ); - assert.equal(result.status, expectedStatus, result.stderr); - return expectedStatus === 0 - ? readFileSync(outputPath, "utf8").trim() - : `status=${result.status}`; + assert.equal(result.signal, null, result.stderr); + assert([expectedStatus].flat().includes(result.status ?? -1), result.stderr); + const output = readFileSync(outputPath, "utf8"); + assert(result.status === 0 || output === "", "failed discovery wrote job outputs"); + return result.status === 0 ? output.trim() : `status=${result.status}`; } finally { rmSync(directory, { force: true, recursive: true }); } @@ -354,6 +357,17 @@ describe.concurrent("generic NVIDIA GPU PR selection", () => { }, ); + it("selects the generic GPU E2E when only its llama.cpp diagnostics owner changes", async ({ + expect, + }) => { + const result = await selectGenericGpuLane([ + "test/e2e/fixtures/llama-cpp-runtime-diagnostics.ts", + ]); + expect(result).toBe( + `base_sha=${BASE_SHA}\nhead_sha=${CANDIDATE_SHA}\npr_number=8748\nselected=true`, + ); + }); + it.for(ARM64_PROOF_AUTHORITY_PATHS)( "does not select the Docker-qualified AMD64 GPU job for ARM64 proof owner %s", async (changedFile, { expect }) => { @@ -391,6 +405,155 @@ describe.concurrent("generic NVIDIA GPU PR selection", () => { ); }); + it.for([ + { + name: "API failure with matching files", + response: '[[{"filename":"scripts/install.sh"}]]', + count: 1, + status: 17, + expectedExit: 17, + }, + { + name: "API failure with nonmatching files", + response: '[[{"filename":"docs/readme.md"}]]', + count: 1, + status: 17, + expectedExit: 17, + }, + { name: "malformed JSON", response: "{", count: 1, status: 0, expectedExit: [4, 5] }, + { name: "non-array pages", response: "null", count: 0, status: 0, expectedExit: 1 }, + { + name: "flat file list", + response: '[{"filename":"scripts/install.sh"}]', + count: 1, + status: 0, + expectedExit: 1, + }, + { name: "non-array page", response: "[[], {}]", count: 0, status: 0, expectedExit: 1 }, + { + name: "non-object file", + response: '[["scripts/install.sh"]]', + count: 1, + status: 0, + expectedExit: 1, + }, + { name: "missing filename", response: "[[{}]]", count: 1, status: 0, expectedExit: 1 }, + { + name: "non-string filename", + response: '[[{"filename":null}]]', + count: 1, + status: 0, + expectedExit: 1, + }, + { + name: "incomplete nonmatching files", + response: '[[{"filename":"docs/readme.md"}]]', + count: 2, + status: 0, + expectedExit: 1, + }, + { + name: "incomplete matching files", + response: '[[{"filename":"scripts/install.sh"}]]', + count: 2, + status: 0, + expectedExit: 1, + }, + { + name: "excess files", + response: '[[{"filename":"scripts/install.sh"}]]', + count: 0, + status: 0, + expectedExit: 1, + }, + { + name: "empty pages with nonzero count", + response: "[]", + count: 1, + status: 0, + expectedExit: 1, + }, + { + name: "empty page with nonzero count", + response: "[[]]", + count: 1, + status: 0, + expectedExit: 1, + }, + { name: "negative count", response: "[[]]", count: -1, status: 0, expectedExit: 4 }, + { name: "fractional count", response: "[[]]", count: 0.5, status: 0, expectedExit: 4 }, + { name: "string count", response: "[[]]", count: "0", status: 0, expectedExit: 4 }, + { name: "null count", response: "[[]]", count: null, status: 0, expectedExit: 4 }, + ])("fails GPU discovery without outputs for $name", async (fixture) => { + await executeSelector( + selectorScript(), + "generic-gpu", + [], + CANDIDATE_SHA, + BASE_SHA, + fixture.status, + fixture.count, + fixture.expectedExit, + fixture.response, + ); + }); + + it("fails GPU discovery without outputs for missing changed_files", async () => { + await executeSelector( + selectorScript(), + "generic-gpu", + [], + CANDIDATE_SHA, + BASE_SHA, + 0, + 0, + 4, + "[[]]", + JSON.stringify({ + number: 8748, + base: { sha: BASE_SHA }, + head: { sha: CANDIDATE_SHA }, + }), + ); + }); + + it.for(["[]", "[[]]"])( + "skips GPU qualification for complete empty pages %s", + async (response, { expect }) => { + const result = await executeSelector( + selectorScript(), + "generic-gpu", + [], + CANDIDATE_SHA, + BASE_SHA, + 0, + 0, + 0, + response, + ); + expect(result).toBe( + `base_sha=${BASE_SHA}\nhead_sha=${CANDIDATE_SHA}\npr_number=8748\nselected=false`, + ); + }, + ); + + it("selects GPU qualification when a later complete page matches", async ({ expect }) => { + const result = await executeSelector( + selectorScript(), + "generic-gpu", + [], + CANDIDATE_SHA, + BASE_SHA, + 0, + 2, + 0, + '[[{"filename":"docs/readme.md"}],[{"filename":"scripts/install.sh"}]]', + ); + expect(result).toBe( + `base_sha=${BASE_SHA}\nhead_sha=${CANDIDATE_SHA}\npr_number=8748\nselected=true`, + ); + }); + it("rejects a copied branch whose commit does not match the current PR head", async ({ expect, }) => { @@ -410,7 +573,7 @@ describe.concurrent("generic NVIDIA GPU PR selection", () => { }) => { assert.match( readFileSync(LLAMA_LIVE_TEST_PATH, "utf8"), - /const agent = await host\.nemoclaw\([\s\S]*await captureManagedRuntimeLogs\([^)]*\);[\s\S]*expect\(agent\.exitCode/u, + /const agent = await host\.nemoclaw\([\s\S]*await captureManagedLlamaCppRuntimeLogs\([^)]*\);[\s\S]*expect\(agent\.exitCode/u, "llama.cpp runtime logs must be captured after the agent request and before its exit assertion", ); expect(workflow().jobs["llama-cpp-generic-gpu"]?.env?.NEMOCLAW_GATEWAY_RUNTIME).toBe("docker"); diff --git a/test/security/admin-approval-helper.test.ts b/test/security/admin-approval-helper.test.ts index 69722e624ac..6ebc4aa47d4 100644 --- a/test/security/admin-approval-helper.test.ts +++ b/test/security/admin-approval-helper.test.ts @@ -22,6 +22,29 @@ import { preApprovalAdminProbeEvidence, } from "../e2e/fixtures/issue-4462-admin-approval-evidence.ts"; +const COMPLETED_CRON_RUN = { + ok: true, + enqueued: true, + runId: "run-1", + completed: true, + status: "ok", + completionStatus: "succeeded", + run: { + jobId: "cron-1", + runId: "run-1", + action: "finished", + status: "ok", + completionStatus: "succeeded", + }, +}; + +const FAILED_CRON_RUN = { + ...COMPLETED_CRON_RUN, + status: "error", + completionStatus: "failed", + run: { ...COMPLETED_CRON_RUN.run, status: "error", completionStatus: "failed" }, +}; + const EXPECTED_REQUEST_ID = "12345678-1234-4123-8123-123456789abc"; const VERSION_ONE_REQUEST_ID = "12345678-1234-1123-8123-123456789abc"; const EXPECTED_OPENSHELL_ARGS = [ @@ -150,8 +173,13 @@ function runAdminApprovalScript( options: { expectedRequestId?: string; verifyCronConsumer?: boolean; + cronRunOutput?: string; + cronRunError?: string; + cronRunExitCode?: number; terminal?: boolean; cleanupFails?: boolean; + removeCaptures?: boolean; + diagnosticFails?: boolean; tamperScript?: boolean; requireNonInteractiveBody?: boolean; omitPreparedWrapper?: boolean; @@ -161,6 +189,10 @@ function runAdminApprovalScript( ): { commands: string[]; capturedApprovalBytes: number; + capturedCronBytes: number; + privateCaptureCount: number; + failureOutputDrained: boolean; + privateCapturesRemaining: string[]; openshellArgs: string[]; stagedScriptRetained: boolean; result: SpawnSyncReturns; @@ -174,6 +206,14 @@ function runAdminApprovalScript( const openshellArgvLogPath = path.join(root, "openshell-argv.log"); const mktempCounterPath = path.join(root, "mktemp-counter"); const stagedScriptReceipt = path.join(root, "staged-script"); + fs.writeFileSync( + path.join(root, "python3"), + `#!/bin/sh +if [ "$FAKE_ADMIN_DIAGNOSTIC_FAIL" = 1 ] && [ "$#" -eq 4 ] && [ "$3" = cron-run ]; then exit 97; fi +exec "$FAKE_REAL_PYTHON" "$@" +`, + { mode: 0o755 }, + ); const stateRoot = writeLocalIdentity(root); const helperPath = writePairingStateHelper(root); const terminalProbe = path.join(root, "terminal-probe.py"); @@ -219,6 +259,7 @@ if [ "$#" -eq 3 ] && [ "$2" = -- ]; then if [ "$FAKE_ADMIN_CLEANUP_FAIL" = 1 ]; then exit 91; fi exec /bin/rm "$@" fi +if [ "$FAKE_REMOVE_CAPTURES" = 1 ]; then exec /bin/rm "$@"; fi `, { mode: 0o755 }, ); @@ -233,16 +274,17 @@ if [ "\${FAKE_OPENCLAW_FAIL:-}" = "$1:$2" ]; then devices:list) printf '%s\\n' 'gateway connection unavailable token=test-gateway-token device=${EXPECTED_DEVICE_ID}' >&2 ;; devices:approve) printf '%s\\n' 'approval denied by policy request=${EXPECTED_REQUEST_ID} publicKey=${EXPECTED_PUBLIC_KEY}' >&2 ;; cron:add) printf '%s\\n' 'scope upgrade pending approval cron=cron-1' >&2 ;; - cron:run) printf '%s\\n' 'request timed out cron=cron-1' >&2 ;; + cron:run) printf '%s\\n' "$FAKE_CRON_RUN_ERROR" >&2 ;; esac python3 -c 'import sys; sys.stderr.buffer.write(b"x" * int(sys.argv[1]))' "$FAKE_FAILURE_OUTPUT_PADDING_BYTES" + : >"$FAKE_FAILURE_OUTPUT_DRAINED" exit 91 fi case "$1:$2" in devices:list) cat "$FAKE_DEVICES_STATE" ;; devices:approve) ;; cron:add) printf '%s\\n' '{"id":"cron-1","name":"admin-cron"}' ;; - cron:run) printf '%s\\n' '{"ok":true,"ran":true}' ;; + cron:run) printf '%s\\n' "$FAKE_CRON_RUN_OUTPUT"; exit "$FAKE_CRON_RUN_EXIT_CODE" ;; *) exit 90 ;; esac `, @@ -261,8 +303,12 @@ esac FAKE_DEVICES_STATE: devicesPath, FAKE_OPENCLAW_FAIL: failureCommand ?? "", FAKE_OPENCLAW_LOG: commandLogPath, + FAKE_CRON_RUN_OUTPUT: options.cronRunOutput ?? JSON.stringify(COMPLETED_CRON_RUN), + FAKE_CRON_RUN_ERROR: options.cronRunError ?? "request timed out cron=cron-1", + FAKE_CRON_RUN_EXIT_CODE: String(options.cronRunExitCode ?? 0), FAKE_OPENSHELL_ARGV_LOG: openshellArgvLogPath, FAKE_FAILURE_OUTPUT_PADDING_BYTES: String(failureOutputPaddingBytes), + FAKE_FAILURE_OUTPUT_DRAINED: path.join(root, "failure-output-drained"), FAKE_MKTEMP_COUNTER: mktempCounterPath, FAKE_MKTEMP_ROOT: root, NEMOCLAW_OPENCLAW_PAIRING_STATE_HELPER: helperPath, @@ -277,6 +323,11 @@ esac ADMIN_REQUIRE_NONINTERACTIVE_BODY: options.requireNonInteractiveBody ? "1" : "0", FAKE_ADMIN_SCRIPT_RECEIPT: stagedScriptReceipt, FAKE_ADMIN_CLEANUP_FAIL: options.cleanupFails ? "1" : "0", + FAKE_REMOVE_CAPTURES: options.removeCaptures ? "1" : "0", + FAKE_ADMIN_DIAGNOSTIC_FAIL: options.diagnosticFails ? "1" : "0", + FAKE_REAL_PYTHON: spawnSync("python3", ["-c", "import sys; print(sys.executable)"], { + encoding: "utf8", + }).stdout.trim(), }; try { const result = spawnSync("bash", [], { @@ -312,7 +363,28 @@ esac const stagedScriptRetained = fs.existsSync(stagedScriptReceipt) && fs.existsSync(fs.readFileSync(stagedScriptReceipt, "utf8")); - return { capturedApprovalBytes, commands, openshellArgs, result, stagedScriptRetained }; + const cronCapturePath = path.join(root, "capture-8"); + const capturedCronBytes = fs.existsSync(cronCapturePath) + ? fs.statSync(cronCapturePath).size + : 0; + const privateCapturesRemaining = fs + .readdirSync(root) + .filter((name) => name.startsWith("capture-")); + const privateCaptureCount = fs.existsSync(mktempCounterPath) + ? Number(fs.readFileSync(mktempCounterPath, "utf8")) + : 0; + const failureOutputDrained = fs.existsSync(path.join(root, "failure-output-drained")); + return { + capturedApprovalBytes, + capturedCronBytes, + privateCaptureCount, + privateCapturesRemaining, + failureOutputDrained, + commands, + openshellArgs, + result, + stagedScriptRetained, + }; } finally { removeStagedAdminScript(stagedScriptReceipt); fs.rmSync(root, { recursive: true, force: true }); @@ -502,6 +574,7 @@ describe("prepared connect-shell administrative approval", () => { expect(result.status, result.stderr).toBe(0); expect(result.stdout).toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); + expect(result.stderr).not.toContain("ADMIN_CRON_"); const output = `${result.stdout}\n${result.stderr}`; expect(output).not.toContain(EXPECTED_REQUEST_ID); expect(output).not.toContain(EXPECTED_DEVICE_ID); @@ -511,11 +584,496 @@ describe("prepared connect-shell administrative approval", () => { expect(commands).toEqual([ "devices list --json", `devices approve ${EXPECTED_REQUEST_ID}`, - "cron add --name admin-cron --every 2h --agent main --session isolated --message hello", - "cron run cron-1", + "cron add --name admin-cron --every 2h --agent main --session isolated --message hello --no-deliver", + "cron run cron-1 --wait --wait-timeout 2m --timeout 10000", ]); }); + it.each([ + ["enqueue only", { ok: true, enqueued: true, runId: "run-1" }], + ["legacy run acknowledgement", { ok: true, ran: true }], + ["unfinished run", { ...COMPLETED_CRON_RUN, completed: false }], + ["missing run identity", { ...COMPLETED_CRON_RUN, runId: undefined }], + ["wrong job", { ...COMPLETED_CRON_RUN, run: { ...COMPLETED_CRON_RUN.run, jobId: "other" } }], + ["wrong run", { ...COMPLETED_CRON_RUN, run: { ...COMPLETED_CRON_RUN.run, runId: "other" } }], + ["failed completion", { ...COMPLETED_CRON_RUN, completionStatus: "failed" }], + ["skipped run", { ...COMPLETED_CRON_RUN, status: "skipped" }], + [ + "nested failed run with outer success", + { ...COMPLETED_CRON_RUN, run: { ...COMPLETED_CRON_RUN.run, status: "error" } }, + ], + [ + "nested skipped completion with outer success", + { ...COMPLETED_CRON_RUN, run: { ...COMPLETED_CRON_RUN.run, completionStatus: "skipped" } }, + ], + ["missing completion record", { ...COMPLETED_CRON_RUN, run: null }], + ])("rejects %s before reporting approval success (#13342)", (_case, response) => { + const { commands, result, stagedScriptRetained } = runAdminApprovalScript( + undefined, + 0, + EXPECTED_REQUEST_ID, + { cronRunOutput: JSON.stringify(response) }, + ); + expect(result.status).toBe(30); + expect(result.stderr).toContain("ADMIN_CRON_RUN_RESULT_INVALID"); + expect(result.stdout).not.toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); + expect(`${result.stdout}\n${result.stderr}`).not.toMatch(/cron-1|run-1|test-gateway-token/); + expect(commands.filter((command) => command.startsWith("cron run "))).toHaveLength(1); + expect(stagedScriptRetained).toBe(false); + }); + + it.each([ + ["wrong job", { ...FAILED_CRON_RUN, run: { ...FAILED_CRON_RUN.run, jobId: "other" } }], + ["wrong run", { ...FAILED_CRON_RUN, run: { ...FAILED_CRON_RUN.run, runId: "other" } }], + ["nonterminal", { ...FAILED_CRON_RUN, completed: false }], + ["contradictory status", { ...FAILED_CRON_RUN, run: { ...FAILED_CRON_RUN.run, status: "ok" } }], + [ + "contradictory completion", + { ...FAILED_CRON_RUN, run: { ...FAILED_CRON_RUN.run, completionStatus: "succeeded" } }, + ], + [ + "unrecognized terminal value", + { + ...FAILED_CRON_RUN, + completionStatus: "private-sentinel", + run: { ...FAILED_CRON_RUN.run, completionStatus: "private-sentinel" }, + }, + ], + ["non-string terminal value", { ...FAILED_CRON_RUN, status: ["error"] }], + ["successful run on CLI failure", COMPLETED_CRON_RUN], + ])("does not attribute %s output to a terminal cron failure", (_case, response) => { + const { result, stagedScriptRetained } = runAdminApprovalScript( + undefined, + 0, + EXPECTED_REQUEST_ID, + { + cronRunOutput: JSON.stringify({ + ...response, + run: { ...response.run, error: "HTTP 503: private-sentinel" }, + }), + cronRunExitCode: 1, + }, + ); + expect(result.status).toBe(29); + expect(result.stderr).toContain("ADMIN_DIAGNOSTIC=command-failed"); + expect(result.stderr).not.toContain("ADMIN_CRON_ERROR_REASON="); + expect(result.stderr).not.toContain("ADMIN_CRON_DIAGNOSTIC_SOURCES="); + expect(result.stderr).not.toContain("ADMIN_CRON_ERROR_SIGNATURES="); + expect(result.stdout).not.toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); + expect(`${result.stdout}\n${result.stderr}`).not.toMatch(/cron-1|run-1|other|private-sentinel/); + expect(stagedScriptRetained).toBe(false); + }); + + it.each([ + ["malformed", '{"private-sentinel":'], + ["deeply nested", `{"private-sentinel":${"[".repeat(2000)}0${"]".repeat(2000)}}`], + ])("preserves a %s cron command failure without raw output", (_case, cronRunOutput) => { + const { result, stagedScriptRetained, privateCaptureCount, privateCapturesRemaining } = + runAdminApprovalScript(undefined, 0, EXPECTED_REQUEST_ID, { + cronRunOutput, + cronRunExitCode: 1, + removeCaptures: true, + }); + expect(result.status).toBe(29); + expect(result.stderr).toContain("ADMIN_DIAGNOSTIC=command-failed"); + expect(result.stderr).not.toContain("ADMIN_CRON_ERROR_REASON="); + expect(result.stderr).not.toContain("ADMIN_CRON_DIAGNOSTIC_SOURCES="); + expect(result.stderr).not.toContain("ADMIN_CRON_ERROR_SIGNATURES="); + expect(`${result.stdout}\n${result.stderr}`).not.toContain("private-sentinel"); + expect(privateCaptureCount).toBe(8); + expect(privateCapturesRemaining).toEqual([]); + expect(stagedScriptRetained).toBe(false); + }); + + it("accepts completed cron output after a CLI banner (#13342)", () => { + const { result } = runAdminApprovalScript(undefined, 0, EXPECTED_REQUEST_ID, { + cronRunOutput: `OpenClaw ready\n${JSON.stringify(COMPLETED_CRON_RUN)}`, + }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); + }); + + it.each([ + "auth", + "auth_permanent", + "format", + "rate_limit", + "overloaded", + "billing", + "server_error", + "timeout", + "tls_certificate", + "context_overflow", + "model_not_found", + "session_expired", + "empty_response", + "no_error_details", + "unclassified", + "unknown", + ])("retains the pinned cron reason %s without private details", (errorReason) => { + const sources = [ + "cron-preflight", + "cron-setup", + "model-preflight", + "agent-run", + "tool", + "exec", + "delivery", + ]; + const entries = sources.map((source, index) => ({ + source, + severity: ["info", "warn", "error"][index % 3], + message: "PRIVATE_PROMPT", + toolName: "PRIVATE_TOOL", + exitCode: 12345, + })); + const response = { + ...FAILED_CRON_RUN, + run: { + ...FAILED_CRON_RUN.run, + errorReason, + error: "test-gateway-token", + summary: "PRIVATE_RESPONSE", + sessionKey: "PRIVATE_SESSION", + diagnostics: { summary: "https://PRIVATE_URL/?token=PRIVATE_TOKEN", entries }, + }, + }; + const { + result, + commands, + privateCaptureCount, + privateCapturesRemaining, + stagedScriptRetained, + } = runAdminApprovalScript(undefined, 0, EXPECTED_REQUEST_ID, { + cronRunOutput: JSON.stringify(response), + cronRunExitCode: 1, + removeCaptures: true, + }); + expect(result.status).toBe(29); + expect(result.stderr).toBe( + `ADMIN_CRON_RUN_FAILED\nADMIN_DIAGNOSTIC=cron-run-error\nADMIN_CRON_ERROR_REASON=${errorReason}\nADMIN_CRON_DIAGNOSTIC_SOURCES=cron-preflight:info,cron-setup:warn,model-preflight:error,agent-run:info,tool:warn,exec:error,delivery:info\nADMIN_CRON_ERROR_SIGNATURES=unavailable\n`, + ); + expect(`${result.stdout}\n${result.stderr}`).not.toMatch( + /PRIVATE_|test-gateway-token|cron-1|run-1|12345/, + ); + expect(result.stdout).toContain("ADMIN_CONNECT_BODY_STATUS=29"); + expect(result.stdout).not.toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); + expect(commands.filter((command) => command.startsWith("cron run "))).toHaveLength(1); + expect(privateCaptureCount).toBe(8); + expect(privateCapturesRemaining).toEqual([]); + expect(stagedScriptRetained).toBe(false); + }); + + it.each([ + undefined, + null, + 42, + {}, + [], + "PRIVATE_VALUE", + "auth\nPRIVATE_VALUE", + "auth\u001b[0m", + "аuth", + ])("omits an invalid cron diagnostic value %j", (value) => { + const response = { + ...FAILED_CRON_RUN, + run: { + ...FAILED_CRON_RUN.run, + errorReason: value, + diagnostics: { + entries: [ + { source: value, severity: "error" }, + { source: "exec", severity: value }, + ], + }, + }, + }; + const { result, privateCapturesRemaining } = runAdminApprovalScript( + undefined, + 0, + EXPECTED_REQUEST_ID, + { cronRunOutput: JSON.stringify(response), cronRunExitCode: 1, removeCaptures: true }, + ); + expect(result.status).toBe(29); + expect(result.stderr).toBe( + "ADMIN_CRON_RUN_FAILED\nADMIN_DIAGNOSTIC=cron-run-error\nADMIN_CRON_ERROR_REASON=unavailable\nADMIN_CRON_DIAGNOSTIC_SOURCES=unavailable\nADMIN_CRON_ERROR_SIGNATURES=unavailable\n", + ); + expect(privateCapturesRemaining).toEqual([]); + }); + + it.each([null, [], { entries: "PRIVATE_VALUE" }, { entries: [null, [], {}, 42] }])( + "omits malformed cron diagnostic entries %j", + (diagnostics) => { + const response = { ...FAILED_CRON_RUN, run: { ...FAILED_CRON_RUN.run, diagnostics } }; + const { result } = runAdminApprovalScript(undefined, 0, EXPECTED_REQUEST_ID, { + cronRunOutput: JSON.stringify(response), + cronRunExitCode: 1, + }); + expect(result.status).toBe(29); + expect(result.stderr).toContain("ADMIN_CRON_DIAGNOSTIC_SOURCES=unavailable\n"); + expect(`${result.stdout}\n${result.stderr}`).not.toContain("PRIVATE_VALUE"); + }, + ); + + it.each([ + ["LLM request timed out.", "message-llm-timeout"], + ["LLM request failed.", "message-llm-failed"], + ["LLM request failed with an unknown error.", "message-llm-unknown"], + ["LLM request failed: provider returned an internal error.", "message-llm-provider-internal"], + [ + "The selected model was not found by the provider. Check the model id or choose a different model.", + "message-model-not-found", + ], + ["Code Mode could not resolve a configured MCP tool.", "message-code-mode-catalog-miss"], + ["HTTP 503: PRIVATE_PROMPT test-gateway-token https://PRIVATE_HOST/", "error-text-http-503"], + ["503 PRIVATE_BODY", "error-text-http-503"], + ["503", "error-text-http-503"], + ["API error (503): PRIVATE_BODY", "error-text-http-503"], + ["status code: 503, message: PRIVATE_BODY", "error-text-http-503"], + ["unexpected status = 503 PRIVATE_BODY", "error-text-http-503"], + ["http status 503 PRIVATE_BODY", "error-text-http-503"], + ["Error: HTTP 503: PRIVATE_BODY", "error-text-http-503"], + ["Error, API error (503): PRIVATE_BODY", "error-text-http-503"], + ["HTTP 400: PRIVATE_BODY", "error-text-http-400"], + ["HTTP 599: PRIVATE_BODY", "error-text-http-599"], + ["HTTP 408: PRIVATE_BODY", "error-text-http-408"], + ["HTTP 429: PRIVATE_BODY", "error-text-http-429"], + ["PRIVATE_PREFIX LLM request timed out.", "unavailable"], + ["LLM request timed out. PRIVATE_SUFFIX", "unavailable"], + ["PRIVATE_BODY contains 503", "unavailable"], + ["503123_PRIVATE_ID", "unavailable"], + ["https://PRIVATE_HOST/503", "unavailable"], + ["HTTP 399: PRIVATE_BODY", "unavailable"], + ["HTTP 600: PRIVATE_BODY", "unavailable"], + ["HTTP ５０３: PRIVATE_BODY", "unavailable"], + ["Provider API error (503): PRIVATE_BODY", "unavailable"], + ["Error: Error: HTTP 503: PRIVATE_BODY", "unavailable"], + [null, "unavailable"], + [{ message: "HTTP 503: PRIVATE_BODY" }, "unavailable"], + ])("projects only a fixed cron error-text signature for %j", (message, signature) => { + const response = { + ...FAILED_CRON_RUN, + run: { + ...FAILED_CRON_RUN.run, + errorReason: "timeout", + error: message, + diagnostics: { + entries: ["tool", "agent-run"].map((source) => ({ source, severity: "error", message })), + }, + }, + }; + const { result, privateCaptureCount, privateCapturesRemaining, stagedScriptRetained } = + runAdminApprovalScript(undefined, 0, EXPECTED_REQUEST_ID, { + cronRunOutput: JSON.stringify(response), + cronRunExitCode: 1, + removeCaptures: true, + }); + const expected = + signature === "unavailable" + ? signature + : ["run", "tool", "agent-run"].map((source) => `${source}:${signature}`).join(","); + expect(result.status).toBe(29); + expect(result.stderr).toContain(`ADMIN_CRON_ERROR_SIGNATURES=${expected}\n`); + expect(result.stderr).toContain("ADMIN_CRON_ERROR_REASON=timeout\n"); + expect(result.stdout).not.toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); + expect(`${result.stdout}\n${result.stderr}`).not.toMatch( + /PRIVATE_|test-gateway-token|cron-1|run-1/, + ); + expect(privateCaptureCount).toBe(8); + expect(privateCapturesRemaining).toEqual([]); + expect(stagedScriptRetained).toBe(false); + }); + + it("ignores cron signatures outside the bound error-message fields", () => { + const message = "HTTP 503: PRIVATE_BODY"; + const response = { + ...FAILED_CRON_RUN, + run: { + ...FAILED_CRON_RUN.run, + summary: message, + arbitrary: { error: message }, + diagnostics: { + summary: message, + entries: [ + { source: "tool", severity: "warn", message }, + { source: "tool", severity: "info", message }, + { source: "PRIVATE_SOURCE", severity: "error", message }, + ...Array.from({ length: 7 }, () => ({ + source: "tool", + severity: "error", + summary: message, + })), + { source: "tool", severity: "error", message }, + ], + }, + }, + }; + const { result, privateCapturesRemaining } = runAdminApprovalScript( + undefined, + 0, + EXPECTED_REQUEST_ID, + { + cronRunOutput: JSON.stringify(response), + cronRunExitCode: 1, + removeCaptures: true, + }, + ); + expect(result.status).toBe(29); + expect(result.stderr).toContain("ADMIN_CRON_ERROR_SIGNATURES=unavailable\n"); + expect(`${result.stdout}\n${result.stderr}`).not.toMatch(/PRIVATE_|error-text-http/); + expect(privateCapturesRemaining).toEqual([]); + }); + + it("limits cron error-text signatures to the run and ten entries", () => { + const entries = Array.from({ length: 11 }, (_, index) => ({ + source: "tool", + severity: "error", + message: `HTTP ${500 + index}: PRIVATE_BODY`, + })); + const response = { + ...FAILED_CRON_RUN, + run: { ...FAILED_CRON_RUN.run, error: "HTTP 429: PRIVATE_BODY", diagnostics: { entries } }, + }; + const { result, privateCapturesRemaining } = runAdminApprovalScript( + undefined, + 0, + EXPECTED_REQUEST_ID, + { + cronRunOutput: JSON.stringify(response), + cronRunExitCode: 1, + removeCaptures: true, + }, + ); + const expected = [ + "run:error-text-http-429", + ...Array.from({ length: 10 }, (_, index) => `tool:error-text-http-${500 + index}`), + ]; + expect(result.status).toBe(29); + expect(result.stderr).toContain(`ADMIN_CRON_ERROR_SIGNATURES=${expected.join(",")}\n`); + expect(result.stderr).not.toMatch(/PRIVATE_|error-text-http-510/); + expect(privateCapturesRemaining).toEqual([]); + }); + + it("bounds and deduplicates cron diagnostic entries while draining output", () => { + const response = { + ...FAILED_CRON_RUN, + run: { + ...FAILED_CRON_RUN.run, + errorReason: "server_error", + diagnostics: { + entries: [ + ...Array.from({ length: 10 }, () => ({ + source: "agent-run", + severity: "error", + message: "HTTP 503: PRIVATE_BODY", + })), + { source: "delivery", severity: "error", message: "HTTP 429: PRIVATE_BODY" }, + ], + }, + }, + }; + const { result, capturedCronBytes, failureOutputDrained } = runAdminApprovalScript( + "cron:run", + 256 * 1024, + EXPECTED_REQUEST_ID, + { cronRunError: JSON.stringify(response) }, + ); + expect(result.status).toBe(29); + expect(capturedCronBytes).toBe(65_536); + expect(failureOutputDrained).toBe(true); + expect(result.stderr).toContain("ADMIN_CRON_DIAGNOSTIC_SOURCES=agent-run:error\n"); + expect(result.stderr).not.toContain("delivery:error"); + expect(result.stderr).toContain("ADMIN_CRON_ERROR_SIGNATURES=agent-run:error-text-http-503\n"); + expect(result.stderr).not.toMatch(/PRIVATE_|error-text-http-429/); + }); + + it("rejects truncated cron JSON after draining the bounded capture", () => { + const { result, capturedCronBytes, failureOutputDrained } = runAdminApprovalScript( + "cron:run", + 256 * 1024, + EXPECTED_REQUEST_ID, + { cronRunError: '{"error":"HTTP 503: PRIVATE_VALUE' }, + ); + expect(result.status).toBe(29); + expect(capturedCronBytes).toBe(65_536); + expect(failureOutputDrained).toBe(true); + expect(result.stderr).toBe("ADMIN_CRON_RUN_FAILED\nADMIN_DIAGNOSTIC=command-failed\n"); + }); + + it("preserves the cron failure and cleans private files when diagnostics fail", () => { + const { + result, + commands, + privateCaptureCount, + privateCapturesRemaining, + stagedScriptRetained, + } = runAdminApprovalScript(undefined, 0, EXPECTED_REQUEST_ID, { + cronRunOutput: JSON.stringify(FAILED_CRON_RUN), + cronRunExitCode: 1, + diagnosticFails: true, + removeCaptures: true, + }); + expect(result.status).toBe(29); + expect(result.stderr).toContain("ADMIN_DIAGNOSTIC=unavailable"); + expect(result.stdout).toContain("ADMIN_CONNECT_BODY_STATUS=29"); + expect(commands.filter((command) => command.startsWith("cron run "))).toHaveLength(1); + expect(privateCaptureCount).toBe(8); + expect(privateCapturesRemaining).toEqual([]); + expect(stagedScriptRetained).toBe(false); + }); + + it.each([ + ["error", "failed", "cron-run-error"], + ["skipped", "failed", "cron-run-skipped"], + ["ok", "failed", "cron-completion-failed"], + ["ok", "unknown", "cron-completion-unknown"], + ])( + "retains the pinned terminal %s/%s category on CLI exit 1", + (status, completionStatus, diagnostic) => { + const response = { + ...COMPLETED_CRON_RUN, + status, + completionStatus, + run: { + ...COMPLETED_CRON_RUN.run, + status, + completionStatus, + error: "private generated text test-gateway-token", + }, + }; + const { commands, result, stagedScriptRetained } = runAdminApprovalScript( + undefined, + 0, + EXPECTED_REQUEST_ID, + { cronRunOutput: `OpenClaw ready\n${JSON.stringify(response)}`, cronRunExitCode: 1 }, + ); + expect(result.status).toBe(29); + expect(result.stderr).toContain("ADMIN_CRON_RUN_FAILED"); + expect(result.stderr).toContain(`ADMIN_DIAGNOSTIC=${diagnostic}`); + expect(result.stdout).not.toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); + expect(`${result.stdout}\n${result.stderr}`).not.toMatch( + /cron-1|run-1|private generated text|test-gateway-token/, + ); + expect(commands.filter((command) => command.startsWith("cron run "))).toHaveLength(1); + expect(stagedScriptRetained).toBe(false); + }, + ); + + it("rejects malformed cron output and removes its staged script (#13342)", () => { + const { result, stagedScriptRetained } = runAdminApprovalScript( + undefined, + 0, + EXPECTED_REQUEST_ID, + { + cronRunOutput: "invalid response token=test-gateway-token", + }, + ); + expect(result.status).toBe(30); + expect(result.stderr).toContain("ADMIN_CRON_RUN_RESULT_INVALID"); + expect(`${result.stdout}\n${result.stderr}`).not.toContain("test-gateway-token"); + expect(stagedScriptRetained).toBe(false); + }); + it.each([ [EXPECTED_REQUEST_ID, 0, []], [VERSION_ONE_REQUEST_ID, 26, ["ADMIN_DIAGNOSTIC=selection-pending-count"]], @@ -565,6 +1123,22 @@ describe("prepared connect-shell administrative approval", () => { expect(commands).toContain(`devices approve ${VERSION_ONE_REQUEST_ID}`); }); + it("distinguishes pinned cron wait exhaustion without exposing its run identity", () => { + const { commands, result, stagedScriptRetained } = runAdminApprovalScript( + "cron:run", + 0, + EXPECTED_REQUEST_ID, + { + cronRunError: "timed out waiting for cron run private-run-id token=test-gateway-token", + }, + ); + expect(result.status).toBe(29); + expect(result.stderr).toContain("ADMIN_DIAGNOSTIC=cron-wait-timeout"); + expect(`${result.stdout}\n${result.stderr}`).not.toMatch(/private-run-id|test-gateway-token/); + expect(commands).toHaveLength(4); + expect(stagedScriptRetained).toBe(false); + }); + it.each([ ["devices:list", 25, "ADMIN_DEVICES_LIST_FAILED", "gateway-unavailable", 1], ["devices:approve", 27, "ADMIN_APPROVE_FAILED", "authorization-rejected", 2], From 3198676cf3e7ff99d4ad697f84824cefb0adbdd4 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter Date: Mon, 19 Oct 2026 03:55:57 -0700 Subject: [PATCH 06/22] fix(e2e): retain PR image contracts for delayed validation (#13417) ## Outcome Manual PR E2E can bind an all-agent managed-image publication for seven days after its run. The contracts previously expired after one day, which blocked #13370 before any live job started. ## Reason The exact-head image run for #13370 finished on October 17. All six all-agent jobs uploaded their contracts, but the October 18 manual E2E run could no longer find them. The separate Pi candidate contracts already use seven-day retention. ### Related issues Refs #13370 and #13362. ## Changes - Retain the six PR all-agent managed-image contract artifacts for seven days. - Record this timing boundary in the owning E2E guidance. ## Verification - [Failed E2E run 39130475566](https://github.com/NVIDIA/NemoClaw/actions/runs/39130475566): `base-image-publication` reported a missing exact artifact; all selected live jobs were skipped. - [Image run 38994806955](https://github.com/NVIDIA/NemoClaw/actions/runs/38994806955): all six all-agent jobs and their contract upload steps passed. The producer finished more than one day before the E2E run began. - Parsed `managed-images.yaml` and confirmed the PR contract upload retention is seven days. - `npx vitest run --project e2e-support test/e2e/support/pr-managed-image-publication.test.ts`: 58 passed. - `npm run validate:pr`: passed, including normal repository hooks. - Reviewed the two-file diff; it contains no secrets, API keys, or credentials. ## Review notes The contracts contain immutable image references and source identity. This change extends their storage lifetime without changing artifact access, credential handling, image publication, or the exact-identity checks. It does not restore artifacts that already expired; #13370 needs a fresh publication before its corrected manual E2E run. --- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> ## Summary by CodeRabbit * **Changes** * Pull request image contracts are now retained for seven days, giving reviewers more time to use them for manual end-to-end testing. * Live end-to-end tests are blocked before candidate execution when the required contract has expired. --------- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Co-authored-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .github/workflows/managed-images.yaml | 2 +- test/e2e/README.md | 2 ++ .../managed/managed-image-publication-workflow.test.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/managed-images.yaml b/.github/workflows/managed-images.yaml index 05df5931c3d..c7c58b1f90a 100644 --- a/.github/workflows/managed-images.yaml +++ b/.github/workflows/managed-images.yaml @@ -1081,7 +1081,7 @@ jobs: path: ${{ runner.temp }}/managed-pr-contract/contract.json if-no-files-found: error overwrite: true - retention-days: 1 + retention-days: 7 pr-managed-activation: name: PR exact all-agent managed runtime activation (${{ matrix.arch }}) diff --git a/test/e2e/README.md b/test/e2e/README.md index bdcb82694aa..f1aec53f98c 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -259,6 +259,8 @@ No matching successful run, invalid or duplicated run metadata, or incomplete, d Manual PR E2E does not fall back to local Dockerfile builds. PR image publication builds each shipped agent on native AMD64 and ARM64 runners. +The workflow retains PR all-agent image contracts for seven days so manual PR E2E can bind them +after review. An expired contract blocks live E2E before candidate execution. Each platform retains its own digest contract and runs the direct startup checks. For OpenClaw, those checks also run the shared agent-reload proof against the image's generated gateway policy and installed distribution. The proof checks live named-agent discovery, gateway diff --git a/test/inference/managed/managed-image-publication-workflow.test.ts b/test/inference/managed/managed-image-publication-workflow.test.ts index 336a3aaac00..1b9f6afbe1a 100644 --- a/test/inference/managed/managed-image-publication-workflow.test.ts +++ b/test/inference/managed/managed-image-publication-workflow.test.ts @@ -584,6 +584,7 @@ describe("complete managed-image publication workflow", () => { expect(contractUpload.with).toMatchObject({ name: "managed-pr-contract-${{ matrix.arch == 'arm64' && 'arm64-' || '' }}${{ github.run_id }}-${{ matrix.agent }}", overwrite: true, + "retention-days": 7, }); expect( step(managedPrActivation(workflow), "Download exact published all-agent contracts").with From 4501ee981d74e2ec88fcd30667625d106575ac82 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 19 Oct 2026 18:14:13 +0700 Subject: [PATCH 07/22] fix(inference): send native local health probes to the recorded provider (#13447) ## Outcome For OpenClaw sandboxes that use a native local provider (Ollama, vLLM or llama.cpp), `status`, `connect` and `start` now send their inference health request to the provider recorded for the sandbox. Before this change they sent it to the shared `inference.local` route, which native-local onboarding does not configure. OpenShell 0.0.116 answers that route with HTTP 503 ("cluster inference is not configured"). `status` still displayed the native endpoint. The GPU Ollama and GPU double-onboard E2Es now use the per-sandbox provider too. ## Reason Native-local onboarding attaches a provider to each sandbox and leaves the shared route empty (#13293). The probe request builder used the native endpoint only when the caller passed an agent that uses native local inference. Some callers did not pass one: - `status` passed the agent only for Deep Agents Code. - `connect` and `start` readiness represent OpenClaw as a null agent. Without an agent, the builder used `https://inference.local/v1/chat/completions`. `doctor` passes the agent, so it passed while `status` and `connect` failed. This matches run 39196254769 on `762aef4c82`: - The authenticated native OpenClaw test failed at `start` after a successful rebuild. - Both managed vLLM cases passed `doctor` and failed `connect --probe-only`. The GPU tests separately hardcoded `inference.local` or expected a shared Ollama route. The 503 response bodies were not retained. The 503 mechanism comes from the pinned OpenShell source (`crates/openshell-supervisor-network/src/proxy.rs`, v0.0.116). The wrong destination is reproduced by the unit tests below. This PR does not claim to explain every historical 503 in #13338. ### Related issues Refs #13338 ## Changes - `src/lib/actions/sandbox/inference-invocation-probe.ts`: when the sandbox has a native local attachment, the probe sends its request to the attachment's recorded endpoint, whichever agent the caller passes. - `src/lib/actions/sandbox/inference-route-health.ts`: native local status reports the endpoint that the failed request used. - GPU E2E: `gpu-e2e` checks the recorded native Ollama endpoint. Both GPU tests send their chat requests there with the OpenShell-issued credential handle (`test/e2e/fixtures/native-local-chat.ts`). - `tools/e2e/target-catalogue.mts`: - Changes to the chat fixture select both GPU targets. - Changes to the probe builder or route health select `inference-routing-openclaw-native`. - Tests: `test/helpers/native-local-probe-shell.ts` runs the generated probe command in a real shell with a recording `curl`. New cases check: - the destination URL and `Authorization` header for vLLM, llama.cpp and Ollama, with the agent omitted, null or `openclaw`; - the legacy null OpenClaw paths through `status` and readiness health; - the endpoint reported on failure; - rejection of a raw credential; - selections without a native attachment still using the shared route. - `ci/platform-matrix.json` and `docs/reference/platform-support.mdx`: new OpenClaw local selections are now described as using sandbox-attached providers. ## Verification - Regression: on the unfixed probe code, the new cases fail (11 failed, 8 passed). With the fix, all 19 pass, and all 182 tests in the three suites pass. - `npm run test:changed` (2525 tests) has 7 failures: - 6 in `native-podman-setup-action.test.ts`, which also fail on unmodified `762aef4c82` in this environment. - 1 in `portable-openclaw-pairing-settlement`, which passes when run alone. - Workflow plan tests: 142 passed. - `tsc -p tsconfig.cli.json --noEmit`, oxlint and oxfmt: passed. Commit and pre-push hooks passed. - `e2e-mock-parity` and `test:e2e-phases:check`: passed. - Live E2E on the final commit: results will be posted on this PR. - The diff contains no secrets, API keys or credentials. ## Review notes - An earlier revision of this PR also derived the agent and passed it from `status`, `connect` and readiness health. It was replaced by the smaller endpoint selection above. - #13354 retries native-local 503s during restart. The cause it describes (a reference-path 503 while `doctor` succeeds) matches the endpoint selection fixed here. A retry does not change the destination. --- Signed-off-by: San Dang ## Summary by CodeRabbit * **New Features** * New OpenClaw selections of Ollama, standalone vLLM, and llama.cpp use their sandbox-attached provider for inference health checks and requests. * Health-check failures now report the endpoint that was actually contacted. * **Compatibility** * Earlier OpenClaw sandboxes without an attached provider, Pi, Local NVIDIA NIM, and other custom or local selections continue to use the shared inference route. * Requests using raw local credentials are not sent. * **Documentation** * Updated provider-routing guidance and platform support details. --------- Signed-off-by: San Dang --- ci/platform-matrix.json | 2 +- docs/get-started/quickstart.mdx | 3 +- docs/reference/platform-support.mdx | 2 +- .../inference-invocation-probe.test.ts | 110 ++++++++++++++++++ .../sandbox/inference-invocation-probe.ts | 3 +- .../actions/sandbox/inference-route-health.ts | 4 +- .../launch-readiness-gateway-health.test.ts | 55 +++++++++ .../status-snapshot-inference-health.test.ts | 51 ++++++++ test/e2e/fixtures/native-local-chat.ts | 19 +++ test/e2e/live/gpu-double-onboard.test.ts | 13 ++- test/e2e/live/gpu-e2e.test.ts | 33 +++--- test/e2e/mock-parity.json | 5 +- test/e2e/support/gpu-e2e-helpers.test.ts | 67 +++++++++++ test/e2e/support/workflow-plan.test.ts | 13 +++ test/helpers/inference-set.ts | 2 + test/helpers/native-local-probe-shell.ts | 69 +++++++++++ tools/e2e/target-catalogue.mts | 4 + 17 files changed, 433 insertions(+), 22 deletions(-) create mode 100644 test/e2e/fixtures/native-local-chat.ts create mode 100644 test/helpers/native-local-probe-shell.ts diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index cc84f687305..43e548bb33e 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -208,7 +208,7 @@ { "name": "Managed inference", "status": "tested", - "notes": "Provider-routed model calls through OpenShell, transparent to the agent. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Deep Agents Code selections of Ollama, standalone vLLM, llama.cpp, and qualifying local compatible endpoints use sandbox-attached providers for their qualified host-local endpoints. Hermes implements the same attachment path, with live qualification for its new local lifecycle pending. OpenClaw, Pi, owned Local NVIDIA NIM, and other custom or local selections retain the shared `inference.local` route. OpenShell keeps provider credentials outside the sandbox. Supports every entry in the Providers table." + "notes": "Provider-routed model calls through OpenShell, transparent to the agent. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Deep Agents Code selections of Ollama, standalone vLLM, llama.cpp, and qualifying local compatible endpoints use sandbox-attached providers for their qualified host-local endpoints. Hermes implements the same attachment path, with live qualification for its new local lifecycle pending. New OpenClaw selections of Ollama, standalone vLLM, and llama.cpp also use sandbox-attached providers, and their `status`, `connect`, and `start` health checks send requests to the recorded provider endpoint. Earlier OpenClaw sandboxes without a sandbox-attached provider, Pi, owned Local NVIDIA NIM, and other custom or local selections retain the shared `inference.local` route. OpenShell keeps provider credentials outside the sandbox. Supports every entry in the Providers table." }, { "name": "Declarative network policy", diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 03d8b724438..3b98b3e215d 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -382,7 +382,8 @@ Use these details when your first-run path needs more control. After confirmation, NemoClaw registers the selected provider with OpenShell. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. - Custom and local providers retain the shared `inference.local` route. + New Ollama, standalone vLLM, and llama.cpp selections also use sandbox-attached providers for their host-local endpoints. + Local NVIDIA NIM, custom endpoints, and earlier local selections without a sandbox-attached provider retain the shared `inference.local` route. The wizard asks whether to enable web search and offers Brave Search or Tavily Search. Provide `BRAVE_API_KEY` for Brave Search or `TAVILY_API_KEY` for Tavily Search when prompted. NemoClaw validates the selected key before it creates the sandbox, registers a sandbox-scoped OpenShell provider, and writes only an OpenShell resolver placeholder into the OpenClaw configuration. diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index 43fef04e495..434c76ff542 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -135,7 +135,7 @@ Each row below is a launch-facing capability claim that NemoClaw makes in docs, |------------|--------|-------| | Guided onboarding | Tested | Single-command interactive wizard (`$$nemoclaw onboard`) that walks the user through inference provider selection, credential setup, and sandbox creation or update. It opens the agent dashboard when the selected runtime provides one. Non-interactive mode is supported with `--non-interactive` and `NEMOCLAW_*` environment variables for CI and scripted installs. | | Sandboxed execution | Tested with limitations | Landlock, seccomp, network namespace isolation, no-new-privileges, privilege dropping, and process limits (nproc 512 at `scripts/lib/sandbox-rlimits.sh:8`) are tested and on by default. OpenShell owns capability enforcement for managed entrypoints, exec processes, and connect shells; those paths do not call NemoClaw capability handling. Direct container runtimes that explicitly override the image user to root retain the existing compatibility drop and required `setpriv` transition. Direct container launches must configure capabilities at the container runtime. | -| Managed inference | Tested | Provider-routed model calls through OpenShell, transparent to the agent. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Deep Agents Code selections of Ollama, standalone vLLM, llama.cpp, and qualifying local compatible endpoints use sandbox-attached providers for their qualified host-local endpoints. Hermes implements the same attachment path, with live qualification for its new local lifecycle pending. OpenClaw, Pi, owned Local NVIDIA NIM, and other custom or local selections retain the shared `inference.local` route. OpenShell keeps provider credentials outside the sandbox. Supports every entry in the Providers table. | +| Managed inference | Tested | Provider-routed model calls through OpenShell, transparent to the agent. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Deep Agents Code selections of Ollama, standalone vLLM, llama.cpp, and qualifying local compatible endpoints use sandbox-attached providers for their qualified host-local endpoints. Hermes implements the same attachment path, with live qualification for its new local lifecycle pending. New OpenClaw selections of Ollama, standalone vLLM, and llama.cpp also use sandbox-attached providers, and their `status`, `connect`, and `start` health checks send requests to the recorded provider endpoint. Earlier OpenClaw sandboxes without a sandbox-attached provider, Pi, owned Local NVIDIA NIM, and other custom or local selections retain the shared `inference.local` route. OpenShell keeps provider credentials outside the sandbox. Supports every entry in the Providers table. | | Declarative network policy | Tested | YAML-defined egress with policy presets. Presets include `slack`, `discord`, `telegram`, `weather`, `openclaw-pricing`, `huggingface`, `npm`, `pypi`, `brew`, and others. Hot-reloadable at runtime with `$$nemoclaw policy add`. | | State persistence and rebuild transfer | Tested | OpenShell preserves the native agent home during routine lifecycle operations. Before destructive replacement, NemoClaw archives and integrity-checks the complete native home/workspace, then restores it into the replacement sandbox. | | Agent skills | Tested | Packaged agent skills are discoverable by Cursor, Claude Code, and other coding assistants under `.agents/skills/`. Skills also install into the sandbox with `$$nemoclaw skill install`. | diff --git a/src/lib/actions/sandbox/inference-invocation-probe.test.ts b/src/lib/actions/sandbox/inference-invocation-probe.test.ts index d4b028e5e50..b2a5f35fd23 100644 --- a/src/lib/actions/sandbox/inference-invocation-probe.test.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.test.ts @@ -27,6 +27,11 @@ import { READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, type SandboxInferenceInvocationInput, } from "./inference-invocation-probe"; +import { + NATIVE_LOCAL_CREDENTIAL_HANDLE, + recordingProbeShell, +} from "../../../../test/helpers/native-local-probe-shell"; +import { nativeLocalTestReceipt } from "../../../../test/helpers/inference-set"; const input = { sandboxName: "dcode-workspace", @@ -977,3 +982,108 @@ it.each([true, undefined])( ).toEqual({ ok: true }); }, ); + +describe("OpenClaw native local probe routing (#13338)", () => { + const PROVIDERS = [ + ["vllm-local", "http://host.openshell.internal:18000/v1"], + ["llama-cpp-local", "http://host.openshell.internal:8081/v1"], + ["ollama-local", "http://host.openshell.internal:11435/v1"], + ] as const; + const okBody = '{"choices":[{"message":{"content":"OK"}}]}'; + + function openClawInput( + provider: (typeof PROVIDERS)[number][0], + endpointUrl: string, + agentName: string | null | undefined, + agent: "openclaw" | null = "openclaw", + ): SandboxInferenceInvocationInput { + return { + sandboxName: "alpha", + gatewayName: "nemoclaw", + ...(agentName === undefined ? {} : { agentName }), + provider, + model: "local-model", + preferredInferenceApi: "openai-completions", + nativeLocalProviderAttachment: nativeLocalTestReceipt( + provider, + endpointUrl, + agent ? { agent } : {}, + ), + }; + } + + it.each( + PROVIDERS.flatMap(([provider, endpointUrl]) => + [undefined, null, "openclaw"].map((agentName) => ({ provider, endpointUrl, agentName })), + ), + )( + "sends $provider to the recorded endpoint when agentName is $agentName", + async ({ provider, endpointUrl, agentName }) => { + const selected = openClawInput(provider, endpointUrl, agentName); + const shell = recordingProbeShell("200", okBody); + await expect( + probeSandboxInferenceInvocation(selected, { execute: shell.execute }), + ).resolves.toEqual({ ok: true }); + expect(shell.destinations()).toEqual([`${endpointUrl}/chat/completions`]); + expect(shell.requests[0]).toContain( + `Authorization: Bearer ${NATIVE_LOCAL_CREDENTIAL_HANDLE}`, + ); + expect(shell.requests[0].join("\n")).not.toContain("inference.local"); + }, + ); + + it("reports the endpoint that actually failed", async () => { + const [provider, endpointUrl] = PROVIDERS[0]; + const shell = recordingProbeShell("503"); + const result = await probeSandboxInferenceInvocation( + openClawInput(provider, endpointUrl, null), + { + execute: shell.execute, + }, + ); + expect(result).toMatchObject({ ok: false, httpStatus: 503 }); + assert(!result.ok); + expect(result.endpoint).toBe(shell.destinations()[0]); + expect(result.endpoint).toBe(`${endpointUrl}/chat/completions`); + }); + + it("refuses a raw credential before calling curl", async () => { + const [provider, endpointUrl] = PROVIDERS[2]; + const shell = recordingProbeShell("200", okBody, { + NEMOCLAW_LOCAL_INFERENCE_TOKEN: "raw-local-secret", + }); + const result = await probeSandboxInferenceInvocation( + openClawInput(provider, endpointUrl, null), + { + execute: shell.execute, + }, + ); + expect(result.ok).toBe(false); + expect(shell.requests).toEqual([]); + }); + + it.each([ + ["ollama-local", undefined], + ["ollama-local", "openclaw"], + ["vllm-local", "openclaw"], + ] as const)( + "keeps a %s selection without native authority (agent %s) on the shared route", + async (provider, agentName) => { + const shell = recordingProbeShell("200", okBody, {}); + await expect( + probeSandboxInferenceInvocation( + { + sandboxName: "alpha", + ...(agentName ? { agentName } : {}), + provider, + model: "local-model", + preferredInferenceApi: "openai-completions", + }, + { execute: shell.execute }, + ), + ).resolves.toEqual({ ok: true }); + expect(shell.destinations()).toEqual(["https://inference.local/v1/chat/completions"]); + expect(shell.requests[0].join("\n")).not.toContain("Authorization"); + }, + ); +}); diff --git a/src/lib/actions/sandbox/inference-invocation-probe.ts b/src/lib/actions/sandbox/inference-invocation-probe.ts index 3577a75280b..60d5031707b 100644 --- a/src/lib/actions/sandbox/inference-invocation-probe.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.ts @@ -145,7 +145,8 @@ function buildProbeRequest(input: SandboxInferenceInvocationInput): { ? nativeHosted.endpoint : isNativeNvidiaProvider(input.provider) || hosted ? "https://inference.local/v1" - : config.inferenceBaseUrl + : // A native local attachment is the route, whichever agent the caller names. + (local?.endpointUrl ?? config.inferenceBaseUrl) ).replace(/\/+$/u, ""); // Gemini's existing OpenAI-compatible API ends in /v1beta/openai, not /v1. const apiBaseUrl = diff --git a/src/lib/actions/sandbox/inference-route-health.ts b/src/lib/actions/sandbox/inference-route-health.ts index d9fe85ddecf..21ac8025a76 100644 --- a/src/lib/actions/sandbox/inference-route-health.ts +++ b/src/lib/actions/sandbox/inference-route-health.ts @@ -438,7 +438,9 @@ export function buildSandboxInferenceRouteHealth( ok: invocation?.ok === true, probed: invocation !== null, providerLabel: "Native local inference", - endpoint: `${context.nativeLocalEndpoint}/chat/completions`, + endpoint: + (invocation?.ok === false && invocation.endpoint) || + `${context.nativeLocalEndpoint}/chat/completions`, detail: invocation?.ok ? "The attached provider served a native local inference request." : invocation diff --git a/src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts b/src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts index fcd20301d5f..3425592de11 100644 --- a/src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts +++ b/src/lib/actions/sandbox/launch-readiness-gateway-health.test.ts @@ -16,6 +16,13 @@ import { requireNativeInferenceHealth, type LaunchReadinessHealthDeps, } from "./launch-readiness/health"; +import { probeSandboxInferenceInvocation } from "./inference-invocation-probe"; +import { runSandboxInferenceInvocationProbe } from "./inference-route-health"; +import { + NATIVE_LOCAL_CREDENTIAL_HANDLE, + recordingProbeShell, +} from "../../../../test/helpers/native-local-probe-shell"; +import { nativeLocalTestReceipt } from "../../../../test/helpers/inference-set"; import { isSandboxGatewayRunningForStatus, waitForStartedHermesGatewayProcess, @@ -448,3 +455,51 @@ describe("Hermes native-local readiness authority", () => { ); }); }); + +describe("OpenClaw native-local readiness identity (#13338)", () => { + it.each([ + { agentName: undefined, agent: "openclaw" }, + { agentName: undefined, agent: undefined }, + { agentName: "openclaw", agent: "openclaw" }, + ])( + "sends the health request to the recorded endpoint (selected $agentName, recorded $agent)", + async ({ agentName, agent }) => { + const receipt = nativeLocalTestReceipt( + "llama-cpp-local", + "http://host.openshell.internal:8081/v1", + { + agent: "openclaw", + gatewayName: "selected", + }, + ); + const shell = recordingProbeShell("200", '{"choices":[{"message":{"content":"OK"}}]}'); + await expect( + requireNativeInferenceHealth({ + sandboxName: "alpha", + gatewayName: "selected", + ...(agentName ? { agentName } : {}), + entry: { + name: "alpha", + ...(agent ? { agent } : {}), + provider: "llama-cpp-local", + model: "local-model", + nativeLocalProviderAttachment: receipt, + }, + deps: { + verifyNativeLocalAttachment: vi.fn(async () => {}), + inferenceInvocationProbe: (input) => + runSandboxInferenceInvocationProbe(input, (selected, _deps, timeoutMs) => + probeSandboxInferenceInvocation(selected, { execute: shell.execute }, timeoutMs), + ), + }, + }), + ).resolves.toBe(true); + expect(shell.destinations()).toEqual([ + "http://host.openshell.internal:8081/v1/chat/completions", + ]); + expect(shell.requests[0]).toContain( + `Authorization: Bearer ${NATIVE_LOCAL_CREDENTIAL_HANDLE}`, + ); + }, + ); +}); diff --git a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts index b00f917e64e..bc75e1b0608 100644 --- a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts @@ -9,6 +9,9 @@ import type { SandboxEntry } from "../../state/registry"; import type { SandboxInferenceInvocationResult } from "./inference-invocation-probe"; import type { SandboxInferenceRouteHealth } from "./inference-route-health"; import { collectSandboxStatusSnapshot } from "./status-snapshot"; +import { probeSandboxInferenceInvocation } from "./inference-invocation-probe"; +import { recordingProbeShell } from "../../../../test/helpers/native-local-probe-shell"; +import { nativeLocalTestReceipt } from "../../../../test/helpers/inference-set"; // Direct, unmocked coverage of `buildSandboxInferenceRouteHealth` through the // exported `collectSandboxStatusSnapshot` entry point (it is not itself @@ -1158,3 +1161,51 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(snapshot.inferenceHealth).toMatchObject({ ok: false, failureLabel: "unhealthy" }); }); }); + +describe("status native local request identity (#13338)", () => { + it.each(["openclaw", undefined] as const)( + "probes and reports the recorded OpenClaw endpoint when the registry agent is %s", + async (agent) => { + const receipt = nativeLocalTestReceipt( + "vllm-local", + "http://host.openshell.internal:18000/v1", + { + agent: "openclaw", + authMode: "sentinel", + }, + ); + const options = snapshotDeps( + null, + null, + { ok: true }, + { + agent, + provider: "vllm-local", + model: "local-model", + gatewayName: "nemoclaw", + nativeLocalProviderAttachment: receipt, + }, + ); + const shell = recordingProbeShell("503"); + const snapshot = await collectSandboxStatusSnapshot("alpha", { + ...options, + deps: { + ...options.deps, + inferenceRouteObserver: { observeInferenceRoute: vi.fn() }, + verifyNativeLocalProviderAttachmentImpl: async () => undefined, + probeSandboxInferenceInvocationImpl: (input, _deps, timeoutMs) => + probeSandboxInferenceInvocation(input, { execute: shell.execute }, timeoutMs), + delayInferenceRecoveryProbe: async () => undefined, + }, + }); + expect(shell.destinations().length).toBeGreaterThan(0); + expect(new Set(shell.destinations())).toEqual( + new Set(["http://host.openshell.internal:18000/v1/chat/completions"]), + ); + expect(snapshot.inferenceHealth).toMatchObject({ + ok: false, + endpoint: shell.destinations()[0], + }); + }, + ); +}); diff --git a/test/e2e/fixtures/native-local-chat.ts b/test/e2e/fixtures/native-local-chat.ts new file mode 100644 index 00000000000..0fd7183e2b3 --- /dev/null +++ b/test/e2e/fixtures/native-local-chat.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { NATIVE_LOCAL_AUTH_HEADER_SCRIPT } from "../../../src/lib/inference/native-local/agent-config.ts"; +import { shellQuote } from "./clients/command.ts"; + +/** + * Sandbox shell command for a chat request to a sandbox-owned native local provider (#13293). + * The auth script accepts only the credential handle that OpenShell issues, so the raw local + * credential never enters the request. The endpoint and body are shell-quoted, so request data + * stays one curl argument. + */ +export function nativeLocalChatCommand( + endpointUrl: string, + body: string, + curlFlags: "-fsS" | "-sS", +): string { + return `${NATIVE_LOCAL_AUTH_HEADER_SCRIPT}; curl ${curlFlags} --max-time 120 ${shellQuote(`${endpointUrl}/chat/completions`)} -H "$AUTH_HEADER" -H 'Content-Type: application/json' --data ${shellQuote(body)}`; +} diff --git a/test/e2e/live/gpu-double-onboard.test.ts b/test/e2e/live/gpu-double-onboard.test.ts index 03777ab4a0e..786ee92beea 100644 --- a/test/e2e/live/gpu-double-onboard.test.ts +++ b/test/e2e/live/gpu-double-onboard.test.ts @@ -4,6 +4,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { normalizeNativeLocalProviderAttachment } from "../../../src/lib/inference/native-local/contract.ts"; +import { getSandbox } from "../../../src/lib/state/registry.ts"; import { containsAnswer } from "../../helpers/e2e-answer-assertions.ts"; import { execTimeout, testTimeout } from "../../helpers/timeouts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -15,6 +17,7 @@ import { resultText } from "../fixtures/clients/command.ts"; import { type HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { nativeLocalChatCommand } from "../fixtures/native-local-chat.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; @@ -140,12 +143,18 @@ async function expectSandboxInference42( model: string, artifactName: string, ): Promise { + // OpenClaw local inference uses a sandbox-owned provider behind the auth proxy, not the shared + // gateway route (#13293). Re-read it after re-onboarding recreates the sandbox. A missing + // attachment yields an unresolvable host, which fails the request assertion below. + const endpoint = + normalizeNativeLocalProviderAttachment(getSandbox(SANDBOX_NAME)?.nativeLocalProviderAttachment) + ?.endpointUrl ?? "http://missing-native-local-attachment.invalid/v1"; const response = await sandbox.exec( SANDBOX_NAME, [ "sh", "-lc", - `curl -fsS --max-time 120 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data '${chatRequest(model)}' | ${parseReplyCommand()}`, + `${nativeLocalChatCommand(endpoint, chatRequest(model), "-fsS")} | ${parseReplyCommand()}`, ], { artifactName, @@ -192,7 +201,7 @@ test( "the persisted Ollama auth-proxy token works after first onboard", "nemoclaw onboard --non-interactive --yes recreates the sandbox", "the running proxy accepts the persisted token after re-onboard and rejects unauthenticated/wrong-token requests", - "sandbox inference.local reaches Ollama after re-onboard", + "the sandbox-owned native provider reaches Ollama after first onboard and re-onboard", ], }); diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index 0c0b895599d..e81a15feedd 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -7,14 +7,17 @@ import path from "node:path"; import { EXPORTED_VLLM_PROFILE_ID } from "../../../src/lib/config/model.ts"; import type { V1Alpha1OllamaProxyService } from "../../../src/lib/config/v1alpha1-export.ts"; import { cleanupLocalModelRuntimes } from "../../../src/lib/inference/local-model-profile/cleanup.ts"; +import { normalizeNativeLocalProviderAttachment } from "../../../src/lib/inference/native-local/contract.ts"; import { HOST_LOCAL_VLLM_CONTAINER_NAME } from "../../../src/lib/inference/serving/vllm-host-local-lifecycle.ts"; import { loadManagedVllmApiKey } from "../../../src/lib/inference/vllm-api-key.ts"; +import { getSandbox } from "../../../src/lib/state/registry.ts"; import { execTimeout, testTimeout } from "../../helpers/timeouts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { parseConfigExport } from "../fixtures/phases/config-export-validation.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { nativeLocalChatCommand } from "../fixtures/native-local-chat.ts"; import { assertAgentExecutionSucceeded, assertGpuInstallProofs, @@ -84,7 +87,7 @@ test( "install Ollama and GPU sandbox", "validate GPU runtime status", "validate Ollama proxy credential boundary", - "run sandbox inference.local chat", + "run sandbox native Ollama chat", "restart Ollama and recover agent inference", ], }, @@ -93,7 +96,7 @@ test( await artifacts.target.declare({ id: "gpu-e2e", boundary: - "GPU host + install.sh Ollama provider + OpenShell sandbox + auth proxy + inference.local", + "GPU host + install.sh Ollama provider + OpenShell sandbox + auth proxy + sandbox-owned native provider", credentialBoundary: "The proxy token remains host/OpenShell-owned and is absent from sandbox env.", remoteInstallerBoundary: @@ -191,12 +194,12 @@ test( expect(retainedSandboxContainer.State).toBe("running"); expect(retainedSandboxContainer.Status).toMatch(/\(healthy\)/i); - const route = await sandbox.openshell(["inference", "get"], { - artifactName: "openshell-inference-route", - env: env(), - timeoutMs: 30_000, - }); - expect(resultText(route)).toMatch(/ollama/i); + // OpenClaw local inference uses a sandbox-owned provider behind the auth proxy, not the + // shared gateway route (#13293). + const nativeEndpoint = normalizeNativeLocalProviderAttachment( + getSandbox(SANDBOX_NAME)?.nativeLocalProviderAttachment, + )?.endpointUrl; + expect(nativeEndpoint).toBe(`http://host.openshell.internal:${PROXY_PORT}/v1`); progress.phase("validate Ollama proxy credential boundary"); const tokenRecord = readTokenFileChecked(ollamaProxyTokenFile()); @@ -238,13 +241,14 @@ test( "OpenShell owns proxy authentication; the host proxy token must not enter sandbox env", ).toBe(""); - progress.phase("run sandbox inference.local chat"); + progress.phase("run sandbox native Ollama chat"); const model = await detectOllamaModel(host); const chat = await sandbox.execShell( SANDBOX_NAME, trustedSandboxShellScript( - `curl -sS --max-time 120 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data '${JSON.stringify( - { + nativeLocalChatCommand( + nativeEndpoint ?? "http://missing-native-local-attachment.invalid/v1", + JSON.stringify({ model, messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], // Keep this assertion about routed inference, not the model's reasoning-token budget. @@ -252,10 +256,11 @@ test( seed: 0, temperature: 0, max_tokens: 32, - }, - )}'`, + }), + "-sS", + ), ), - { artifactName: "sandbox-inference-local-chat", env: env(), timeoutMs: 150_000 }, + { artifactName: "sandbox-native-ollama-chat", env: env(), timeoutMs: 150_000 }, ); expect(chatContent(chat.stdout)).toMatch(/pong/i); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 944c8ddccf2..2c8e2653ab3 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -602,9 +602,11 @@ }, { "live": "test/e2e/live/gpu-double-onboard.test.ts", + "liveSources": ["test/e2e/fixtures/native-local-chat.ts"], "fast": [ "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/gpu-e2e-helpers.test.ts", "test/helpers/e2e-answer-assertions.test.ts" ] }, @@ -612,7 +614,8 @@ "live": "test/e2e/live/gpu-e2e.test.ts", "liveSources": [ "test/e2e/live/gpu-e2e-helpers.ts", - "test/e2e/live/hermes-cli-adapter-live.ts" + "test/e2e/live/hermes-cli-adapter-live.ts", + "test/e2e/fixtures/native-local-chat.ts" ], "fast": [ "src/lib/adapters/config/live-export-source-ollama.test.ts", diff --git a/test/e2e/support/gpu-e2e-helpers.test.ts b/test/e2e/support/gpu-e2e-helpers.test.ts index 3eca30364b1..8ce1227225b 100644 --- a/test/e2e/support/gpu-e2e-helpers.test.ts +++ b/test/e2e/support/gpu-e2e-helpers.test.ts @@ -11,6 +11,7 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { nativeLocalChatCommand } from "../fixtures/native-local-chat.ts"; import { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import { CleanupRegistry } from "../fixtures/cleanup.ts"; @@ -854,3 +855,69 @@ describe("GPU E2E helpers", () => { } }); }); + +describe("GPU native local chat command", () => { + const endpointUrl = "http://host.openshell.internal:11435/v1"; + const body = JSON.stringify({ model: GPU_MODEL, messages: [{ role: "user", content: "hi" }] }); + + function runWithRecordingCurl(token: string, curlFlags: "-fsS" | "-sS", requestBody = body) { + const root = mkdtempSync(path.join(tmpdir(), "gpu-native-chat-")); + try { + const bin = path.join(root, "bin"); + mkdirSync(bin); + writeShellCommands(bin, [["curl", 'for arg in "$@"; do printf "%s\\n" "$arg"; done\n']]); + try { + const stdout = execFileSync( + "sh", + ["-c", nativeLocalChatCommand(endpointUrl, requestBody, curlFlags)], + { + ...SYNC_E2E_CHILD_OPTIONS, + encoding: "utf8", + env: { + PATH: `${bin}:${process.env.PATH ?? ""}`, + NEMOCLAW_LOCAL_INFERENCE_TOKEN: token, + }, + }, + ); + return { status: 0, args: stdout.split("\n").filter(Boolean) }; + } catch (error) { + return { status: (error as { status?: number }).status, args: [] as string[] }; + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + } + + it("sends the OpenShell-issued handle to the recorded endpoint", () => { + const handle = "openshell:resolve:env:v7_NEMOCLAW_LOCAL_INFERENCE_TOKEN"; + expect(runWithRecordingCurl(handle, "-fsS")).toEqual({ + status: 0, + args: [ + "-fsS", + "--max-time", + "120", + `${endpointUrl}/chat/completions`, + "-H", + `Authorization: Bearer ${handle}`, + "-H", + "Content-Type: application/json", + "--data", + body, + ], + }); + expect(runWithRecordingCurl(handle, "-sS").args[0]).toBe("-sS"); + }); + + it("refuses a raw or missing credential before calling curl", () => { + expect(runWithRecordingCurl("raw-proxy-token-value", "-fsS")).toEqual({ status: 2, args: [] }); + expect(runWithRecordingCurl("", "-fsS")).toEqual({ status: 2, args: [] }); + }); + + it("keeps request data with shell syntax as one curl argument", () => { + const handle = "openshell:resolve:env:v7_NEMOCLAW_LOCAL_INFERENCE_TOKEN"; + const hostileBody = JSON.stringify({ model: "x'; echo injected; '$(echo also)", messages: [] }); + const result = runWithRecordingCurl(handle, "-fsS", hostileBody); + expect(result.status).toBe(0); + expect(result.args.slice(-2)).toEqual(["--data", hostileBody]); + }); +}); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 3b739488ea3..6a4e9879a0a 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -696,6 +696,19 @@ describe("E2E workflow plan", () => { expect(selectedWorkflowJobs(plan)).toEqual(["catalogue-standard", "jetson-nvmap-gpu"]); }); + it.each<[string, string[]]>([ + ["test/e2e/fixtures/native-local-chat.ts", ["gpu-double-onboard", "gpu-e2e"]], + [ + "src/lib/actions/sandbox/inference-invocation-probe.ts", + ["inference-routing-openclaw-native"], + ], + ["src/lib/actions/sandbox/inference-route-health.ts", ["inference-routing-openclaw-native"]], + ])("selects the live owners of %s", (changedFile, targets) => { + expect(catalogueTargetsForChangedFiles([changedFile]).map((target) => target.id)).toEqual( + expect.arrayContaining(targets), + ); + }); + it("selects Hermes GPU startup when its output proof changes", () => { const plan = buildE2eWorkflowPlan( {}, diff --git a/test/helpers/inference-set.ts b/test/helpers/inference-set.ts index 1e6945e61a6..ca4af828f4d 100644 --- a/test/helpers/inference-set.ts +++ b/test/helpers/inference-set.ts @@ -292,6 +292,7 @@ export type CaptureOpenshell = ( export function nativeLocalTestReceipt( provider: NativeLocalBinding["provider"] = "ollama-local", endpointUrl = "http://host.openshell.internal:11435/v1", + overrides: Partial = {}, ) { const binding: NativeLocalBinding = { provider, @@ -300,6 +301,7 @@ export function nativeLocalTestReceipt( gatewayName: "nemoclaw", credentialEnv: "NEMOCLAW_LOCAL_INFERENCE_TOKEN", authMode: "authenticated", + ...overrides, }; return { ...binding, diff --git a/test/helpers/native-local-probe-shell.ts b/test/helpers/native-local-probe-shell.ts new file mode 100644 index 00000000000..977d7ac5576 --- /dev/null +++ b/test/helpers/native-local-probe-shell.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import type { SandboxInferenceInvocationDeps } from "../../src/lib/actions/sandbox/inference-invocation-probe"; + +/** The shape OpenShell issues for an attached native local provider credential. */ +export const NATIVE_LOCAL_CREDENTIAL_HANDLE = + "openshell:resolve:env:v7_NEMOCLAW_LOCAL_INFERENCE_TOKEN"; + +/** + * Run each generated probe command in a real shell whose curl records its + * arguments and answers with a fixed HTTP status and body. The recorded + * arguments show the actual request destination and headers. + */ +export function recordingProbeShell( + code: string, + body = "", + environment: NodeJS.ProcessEnv = { + NEMOCLAW_LOCAL_INFERENCE_TOKEN: NATIVE_LOCAL_CREDENTIAL_HANDLE, + }, +) { + const requests: string[][] = []; + const execute: NonNullable = async ( + _sandboxName, + command, + ) => { + const dir = mkdtempSync(path.join(tmpdir(), "nemoclaw-probe-route-")); + try { + const bin = path.join(dir, "bin"); + const argv = path.join(dir, "argv.txt"); + mkdirSync(bin); + writeFileSync(path.join(dir, "body.txt"), body); + writeFileSync( + path.join(bin, "curl"), + [ + "#!/bin/sh", + `printf '%s\\n' "$@" > ${JSON.stringify(argv)}`, + 'out=""; prev=""', + 'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done', + `cat ${JSON.stringify(path.join(dir, "body.txt"))} > "$out"`, + `printf '%s' ${JSON.stringify(code)}`, + ].join("\n"), + { mode: 0o755 }, + ); + const run = spawnSync("/bin/sh", ["-c", command], { + encoding: "utf8", + env: { PATH: `${bin}:${process.env.PATH ?? ""}`, ...environment }, + }); + try { + requests.push(readFileSync(argv, "utf8").trimEnd().split("\n")); + } catch { + /* curl was not reached */ + } + return { status: run.status ?? 1, stdout: run.stdout ?? "", stderr: run.stderr ?? "" }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + return { + execute, + requests, + destinations: () => requests.map((argv) => argv.at(-1)), + }; +} diff --git a/tools/e2e/target-catalogue.mts b/tools/e2e/target-catalogue.mts index 894ea84cd39..e8546a80e1d 100644 --- a/tools/e2e/target-catalogue.mts +++ b/tools/e2e/target-catalogue.mts @@ -816,6 +816,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ installMode: "authenticated", restoreCli: true, exposeCliBin: true, + owningPaths: ["test/e2e/fixtures/native-local-chat.ts"], environment: { ...nonInteractive, NEMOCLAW_MODEL: "qwen3.5:9b", @@ -836,6 +837,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ exposeCliBin: true, owningPaths: [ "test/e2e/live/gpu-e2e-helpers.ts", + "test/e2e/fixtures/native-local-chat.ts", "test/e2e/live/hermes-cli-adapter-live.ts", "src/lib/inference/ollama/proxy.ts", "src/lib/inference/ollama/proxy-observation.ts", @@ -1058,6 +1060,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ "src/lib/onboard/openclaw/initial-inference-route.ts", "src/lib/onboard/setup-inference.ts", "src/lib/actions/sandbox/rebuild-preflight-guards.ts", + "src/lib/actions/sandbox/inference-invocation-probe.ts", + "src/lib/actions/sandbox/inference-route-health.ts", "test/e2e/fixtures/inference-routing-openclaw-native.ts", "tools/e2e/onboard-timeout-contract.mts", ], From b01bb7ffc1bc0e1e7c8b015c7103039998aa9a6e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter Date: Mon, 19 Oct 2026 05:57:08 -0700 Subject: [PATCH 08/22] test(e2e): retain portable registry readiness evidence (#13352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Outcome Portable Hermes finalization now retains bounded managed-registry readiness evidence and failure diagnostics before cleanup. The live scenario still requires an unauthenticated HTTP 200 from the local registry before build and publication continue. ## Reason The canonical-main finalization run for the prior correction stopped when the managed registry never became reachable, but cleanup removed the container and port-forwarding state needed to classify the failure. This change makes a recurrence diagnosable without broadening the accepted readiness contract or retrying mutation. ## Changes - Replace the ad hoc registry polling loop with the repository retry-evidence helper. The Portable Hermes live scenario is the sole consumer, and only connection refusal, reset, and timeout are retryable within the existing bounded startup window. - Retain the attempt receipt on every run. On failure, retain only bounded container state, published port mapping, and the last 50 registry log lines before cleanup; a direct readiness assertion remains in the live scenario. - Add focused E2E-support coverage for first-attempt success, allowed transient retries, deterministic HTTP and transport failures, and exhausted timeout evidence. ## Verification - `vitest run --project e2e-support test/e2e/support/portable-registry-readiness.test.ts` — 5 tests passed - `vitest run --project integration test/automation/pull-requests/growth-guardrails.test.ts` — 7 tests passed - `npm run test:projects:check` — exact membership passed for 2,795 files across 7 projects - `npm run e2e:assertions:check` — live E2E assertion ratchet passed - `npm run test:e2e-phases:check` — semantic phase coverage passed for 104 tests across 78 files - `npm run typecheck:cli` — passed - `npm run checks:repository` — all 18 repository checks passed - Commit hooks — formatting, lint, secret scanning, semantic phases, source shape, growth guardrails, and commit message checks passed - Diff inspection confirmed no secrets, API keys, or credentials --- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> ## Summary by CodeRabbit * **Bug Fixes** * Portable profile checks now handle temporary registry connection issues with bounded retries and clearer failure reporting. When readiness checks fail, available diagnostic information is retained without hiding the failure. * **Documentation** * Clarified how registry readiness checks behave and what diagnostic information may be collected when they fail. --------- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Co-authored-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .github/workflows/portable-profile-e2e.yaml | 4 + test/e2e/README.md | 4 + .../fixtures/portable-registry-readiness.ts | 192 ++++++++++++++++ .../portable-profile-rootless-linux.test.ts | 40 ++-- test/e2e/mock-parity.json | 2 + ...-profile-rootless-runtime-workflow.test.ts | 6 + .../portable-registry-readiness.test.ts | 214 ++++++++++++++++++ 7 files changed, 439 insertions(+), 23 deletions(-) create mode 100644 test/e2e/fixtures/portable-registry-readiness.ts create mode 100644 test/e2e/support/portable-registry-readiness.test.ts diff --git a/.github/workflows/portable-profile-e2e.yaml b/.github/workflows/portable-profile-e2e.yaml index 7578389197f..dfdd36d49a8 100644 --- a/.github/workflows/portable-profile-e2e.yaml +++ b/.github/workflows/portable-profile-e2e.yaml @@ -87,6 +87,8 @@ on: - "src/lib/core/retry.ts" - "test/e2e/live/portable-profile-rootless-linux.test.ts" - "test/e2e/fixtures/portable-hermes-transaction-live.ts" + - "test/e2e/fixtures/portable-registry-readiness.ts" + - "test/e2e/support/portable-registry-readiness.test.ts" - "test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts" push: branches: @@ -131,6 +133,8 @@ on: - "test/e2e/fixtures/portable-profile-systemctl-shim.sh" - "test/e2e/fixtures/portable-profile-systemctl.ts" - "test/e2e/fixtures/portable-hermes-transaction-live.ts" + - "test/e2e/fixtures/portable-registry-readiness.ts" + - "test/e2e/support/portable-registry-readiness.test.ts" - "test/e2e/live/full-e2e.test.ts" - "test/e2e/live/launch-agent-turn.ts" - "test/e2e/live/portable-profile-gateway-proof.ts" diff --git a/test/e2e/README.md b/test/e2e/README.md index f1aec53f98c..2a7907ef04e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -69,6 +69,10 @@ The install refuses package removals, preserving the runner's Docker and contain `jobs=portable-hermes-finalization`; the job selects Podman 5.7 regardless of gateway-runtime inputs. The live scenario also runs the receipt-bound Hermes build, registry publication, and OpenShell create transaction against those services. + Its managed-registry HTTP readiness check uses a bounded read-only policy with + attempt evidence. A failure attempts to retain redacted container state, the + published port mapping, and a bounded registry log tail before cleanup. + Unavailable diagnostics do not suppress the readiness failure. - `.github/workflows/podman-cpu-proof.yaml` publishes PR-only experimental runtime evidence. - `.github/workflows/sandbox-images.yaml` provides reusable sandbox-image build and test evidence. `.github/workflows/e2e.yaml` selects free-standing jobs, including `whatsapp-qr-compact` and `ollama-auth-proxy`. diff --git a/test/e2e/fixtures/portable-registry-readiness.ts b/test/e2e/fixtures/portable-registry-readiness.ts new file mode 100644 index 00000000000..cf396dbac68 --- /dev/null +++ b/test/e2e/fixtures/portable-registry-readiness.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import http from "node:http"; + +import { + runBoundedRetry, + type BoundedRetryResult, + type RetryEvidence, +} from "../../../tools/e2e/retry-evidence.mts"; +import type { ArtifactSink } from "./artifacts.ts"; +import { buildChildEnv } from "./redaction.ts"; +import { type ShellProbe, trustedShellCommand } from "./shell-probe.ts"; + +export type PortableRegistryReadinessObservation = + | { readonly kind: "ready"; readonly statusCode: 200 } + | { readonly kind: "http-status"; readonly statusCode: number } + | { + readonly kind: "transport"; + readonly code: "connection-refused" | "connection-reset" | "timeout" | "unexpected"; + }; + +export const PORTABLE_REGISTRY_READINESS_MAX_ATTEMPTS = 10; +export const PORTABLE_REGISTRY_READINESS_DELAY_MS = 1_000; + +const PORTABLE_REGISTRY_DIAGNOSTIC_ENV_KEYS = [ + "CONTAINERS_CONF", + "CONTAINERS_STORAGE_CONF", + "DOCKER_HOST", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_RUNTIME_DIR", +] as const; + +function retryableTransport(observation: PortableRegistryReadinessObservation): boolean { + return observation.kind === "transport" && observation.code !== "unexpected"; +} + +/** + * Wait for the managed registry through a bounded read-only policy. + * + * Connection refusal, reset, and timeout are the only transient startup + * signatures. An HTTP response other than the unauthenticated registry 200, + * or an unknown transport failure, stops immediately for diagnostics. + */ +export async function runPortableRegistryReadinessPolicy(options: { + readonly probe: (attempt: number) => Promise; + readonly onEvidence?: (evidence: RetryEvidence) => Promise | void; + readonly maxAttempts?: number; + readonly delayMs?: number; + readonly sleep?: (milliseconds: number) => Promise; +}): Promise> { + return runBoundedRetry({ + operation: "portable-registry.http-readiness", + owner: "portable-profile-rootless-linux", + idempotence: "read-only", + maxAttempts: options.maxAttempts ?? PORTABLE_REGISTRY_READINESS_MAX_ATTEMPTS, + delayMs: options.delayMs ?? PORTABLE_REGISTRY_READINESS_DELAY_MS, + run: options.probe, + classify: (observation, error) => { + if (error !== undefined || observation === undefined) { + return { outcome: "failed", failureClass: "deterministic" }; + } + if (observation.kind === "ready") return { outcome: "passed" }; + return { + outcome: "failed", + failureClass: retryableTransport(observation) ? "transient-external" : "deterministic", + }; + }, + ...(options.sleep ? { sleep: options.sleep } : {}), + ...(options.onEvidence ? { onEvidence: options.onEvidence } : {}), + }); +} + +export async function probePortableRegistry( + url = "http://127.0.0.1:5000/v2/", +): Promise { + return new Promise((resolve) => { + let settled = false; + const settle = (observation: PortableRegistryReadinessObservation): void => { + if (settled) return; + settled = true; + resolve(observation); + }; + const request = http.get(url, (response) => { + const responseFailure = (): void => { + settle({ kind: "transport", code: "unexpected" }); + }; + response.once("aborted", responseFailure); + response.once("error", responseFailure); + response.once("close", () => { + if (!response.complete) responseFailure(); + }); + response.resume(); + response.once("end", () => + settle( + response.statusCode === 200 + ? { kind: "ready", statusCode: 200 } + : { kind: "http-status", statusCode: response.statusCode ?? 0 }, + ), + ); + }); + request.once("error", (error: NodeJS.ErrnoException) => { + const code = + error.code === "ECONNREFUSED" + ? "connection-refused" + : error.code === "ECONNRESET" + ? "connection-reset" + : error.code === "ETIMEDOUT" + ? "timeout" + : "unexpected"; + settle({ kind: "transport", code }); + }); + request.setTimeout(500, () => { + settle({ kind: "transport", code: "timeout" }); + request.destroy(); + }); + }); +} + +async function captureRegistryFailureDiagnostics(shellProbe: ShellProbe): Promise { + const env = buildChildEnv(process.env, { + additionalAllowedEnv: PORTABLE_REGISTRY_DIAGNOSTIC_ENV_KEYS, + fixtureOverlay: {}, + }); + const commands = [ + { + artifactName: "portable-registry-container-inspect", + args: [ + "container", + "inspect", + "--format", + [ + "id={{.Id}}", + "name={{.Name}}", + "status={{.State.Status}}", + "running={{.State.Running}}", + "exit_code={{.State.ExitCode}}", + "error={{.State.Error}}", + ].join("\\n"), + "nemoclaw-portable-registry", + ], + reason: "capture the managed registry container state after readiness failure", + }, + { + artifactName: "portable-registry-port-mapping", + args: ["port", "nemoclaw-portable-registry", "5000/tcp"], + reason: "capture the managed registry published port after readiness failure", + }, + { + artifactName: "portable-registry-log-tail", + args: ["logs", "--tail", "50", "nemoclaw-portable-registry"], + reason: "capture a bounded managed registry log tail after readiness failure", + }, + ] as const; + await Promise.allSettled( + commands.map((command) => + shellProbe.run( + trustedShellCommand({ + command: "podman", + args: command.args, + reason: command.reason, + }), + { + artifactName: command.artifactName, + captureLimitBytes: 32 * 1024, + env, + timeoutMs: 10_000, + }, + ), + ), + ); +} + +export async function collectPortableRegistryReadinessFailure(options: { + readonly artifacts: ArtifactSink; + readonly probe?: (attempt: number) => Promise; + readonly shellProbe: ShellProbe; +}): Promise { + const result = await runPortableRegistryReadinessPolicy({ + probe: options.probe ?? (() => probePortableRegistry()), + onEvidence: async (evidence) => { + await options.artifacts.writeJson("portable-registry-readiness-retry.json", evidence); + }, + }); + if (result.outcome === "passed") return undefined; + await options.artifacts.writeJson("portable-registry-readiness-failure.json", { + observation: result.value ?? null, + }); + await captureRegistryFailureDiagnostics(options.shellProbe); + return result.value; +} diff --git a/test/e2e/live/portable-profile-rootless-linux.test.ts b/test/e2e/live/portable-profile-rootless-linux.test.ts index 54f7f1af8a2..1cd64dac1c8 100644 --- a/test/e2e/live/portable-profile-rootless-linux.test.ts +++ b/test/e2e/live/portable-profile-rootless-linux.test.ts @@ -5,7 +5,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; -import http from "node:http"; import os from "node:os"; import path from "node:path"; @@ -59,13 +58,16 @@ import type { SandboxEntry } from "../../../src/lib/state/registry/types.ts"; import { retryUntil } from "../../../src/lib/core/retry.ts"; import { streamSandboxCreate } from "../../../src/lib/sandbox/create-stream.ts"; import { test } from "../fixtures/e2e-test.ts"; +import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { OPENSHELL_V0116_QUALIFICATION } from "../fixtures/openshell-v0116-qualification.ts"; +import { collectPortableRegistryReadinessFailure } from "../fixtures/portable-registry-readiness.ts"; import { cleanupPortableHostGatewayAlias, cleanupPortableProfileRootlessFixture, installPortableProfileSystemctlShim, } from "../fixtures/portable-profile-systemctl.ts"; import type { TestProgress } from "../fixtures/progress.ts"; +import type { ShellProbe } from "../fixtures/shell-probe.ts"; import { getSandboxConfigRequest, mintSandboxJwt, @@ -335,24 +337,12 @@ function parseOnePodmanRecord(raw: string, label: string): Record; } -async function waitForRegistry(attempt = 0): Promise { - assert.ok(attempt < 60, "The managed local registry did not become ready."); - const ready = await new Promise((resolve) => { - const request = http.get("http://127.0.0.1:5000/v2/", (response) => { - response.resume(); - response.once("end", () => resolve(response.statusCode === 200)); - }); - request.once("error", () => resolve(false)); - request.setTimeout(500, () => { - request.destroy(); - resolve(false); - }); - }); - return ready - ? undefined - : new Promise((resolve) => setTimeout(resolve, 250)).then(() => - waitForRegistry(attempt + 1), - ); +async function waitForRegistry(artifacts: ArtifactSink, shellProbe: ShellProbe): Promise { + assert.equal( + await collectPortableRegistryReadinessFailure({ artifacts, shellProbe }), + undefined, + "The managed local registry did not become ready. See the bounded registry readiness artifacts.", + ); } function selectInstallerPodmanRuntime(repoRoot: string): string { @@ -919,7 +909,11 @@ async function proveHistoricalHermesPortableLifecycle(input: { return primaryFailed || cleanupFailed ? Promise.reject(failure) : lifecycleEvidence!; } -async function main(progress: TestProgress): Promise { +async function main( + progress: TestProgress, + artifacts: ArtifactSink, + shellProbe: ShellProbe, +): Promise { assert.equal(process.platform, "linux", "portable profile E2E requires Linux"); assert.notEqual(process.getuid?.(), 0, "portable profile E2E must run without root privileges"); const sourceRevision = process.env.E2E_SOURCE_REVISION ?? ""; @@ -997,7 +991,7 @@ async function main(progress: TestProgress): Promise { const podmanInfo = JSON.parse(run("podman", ["info", "--format", "json"])); assert.equal(podmanInfo.host?.security?.rootless, true, "Podman must be rootless"); run("docker", ["version"]); - await waitForRegistry(); + await waitForRegistry(artifacts, shellProbe); progress.phase("verify immutable non-force network removal"); const verifiedPodmanUrl = String(process.env.DOCKER_HOST); @@ -1457,7 +1451,7 @@ test( meta: { e2ePhases: PORTABLE_PROFILE_E2E_PHASES }, timeout: 900_000, }, - async ({ progress }) => { - await main(progress); + async ({ artifacts, progress, shellProbe }) => { + await main(progress, artifacts, shellProbe); }, ); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 2c8e2653ab3..67984038a03 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -33,6 +33,7 @@ }, { "live": "test/e2e/live/portable-profile-rootless-linux.test.ts", + "liveSources": ["test/e2e/fixtures/portable-registry-readiness.ts"], "fast": [ "src/lib/onboard/experimental/hermes-portable-container.test.ts", "src/lib/onboard/experimental/hermes-portable-lifecycle.test.ts", @@ -41,6 +42,7 @@ "src/lib/onboard/experimental/portable-host-preparation.test.ts", "src/lib/onboard/sandbox-prebuild.test.ts", "test/agents/hermes/hermes-start.test.ts", + "test/e2e/support/portable-registry-readiness.test.ts", "test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts" ] }, diff --git a/test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts b/test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts index ae982f872e9..5e73d38d21f 100644 --- a/test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts +++ b/test/e2e/support/portable-profile-rootless-runtime-workflow.test.ts @@ -318,8 +318,14 @@ ${serviceIdentityCheck}`, const selects = (event: "pull_request" | "push", changedPath: string) => workflow.on[event].paths.some((pattern) => matchesGlob(changedPath, pattern)); const transactionFixture = "test/e2e/fixtures/portable-hermes-transaction-live.ts"; + const readinessFixture = "test/e2e/fixtures/portable-registry-readiness.ts"; + const readinessSupportTest = "test/e2e/support/portable-registry-readiness.test.ts"; expect.soft(selects("pull_request", transactionFixture)).toBe(true); expect.soft(selects("push", transactionFixture)).toBe(true); + expect.soft(selects("pull_request", readinessFixture)).toBe(true); + expect.soft(selects("push", readinessFixture)).toBe(true); + expect.soft(selects("pull_request", readinessSupportTest)).toBe(true); + expect.soft(selects("push", readinessSupportTest)).toBe(true); expect.soft(selects("pull_request", "src/lib/domain/sandbox/image-tag.ts")).toBe(true); expect.soft(selects("push", "src/lib/domain/sandbox/image-tag.ts")).toBe(true); expect.soft(selects("pull_request", "src/lib/onboard/session-bootstrap.ts")).toBe(true); diff --git a/test/e2e/support/portable-registry-readiness.test.ts b/test/e2e/support/portable-registry-readiness.test.ts new file mode 100644 index 00000000000..4eb4154217a --- /dev/null +++ b/test/e2e/support/portable-registry-readiness.test.ts @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { once } from "node:events"; +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { ArtifactSink } from "../fixtures/artifacts.ts"; +import { + collectPortableRegistryReadinessFailure, + probePortableRegistry, + runPortableRegistryReadinessPolicy, + type PortableRegistryReadinessObservation, +} from "../fixtures/portable-registry-readiness.ts"; +import type { + ShellProbe, + ShellProbeResult, + ShellProbeRunOptions, + TrustedShellCommand, +} from "../fixtures/shell-probe.ts"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("portable registry readiness policy", () => { + it("records first-attempt readiness without sleeping", async () => { + const sleep = vi.fn(async () => {}); + const onEvidence = vi.fn(); + + const result = await runPortableRegistryReadinessPolicy({ + probe: async () => ({ kind: "ready", statusCode: 200 }), + sleep, + onEvidence, + }); + + expect(result.outcome).toBe("passed"); + expect(result.evidence.outcome).toBe("passed-first-attempt"); + expect(result.evidence.attempts).toEqual([ + { attempt: 1, outcome: "passed", retryScheduled: false }, + ]); + expect(sleep).not.toHaveBeenCalled(); + expect(onEvidence).toHaveBeenCalledWith(result.evidence); + }); + + it("retries only the named transport startup failures", async () => { + const observations: PortableRegistryReadinessObservation[] = [ + { kind: "transport", code: "connection-refused" }, + { kind: "transport", code: "connection-reset" }, + { kind: "ready", statusCode: 200 }, + ]; + const probe = vi.fn(async () => observations.shift()!); + const sleep = vi.fn(async () => {}); + + const result = await runPortableRegistryReadinessPolicy({ probe, sleep, delayMs: 25 }); + + expect(result.outcome).toBe("passed"); + expect(result.evidence.outcome).toBe("passed-after-retry"); + expect(result.evidence.attempts).toEqual([ + { + attempt: 1, + outcome: "failed", + failureClass: "transient-external", + retryScheduled: true, + }, + { + attempt: 2, + outcome: "failed", + failureClass: "transient-external", + retryScheduled: true, + }, + { attempt: 3, outcome: "passed", retryScheduled: false }, + ]); + expect(sleep).toHaveBeenNthCalledWith(1, 25); + expect(sleep).toHaveBeenNthCalledWith(2, 25); + }); + + it.each([ + { kind: "http-status", statusCode: 503 } as const, + { kind: "transport", code: "unexpected" } as const, + ])("stops without retrying a deterministic $kind failure", async (observation) => { + const probe = vi.fn(async () => observation); + const sleep = vi.fn(async () => {}); + + const result = await runPortableRegistryReadinessPolicy({ probe, sleep }); + + expect(result.outcome).toBe("failed"); + expect(result.value).toEqual(observation); + expect(result.evidence.outcome).toBe("failed-no-retry"); + expect(result.evidence.attempts).toEqual([ + { + attempt: 1, + outcome: "failed", + failureClass: "deterministic", + retryScheduled: false, + }, + ]); + expect(sleep).not.toHaveBeenCalled(); + }); + + it("retains exhausted timeout attempt evidence", async () => { + const onEvidence = vi.fn(); + const result = await runPortableRegistryReadinessPolicy({ + probe: async () => ({ kind: "transport", code: "timeout" }), + maxAttempts: 2, + delayMs: 0, + onEvidence, + }); + + expect(result.outcome).toBe("failed"); + expect(result.evidence.outcome).toBe("exhausted"); + expect(result.evidence.attempts).toEqual([ + { + attempt: 1, + outcome: "failed", + failureClass: "transient-external", + retryScheduled: true, + }, + { + attempt: 2, + outcome: "failed", + failureClass: "transient-external", + retryScheduled: false, + }, + ]); + expect(onEvidence).toHaveBeenCalledWith(result.evidence); + }); + + it("retains failure evidence and bounded diagnostics after a response abort", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "portable-registry-readiness-")); + const server = http.createServer((_request, response) => { + response.writeHead(200, { "content-length": "8" }); + response.flushHeaders(); + response.write("partial"); + setImmediate(() => response.destroy()); + }); + try { + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert(address && typeof address !== "string", "HTTP server has no TCP port"); + vi.stubEnv("DOCKER_HOST", "unix:///run/user/1000/podman/podman.sock"); + vi.stubEnv("XDG_RUNTIME_DIR", "/run/user/1000"); + vi.stubEnv("CONTAINERS_CONF", "/tmp/containers.conf"); + vi.stubEnv("UNRELATED_API_KEY", "must-not-reach-podman"); + const run = vi.fn( + async ( + _command: TrustedShellCommand, + _options: ShellProbeRunOptions, + ): Promise => ({ + command: [], + exitCode: 0, + signal: null, + timedOut: false, + stdout: "", + stderr: "", + artifacts: { stdout: "", stderr: "", result: "" }, + }), + ); + const result = await collectPortableRegistryReadinessFailure({ + artifacts: new ArtifactSink(root), + probe: async () => probePortableRegistry(`http://127.0.0.1:${address.port}/v2/`), + shellProbe: { run } as unknown as ShellProbe, + }); + + expect(result).toEqual({ kind: "transport", code: "unexpected" }); + expect( + JSON.parse( + fs.readFileSync(path.join(root, "portable-registry-readiness-retry.json"), "utf8"), + ), + ).toMatchObject({ + outcome: "failed-no-retry", + attempts: [{ attempt: 1, retryScheduled: false }], + }); + expect( + JSON.parse( + fs.readFileSync(path.join(root, "portable-registry-readiness-failure.json"), "utf8"), + ), + ).toEqual({ observation: { kind: "transport", code: "unexpected" } }); + expect(run).toHaveBeenCalledTimes(3); + expect(run.mock.calls.map(([command]) => command.args[0])).toEqual([ + "container", + "port", + "logs", + ]); + const expectedOptions = expect.objectContaining({ + captureLimitBytes: 32 * 1024, + env: expect.objectContaining({ + CONTAINERS_CONF: "/tmp/containers.conf", + DOCKER_HOST: "unix:///run/user/1000/podman/podman.sock", + XDG_RUNTIME_DIR: "/run/user/1000", + }), + timeoutMs: 10_000, + }); + expect(run).toHaveBeenNthCalledWith(1, expect.anything(), expectedOptions); + expect(run).toHaveBeenNthCalledWith(2, expect.anything(), expectedOptions); + expect(run).toHaveBeenNthCalledWith(3, expect.anything(), expectedOptions); + expect(run.mock.calls[0]![1].env).not.toHaveProperty("UNRELATED_API_KEY"); + expect(run.mock.calls[1]![1].env).not.toHaveProperty("UNRELATED_API_KEY"); + expect(run.mock.calls[2]![1].env).not.toHaveProperty("UNRELATED_API_KEY"); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); From 0a9ca6d8178eb27b0ede47094fa76c5674ff60a6 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:30:12 -0700 Subject: [PATCH 09/22] fix(pi): stage managed settings before replacement --- agents/pi/generate-config.ts | 39 +++++++-- .../pi-candidate-runtime-artifacts.test.ts | 84 +++++++++++++++++++ 2 files changed, 117 insertions(+), 6 deletions(-) diff --git a/agents/pi/generate-config.ts b/agents/pi/generate-config.ts index 191f2279bbc..01377f6a912 100644 --- a/agents/pi/generate-config.ts +++ b/agents/pi/generate-config.ts @@ -17,9 +17,21 @@ import { NVIDIA_HOSTED_NATIVE_ENDPOINT, } from "../../src/lib/inference/native-nvidia/contract.ts"; -import { chmodSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { + chmodSync, + closeSync, + fchmodSync, + mkdirSync, + openSync, + readFileSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; const SUPPORTED_INFERENCE_API = "openai-completions"; const MANAGED_PROVIDER_ID = "openshell"; @@ -188,6 +200,7 @@ function syncManagedDefaults(configDir: string, model: string, nativeLocal: bool const settingsPath = join(configDir, "settings.json"); const defaults = { defaultProvider: MANAGED_PROVIDER_ID, defaultModel: model }; let existing: Record; + let destination = settingsPath; try { const content = readFileSync(settingsPath, "utf8"); const parsed: unknown = content.length === 0 ? {} : JSON.parse(content); @@ -195,6 +208,7 @@ function syncManagedDefaults(configDir: string, model: string, nativeLocal: bool throw new Error("Pi settings.json must contain an object."); } existing = parsed as Record; + destination = realpathSync(settingsPath); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; // Hosted startup did not previously create settings.json. Native-local @@ -202,10 +216,23 @@ function syncManagedDefaults(configDir: string, model: string, nativeLocal: bool if (!nativeLocal) return; existing = {}; } - writeFileSync(settingsPath, `${JSON.stringify({ ...existing, ...defaults }, null, 2)}\n`, { - mode: 0o600, - }); - chmodSync(settingsPath, 0o600); + const stagedPath = join(dirname(destination), `.settings.json.${randomUUID()}.tmp`); + let staged = false; + let descriptor: number | null = null; + try { + descriptor = openSync(stagedPath, "wx", 0o600); + staged = true; + writeFileSync(descriptor, `${JSON.stringify({ ...existing, ...defaults }, null, 2)}\n`); + fchmodSync(descriptor, 0o600); + const completedDescriptor = descriptor; + descriptor = null; + closeSync(completedDescriptor); + renameSync(stagedPath, destination); + staged = false; + } finally { + if (descriptor !== null) closeSync(descriptor); + if (staged) rmSync(stagedPath, { force: true }); + } } function main(): void { diff --git a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts index a1892fa9e6c..fed9dde2adf 100644 --- a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts +++ b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts @@ -299,6 +299,90 @@ describe("Pi managed model catalog generation", () => { } }); + it("preserves both Pi files when settings permission update fails (#13362)", () => { + const selected = { + NEMOCLAW_MODEL: "previous-model", + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", + NEMOCLAW_INFERENCE_BASE_URL: "http://host.openshell.internal:11434/v1", + }; + const { home, status, stderr } = generate(selected); + try { + expect(status, stderr).toBe(0); + const configDir = path.join(home, ".pi/agent"); + const modelsPath = path.join(configDir, "models.json"); + const settingsPath = path.join(configDir, "settings.json"); + const previousCatalog = fs.readFileSync(modelsPath); + const previousSettings = fs.readFileSync(settingsPath); + const shimPath = path.join(home, "fail-settings-permissions.cjs"); + fs.writeFileSync( + shimPath, + `const fs = require("node:fs"); +const { syncBuiltinESMExports } = require("node:module"); +const chmodSync = fs.chmodSync; +fs.chmodSync = (target, mode) => { + if (String(target).includes("settings.json")) throw new Error("simulated settings chmod failure"); + return chmodSync(target, mode); +}; +fs.fchmodSync = () => { throw new Error("simulated settings chmod failure"); }; +syncBuiltinESMExports();\n`, + ); + + const failed = spawnSync( + process.execPath, + ["--require", shimPath, path.join(root, "agents/pi/generate-config.ts")], + { + cwd: root, + encoding: "utf8", + env: { + PATH: process.env.PATH ?? "", + HOME: home, + ...selected, + NEMOCLAW_MODEL: "next-model", + }, + }, + ); + expect(failed.status).not.toBe(0); + expect(failed.stderr).toContain("simulated settings chmod failure"); + expect(fs.readFileSync(modelsPath)).toEqual(previousCatalog); + expect(fs.readFileSync(settingsPath)).toEqual(previousSettings); + expect(fs.readdirSync(configDir).sort()).toEqual(["models.json", "settings.json"]); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + + it("updates the target of an existing Pi settings symlink (#13362)", () => { + const { home, status, stderr } = generate( + { + NEMOCLAW_MODEL: "selected-model", + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", + NEMOCLAW_INFERENCE_BASE_URL: "http://host.openshell.internal:11434/v1", + }, + (home) => { + const configDir = path.join(home, ".pi/agent"); + fs.mkdirSync(configDir, { recursive: true }); + fs.writeFileSync(path.join(home, "user-settings.json"), '{"theme":"dark"}\n'); + fs.symlinkSync( + path.join(home, "user-settings.json"), + path.join(configDir, "settings.json"), + ); + }, + ); + try { + expect(status, stderr).toBe(0); + const settingsPath = path.join(home, ".pi/agent/settings.json"); + expect(fs.lstatSync(settingsPath).isSymbolicLink()).toBe(true); + expect(JSON.parse(fs.readFileSync(path.join(home, "user-settings.json"), "utf8"))).toEqual({ + theme: "dark", + defaultProvider: "openshell", + defaultModel: "selected-model", + }); + expect(fs.statSync(settingsPath).mode & 0o777).toBe(0o600); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("updates Pi's managed defaults when the native-local model changes and preserves other settings", () => { const selected = { NEMOCLAW_MODEL: "selected-local-model", From 90f184bec993a755fd6dd110b4a1f6d08366629c Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:24:36 -0700 Subject: [PATCH 10/22] chore(pi): refresh paired qualification receipts --- ci/pi-agent-qualification-v1-linux-amd64.json | 8 ++++---- ci/pi-agent-qualification-v1-linux-arm64.json | 8 ++++---- src/lib/agent/candidate-authority.ts | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/pi-agent-qualification-v1-linux-amd64.json b/ci/pi-agent-qualification-v1-linux-amd64.json index ab071ab179e..8252283bdb4 100644 --- a/ci/pi-agent-qualification-v1-linux-amd64.json +++ b/ci/pi-agent-qualification-v1-linux-amd64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/amd64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:35683a18dff9233b4465f6de83f7749f6bcd02a284765fc77a822d7188b51a38", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:35683a18dff9233b4465f6de83f7749f6bcd02a284765fc77a822d7188b51a38", + "digest": "sha256:a789c4b50b25e02a990389c7d0b2a8dcf8b801036eb4cda43ee6a7574ab409ce", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:a789c4b50b25e02a990389c7d0b2a8dcf8b801036eb4cda43ee6a7574ab409ce", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "f4bc4efd9e29961624fc82bc9866862af98bb5d2", + "revision": "0a9ca6d8178eb27b0ede47094fa76c5674ff60a6", "release": "v0.1.0", - "cohort": "ghrun-38991416051-1" + "cohort": "ghrun-39286562149-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/ci/pi-agent-qualification-v1-linux-arm64.json b/ci/pi-agent-qualification-v1-linux-arm64.json index 4308e7ef690..eb1d8764659 100644 --- a/ci/pi-agent-qualification-v1-linux-arm64.json +++ b/ci/pi-agent-qualification-v1-linux-arm64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/arm64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:46108cfed56bad0ca341147cd7b7f6f1ae410fe7e50be9db7c19c2883bea8bfb", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:46108cfed56bad0ca341147cd7b7f6f1ae410fe7e50be9db7c19c2883bea8bfb", + "digest": "sha256:75bbc5644e9119dc9824e4ad30637ca27032e352d10584e8a624cc1a9c9f5378", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:75bbc5644e9119dc9824e4ad30637ca27032e352d10584e8a624cc1a9c9f5378", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "f4bc4efd9e29961624fc82bc9866862af98bb5d2", + "revision": "0a9ca6d8178eb27b0ede47094fa76c5674ff60a6", "release": "v0.1.0", - "cohort": "ghrun-38991416051-1" + "cohort": "ghrun-39286562149-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/src/lib/agent/candidate-authority.ts b/src/lib/agent/candidate-authority.ts index 58185cbd898..dfd5caaa71c 100644 --- a/src/lib/agent/candidate-authority.ts +++ b/src/lib/agent/candidate-authority.ts @@ -13,8 +13,8 @@ export const CANDIDATE_QUALIFICATION_RECEIPT_DIGESTS: Readonly< Record > = Object.freeze({ pi: Object.freeze([ - "8ff0bc51fa60d58057cc5505d95261c9833b934fcd55d1f900967a66c028090c", - "0a346869512ea536379c2c2ea58209ddffd7b37654a690ba95085544c14d8ecc", + "0a76f266e2831b50b86635f72cf62697932e2f5822a7b291131809a17eb3a87b", + "b785ac087a120b75c06614b419c5c35ee869eaecb2709919af3ef5d7437cc6eb", ]), }); From af1b779c7c53b015335aa838fd6b71a6994a4956 Mon Sep 17 00:00:00 2001 From: hunglp6d <89095484+hunglp6d@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:33:35 +0700 Subject: [PATCH 11/22] fix(onboard): repair DGX Station Express install and resume (#13480) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Outcome DGX Station Express now accepts the serving catalog default and carries its receipt into onboarding. Interrupted setup can retain its GPU/model choices, validate an existing owned vLLM server, and reuse an unchanged sandbox inference route. ## Reason Live installation on a GB300 Station exposed inconsistent contracts between the installer, catalog selection, saved session, model loader, and route reservation. Each failure blocked a later stage after the previous one was repaired. ## Changes - Resolve an unset Station model through the catalog, accept canonical `model=auto` receipts, and export the fresh receipt generation to the CLI. - Accept the downloader's shared Hub blob layout while retaining metadata hashes, canonical directories, offline copies, and cache-escape refusals. - Preserve GPU mode/device before sandbox registration; a boolean cannot distinguish automatic selection from explicit enablement. Saved-preflight regression tests cover this state. - Recognize aliases of the recorded model, validate automatic reuse of an owned reviewed vLLM snapshot, and retry unfinished Station provider selection before binding its checkpoint. - Compare reservations using the existing route contract so retained model-selection provenance does not cause false drift. Session, gateway, attachment, endpoint, model, and API checks remain enforced. - Isolate recovery-test session persistence in private temporary homes established before module imports; reset the module cache, restore the test environment, and remove only owned fixtures. ## Verification Publication candidate: `8e05ebfab808afc84d193bfe4b14df4a3175704c`, based on `762aef4c82dec358e307a771d2b66da352a904e9`. The latest commit changes only two recovery-test fixtures. Production source is unchanged from `498ee02d172c864b2777a86d1c73b61d9f7e3860`; earlier implementation evidence below is retained on that basis. - Session-isolation repair on macOS ARM64, Node 24.14.1, Vitest 4.1.9: ran `vitest run --project cli` separately for `preflight.test.ts` and `provider-inference-station-express.test.ts`. All **25 + 3 tests passed** before and after the fix. Each run started with a caller session created through the real `saveSession()` path in a disposable HOME. The unchanged parent tests deleted that session; the repaired fixtures preserved its bytes exactly. No test cases were added. CLI TypeScript, lint, formatting, and diff checks passed. - Test-reduction checkpoint: **137 passed in six CLI files** on Ubuntu 24.04.4 ARM64 with Node 22.23.3. Substituting the six original production owners reproduced ten expected regression failures; the rest of the patched source remained present during that comparison. - Earlier shared-cache fixture repair on macOS ARM64 with Node 24.14.1: **29 loader tests and all 7 required growth guardrails passed**, along with lint and CLI TypeScript. Shared-cache inputs use independent `test.each` cases without new conditionals, loops, helpers, or budget exceptions. - Earlier focused installer handoff/cleanup, GPU session persistence, provider binding, and reservation checks passed. Overlapping runs are not counted as unique coverage. The tests retain real installer/session paths and denial cases while removing repeated wrappers and fixture combinations. - `node_modules/.bin/tsc -p tsconfig.cli.json`, CLI/plugin builds, formatting, and `git diff --check` passed in their recorded validation stages. The current commit passed all applicable pre-commit hooks, including growth guardrails, and commitlint. The parent commit passed pre-push publication validation and plugin/config/CLI TypeScript checks. Pre-push validation and CI for the current commit remain pending at body preparation time. - Live run: started with `NEMOCLAW_REPO_ROOT="$(pwd)" bash install.sh`, applied incremental fixes, and used native session recovery, including one `--fresh` attempt. The final `nemoclaw onboard --resume --name my-assistant` exited **0**, reused the sandbox, and verified local inference from inside it. Gateway and dashboard were healthy; OpenClaw reported 2026.9.2. This used the same production changes with an earlier test revision. - The diff contains no secrets, API keys, or credentials. No dependency, model, image, or driver pin was upgraded. ## Review notes The [P1 review](https://github.com/NVIDIA/NemoClaw/pull/13480#issuecomment-6271124951) identified caller-state loss in the two new persistence tests. The repair confines session writes and cleanup to test-owned homes. Prior validation used disposable outer homes, which protected the host but did not establish that the tests themselves preserved caller state; the before/after check above closes that evidence gap. This test-safety update is being published before automated evaluation of `498ee02d172c864b2777a86d1c73b61d9f7e3860` has completed. Its [CI run](https://github.com/NVIDIA/NemoClaw/actions/runs/39277544878) and Advisor collection remain incomplete at preparation time. A manual Advisor run will be requested after this push; independent CI and review remain required for the new candidate. This remains a draft pending independent review and CI. The reviewed candidate is NVIDIA/NemoClaw commit `8e05ebfab808afc84d193bfe4b14df4a3175704c`. Sensitive changes are in `scripts/install.sh`, `src/lib/inference/**`, and `src/lib/onboard/**`, with an additional comparison in `src/lib/state/registry.ts`. Pre-publication context is the author's review of the local diff plus assistant source inspection, regression checks, and the manual Station run; this is not independent maintainer approval. The live run used a single GB300 Station with the colossus-baseos profile, driver 595.58.03, Docker 29.2.1, and OpenShell 0.0.116. The managed gateway user service fell back to a standalone gateway. NIC temperature reached 106°C against a 105°C critical limit; the earlier host freeze remains unexplained. This is functional onboarding evidence, not hardware stability or LKG qualification. A clean installation of the final patch, reboot persistence, actual agent tool execution, and the full test suite were not run. The native `--fresh` attempt cleared the original installer receipt, so its final absence is not evidence of successful-completion retirement; fresh receipt handoff/cleanup is covered by the installer tests. The reused sandbox's individual step marker remained pending while the overall session completed; that observation is retained separately from runtime health. --- Signed-off-by: Hung Le ## Summary by CodeRabbit * **Bug Fixes** * Resuming onboarding now retains the selected sandbox GPU mode and device. * vLLM model aliases are recognized consistently during installation and resume, while conflicting checkpoint selections are rejected. * Station Express resumes now handle default model selection and interrupted provider reviews more reliably. * Verified model files can be loaded from the shared Hugging Face cache; files outside approved locations are rejected. * Reusing a sandbox inference route preserves its model-selection details and rejects conflicting changes. --------- Signed-off-by: Hung Le --- scripts/install.sh | 1 + .../serving/reviewed-model-code.test.ts | 75 +++++++++++- .../inference/serving/reviewed-model-code.ts | 8 +- .../vllm-fixed-catalog-install.test.ts | 32 +++++- src/lib/inference/vllm.ts | 5 +- .../machine/handlers/preflight.test.ts | 93 ++++++++++++++- src/lib/onboard/machine/handlers/preflight.ts | 14 ++- ...provider-inference-station-express.test.ts | 107 +++++++++++++++++- .../machine/handlers/provider-inference.ts | 9 ++ src/lib/onboard/resume-config.test.ts | 92 +++++++++++++++ src/lib/onboard/resume-config.ts | 33 +++++- src/lib/onboard/sandbox-gpu-mode.test.ts | 4 +- src/lib/onboard/sandbox-gpu-mode.ts | 8 +- src/lib/onboard/setup-nim-vllm.test.ts | 32 ++++-- src/lib/onboard/setup-nim-vllm.ts | 4 +- .../onboard/station-express-resume.test.ts | 53 +++++---- src/lib/onboard/station-express-resume.ts | 20 +++- src/lib/state/onboard-session.test.ts | 12 ++ src/lib/state/onboard-session.ts | 14 +++ ...egistry-route-reservation-security.test.ts | 42 +++++++ src/lib/state/registry.ts | 4 +- .../install-express-noninteractive.test.ts | 50 ++++++++ 22 files changed, 649 insertions(+), 63 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index fa4c622ef22..7715eb646bf 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -6663,6 +6663,7 @@ save_station_express_resume() { assert_station_express_resume_file_safe "$state_file" _STATION_EXPRESS_RESUME_REVISION="$revision" _STATION_EXPRESS_RESUME_GENERATION="$generation" + export NEMOCLAW_STATION_EXPRESS_RECEIPT_GENERATION="$generation" _STATION_EXPRESS_RESUME_AGENT="$agent" _STATION_EXPRESS_RESUME_SANDBOX="$sandbox" _STATION_EXPRESS_RESUME_POLICY_TIER="$policy_tier" diff --git a/src/lib/inference/serving/reviewed-model-code.test.ts b/src/lib/inference/serving/reviewed-model-code.test.ts index fdc989a44f7..a15014b8bb2 100644 --- a/src/lib/inference/serving/reviewed-model-code.test.ts +++ b/src/lib/inference/serving/reviewed-model-code.test.ts @@ -169,6 +169,47 @@ describe("reviewed model code", () => { symlinkSync(path.join(root, "outside.py"), path.join(snapshot, "model.py")); }, }, + { + kind: "changed shared metadata", + change: (snapshot: string, root: string) => { + const blobs = path.join(root, "hub/blobs"); + mkdirSync(blobs); + const blob = path.join(blobs, "reviewed-code"); + writeFileSync(blob, "different code"); + rmSync(path.join(snapshot, "model.py")); + symlinkSync(blob, path.join(snapshot, "model.py")); + }, + }, + { + kind: "redirected shared blob directory", + change: (snapshot: string, root: string) => { + const outside = path.join(root, "outside-blobs"); + mkdirSync(outside); + renameSync(path.join(snapshot, "model.py"), path.join(outside, "reviewed-code")); + const blobs = path.join(root, "hub/blobs"); + symlinkSync(outside, blobs); + symlinkSync(path.join(blobs, "reviewed-code"), path.join(snapshot, "model.py")); + }, + }, + { + kind: "another model's cache", + change: (snapshot: string, root: string) => { + const sibling = path.join(root, "hub/models--other--model"); + mkdirSync(sibling); + const blob = path.join(sibling, "model.py"); + renameSync(path.join(snapshot, "model.py"), blob); + symlinkSync(blob, path.join(snapshot, "model.py")); + }, + }, + { + kind: "nonregular shared weight blob", + change: (snapshot: string, root: string) => { + const blob = path.join(root, "hub/blobs/weight-directory"); + mkdirSync(blob, { recursive: true }); + rmSync(path.join(snapshot, "weights.safetensors")); + symlinkSync(blob, path.join(snapshot, "weights.safetensors")); + }, + }, { kind: "directory", change: (snapshot: string) => mkdirSync(path.join(snapshot, "package")) }, ])("rejects $kind before starting the server", ({ change }) => { fixture((root, snapshot) => { @@ -205,10 +246,38 @@ describe("reviewed model code", () => { }); }); - it("rejects a snapshot directory replaced by a symlink", () => { + it.each(["config.json", "model.py", "weights.safetensors"])( + "accepts %s through the shared Hugging Face blob cache", + (name) => { + fixture((root, snapshot) => { + const modelBlobs = path.join(snapshot, "../../blobs"); + const sharedBlobs = path.join(root, "hub/blobs/ab"); + mkdirSync(modelBlobs); + mkdirSync(sharedBlobs, { recursive: true }); + const source = path.join(snapshot, name); + const sharedBlob = path.join(sharedBlobs, name); + const modelBlob = path.join(modelBlobs, name); + renameSync(source, sharedBlob); + symlinkSync(path.relative(modelBlobs, sharedBlob), modelBlob); + symlinkSync(path.relative(snapshot, modelBlob), source); + const result = probe(root); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + config: "{}", + weight: "fixture weights", + }); + }); + }, + ); + + it.each([ + ["snapshot", "."], + ["model cache", "../.."], + ] as const)("rejects a redirected %s directory", (_name, relative) => { fixture((root, snapshot) => { - renameSync(snapshot, snapshot + "-moved"); - symlinkSync(snapshot + "-moved", snapshot); + const source = path.resolve(snapshot, relative); + renameSync(source, source + "-moved"); + symlinkSync(source + "-moved", source); const result = probe(root); expect(result.status).not.toBe(0); expect(result.stderr).toContain("not a canonical directory"); diff --git a/src/lib/inference/serving/reviewed-model-code.ts b/src/lib/inference/serving/reviewed-model-code.ts index aedb83bfcf3..93981bcf3b4 100644 --- a/src/lib/inference/serving/reviewed-model-code.ts +++ b/src/lib/inference/serving/reviewed-model-code.ts @@ -75,8 +75,10 @@ import hashlib, json, os, pathlib, re, shutil, signal, subprocess, sys, tempfile model_id, revision, encoded = sys.argv[1:4] manifest = json.loads(encoded) -root = pathlib.Path(os.environ['HF_HOME']) / 'hub' / ('models--' + model_id.replace('/', '--')) -root = root.resolve(strict=True) +hub = (pathlib.Path(os.environ['HF_HOME']) / 'hub').resolve(strict=True) +root = hub / ('models--' + model_id.replace('/', '--')) +# Do not resolve this directory; redirected blob directories must not expand the trusted cache. +shared_blobs = hub / 'blobs' source = root / 'snapshots' / revision if source.is_symlink() or source.resolve(strict=True) != source: raise RuntimeError('reviewed model snapshot is not a canonical directory') @@ -121,7 +123,7 @@ try: signal.signal(signum, forward_signal) for path in entries: target = path.resolve(strict=True) - if not target.is_relative_to(root) or not target.is_file(): + if not (target.is_relative_to(root) or target.is_relative_to(shared_blobs)) or not target.is_file(): raise RuntimeError('reviewed model file escapes its cache or is not regular') destination = snapshot / path.name if path.name in expected: diff --git a/src/lib/inference/vllm-fixed-catalog-install.test.ts b/src/lib/inference/vllm-fixed-catalog-install.test.ts index c781d64de92..76e84a3d75d 100644 --- a/src/lib/inference/vllm-fixed-catalog-install.test.ts +++ b/src/lib/inference/vllm-fixed-catalog-install.test.ts @@ -329,10 +329,14 @@ describe("fixed catalog vLLM installs", () => { expect(mocks.dockerRunDetached).toHaveBeenCalledOnce(); }); - it.each(["", "nemotron-3.5-super-vl-120b"])( - "carries the reviewed model guard through native installation with model intent '%s'", - async (modelIntent) => { - process.env.NEMOCLAW_VLLM_MODEL = modelIntent; + it.each([ + ["", undefined], + ["nemotron-3.5-super-vl-120b", undefined], + ["nemotron-3.5-super-vl-120b", "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4"], + ])( + "carries the reviewed model guard for selection '%s' and checkpoint '%s'", + async (requestedModel, checkpointModel) => { + process.env.NEMOCLAW_VLLM_MODEL = requestedModel; process.env.HF_TOKEN = "hf_fixture_download_only"; mocks.getGpuIndicesByName.mockReturnValue([0]); const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; @@ -356,6 +360,7 @@ describe("fixed catalog vLLM installs", () => { hasImage: true, nonInteractive: true, promptFn: vi.fn(), + modelIntent: checkpointModel, readinessReports, resolveManagedBridgeHost: () => "172.18.0.1", }); @@ -494,6 +499,25 @@ describe("fixed catalog vLLM installs", () => { expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); }); + it("rejects a different model override before resuming vLLM", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + mocks.getGpuIndicesByName.mockReturnValue([0]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + modelIntent: "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", + }); + + expect(result).toEqual({ ok: false }); + expect(spies.errSpy).toHaveBeenCalledWith( + expect.stringContaining("the resumed model conflicts with NEMOCLAW_VLLM_MODEL"), + ); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + }); + it("still rejects an operator model override against a serving preset", async () => { const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; const presetModel = "muse-glimmer-30b"; diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 038043d76c8..3f6ab87b3de 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -64,6 +64,7 @@ import { NEMOCLAW_VLLM_GPU_DEVICE_ENV, normalizeVllmGpuDevice, parseVllmExtraServeArgs, + resolveVllmModelAlias, resolveVllmGpuMemoryUtilization, VLLM_EXTRA_ARGS_ENV, VLLM_MODELS, @@ -1877,10 +1878,12 @@ function resolveVllmInstallSelectionEnv( ): VllmInstallSelectionEnv { const resumedModel = String(modelIntent ?? "").trim(); const environmentModel = String(env.NEMOCLAW_VLLM_MODEL ?? "").trim(); + const registeredModel = resolveVllmModelAlias(resumedModel); if ( resumedModel && environmentModel && - resumedModel.toLowerCase() !== environmentModel.toLowerCase() + resumedModel.toLowerCase() !== environmentModel.toLowerCase() && + !(registeredModel && vllmModelMatchesAlias(registeredModel, environmentModel)) ) { return { ok: false }; } diff --git a/src/lib/onboard/machine/handlers/preflight.test.ts b/src/lib/onboard/machine/handlers/preflight.test.ts index 5ca6f1a5a7f..f1d38d9f741 100644 --- a/src/lib/onboard/machine/handlers/preflight.test.ts +++ b/src/lib/onboard/machine/handlers/preflight.test.ts @@ -1,16 +1,52 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; + +const sessionHome = await vi.hoisted(async () => { + const fs = await import("node:fs"); + const os = await import("node:os"); + const path = await import("node:path"); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-preflight-session-")); + vi.stubEnv("HOME", directory); + vi.resetModules(); + return directory; +}); import type { GpuDetection } from "../../../inference/nim"; -import { createSession, type Session } from "../../../state/onboard-session"; +import { + clearSession, + createSession, + loadSession, + markStepComplete, + saveSession, + updateSession, + type Session, +} from "../../../state/onboard-session"; import { persistedProviderNameToSelectionKey } from "../../inference-providers/provider-selection-keys"; -import { resolveSandboxGpuConfig } from "../../sandbox-gpu-mode"; +import { + getResumeSandboxGpuOverrides as resolveResumeSandboxGpuOverrides, + resolveSandboxGpuConfig, + type SandboxGpuMode, +} from "../../sandbox-gpu-mode"; import { handlePreflightState, type PreflightStateOptions } from "./preflight"; +beforeEach(() => { + vi.stubEnv("HOME", sessionHome); +}); + +afterAll(() => { + vi.unstubAllEnvs(); + fs.rmSync(sessionHome, { recursive: true, force: true }); +}); + type Gpu = GpuDetection | null; -type SandboxEntry = { sandboxGpuEnabled?: boolean }; +type SandboxEntry = { + sandboxGpuEnabled?: boolean; + sandboxGpuMode?: SandboxGpuMode; + sandboxGpuDevice?: string | null; +}; type Host = { cdiNvidiaGpuSpecMissing?: boolean }; function createDeps( @@ -45,9 +81,14 @@ function createDeps( getResumeSandboxGpuOverrides: ( sandbox: SandboxEntry | null, sessionGpuPassthrough: boolean | null | undefined, + sessionSelection?: Session["sandboxGpuSelection"], ) => { if (overrides.getResumeSandboxGpuOverrides) { - return overrides.getResumeSandboxGpuOverrides(sandbox, sessionGpuPassthrough); + return overrides.getResumeSandboxGpuOverrides( + sandbox, + sessionGpuPassthrough, + sessionSelection, + ); } return { flag: "enable" as const, device: "0" }; }, @@ -606,7 +647,11 @@ describe("handlePreflightState", () => { }); expect(getSandbox).toHaveBeenCalledWith("saved"); - expect(getResumeSandboxGpuOverrides).toHaveBeenCalledWith({ sandboxGpuEnabled: true }, true); + expect(getResumeSandboxGpuOverrides).toHaveBeenCalledWith( + { sandboxGpuEnabled: true }, + true, + undefined, + ); expect(result.resumeHasResolvedGpuIntent).toBe(true); expect(result.effectiveSandboxGpuFlag).toBe("enable"); expect(result.effectiveSandboxGpuDevice).toBe("1"); @@ -632,4 +677,40 @@ describe("handlePreflightState", () => { expect(result.session?.gpuPassthrough).toBe(true); expect(harness.deps.updateSession).toHaveBeenCalledOnce(); }); + + it.each([ + { mode: "auto", flag: null, device: null }, + { mode: "1", flag: "enable", device: "nvidia.com/gpu=0" }, + { mode: "0", flag: "disable", device: null }, + ] as const)( + "retains GPU mode $mode through a saved preflight before sandbox creation", + async (choice) => { + const initial = saveSession(createSession({ sandboxName: "my-assistant" })); + const harness = createDeps({ + getSandbox: () => null, + getResumeSandboxGpuOverrides: (entry, passthrough, selection) => + resolveResumeSandboxGpuOverrides(entry, passthrough ?? undefined, selection), + resolveSandboxGpuConfig, + recordStepComplete: async () => markStepComplete("preflight"), + recordStateSkipped: async () => loadSession()!, + updateSession, + }); + try { + const first = await handlePreflightState({ + ...baseOptions(harness.deps, initial), + explicitSandboxGpuFlag: choice.flag, + sandboxGpuDevice: choice.device, + }); + const saved = loadSession()!; + const resumed = await handlePreflightState({ + ...baseOptions(harness.deps, saved), + resume: true, + }); + expect(resumed.sandboxGpuConfig).toEqual(first.sandboxGpuConfig); + expect(resumed.sandboxGpuConfig.mode).toBe(choice.mode); + } finally { + clearSession(); + } + }, + ); }); diff --git a/src/lib/onboard/machine/handlers/preflight.ts b/src/lib/onboard/machine/handlers/preflight.ts index 19a4f55fdeb..01b622b7fd2 100644 --- a/src/lib/onboard/machine/handlers/preflight.ts +++ b/src/lib/onboard/machine/handlers/preflight.ts @@ -5,6 +5,7 @@ import type { Session } from "../../../state/onboard-session"; import { isN1xManagedVllmProviderModel } from "../../../domain/sandbox/n1x-managed-vllm-rebuild"; import { hasExplicitDeferredN1xOnboardingIntent } from "../../../readiness/onboard-admission"; import { isN1xOnboardingProviderKey } from "../../inference-providers/provider-selection-keys"; +import { normalizeSandboxGpuMode } from "../../sandbox-gpu-mode"; import { withPreflightTrace } from "../../tracing"; import { advanceTo, type OnboardStateTransitionResult } from "../result"; @@ -45,6 +46,7 @@ export interface PreflightStateOptions< getResumeSandboxGpuOverrides( sandbox: SandboxEntry | null, sessionGpuPassthrough: boolean | null | undefined, + sessionSelection?: Session["sandboxGpuSelection"], ): PreflightSandboxGpuOverrides; /** Observe GPU state without running a container-backed proof. */ detectGpuForReadiness(): Gpu; @@ -163,6 +165,7 @@ export async function handlePreflightState< ? deps.getResumeSandboxGpuOverrides( resumeSandboxNameForGpu ? deps.getSandbox(resumeSandboxNameForGpu) : null, session?.gpuPassthrough, + session?.sandboxGpuSelection, ) : { flag: null, device: null }; const effectiveSandboxGpuFlag = explicitSandboxGpuFlag ?? resumedSandboxGpuOverrides.flag; @@ -262,9 +265,18 @@ export async function handlePreflightState< env, }); const gpuPassthrough = sandboxGpuConfig.sandboxGpuEnabled; - if (session && session.gpuPassthrough !== gpuPassthrough) { + const mode = normalizeSandboxGpuMode(sandboxGpuConfig.mode); + const device = sandboxGpuConfig.sandboxGpuDevice ?? null; + if ( + session && + (session.gpuPassthrough !== gpuPassthrough || + (mode && + (session.sandboxGpuSelection?.mode !== mode || + session.sandboxGpuSelection?.device !== device))) + ) { session = deps.updateSession((current) => { current.gpuPassthrough = gpuPassthrough; + if (mode) current.sandboxGpuSelection = { mode, device }; return current; }); } diff --git a/src/lib/onboard/machine/handlers/provider-inference-station-express.test.ts b/src/lib/onboard/machine/handlers/provider-inference-station-express.test.ts index af69d76cdf0..217be90502a 100644 --- a/src/lib/onboard/machine/handlers/provider-inference-station-express.test.ts +++ b/src/lib/onboard/machine/handlers/provider-inference-station-express.test.ts @@ -1,14 +1,115 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { createSession } from "../../../state/onboard-session"; +const sessionHome = await vi.hoisted(async () => { + const fs = await import("node:fs"); + const os = await import("node:os"); + const path = await import("node:path"); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-provider-session-")); + vi.stubEnv("HOME", directory); + vi.resetModules(); + return directory; +}); + +import { + clearSession, + createSession, + loadSession, + markStepComplete, + markStepFailed, + markStepStarted, + saveSession, + updateSession, +} from "../../../state/onboard-session"; +import { checkpointSandboxName } from "../../session-bootstrap"; import { withStationExpressResumeEnvironment } from "../../station-express-resume"; import { handleProviderInferenceState } from "./provider-inference"; -import { baseOptions, createDeps } from "./provider-inference.test-support"; +import { baseOptions, baseSelection, createDeps } from "./provider-inference.test-support"; + +beforeEach(() => { + vi.stubEnv("HOME", sessionHome); +}); + +afterAll(() => { + vi.unstubAllEnvs(); + fs.rmSync(sessionHome, { recursive: true, force: true }); +}); describe("Station Express provider binding (#7048)", () => { + it("revalidates the model after selection fails while committing a reviewed provider", async () => { + const selection = { + ...baseSelection, + provider: "vllm-local", + model: "nvidia-nemotron-3.5-super-vl-120b-a12b-nvfp4", + endpointUrl: null, + credentialEnv: null, + preferredInferenceApi: "openai-completions", + vllmModelIdentity: "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", + }; + const setupNim = vi + .fn() + .mockResolvedValue(selection) + .mockResolvedValueOnce({ + ...selection, + vllmModelIdentity: "/tmp/nemoclaw-reviewed-model/snapshot", + }); + const { deps, calls } = createDeps({ + setupNim, + startRecordedStep: async (step, updates) => { + markStepStarted(step); + updateSession((current) => Object.assign(current, updates)); + }, + recordStepComplete: async (step, updates) => markStepComplete(step, updates), + checkpointSandboxIdentity: (name, agent) => checkpointSandboxName(name, agent, updateSession), + }); + const session = saveSession( + createSession({ + mode: "non-interactive", + stationExpressIntent: { + version: 1, + model: "nemotron-3.5-super-vl-120b", + sandboxName: "my-assistant", + }, + }), + ); + try { + await expect( + handleProviderInferenceState({ + ...baseOptions(deps, session), + sandboxName: "my-assistant", + }), + ).rejects.toThrow("Cannot record an invalid DGX Station Express provider selection."); + markStepFailed("provider_selection", "Model identity rejected"); + const saved = loadSession()!; + expect(saved).toMatchObject({ + provider: selection.provider, + model: selection.model, + sandboxPromptProgress: { sandboxName: true }, + steps: { provider_selection: { status: "failed" } }, + }); + + await handleProviderInferenceState({ + ...baseOptions(deps, saved), + resume: true, + sandboxName: "my-assistant", + }); + expect(setupNim).toHaveBeenCalledTimes(2); + expect(calls.skipped).not.toHaveBeenCalledWith("provider_selection", expect.anything()); + expect(loadSession()).toMatchObject({ + steps: { provider_selection: { status: "complete" } }, + stationExpressIntent: { + servedModel: selection.model, + checkpointModel: selection.vllmModelIdentity, + }, + }); + } finally { + clearSession(); + } + }); + it("carries validated vLLM checkpoint identity into the atomic session update", async () => { const setupNim = vi.fn(async () => ({ model: "nemotron-ultra", diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 0a8f2b8d9e3..d0380006090 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -1084,6 +1084,15 @@ function canResumeProviderSelection( provider: string | null, model: string | null, ): boolean { + // An interrupted Station review has a provider/model but no validated model + // identity yet. Retry selection so its checkpoint can bind that identity. + if ( + session?.stationExpressIntent && + session.stationExpressIntent.kind !== "spark" && + session.steps.provider_selection.status !== "complete" + ) { + return false; + } return ( !forceProviderSelection && effectiveResume && diff --git a/src/lib/onboard/resume-config.test.ts b/src/lib/onboard/resume-config.test.ts index d69c965269f..477fd445c01 100644 --- a/src/lib/onboard/resume-config.test.ts +++ b/src/lib/onboard/resume-config.test.ts @@ -196,4 +196,96 @@ describe("authoritative rebuild resume config", () => { recorded: "vllm-local", }); }); + + it.each(["nemotron-3.5-super-vl-120b", "nvidia-nemotron-3.5-super-vl-120b-a12b-nvfp4"])( + "resumes the recorded vLLM checkpoint through its catalog alias %s", + (alias) => { + vi.stubEnv("NEMOCLAW_PROVIDER", "install-vllm"); + vi.stubEnv("NEMOCLAW_VLLM_MODEL", alias); + + expect( + getResumeConfigConflicts( + { + provider: null, + model: null, + vllmInstallModel: "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", + }, + { nonInteractive: true }, + ), + ).toEqual([]); + }, + ); + + it("keeps hosted model names distinct from managed vLLM aliases", () => { + vi.stubEnv("NEMOCLAW_PROVIDER", "build"); + vi.stubEnv("NEMOCLAW_MODEL", "nemotron-3.5-super-vl-120b"); + const recorded = "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4"; + + expect( + getResumeConfigConflicts( + { provider: "nvidia-prod", model: recorded }, + { nonInteractive: true }, + ), + ).toContainEqual({ + field: "model", + requested: "nemotron-3.5-super-vl-120b", + recorded, + }); + }); + + const stationSelection = { + provider: "vllm-local", + model: "nvidia-nemotron-3.5-super-vl-120b-a12b-nvfp4", + stationExpressIntent: { + version: 1, + model: "nemotron-3.5-super-vl-120b", + sandboxName: "my-assistant", + }, + }; + + it.each(["pending", "in_progress", "failed"])( + "resumes the same Station selection while provider selection is %s", + (status) => { + const session = { ...stationSelection, steps: { provider_selection: { status } } }; + vi.stubEnv("NEMOCLAW_PROVIDER", "install-vllm"); + vi.stubEnv("NEMOCLAW_MODEL", session.model); + expect(getResumeConfigConflicts(session, { nonInteractive: true })).toEqual([]); + + vi.stubEnv("NEMOCLAW_MODEL", "other-model"); + expect(getResumeConfigConflicts(session, { nonInteractive: true })).toContainEqual({ + field: "model", + requested: "other-model", + recorded: session.model, + }); + vi.stubEnv("NEMOCLAW_PROVIDER", "build"); + expect(getResumeConfigConflicts(session, { nonInteractive: true })).toContainEqual({ + field: "provider", + requested: "nvidia-prod", + recorded: session.provider, + }); + }, + ); + + it.each([ + ["failed", undefined], + ["failed", { version: 1 }], + ["failed", { version: 1, kind: "spark", sandboxName: "my-assistant" }], + ["complete", stationSelection.stationExpressIntent], + [undefined, stationSelection.stationExpressIntent], + ] as const)( + "rejects install-vllm for status %s and intent %j", + (status, stationExpressIntent) => { + vi.stubEnv("NEMOCLAW_PROVIDER", "install-vllm"); + const session = { + ...stationSelection, + stationExpressIntent, + steps: { provider_selection: { status } }, + }; + expect(getResumeConfigConflicts(session, { nonInteractive: true })).toContainEqual({ + field: "provider", + requested: "install-vllm", + recorded: session.provider, + }); + }, + ); }); diff --git a/src/lib/onboard/resume-config.ts b/src/lib/onboard/resume-config.ts index 260056ef956..5f32cc27020 100644 --- a/src/lib/onboard/resume-config.ts +++ b/src/lib/onboard/resume-config.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import path from "node:path"; +import { resolveVllmModelAlias, vllmModelMatchesAlias } from "../inference/vllm-models"; import { isDecisionSelected } from "../state/onboard-checkpoint-decision"; import { hasInvalidSessionHostMounts, @@ -9,6 +10,10 @@ import { } from "../state/onboard-session"; import type { SandboxHostMount } from "../state/registry/types"; import { normalizeToolDisclosure, type ToolDisclosure } from "../tool-disclosure"; +import { + isValidStationExpressProviderState, + parseStationExpressResumeIntent, +} from "./station-express-resume"; import { preflightVllmModelEnvOrExit } from "./vllm-model-preflight"; const onboardProviders = require("./providers"); @@ -18,6 +23,7 @@ export interface ResumeSessionLike { provider?: string | null; model?: string | null; vllmInstallModel?: string | null; + stationExpressIntent?: unknown; agent?: string | null; toolDisclosure?: ToolDisclosure; observabilityEnabled?: boolean; @@ -26,7 +32,10 @@ export interface ResumeSessionLike { fromImage?: string | null; hostMounts?: SandboxHostMount[]; } | null; - steps?: { sandbox?: { status?: string | null } | null } | null; + steps?: { + sandbox?: { status?: string | null } | null; + provider_selection?: { status?: string | null } | null; + } | null; checkpoint?: { sandboxIdentity?: import("../state/onboard-checkpoint-types").CheckpointDecision< import("../state/onboard-checkpoint-types").CheckpointSandboxIdentity @@ -167,8 +176,22 @@ export function getResumeConfigConflicts( } const requestedProvider = getRequestedProviderHint(nonInteractive, allowHostedInferenceStaging); + const stationIntent = parseStationExpressResumeIntent(session?.stationExpressIntent); + const providerStepStatus = session?.steps?.provider_selection?.status; + // Express replays install-vllm until provider selection completes, including + // retries that already selected an owned vLLM server without installing it. + const retryingStationProvider = + stationIntent?.kind !== "spark" && + stationIntent !== null && + ["pending", "in_progress", "failed"].includes(providerStepStatus ?? "") && + isValidStationExpressProviderState( + stationIntent, + providerStepStatus, + session?.provider, + session?.model, + ); const effectiveRequestedProvider = - session?.vllmInstallModel && requestedProvider === "install-vllm" + (session?.vllmInstallModel || retryingStationProvider) && requestedProvider === "install-vllm" ? "vllm-local" : onboardProviders.getEffectiveProviderName(requestedProvider); const recordedProvider = session?.vllmInstallModel ? "vllm-local" : session?.provider; @@ -190,10 +213,14 @@ export function getResumeConfigConflicts( ? requestedVllmModel : getRequestedModelHint(nonInteractive, allowHostedInferenceStaging); const recordedModel = session?.vllmInstallModel ?? session?.model; + const recordedVllmModel = session?.vllmInstallModel + ? resolveVllmModelAlias(session.vllmInstallModel) + : null; if ( requestedModel && recordedModel && - requestedModel.toLowerCase() !== recordedModel.toLowerCase() + requestedModel.toLowerCase() !== recordedModel.toLowerCase() && + !(recordedVllmModel && vllmModelMatchesAlias(recordedVllmModel, requestedModel)) ) { conflicts.push({ field: "model", diff --git a/src/lib/onboard/sandbox-gpu-mode.test.ts b/src/lib/onboard/sandbox-gpu-mode.test.ts index 05d36dffd3e..3df6f3867e0 100644 --- a/src/lib/onboard/sandbox-gpu-mode.test.ts +++ b/src/lib/onboard/sandbox-gpu-mode.test.ts @@ -138,8 +138,10 @@ describe("sandbox GPU mode helpers", () => { const resumedDisabled = getResumeSandboxGpuOverrides( { sandboxGpuMode: "0", sandboxGpuDevice: null }, - false, + true, + { mode: "1", device: "nvidia.com/gpu=0" }, ); + expect(resumedDisabled).toEqual({ flag: "disable", device: null }); expect(resolveSandboxGpuConfig(gpu(), { ...resumedDisabled, env: {} }).sandboxGpuEnabled).toBe( false, ); diff --git a/src/lib/onboard/sandbox-gpu-mode.ts b/src/lib/onboard/sandbox-gpu-mode.ts index 8f89161c9a6..ad302e7cf33 100644 --- a/src/lib/onboard/sandbox-gpu-mode.ts +++ b/src/lib/onboard/sandbox-gpu-mode.ts @@ -3,6 +3,7 @@ import type { GpuDetection } from "../inference/nim"; import type { SandboxGpuProofResult } from "../state/registry"; +import type { Session } from "../state/onboard-session"; import { normalizeSandboxGpuDeviceForCdi } from "./sandbox-gpu-create"; export type SandboxGpuMode = "auto" | "1" | "0"; @@ -108,10 +109,13 @@ export function getResumeSandboxGpuOverrides( | null | undefined, sessionGpuPassthrough: boolean | undefined, + sessionSelection?: Session["sandboxGpuSelection"], ): ResumeSandboxGpuOverrides { - const recordedMode = normalizeSandboxGpuMode(entry?.sandboxGpuMode); + const registryMode = normalizeSandboxGpuMode(entry?.sandboxGpuMode); + const recordedMode = registryMode ?? sessionSelection?.mode; + const recordedDevice = registryMode ? entry?.sandboxGpuDevice : sessionSelection?.device; if (recordedMode === "1") { - return { flag: "enable", device: entry?.sandboxGpuDevice || null }; + return { flag: "enable", device: recordedDevice || null }; } if (recordedMode === "0") { return { flag: "disable", device: null }; diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index 52272bb2612..d855ee8130c 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -63,6 +63,7 @@ afterEach(() => vi.restoreAllMocks()); function reviewedStationSelection( options: { + automatic?: boolean; unmanaged?: boolean; missingReceipt?: boolean; changedField?: "catalogDigest" | "presetId" | "presetDigest" | "recipeId" | "recipeDigest"; @@ -89,7 +90,7 @@ function reviewedStationSelection( }, ], }); - const selection = state(variant.servedModelId); + const selection = state(options.automatic ? null : variant.servedModelId); const handler = createSetupNimVllmHandler( deps({ getManagedVllmProviderBinding: () => @@ -116,12 +117,18 @@ function reviewedStationSelection( } describe("setupNim vLLM route containment", () => { - it("binds a reviewed Station snapshot through its verified recipe identity", async () => { - const { handler, selection, bind, model, variant } = reviewedStationSelection(); - await expect(handler(selection, { managedInstall: true })).resolves.toBe("selected"); - expect(selection.vllmModelIdentity).toBe(model.id); - expect(bind()).toMatchObject({ servedModel: variant.servedModelId, checkpointModel: model.id }); - }); + it.each([false, true])( + "binds a reviewed Station snapshot through its verified recipe identity (automatic=%s)", + async (automatic) => { + const { handler, selection, bind, model, variant } = reviewedStationSelection({ automatic }); + await expect(handler(selection, { managedInstall: !automatic })).resolves.toBe("selected"); + expect(selection.vllmModelIdentity).toBe(model.id); + expect(bind()).toMatchObject({ + servedModel: variant.servedModelId, + checkpointModel: model.id, + }); + }, + ); it.each([ ["unmanaged endpoint", { unmanaged: true }], @@ -132,9 +139,16 @@ describe("setupNim vLLM route containment", () => { ["different recipe", { changedField: "recipeId" }], ["different recipe digest", { changedField: "recipeDigest" }], ["different checkpoint", { root: "other/checkpoint" }], + ["automatic unmanaged endpoint", { automatic: true, unmanaged: true }], + ["automatic missing receipt", { automatic: true, missingReceipt: true }], + ["automatic different recipe", { automatic: true, changedField: "recipeDigest" }], + ["automatic different checkpoint", { automatic: true, root: "other/checkpoint" }], ] as const)("refuses Station snapshot binding for a %s", async (_name, options) => { - const { handler, selection, bind } = reviewedStationSelection(options); - await expect(handler(selection, { managedInstall: true })).resolves.toBe("selected"); + const selectionOptions = { automatic: false, ...options }; + const { handler, selection, bind } = reviewedStationSelection(selectionOptions); + await expect(handler(selection, { managedInstall: !selectionOptions.automatic })).resolves.toBe( + "selected", + ); expect(bind).toThrow("Cannot record an invalid DGX Station Express provider selection"); }); diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index aec6a6596e1..c97f724cc74 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -177,10 +177,10 @@ function validatedVllmModelIdentity( managedBinding: ManagedVllmProviderBinding | null | undefined, ): string | null { const root = reportedModelRoot(findVllmModelEntry(models, detectedModel)); - const registeredModel = requestedModel ? resolveVllmModelAlias(requestedModel) : null; + const registeredModel = resolveVllmModelAlias(requestedModel ?? detectedModel); if ( root?.startsWith("/") && - requestedModel === detectedModel && + (requestedModel === null || requestedModel === detectedModel) && managedBinding?.serving && registeredModel?.runtimeVariants?.some( (variant) => diff --git a/src/lib/onboard/station-express-resume.test.ts b/src/lib/onboard/station-express-resume.test.ts index 59d74b6fb99..a41ff1dd3c2 100644 --- a/src/lib/onboard/station-express-resume.test.ts +++ b/src/lib/onboard/station-express-resume.test.ts @@ -111,6 +111,20 @@ describe("DGX Station Express resume (#7048)", () => { }); }); + it.each(["auto", "unknown-model", "qwen3.6-35b-a3b-nvfp4"])( + "rejects explicit model %s instead of selecting the Station default", + (model) => { + const env = expressEnv(); + env.NEMOCLAW_VLLM_MODEL = model; + delete env.NEMOCLAW_MODEL; + + expect(getStationExpressResumeIntent(env, "my-assistant")).toEqual({ + ok: false, + message: "DGX Station Express requires a registered Station vLLM model and sandbox name.", + }); + }, + ); + it("captures the qualified dual-Station served alias in the sealed intent", () => { expect(getStationExpressResumeIntent(dualExpressEnv(), "my-assistant")).toEqual({ ok: true, @@ -892,26 +906,27 @@ describe("DGX Station Express resume (#7048)", () => { } }); - it("rejects an arbitrary served alias in an installer receipt", () => { - const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-receipt-alias-")); - const stateDir = path.join(home, ".nemoclaw"); - const receipt = path.join(stateDir, "station-express-resume"); - fs.mkdirSync(stateDir, { mode: 0o700 }); - fs.writeFileSync(receipt, receiptText(receiptGeneration, "nvidia/nemotron-3-ultra-550b-a55b"), { - mode: 0o600, - }); + it.each(["nvidia/nemotron-3-ultra-550b-a55b", "AUTO", " auto"])( + "rejects noncanonical model %s in an installer receipt", + (model) => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-receipt-alias-")); + const stateDir = path.join(home, ".nemoclaw"); + const receipt = path.join(stateDir, "station-express-resume"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync(receipt, receiptText(receiptGeneration, model), { + mode: 0o600, + }); - try { - expect(() => - retireStationExpressInstallerResume(receiptGeneration, { env: { HOME: home } }), - ).toThrow("installer resume state is malformed"); - expect(fs.readFileSync(receipt, "utf8")).toBe( - receiptText(receiptGeneration, "nvidia/nemotron-3-ultra-550b-a55b"), - ); - } finally { - fs.rmSync(home, { recursive: true, force: true }); - } - }); + try { + expect(() => + retireStationExpressInstallerResume(receiptGeneration, { env: { HOME: home } }), + ).toThrow("installer resume state is malformed"); + expect(fs.readFileSync(receipt, "utf8")).toBe(receiptText(receiptGeneration, model)); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }, + ); it("never unlinks a replacement receipt introduced before its atomic claim", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-receipt-race-")); diff --git a/src/lib/onboard/station-express-resume.ts b/src/lib/onboard/station-express-resume.ts index 17273437b54..c254be193d2 100644 --- a/src/lib/onboard/station-express-resume.ts +++ b/src/lib/onboard/station-express-resume.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { isErrnoException } from "../core/errno"; import { detectNvidiaPlatform, type NvidiaPlatform } from "../inference/nim"; import { + defaultVllmModelForPlatform, selectVllmModelFromEnv, STATION_PAIR_OPTIONAL_ORCHESTRATION, type VllmModelDef, @@ -415,10 +416,10 @@ function readStationExpressInstallerResumeGeneration(stateFile: string): string const generation = lines[2]?.startsWith("generation=") ? lines[2].slice("generation=".length) : ""; - const canonicalModel = canonicalStationModelValue(modelValue); + const validModel = modelValue === "auto" || canonicalStationModelValue(modelValue) !== null; if ( !STATION_EXPRESS_RECEIPT_REVISION_PATTERN.test(revision) || - !canonicalModel || + !validModel || !isValidStationExpressReceiptGeneration(generation) ) { throw new Error("DGX Station Express installer resume state is malformed."); @@ -994,8 +995,19 @@ export function getStationExpressResumeIntent( return { ok: false, message: `${STATION_EXPRESS_ENV} must be 1 when set.` }; } - const model = stationModel(env.NEMOCLAW_VLLM_MODEL); - if (!model || !sandboxName || !validSandboxName(sandboxName)) { + let model: VllmModelDef | null = null; + try { + // Capture the same catalog default as the Station runtime before persisting resume intent. + model = selectVllmModelFromEnv(env) ?? defaultVllmModelForPlatform("station", "arm64"); + } catch { + // Invalid explicit selectors and catalog failures must still reject the intent. + } + if ( + !model || + !model.platforms.includes("station") || + !sandboxName || + !validSandboxName(sandboxName) + ) { return { ok: false, message: "DGX Station Express requires a registered Station vLLM model and sandbox name.", diff --git a/src/lib/state/onboard-session.test.ts b/src/lib/state/onboard-session.test.ts index 218ba5c22b5..e6810d28d39 100644 --- a/src/lib/state/onboard-session.test.ts +++ b/src/lib/state/onboard-session.test.ts @@ -974,6 +974,18 @@ describe("onboard session", () => { expect(loaded.metadata.hostMounts).not.toBe(hostMounts); }); + it("keeps only the selected GPU mode and device in persisted GPU intent", () => { + session.saveSession(session.createSession()); + const persisted = JSON.parse(fs.readFileSync(session.SESSION_FILE, "utf8")); + persisted.sandboxGpuSelection = { mode: "auto", device: null, token: "do-not-retain" }; + fs.writeFileSync(session.SESSION_FILE, JSON.stringify(persisted)); + + const loaded = requireLoadedSession(session.loadSession()); + expect(loaded.sandboxGpuSelection).toEqual({ mode: "auto", device: null }); + session.saveSession(loaded); + expect(fs.readFileSync(session.SESSION_FILE, "utf8")).not.toContain("do-not-retain"); + }); + it("preserves a fail-closed marker for malformed host mount metadata", () => { const malformed = session.createSession(); fs.mkdirSync(path.dirname(session.SESSION_FILE), { recursive: true }); diff --git a/src/lib/state/onboard-session.ts b/src/lib/state/onboard-session.ts index cd20f67f501..dabb1f4298d 100644 --- a/src/lib/state/onboard-session.ts +++ b/src/lib/state/onboard-session.ts @@ -337,6 +337,8 @@ export interface Session { /** Non-secret names of credential providers registered before sandbox setup completed. */ stagedCredentialProviders: string[]; gpuPassthrough: boolean; + /** Requested GPU mode/device, retained before a sandbox registry entry exists. */ + sandboxGpuSelection?: { mode: "auto" | "1" | "0"; device: string | null }; telegramConfig: TelegramConfig | null; wechatConfig: WechatConfig | null; metadata: SessionMetadata; @@ -715,6 +717,14 @@ function parseSessionMetadata(value: SessionJsonValue | undefined): SessionMetad }; } +function parseSandboxGpuSelection(value: unknown): Session["sandboxGpuSelection"] { + if (!isObject(value)) return undefined; + const mode = value.mode; + if (mode !== "auto" && mode !== "1" && mode !== "0") return undefined; + if (value.device != null && typeof value.device !== "string") return undefined; + return { mode, device: readString(value.device) }; +} + function parseStepState(value: SessionJsonValue | undefined): StepState | null { if (!isObject(value)) return null; const status = readStepStatus(value.status); @@ -1051,6 +1061,9 @@ export function createSession(overrides: Partial = {}): Session { messagingPlan: parseSandboxMessagingPlan(overrides.messagingPlan), stagedCredentialProviders: readStringArray(overrides.stagedCredentialProviders) ?? [], gpuPassthrough: overrides.gpuPassthrough === true, + ...(parseSandboxGpuSelection(overrides.sandboxGpuSelection) + ? { sandboxGpuSelection: parseSandboxGpuSelection(overrides.sandboxGpuSelection) } + : {}), telegramConfig: parseTelegramConfig(overrides.telegramConfig), wechatConfig: parseWechatConfig(overrides.wechatConfig), metadata: { @@ -1193,6 +1206,7 @@ export function normalizeSession(data: Session | SessionJsonValue | undefined): messagingPlan: parseSandboxMessagingPlan(data.messagingPlan), stagedCredentialProviders: readStringArray(data.stagedCredentialProviders) ?? [], gpuPassthrough: data.gpuPassthrough === true, + sandboxGpuSelection: parseSandboxGpuSelection(data.sandboxGpuSelection), telegramConfig: parseTelegramConfig(data.telegramConfig), wechatConfig: parseWechatConfig(data.wechatConfig), lastStepStarted: readString(data.lastStepStarted), diff --git a/src/lib/state/registry-route-reservation-security.test.ts b/src/lib/state/registry-route-reservation-security.test.ts index 2d35532b53f..cd88875a0f6 100644 --- a/src/lib/state/registry-route-reservation-security.test.ts +++ b/src/lib/state/registry-route-reservation-security.test.ts @@ -171,6 +171,48 @@ describe("sandbox inference route reservation security", () => { } }); + it("reuses a published model's reservation without treating selection provenance as route drift", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-provenance-reservation-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("./registry"); + const route = { + ...EXACT_ROUTE_SELECTION, + gatewayName: "nemoclaw", + reservationSessionId: "session-owner", + } as const; + const provenance = { + schemaVersion: 1, + provider: route.provider, + model: route.model, + providerProfile: "ollama", + modelSource: "local", + apiFamily: "openai-completions", + } as const; + registry.registerSandbox({ name: "alpha", ...route, modelSelectionProvenance: provenance }); + expect(registry.reserveSandboxInferenceRoute("alpha", route)).toBe(true); + const before = await fs.readFile(registry.REGISTRY_FILE, "utf8"); + expect(registry.reserveSandboxInferenceRoute("alpha", route)).toBe(true); + expect(registry.getSandbox("alpha")?.modelSelectionProvenance).toEqual(provenance); + expect(() => + registry.reserveSandboxInferenceRoute("alpha", { + ...route, + endpointUrl: "http://127.0.0.1:11435/v1", + }), + ).toThrow(/cannot change before the owning create transaction completes/u); + expect(() => + registry.reserveSandboxInferenceRoute("alpha", { + ...route, + preferredInferenceApi: "openai-responses", + }), + ).toThrow(/cannot change before the owning create transaction completes/u); + expect(await fs.readFile(registry.REGISTRY_FILE, "utf8")).toBe(before); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + it("preserves a replacement when cleanup holds an earlier reservation snapshot (#9833)", async () => { const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-route-reservation-")); vi.stubEnv("HOME", home); diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index fe31aefa34d..e2ffac82313 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -763,8 +763,8 @@ export function reserveSandboxInferenceRoute( nativeNvidiaProviderAttachment ?? existing.nativeNvidiaProviderAttachment, ) && isDeepStrictEqual( - normalizeInferenceSelection(existing), - normalizeInferenceSelection(route), + normalizeSandboxInferenceRouteSelection(normalizeInferenceSelection(existing)), + normalizeSandboxInferenceRouteSelection(normalizeInferenceSelection(route)), )); if (!sameReservation) { if (existing.pendingCreateIdentity) { diff --git a/test/installer-integration/install-express-noninteractive.test.ts b/test/installer-integration/install-express-noninteractive.test.ts index 44694307312..a1a34bdf48c 100644 --- a/test/installer-integration/install-express-noninteractive.test.ts +++ b/test/installer-integration/install-express-noninteractive.test.ts @@ -6,10 +6,60 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it, onTestFinished } from "vitest"; +import { defaultVllmModelForPlatform } from "../../src/lib/inference/vllm-models"; +import { + assertStationExpressInstallerResumeMatches, + getStationExpressResumeIntent, + retireStationExpressInstallerResume, +} from "../../src/lib/onboard/station-express-resume"; import { runInstallerSourced } from "../helpers/installer-express-prompt-harness"; import { INSTALLER_PAYLOAD } from "../helpers/installer-sourced-env"; describe("non-interactive Station Express selection", () => { + it.each(["fresh", "resumed"])( + "passes automatic Station Express selection to onboarding on %s entry", + (entry) => { + const { home, result, output } = runInstallerSourced( + ` +station_installer_revision() { printf 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; } +activate_express_install 'DGX Station' +save_station_express_resume +if [[ "$TEST_ENTRY" == resumed ]]; then + unset NEMOCLAW_STATION_EXPRESS NEMOCLAW_PROVIDER NEMOCLAW_SANDBOX_NAME + load_station_express_resume + resume_loaded_station_install 'DGX Station' +fi +"$TEST_NODE" -e ' + const env = Object.fromEntries(Object.entries(process.env).filter( + ([name]) => name === "HOME" || name.startsWith("NEMOCLAW_") + )); + console.log("ONBOARD_ENV=" + JSON.stringify(env)); +' +`, + { TEST_ENTRY: entry, TEST_NODE: process.execPath }, + ); + onTestFinished(() => fs.rmSync(home, { recursive: true, force: true })); + expect(result.status, output).toBe(0); + const serialized = result.stdout.split("\n").find((line) => line.startsWith("ONBOARD_ENV=")); + expect(serialized, output).toBeDefined(); + const env = JSON.parse(serialized!.slice("ONBOARD_ENV=".length)) as NodeJS.ProcessEnv; + const receipt = path.join(home, ".nemoclaw/station-express-resume"); + const generation = fs.readFileSync(receipt, "utf8").match(/^generation=(.+)$/mu)?.[1] ?? ""; + expect(getStationExpressResumeIntent(env, env.NEMOCLAW_SANDBOX_NAME ?? null)).toEqual({ + ok: true, + intent: { + version: 1, + model: defaultVllmModelForPlatform("station", "arm64").envValue, + sandboxName: "my-assistant", + receiptGeneration: generation, + }, + }); + assertStationExpressInstallerResumeMatches(generation, env); + retireStationExpressInstallerResume(generation, { env }); + expect(fs.existsSync(receipt)).toBe(false); + }, + ); + it.each(["express", "provider"])( "leaves the catalog default unforced across %s reboot resume", (mode) => { From f421e01e62b4d03a1410157dcce11c3a1aaf3a7c Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:03:44 -0700 Subject: [PATCH 12/22] fix(pi): retain staging failures through cleanup --- agents/pi/generate-config.ts | 26 +++++- .../pi-candidate-runtime-artifacts.test.ts | 79 +++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/agents/pi/generate-config.ts b/agents/pi/generate-config.ts index 01377f6a912..d23c4251411 100644 --- a/agents/pi/generate-config.ts +++ b/agents/pi/generate-config.ts @@ -229,9 +229,29 @@ function syncManagedDefaults(configDir: string, model: string, nativeLocal: bool closeSync(completedDescriptor); renameSync(stagedPath, destination); staged = false; - } finally { - if (descriptor !== null) closeSync(descriptor); - if (staged) rmSync(stagedPath, { force: true }); + } catch (error) { + const cleanupFailures: unknown[] = []; + if (descriptor !== null) { + try { + closeSync(descriptor); + } catch (cleanupError) { + cleanupFailures.push(cleanupError); + } + } + if (staged) { + try { + rmSync(stagedPath, { force: true }); + } catch (cleanupError) { + cleanupFailures.push(cleanupError); + } + } + if (cleanupFailures.length > 0) { + throw new AggregateError( + [error, ...cleanupFailures], + `Pi settings staging failed; cleanup also failed for ${stagedPath}.`, + ); + } + throw error; } } diff --git a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts index fed9dde2adf..20032e0f126 100644 --- a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts +++ b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts @@ -351,6 +351,85 @@ syncBuiltinESMExports();\n`, } }); + it.each(["close", "remove"])( + "retains the staging failure when %s cleanup fails (#13362)", + (failedCleanup) => { + const selected = { + NEMOCLAW_MODEL: "previous-model", + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", + NEMOCLAW_INFERENCE_BASE_URL: "http://host.openshell.internal:11434/v1", + }; + const { home, status, stderr } = generate(selected); + try { + expect(status, stderr).toBe(0); + const configDir = path.join(home, ".pi/agent"); + const modelsPath = path.join(configDir, "models.json"); + const settingsPath = path.join(configDir, "settings.json"); + const previousCatalog = fs.readFileSync(modelsPath); + const previousSettings = fs.readFileSync(settingsPath); + const removalMarker = path.join(home, "staged-removal-attempt.txt"); + const shimPath = path.join(home, "fail-staged-cleanup.cjs"); + fs.writeFileSync( + shimPath, + `const fs = require("node:fs"); +const { syncBuiltinESMExports } = require("node:module"); +const originalOpen = fs.openSync; +const originalClose = fs.closeSync; +const originalRemove = fs.rmSync; +let stagedDescriptor; +fs.openSync = (target, ...args) => { + const descriptor = originalOpen(target, ...args); + if (String(target).includes(".settings.json.")) stagedDescriptor = descriptor; + return descriptor; +}; +fs.fchmodSync = () => { throw new Error("simulated staging failure"); }; +fs.closeSync = (descriptor) => { + if (${JSON.stringify(failedCleanup)} === "close" && descriptor === stagedDescriptor) { + throw new Error("simulated close failure"); + } + return originalClose(descriptor); +}; +fs.rmSync = (target, options) => { + if (String(target).includes(".settings.json.")) { + fs.writeFileSync(${JSON.stringify(removalMarker)}, String(target)); + if (${JSON.stringify(failedCleanup)} === "remove") { + throw new Error("simulated remove failure"); + } + } + return originalRemove(target, options); +}; +syncBuiltinESMExports();\n`, + ); + + const failed = spawnSync( + process.execPath, + ["--require", shimPath, path.join(root, "agents/pi/generate-config.ts")], + { + cwd: root, + encoding: "utf8", + env: { + PATH: process.env.PATH ?? "", + HOME: home, + ...selected, + NEMOCLAW_MODEL: "next-model", + }, + }, + ); + expect(failed.status).not.toBe(0); + expect(failed.stderr).toContain("simulated staging failure"); + expect(failed.stderr).toContain(`simulated ${failedCleanup} failure`); + expect(fs.readFileSync(modelsPath)).toEqual(previousCatalog); + expect(fs.readFileSync(settingsPath)).toEqual(previousSettings); + const stagedPath = fs.readFileSync(removalMarker, "utf8"); + expect(stagedPath).toContain(".settings.json."); + expect(failed.stderr).toContain(stagedPath); + expect(fs.existsSync(stagedPath)).toBe(failedCleanup === "remove"); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }, + ); + it("updates the target of an existing Pi settings symlink (#13362)", () => { const { home, status, stderr } = generate( { From ae0dcf273d0ec2d6806d4e9c5ef42281d89ea220 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter Date: Mon, 19 Oct 2026 12:19:49 -0700 Subject: [PATCH 13/22] test(dcode): validate native inference for issue 13164 (#13363) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Outcome The managed DCode resolver now passes the attached compatible provider reference to its model constructor. The DCode live target checks inference through the native endpoint and provider attached during onboarding. ## Reason Issue #13164 reports a Ready DCode sandbox with a healthy upstream endpoint but HTTP 503 on the managed route. OpenShell's provider contract routes requests to the native endpoint through an attached profile. The DCode lifecycle checks still used the retired managed route, and the installed resolver returned a synthetic managed credential for a direct compatible endpoint. ### Related issues Fixes #13164. ## Changes - Normalize the expected compatible endpoint in cloud checks 03, 04, and 07 using the onboarding URL normalizer, after rejecting unsafe raw input. A shared CLI helper keeps all three consumers consistent; 32 regression cases cover canonicalization, unsafe inputs, and strict route/key rejection. - Derive the actual OpenShell provider name, credential key, and completion URL from the recorded native attachment before the DCode lifecycle test rotates a credential or sends a request. Reject absent or mismatched attachments. The lifecycle support suite covers this binding and the existing rejected, restored, and successful rebuild paths. - Update the DCode Ultra profile, fresh re-onboard, and headless cloud checks to recognize the attached compatible endpoint and its placeholder. The headless route classifier still rejects mixed route and credential pairs. - Forward the onboarding-selected compatible endpoint into cloud check child processes so fresh re-onboarding checks the same route. - Match the headless DCode route against the selected compatible endpoint, including custom `NEMOCLAW_ENDPOINT_URL` values. Reject mismatched routes and placeholders. - Use the issued `COMPATIBLE_API_KEY` reference for a directly attached compatible endpoint. Reject missing, raw, and wrong-key values while retaining the legacy shared, native local, NVIDIA, and OpenRouter credential paths. - Correct the Deep Agents route guidance for hosted compatible endpoints and retained shared routes. - Merge current `main` and preserve both Pi native local-route guidance and Deep Agents hosted compatible guidance in the two overlapping inference pages. - Correct lifecycle, status, policy, architecture, troubleshooting, and support guidance that still classified new Deep Agents Code hosted compatible endpoints as shared routes. - Correct the overview and generated platform support claims for the same native route, and limit shared-route language to compatible endpoints without an attachment and retained local selections. - Merge current `main` at `1e82499a048d5a4d85b0876d8c07bd93c8a9a5de`, preserving its Hermes native local guidance and this PR's Deep Agents hosted compatible guidance in the platform matrix and 12 documentation pages. - Correct remaining Deep Agents state, ecosystem, security, status, and monitoring guidance that described new hosted compatible endpoints as shared routes. - Preserve the compatible provider credential when its endpoint URL equals the native NVIDIA URL in both the generated config and managed model constructor. Make the headless route check require an active compatible key in the selected provider table, and correct shared-route and Hermes-specific documentation. - Reject TOML array-table headers when checking the selected Deep Agents Code provider, so credentials from another table cannot satisfy the E2E route check. - Merge current `main` at `b01bb7ffc1bc0e1e7c8b015c7103039998aa9a6e`, preserving Deep Agents Code hosted compatible guidance and OpenClaw native local health probes in the generated platform matrix and support table. ## Verification - Endpoint normalization repair `5fa267e21116f80621ecd0a48e138f4498202866`: 262 focused tests passed. The new initialization regressions had 24 failures before the fix and all 30 passed afterward; two additional classifier cases passed. Signed commit hooks and credential-isolated pre-push publication validation and CLI TypeScript checks passed. GitHub reports the commit as Verified. No live E2E run was performed for this repair. - `npx vitest run --project e2e-support test/e2e/support/e2e-phase-lifecycle.test.ts` — 35 passed. - `npx vitest run --project integration test/agents/deepagents/langchain-deepagents-code-image.test.ts` — 142 passed, including custom and mismatched endpoint cases. - The combined image and managed resolver integration run — 148 passed, including direct constructor binding and wrong-key rejection. - The adjacent DCode config generator suite — 34 passed. The other two adjacent credential and patch suites — 314 passed, 2 skipped. - `npm run docs` — passed after resolving the ninth-commit merge; Fern reported zero errors and two warnings. - `python3 scripts/generate-platform-docs.py --check` — all generated platform tables matched `ci/platform-matrix.json`. - The focused `builds a minimal cloud-experimental child environment` E2E support test passed with a custom endpoint. The wider file had one unrelated 5-second observability-test timeout; that test passed in isolation. - `NODE_OPTIONS=--max-old-space-size=8192 npm run typecheck:cli` — passed. - `bash -n` on the three changed cloud check scripts — passed. - Commit hooks, including shellcheck, shfmt, Oxfmt, Oxlint, repository checks, secret scan, and commitlint — passed. - The credential-isolated pre-push hook passed publication validation and CLI TypeScript checks for commit `f32b4e07ce660caf8f33e74cc13f7f71c9f6b09b`. - The signed ninth merge commit `2ed12918872a5c0e5678585120a5bb38f4033d21` passed `npm run validate:pr`, including repository checks, builds, publication validation, and CLI TypeScript checks. Its credential-isolated pre-push hook also passed. GitHub reports the commit as Verified. - The signed tenth commit `1eb7f6fea7527ad89ad24ed47ec02ebb5a58ace4` passed the docs build, signed commit hooks, full `npm run validate:pr`, and the credential-isolated pre-push hook. GitHub reports the commit as Verified. - The twelfth commit `eef9e373054439b14edd30b2d259490c00ea8251` repaired a CodeRabbit finding in the E2E TOML parser. The new regression failed before repair; all 146 image contract tests passed after repair. Signed hooks, full `npm run validate:pr`, credential-isolated pre-push validation, and exact-old-head guarded publication passed. GitHub marked it Verified. - The signed thirteenth merge commit `7c378e9feac006c3cc12347204e9d2a6af8facfc` passed platform matrix sync, all 146 image contract tests, `npm run docs` with zero errors and two existing warnings, signed hooks, full credential-isolated `npm run validate:pr`, and credential-isolated pre-push validation. Publication used an exact-old-head guard and GitHub marked it Verified. All exact-head CI checks passed or were legitimately skipped after one full managed-image workflow rerun. Its first amd64 activation attempt stopped before testing on an `npm ci` network `ECONNRESET`; the second attempt passed amd64, arm64, and rootless Podman activation. [Advisor run 39270711349](https://github.com/NVIDIA/NemoClaw/actions/runs/39270711349) passed all nine specialists with no findings and no additional E2E selector; the [selection record](https://github.com/NVIDIA/NemoClaw/pull/13363#issuecomment-6270339168) names the exact commit. CodeRabbit marked its exact-head review complete with no new comments. - All nine Advisor specialists reviewed commit `146a5e86ab268ad0f900325098618f0679129b9c`. Two P1 findings identified the same remaining documentation gap; this commit corrects both affected pages and the platform matrix source. All nine specialists also cleared commit `dac4958e3637d0526cb816561f515a6ac3dfee2e` with no additional E2E selector. No manual PR E2E run was required; the [selection record](https://github.com/NVIDIA/NemoClaw/pull/13363#issuecomment-6227181747) names the commit and result. The diff contains no secrets, API keys, or credentials. ## Review notes The endpoint-normalization P2 is fixed in `5fa267e21116f80621ecd0a48e138f4498202866`. Self-review of the published repair found no further actionable issue; independent CI and automated review for this commit are pending. `agents/langchain-deepagents-code/managed-dcode-runtime.py` is a sensitive runtime path in this PR. The eleventh commit repaired the credential collision and related route findings. The twelfth commit repaired CodeRabbit's TOML array-table false pass. CodeRabbit's later suggestion to accept trailing inline comments is non-blocking for this E2E check because it reads NemoClaw-generated complete scalar lines. The thirteenth head passed all nine Advisor specialists and CodeRabbit's review completed without new comments. --- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Signed-off-by: Prekshi Vyas ## Summary by CodeRabbit * **New Features** * Deep Agents Code supports hosted OpenAI-compatible endpoints through sandbox-attached providers, using the endpoint’s configured credentials. * Supported host-local compatible endpoints can also use attached providers. OpenClaw supports attached-provider routing for Ollama, standalone vLLM, and llama.cpp. * **Bug Fixes** * Hosted compatible endpoints use their configured credentials instead of the managed inference credential. Endpoints without an attachment and retained shared routes continue to use the shared inference route. * **Documentation** * Updated setup, routing, security, and troubleshooting guidance to clarify which provider selections use attached providers versus the shared route. --------- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Signed-off-by: Prekshi Vyas Co-authored-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Co-authored-by: Prekshi Vyas --- .../generate-config.ts | 7 +- .../managed-dcode-runtime.py | 7 +- ci/platform-matrix.json | 4 +- docs/about/ecosystem-deepagents.mdx | 4 +- docs/about/how-it-works.mdx | 5 +- docs/about/overview.mdx | 2 +- docs/deployment/deploy-to-headless-server.mdx | 4 +- .../quickstart-langchain-deepagents-code.mdx | 22 ++++- docs/inference/custom-endpoint-security.mdx | 9 +- .../inference/how-inference-routing-works.mdx | 10 ++- .../set-up-openai-compatible-endpoint.mdx | 25 ++++-- .../understand-provider-validation.mdx | 4 +- docs/inference/use-shared-gateway-routes.mdx | 4 +- .../inference/view-active-inference-route.mdx | 13 ++- docs/manage-sandboxes/workspace-files.mdx | 2 +- docs/monitoring/monitor-sandbox-activity.mdx | 2 +- docs/reference/architecture.mdx | 4 +- docs/reference/commands.mdx | 18 ++-- docs/reference/enterprise-readiness.mdx | 4 +- docs/reference/network-policies.mdx | 7 +- docs/reference/platform-support.mdx | 4 +- docs/reference/troubleshooting.mdx | 2 +- docs/security/best-practices.mdx | 8 +- .../langchain-deepagents-code-config.test.ts | 11 +++ .../langchain-deepagents-code-image.test.ts | 64 ++++++++++++++ ...epagents-code-managed-model-params.test.ts | 73 +++++++++++++++- ...-deepagents-code-nemotron-ultra-profile.sh | 25 ++++-- .../04-deepagents-code-fresh-reonboard.sh | 11 +-- .../07-deepagents-code-headless-inference.sh | 56 +++++++++++- .../normalize-hosted-endpoint.ts | 13 +++ .../lifecycle-dcode-invalid-credential.ts | 59 +++++++++---- test/e2e/live/cloud-experimental-checks.ts | 2 + .../e2e/support/cloud-hosted-endpoint.test.ts | 85 +++++++++++++++++++ test/e2e/support/e2e-phase-lifecycle.test.ts | 52 +++++++++++- ...platform-parity-cloud-experimental.test.ts | 2 + .../langchain-deepagents-code-headless.ts | 14 ++- 36 files changed, 542 insertions(+), 96 deletions(-) create mode 100644 test/e2e/e2e-cloud-experimental/normalize-hosted-endpoint.ts create mode 100644 test/e2e/support/cloud-hosted-endpoint.test.ts diff --git a/agents/langchain-deepagents-code/generate-config.ts b/agents/langchain-deepagents-code/generate-config.ts index 7c99d4776f7..9d5ef3e04e1 100644 --- a/agents/langchain-deepagents-code/generate-config.ts +++ b/agents/langchain-deepagents-code/generate-config.ts @@ -172,9 +172,10 @@ function providerConfigLines( const nativeHosted = nativeHostedAgentConfig(upstreamProvider, baseUrl); const apiKeyEnv = nativeLocalCredentialReference(upstreamProvider, baseUrl) ? NATIVE_LOCAL_CREDENTIAL_ENV - : baseUrl === NVIDIA_HOSTED_NATIVE_ENDPOINT - ? ATTACHED_PROVIDER_API_KEY_ENV - : (nativeHosted?.credentialEnv ?? MANAGED_INFERENCE_API_KEY_ENV); + : (nativeHosted?.credentialEnv ?? + (baseUrl === NVIDIA_HOSTED_NATIVE_ENDPOINT + ? ATTACHED_PROVIDER_API_KEY_ENV + : MANAGED_INFERENCE_API_KEY_ENV)); return [ `[models.providers.${provider}]`, `models = ${tomlArray([model])}`, diff --git a/agents/langchain-deepagents-code/managed-dcode-runtime.py b/agents/langchain-deepagents-code/managed-dcode-runtime.py index 2899d4e068b..761c902ada3 100644 --- a/agents/langchain-deepagents-code/managed-dcode-runtime.py +++ b/agents/langchain-deepagents-code/managed-dcode-runtime.py @@ -1102,7 +1102,7 @@ def managed_inference_api_key(base_url: str) -> str: ) is None: raise RuntimeError("native local inference requires an issued OpenShell credential reference") return value - if base_url == "https://integrate.api.nvidia.com/v1": + if base_url == "https://integrate.api.nvidia.com/v1" and upstream_provider != "compatible-endpoint": name = "NVIDIA_INFERENCE_API_KEY" provider = "NVIDIA" elif base_url == "https://openrouter.ai/api/v1" and upstream_provider in ( @@ -1114,6 +1114,11 @@ def managed_inference_api_key(base_url: str) -> str: else "OPENROUTER_API_KEY" ) provider = "OpenRouter" + elif upstream_provider == "compatible-endpoint" and host not in { + "inference.local", "localhost", "127.0.0.1", "::1" + }: + name = "COMPATIBLE_API_KEY" + provider = "compatible endpoint" else: return "nemoclaw-managed-inference" value = os.environ.get(name, "") diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 43e548bb33e..bbae2759840 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -190,7 +190,7 @@ "name": "LangChain Deep Agents Code", "status": "tested", "default": false, - "notes": "Terminal-oriented coding agent runtime with no in-sandbox gateway or dashboard, built on the Deep Agents SDK. Its manifest is `agents/langchain-deepagents-code/manifest.yaml`, and its binary is `dcode`. Onboard with `nemo-deepagents onboard` or `nemoclaw onboard --agent langchain-deepagents-code`, and follow [the quickstart](/user-guide/deepagents/get-started/quickstart). NemoClaw runs it as a managed agent runtime: unmanaged sandbox, Model Context Protocol (MCP) server, and shell overrides are rejected, and credential-bearing proxy URLs are dropped from persisted shell environment values. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use sandbox-attached providers for qualified host-local endpoints. Owned Local NVIDIA NIM and other custom or local selections retain the shared `inference.local` route. Live runtime acceptance, broader launch material, and terminal-agent diagnostics are tracked at open issue #4861." + "notes": "Terminal-oriented coding agent runtime with no in-sandbox gateway or dashboard, built on the Deep Agents SDK. Its manifest is `agents/langchain-deepagents-code/manifest.yaml`, and its binary is `dcode`. Onboard with `nemo-deepagents onboard` or `nemoclaw onboard --agent langchain-deepagents-code`, and follow [the quickstart](/user-guide/deepagents/get-started/quickstart). NemoClaw runs it as a managed agent runtime: unmanaged sandbox, Model Context Protocol (MCP) server, and shell overrides are rejected, and credential-bearing proxy URLs are dropped from persisted shell environment values. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Ollama, standalone vLLM, llama.cpp, hosted compatible, and qualifying host-local compatible selections use sandbox-attached providers. Owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared local selections use `inference.local`. Live runtime acceptance, broader launch material, and terminal-agent diagnostics are tracked at open issue #4861." } ], @@ -208,7 +208,7 @@ { "name": "Managed inference", "status": "tested", - "notes": "Provider-routed model calls through OpenShell, transparent to the agent. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Deep Agents Code selections of Ollama, standalone vLLM, llama.cpp, and qualifying local compatible endpoints use sandbox-attached providers for their qualified host-local endpoints. Hermes implements the same attachment path, with live qualification for its new local lifecycle pending. New OpenClaw selections of Ollama, standalone vLLM, and llama.cpp also use sandbox-attached providers, and their `status`, `connect`, and `start` health checks send requests to the recorded provider endpoint. Earlier OpenClaw sandboxes without a sandbox-attached provider, Pi, owned Local NVIDIA NIM, and other custom or local selections retain the shared `inference.local` route. OpenShell keeps provider credentials outside the sandbox. Supports every entry in the Providers table." + "notes": "Provider-routed model calls through OpenShell, transparent to the agent. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Deep Agents Code selections of Ollama, standalone vLLM, llama.cpp, hosted compatible endpoints, and qualified host-local compatible endpoints use sandbox-attached providers. Hermes uses attached providers for qualified host-local Ollama, standalone vLLM, llama.cpp, and compatible selections; live qualification for its new local lifecycle remains pending. New OpenClaw selections of Ollama, standalone vLLM, and llama.cpp also use attached providers, and their `status`, `connect`, and `start` health checks send requests to the recorded provider endpoint. Earlier OpenClaw sandboxes without a sandbox-attached provider, Pi, owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared local selections use `inference.local`. OpenShell keeps provider credentials outside the sandbox. Supports every entry in the Providers table." }, { "name": "Declarative network policy", diff --git a/docs/about/ecosystem-deepagents.mdx b/docs/about/ecosystem-deepagents.mdx index 5436e5e141c..cb206936fec 100644 --- a/docs/about/ecosystem-deepagents.mdx +++ b/docs/about/ecosystem-deepagents.mdx @@ -74,10 +74,10 @@ The following table compares custom OpenShell integration with `nemo-deepagents | Capability | Custom OpenShell + Deep Agents | `nemo-deepagents onboard` | |---|---|---| | Sandbox isolation | You define and apply OpenShell seccomp, Landlock, network namespace isolation, and no-new-privileges policy. | NemoClaw applies the managed Deep Agents policy through the blueprint and requires the runtime checks that the managed agent runtime depends on. | -| Credential handling | You create OpenShell providers manually and decide how `dcode` receives endpoint credentials. | NemoClaw keeps provider credentials outside the sandbox. New NVIDIA Endpoints, supported fixed hosted providers, Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use sandbox-attached native providers. Owned Local NVIDIA NIM and other routes retain `inference.local`. | +| Credential handling | You create OpenShell providers manually and decide how `dcode` receives endpoint credentials. | NemoClaw keeps provider credentials outside the sandbox. New NVIDIA Endpoints, supported fixed hosted providers, hosted compatible endpoints, Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections use sandbox-attached providers. Owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared routes use `inference.local`. | | Image hardening | Depends on your base image and install steps. | NemoClaw installs the pinned Deep Agents Code package in `/opt/venv`, uses managed launchers, and rejects unsupported runtime paths such as unmanaged MCP autoloading, nested sandbox providers, and alternate model routes. | | Filesystem policy | You define read-only and read-write paths in policy YAML. | NemoClaw defines a targeted layout: system paths are read-only, `/sandbox` and `/tmp` are writable, and Deep Agents state lives under `/sandbox/.deepagents`. | -| Inference setup | You configure OpenShell inference routing and Deep Agents provider settings manually. | NemoClaw validates the selected provider from the host, configures the OpenShell route, and writes `/sandbox/.deepagents/config.toml` for the managed OpenAI-compatible path. | +| Inference setup | You configure OpenShell inference routing and Deep Agents provider settings manually. | NemoClaw validates the selected provider from the host, configures its sandbox attachment or shared OpenShell route, and writes `/sandbox/.deepagents/config.toml` for the managed OpenAI-compatible path. | | Managed MCP | You decide how MCP servers enter the sandbox and how they receive credentials. | NemoClaw validates and updates the agent-native Deep Agents MCP configuration plus live OpenShell policy, providers, attachments, and credential placeholders through host-side commands. | | Optional integrations | You decide how web search, tracing, and other integrations enter the sandbox. | NemoClaw supports an explicit Tavily policy and credential path plus opt-in bounded trace export to an operator-managed host collector. Native LangSmith tracing and ambient OpenTelemetry exporter configuration remain disabled inside the sandbox. | | Blueprint versioning | No NemoClaw blueprint; your image tag is whatever you built locally. | NemoClaw downloads the blueprint artifact, checks version compatibility, and verifies its digest before applying. | diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index 62796acfd56..2c6f146dc42 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -212,9 +212,10 @@ Refer to [Recover and Rebuild Sandboxes](../manage-sandboxes/operate-sandboxes/r New NVIDIA Endpoints, OpenAI, Anthropic, Google Gemini, OpenRouter, and Hermes Provider selections use sandbox-attached OpenShell providers. The agent calls the selected provider API hostname, and OpenShell permits only the approved destination and request paths. -New Deep Agents Code and Hermes selections also use native providers for Ollama, standalone vLLM, llama.cpp, and qualifying local compatible endpoints. +New Deep Agents Code and Hermes selections also use native providers for Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible endpoints. +Deep Agents Code also uses native providers for hosted compatible endpoints. New OpenClaw Ollama, standalone vLLM, and llama.cpp selections also use sandbox-attached native providers. -Owned Local NVIDIA NIM, other custom shared routes, Pi local selections, retained OpenClaw shared local selections, and Model Router retain `inference.local`. +Owned Local NVIDIA NIM, compatible endpoints without a native attachment, retained Pi local selections, retained Hermes local selections without an attachment, retained OpenClaw shared local selections, and Model Router retain `inference.local`. Hermes live qualification for the new local lifecycle remains pending. Legacy Deep Agents Code standalone local sandboxes require backup and fresh recreation before native-provider onboarding; NemoClaw blocks their resume and rebuild. An older OpenClaw Ollama, standalone vLLM, or llama.cpp sandbox without an attachment can retain its supported shared-route model switches, but cannot resume or rebuild. Back it up and re-onboard with `--fresh --name --recreate-sandbox`. Owned Local NIM sandboxes retain the shared route and can resume or rebuild without a native attachment. Refer to [About Inference Routing](../inference/about-inference-routing) for endpoints and route inspection. diff --git a/docs/about/overview.mdx b/docs/about/overview.mdx index e533b1ca9af..2072ed3824d 100644 --- a/docs/about/overview.mdx +++ b/docs/about/overview.mdx @@ -37,7 +37,7 @@ NemoClaw groups its supported capabilities around running, connecting, controlli | Area | Capability | |---|---| | Run supported agents | Onboards OpenClaw, Hermes, or LangChain Deep Agents Code into an agent-specific sandbox image and configuration. | -| Route inference | Connects supported hosted and local inference providers through OpenShell-managed routes. NVIDIA Endpoints and supported fixed hosted providers use sandbox-attached native providers. New Deep Agents Code and Hermes selections also use native providers for Ollama, standalone vLLM, llama.cpp, and qualifying local compatible endpoints; new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use native providers too. Owned Local NVIDIA NIM, other custom shared routes, and retained shared local routes use `inference.local`. Hermes live qualification for the new local lifecycle remains pending. OpenShell keeps provider credentials outside the sandbox. | +| Route inference | Connects supported hosted and local inference providers through OpenShell-managed routes. NVIDIA Endpoints and supported fixed hosted providers use sandbox-attached native providers. New Deep Agents Code and Hermes selections also use native providers for Ollama, standalone vLLM, llama.cpp and qualified host-local compatible endpoints. Deep Agents Code also uses native providers for hosted compatible endpoints. New OpenClaw Ollama, standalone vLLM, and llama.cpp selections use native providers too. Owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared local selections without an attachment use `inference.local`. Hermes live qualification for the new local lifecycle remains pending. OpenShell keeps provider credentials outside the sandbox. | | Control sandbox access | Applies network policy, filesystem policy, process controls, and managed inference through OpenShell. | | Operate the lifecycle | Reports host readiness, creates or updates sandboxes, reports status and logs, and rebuilds or recovers sandboxes while preserving the native agent home. | | Connect Model Context Protocol (MCP) servers | Manages authenticated HTTPS Streamable HTTP MCP servers for all supported agent runtimes. OpenShell replaces credential placeholders at approved egress boundaries. | diff --git a/docs/deployment/deploy-to-headless-server.mdx b/docs/deployment/deploy-to-headless-server.mdx index c54d2d6f9cc..38135a7df3f 100644 --- a/docs/deployment/deploy-to-headless-server.mdx +++ b/docs/deployment/deploy-to-headless-server.mdx @@ -206,7 +206,7 @@ $$nemoclaw headless-agent status $$nemoclaw headless-agent connect --probe-only ``` -`$$nemoclaw headless-agent status` exits nonzero when the sandbox, gateway, local container, or authoritative inference route is not verified. Its main `Inference` line probes the selected provider path from inside the sandbox. Shared providers use `https://inference.local/v1/models`, then send an inference request over the same route when that probe reports the route reachable. NVIDIA Endpoints, supported fixed hosted providers, and qualified Deep Agents local selections verify the recorded sandbox-attached OpenShell provider with an inference request. Status repeats inference requests up to three total attempts when they return a transient gateway status; refer to the [CLI commands reference](../reference/commands) for the retried statuses and their token cost. The line reports `healthy` when the route served the request, `unauthorized` when the route rejected it with HTTP `401` or `403`, and `unhealthy` after a final failure or after all transient attempts fail. +`$$nemoclaw headless-agent status` exits nonzero when the sandbox, gateway, local container, or authoritative inference route is not verified. Its main `Inference` line probes the selected provider path from inside the sandbox. Shared providers use `https://inference.local/v1/models`, then send an inference request over the same route when that probe reports the route reachable. NVIDIA Endpoints, supported fixed hosted providers, new Deep Agents Code hosted compatible endpoints, and qualified Deep Agents Code local selections verify the recorded sandbox-attached OpenShell provider with an inference request. Status repeats inference requests up to three total attempts when they return a transient gateway status; refer to the [CLI commands reference](../reference/commands) for the retried statuses and their token cost. The line reports `healthy` when the route served the request, `unauthorized` when the route rejected it with HTTP `401` or `403`, and `unhealthy` after a final failure or after all transient attempts fail. @@ -269,7 +269,7 @@ unset TOKEN -Deep Agents Code does not expose a dashboard URL or gateway token. NVIDIA Endpoints, supported fixed hosted providers, and new Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, retained legacy and other custom shared routes use the OpenShell-managed `inference.local` route. +Deep Agents Code does not expose a dashboard URL or gateway token. NVIDIA Endpoints, supported fixed hosted providers, and new Ollama, standalone vLLM, llama.cpp, hosted compatible, and qualifying host-local compatible selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, retained legacy routes, and other custom routes without an attachment use the OpenShell-managed `inference.local` route. diff --git a/docs/get-started/quickstart-langchain-deepagents-code.mdx b/docs/get-started/quickstart-langchain-deepagents-code.mdx index b9eeef8e599..54138a215ca 100644 --- a/docs/get-started/quickstart-langchain-deepagents-code.mdx +++ b/docs/get-started/quickstart-langchain-deepagents-code.mdx @@ -165,7 +165,27 @@ The wizard asks for an inference provider, model, required credential, and sandb When you edit inference, NemoClaw clears the credential staged for the discarded selection. NemoClaw preserves the sandbox name. When you edit the sandbox name, NemoClaw preserves the inference selection. The sandbox prompt shows the prior name as its default. After you apply the configuration, routine editing ends. If inference setup fails and offers a `back` recovery action, you can return to provider and model selection and then review the updated configuration again. Provider registration, inference setup, policy selection, and sandbox creation then continue forward. The default Deep Agents sandbox name is `deepagents-code`. Use a distinct name, such as `my-deepagents`, when you run Deep Agents, Hermes, and OpenClaw sandboxes side by side. Refer to [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for provider-specific prompts. -The image installs hash-locked Deep Agents Code `0.1.71` with NVIDIA provider support. After the terminal smoke checks, onboarding runs `dcode --version` and compares the result with the version required by the agent manifest. Fresh and resumed onboarding exit nonzero instead of reporting the runtime ready when the installed version is too old, uses an incompatible version scheme, or cannot be verified. If the version check fails, review the reported version error and run `nemo-deepagents rebuild` before resuming onboarding. NemoClaw writes `/sandbox/.deepagents/config.toml` with an OpenAI-compatible provider and sets `use_responses_api = false` for Chat Completions compatibility. NVIDIA Endpoints uses `https://integrate.api.nvidia.com/v1` through a least-privilege OpenShell provider attached to the sandbox. Supported fixed hosted providers also use their native endpoints through sandbox-attached providers. New Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use their qualified host-local endpoint through a sandbox-attached OpenShell provider. Other custom endpoints and owned Local NVIDIA NIM retain `https://inference.local/v1`. Both paths put a scoped placeholder in the agent config instead of the real provider key. When onboarding records a reasoning effort on a `compatible-endpoint` route that uses `openai-completions`, managed startup writes that value to a root-owned file and Deep Agents Code model requests carry it as an `extra_body.reasoning_effort` request parameter. Leave `NEMOCLAW_REASONING_EFFORT` unset to keep the endpoint's own default. Deep Agents Code has no runtime `inference set` path, so re-onboard the sandbox with `nemo-deepagents onboard --fresh --name --recreate-sandbox` to change the recorded effort. When you use NVIDIA Endpoints without selecting another model, new Deep Agents Code sandboxes default to `nvidia/nemotron-3-ultra-550b-a55b`. For this model, the managed image maps the OpenAI-compatible route to Deep Agents `0.7.15`'s native Nemotron 3 Ultra harness profile, including model-specific tool-calling, filesystem, retry, context, and final-answer safeguards. Rebuild existing Deep Agents Code sandboxes after upgrading NemoClaw so their image includes the pinned runtime and profile. This agent-specific default does not change the shared Nemotron 3 Super default for OpenClaw and Hermes. The image launcher normalizes the runtime proxy environment for interactive, login-shell, and direct-exec paths and removes inherited proxy credentials and bypass entries before `dcode` starts. Managed interactive sessions pre-complete Deep Agents Code's optional first-run onboarding, skip its dependency and model selection screens, then open the TUI with the model selected during NemoClaw onboarding. The image includes `ripgrep` and `dos2unix`, and ordinary sessions suppress the optional Tavily warning unless web search is configured or invoked. +The image installs hash-locked Deep Agents Code `0.1.71` with NVIDIA provider support. +After the terminal smoke checks, onboarding runs `dcode --version` and compares the result with the version required by the agent manifest. +Fresh and resumed onboarding exit nonzero instead of reporting the runtime ready when the installed version is too old, uses an incompatible version scheme, or cannot be verified. +If the version check fails, review the reported version error and run `nemo-deepagents rebuild` before resuming onboarding. +NemoClaw writes `/sandbox/.deepagents/config.toml` with an OpenAI-compatible provider and sets `use_responses_api = false` for Chat Completions compatibility. +NVIDIA Endpoints uses `https://integrate.api.nvidia.com/v1` through a least-privilege OpenShell provider attached to the sandbox. +Supported fixed hosted providers also use their native endpoints through sandbox-attached providers. +New Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use their qualified host-local endpoint through a sandbox-attached OpenShell provider. +New hosted compatible endpoints use their recorded endpoint through a sandbox-attached provider. +Retained shared selections and owned Local NVIDIA NIM use `https://inference.local/v1`. +Each route puts a scoped placeholder in the agent config instead of the real provider key. +When onboarding records a reasoning effort on a `compatible-endpoint` route that uses `openai-completions`, managed startup writes that value to a root-owned file and Deep Agents Code model requests carry it as an `extra_body.reasoning_effort` request parameter. +Leave `NEMOCLAW_REASONING_EFFORT` unset to keep the endpoint's own default. +Deep Agents Code has no runtime `inference set` path, so re-onboard the sandbox with `nemo-deepagents onboard --fresh --name --recreate-sandbox` to change the recorded effort. +When you use NVIDIA Endpoints without selecting another model, new Deep Agents Code sandboxes default to `nvidia/nemotron-3-ultra-550b-a55b`. +For this model, the managed image maps the OpenAI-compatible route to Deep Agents `0.7.15`'s native Nemotron 3 Ultra harness profile, including model-specific tool-calling, filesystem, retry, context, and final-answer safeguards. +Rebuild existing Deep Agents Code sandboxes after upgrading NemoClaw so their image includes the pinned runtime and profile. +This agent-specific default does not change the shared Nemotron 3 Super default for OpenClaw and Hermes. +The image launcher normalizes the runtime proxy environment for interactive, login-shell, and direct-exec paths and removes inherited proxy credentials and bypass entries before `dcode` starts. +Managed interactive sessions pre-complete Deep Agents Code's optional first-run onboarding, skip its dependency and model selection screens, then open the TUI with the model selected during NemoClaw onboarding. +The image includes `ripgrep` and `dos2unix`, and ordinary sessions suppress the optional Tavily warning unless web search is configured or invoked. diff --git a/docs/inference/custom-endpoint-security.mdx b/docs/inference/custom-endpoint-security.mdx index a24a4fc6e23..32d85dd923c 100644 --- a/docs/inference/custom-endpoint-security.mdx +++ b/docs/inference/custom-endpoint-security.mdx @@ -16,8 +16,13 @@ NemoClaw keeps provider credentials on the host and validates explicit custom en The agent inside the sandbox sends requests to `inference.local` instead of connecting to the upstream endpoint directly. - -New Deep Agents Code and Hermes sandboxes use the recorded endpoint through a sandbox-attached OpenShell provider when the host-local selection qualifies. + +New Deep Agents Code sandboxes use the recorded endpoint through a sandbox-attached OpenShell provider for hosted compatible endpoints and qualified host-local endpoints. +Retained shared routes and owned Local NVIDIA NIM use `inference.local`. +Refer to [How Inference Routing Works](../about-inference-routing) for the qualification boundary. + + +New Hermes sandboxes use the recorded endpoint through a sandbox-attached OpenShell provider for qualified host-local endpoints. Other custom endpoints retain the shared `inference.local` route. Refer to [How Inference Routing Works](../about-inference-routing) for the qualification boundary. diff --git a/docs/inference/how-inference-routing-works.mdx b/docs/inference/how-inference-routing-works.mdx index 470c0a7b03f..5df86746411 100644 --- a/docs/inference/how-inference-routing-works.mdx +++ b/docs/inference/how-inference-routing-works.mdx @@ -39,8 +39,9 @@ A different destination cannot gain access through a later DNS change. Existing sandboxes without a native attachment retain their recorded shared route. Legacy local routes have the recovery restrictions described under [Credential Boundary](#credential-boundary). New Deep Agents Code, Hermes, OpenClaw, and Pi selections for Ollama, standalone vLLM, and llama.cpp use a sandbox-attached provider for the qualified host-local endpoint. -Deep Agents Code, Hermes, and Pi also use attached providers for qualified host-local compatible endpoints. -Local NVIDIA NIM, other custom endpoints, Model Router, and unchanged published Pi shared local selections retain the shared route. +Deep Agents Code also uses an attached provider for hosted compatible endpoints. +Deep Agents Code, Hermes, and Pi use attached providers for qualified host-local compatible endpoints. +OpenClaw compatible endpoints, other custom endpoints without an attachment, Local NVIDIA NIM, Model Router, and unchanged published Pi and Hermes shared local selections retain the shared route. Hermes live qualification for the new local lifecycle remains pending. ```text @@ -71,7 +72,10 @@ When a rebuild reuses an automatically bridged compatible-endpoint route without ## Deep Agents Configuration -For Deep Agents, NemoClaw writes `/sandbox/.deepagents/config.toml` with a managed OpenAI-compatible provider, a scoped placeholder API key, and `use_responses_api = false`. Supported native hosted selections use the endpoints above. New Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use their qualified host-local endpoint through an attached provider. Owned Local NVIDIA NIM and other shared-route selections retain `https://inference.local/v1`. +For Deep Agents, NemoClaw writes `/sandbox/.deepagents/config.toml` with a managed OpenAI-compatible provider, a scoped placeholder API key, and `use_responses_api = false`. +Supported native hosted selections use the endpoints above, and hosted compatible selections use their recorded endpoint. +New Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use their qualified host-local endpoint through an attached provider. +Owned Local NVIDIA NIM and retained shared-route selections use `https://inference.local/v1`. The managed `dcode` runtime uses Chat Completions through OpenShell even when a compatible endpoint also supports the Responses API. diff --git a/docs/inference/set-up-openai-compatible-endpoint.mdx b/docs/inference/set-up-openai-compatible-endpoint.mdx index 7e6e6f4ea25..4d21f2b837e 100644 --- a/docs/inference/set-up-openai-compatible-endpoint.mdx +++ b/docs/inference/set-up-openai-compatible-endpoint.mdx @@ -17,10 +17,16 @@ Examples include vLLM, TensorRT-LLM, llama.cpp, LocalAI, and other compatible se The agent connects to `inference.local` inside the sandbox. OpenShell forwards that traffic to the endpoint configured during onboarding. - -New Deep Agents Code and Hermes sandboxes use sandbox-attached OpenShell providers for qualified host-local endpoints. + +New Deep Agents Code sandboxes use sandbox-attached OpenShell providers for hosted compatible endpoints and qualified host-local endpoints. The agent uses the recorded endpoint and a credential reference; OpenShell supplies the credential at egress. -Other compatible endpoints retain the shared `inference.local` route. +Retained shared routes and owned Local NVIDIA NIM use `inference.local`. +Refer to [How Inference Routing Works](../about-inference-routing) for the qualification boundary. + + +New Hermes sandboxes use sandbox-attached OpenShell providers for qualified host-local endpoints. +The agent uses the recorded endpoint and a credential reference; OpenShell supplies the credential at egress. +Retained shared routes and owned Local NVIDIA NIM use `inference.local`. Refer to [How Inference Routing Works](../about-inference-routing) for the qualification boundary. @@ -72,12 +78,19 @@ Enter the server base URL and the model ID reported by the server. Use a host-routable URL such as `http://localhost:8000/v1` when you want onboarding to verify the API, tool-calling, and streaming paths before gateway registration. To qualify for automatic rewriting, an HTTP endpoint URL must use the loopback host `localhost`, `127.0.0.1`, or `[::1]`. Automatic rewriting is limited to the port selected by `NEMOCLAW_VLLM_PORT` (`8000` by default) and ports `11434` and `11435`. -NemoClaw validates the entered URL from the host and registers the OpenShell gateway route through `host.openshell.internal:` for sandbox traffic. +NemoClaw validates the entered URL from the host. +Shared selections use `https://inference.local/v1` inside the sandbox. Qualified host-local selections use a sandbox-attached provider for the recorded endpoint, which may use `host.openshell.internal:` to reach the host. Sandbox inference requests continue to use the base `inference.local` policy, so the managed compatible-endpoint route does not require adding the `local-inference` preset. - -For qualified host-local endpoints, the attached provider profile authorizes the recorded endpoint and the selected agent runtime binaries. + +For qualified host-local endpoints, the attached provider profile authorizes the recorded endpoint and the managed DCode runtime binaries. +Hosted compatible endpoints use their recorded endpoint through an attached provider. +Retained shared routes and owned Local NVIDIA NIM use the base `inference.local` policy. +These routes do not require the `local-inference` preset. + + +For qualified host-local endpoints, the attached provider profile authorizes the recorded endpoint and the Hermes runtime binaries. Other compatible endpoints use the base `inference.local` policy. Neither path requires adding the `local-inference` preset. diff --git a/docs/inference/understand-provider-validation.mdx b/docs/inference/understand-provider-validation.mdx index 440d0bbd294..00cf1750b00 100644 --- a/docs/inference/understand-provider-validation.mdx +++ b/docs/inference/understand-provider-validation.mdx @@ -52,7 +52,9 @@ If the last retry fails or times out, validation stops without another Chat Comp -The managed Deep Agents runtime keeps `use_responses_api = false` and uses Chat Completions. NVIDIA Endpoints, supported fixed hosted providers, and new Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, retained legacy and other custom shared routes use `inference.local`. +The managed Deep Agents runtime keeps `use_responses_api = false` and uses Chat Completions. +NVIDIA Endpoints, supported fixed hosted providers, hosted compatible endpoints, and new Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections use native endpoints through sandbox-attached OpenShell providers. +Owned Local NVIDIA NIM and retained legacy shared routes use `inference.local`. `NEMOCLAW_PREFERRED_API` does not change that runtime selection. diff --git a/docs/inference/use-shared-gateway-routes.mdx b/docs/inference/use-shared-gateway-routes.mdx index 456e1875e79..cfb30d5f070 100644 --- a/docs/inference/use-shared-gateway-routes.mdx +++ b/docs/inference/use-shared-gateway-routes.mdx @@ -10,8 +10,8 @@ content: type: "how_to" --- -These rules apply to providers that still use the shared route, including other custom shared routes, Local NVIDIA NIM, and retained shared local selections. -NVIDIA Endpoints, fixed hosted providers, new Deep Agents Code, Hermes, and Pi qualified local selections, and new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use sandbox-attached providers. +These rules apply to providers that still use the shared route, including compatible endpoints without an attachment, Local NVIDIA NIM, and retained shared local selections. +NVIDIA Endpoints, fixed hosted providers, new Deep Agents Code hosted compatible selections, new Deep Agents Code, Hermes, and Pi qualified local selections, and new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use sandbox-attached providers. Their selected model or provider does not re-point the shared route. Unchanged published Pi local selections retain their recorded shared route during resume and rebuild. diff --git a/docs/inference/view-active-inference-route.mdx b/docs/inference/view-active-inference-route.mdx index 96b296e9c98..d8873635a2f 100644 --- a/docs/inference/view-active-inference-route.mdx +++ b/docs/inference/view-active-inference-route.mdx @@ -18,12 +18,17 @@ For a compatible custom provider on a shared route, the command also reports its Native hosted providers are OpenAI, Anthropic, Gemini, OpenRouter, and Hermes Provider. New OpenClaw selections for Ollama, standalone vLLM, and llama.cpp also report their sandbox-attached local provider. -Local NVIDIA NIM and generic compatible endpoints keep shared-route readback. +Local NVIDIA NIM and OpenClaw compatible endpoints keep shared-route readback. - -Deep Agents Code and Hermes also report a recorded native local attachment for qualified Ollama, vLLM, llama.cpp, and host-local compatible endpoints. + +Deep Agents Code also reports a recorded native attachment for hosted compatible endpoints and qualified Ollama, vLLM, llama.cpp, and host-local compatible endpoints. The displayed endpoint is the host-reachable address recorded for that sandbox. -Nonqualifying custom endpoints and Local NVIDIA NIM retain shared-route readback. +Legacy selections without an attachment and owned Local NVIDIA NIM retain shared-route readback. + + +Hermes also reports a recorded native local attachment for qualified Ollama, vLLM, llama.cpp, and host-local compatible endpoints. +The displayed endpoint is the host-reachable address recorded for that sandbox. +Legacy selections without an attachment and owned Local NVIDIA NIM retain shared-route readback. An invalid recorded native attachment produces an error instead of falling back to the shared route. This command reports configuration state and does not authenticate a model request. diff --git a/docs/manage-sandboxes/workspace-files.mdx b/docs/manage-sandboxes/workspace-files.mdx index 42a68bd434c..7b90b972e2e 100644 --- a/docs/manage-sandboxes/workspace-files.mdx +++ b/docs/manage-sandboxes/workspace-files.mdx @@ -283,7 +283,7 @@ Deep Agents Code normally creates the `/sandbox/.deepagents/agent/` paths below | Path | Purpose | |---|---| -| `/sandbox/.deepagents/config.toml` | NemoClaw-generated model and provider configuration. New NVIDIA Endpoints, supported fixed hosted providers, Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use native endpoints with scoped credential references. Owned Local NVIDIA NIM and other routes use `inference.local`. | +| `/sandbox/.deepagents/config.toml` | NemoClaw-generated model and provider configuration. New NVIDIA Endpoints, supported fixed hosted providers, hosted compatible endpoints, Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections use sandbox-attached providers with scoped credential references. Owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared routes use `inference.local`. | | `/sandbox/.deepagents/.state/` | Deep Agents Code runtime state, including persisted session and MCP-related state. | | `/sandbox/.deepagents/agent/AGENTS.md` | Global memory file for the default Deep Agents Code agent, loaded at session start. | | `/sandbox/.deepagents/agent/memories/` | Topic-specific markdown memories that Deep Agents Code can read and update across sessions. | diff --git a/docs/monitoring/monitor-sandbox-activity.mdx b/docs/monitoring/monitor-sandbox-activity.mdx index ec18194f174..4db2285c1c1 100644 --- a/docs/monitoring/monitor-sandbox-activity.mdx +++ b/docs/monitoring/monitor-sandbox-activity.mdx @@ -27,7 +27,7 @@ Run the status command to view sandbox state, gateway health, and the active inf $$nemoclaw status ``` -`$$nemoclaw status` probes the selected provider path from inside the sandbox as the authoritative inference check. Shared providers use `https://inference.local/v1/models`. NVIDIA Endpoints and supported fixed hosted providers verify the recorded sandbox-attached OpenShell provider with an inference request to its native endpoint. +`$$nemoclaw status` probes the selected provider path from inside the sandbox as the authoritative inference check. Shared providers use `https://inference.local/v1/models`. NVIDIA Endpoints, supported fixed hosted providers, new Deep Agents Code hosted compatible endpoints, and qualified local selections verify the recorded sandbox-attached OpenShell provider with an inference request to its native endpoint. For local Ollama and local vLLM routes, it also prints labeled host-side backend diagnostics that help identify which hop failed. Review these output fields. diff --git a/docs/reference/architecture.mdx b/docs/reference/architecture.mdx index f230f4e4579..d3312a4b375 100644 --- a/docs/reference/architecture.mdx +++ b/docs/reference/architecture.mdx @@ -359,13 +359,13 @@ Inside the sandbox: ## Inference Routing Agent inference follows one of two OpenShell-managed paths. -Owned Local NVIDIA NIM, other custom shared routes, Pi local selections, and retained legacy shared local routes use `inference.local` through the OpenShell gateway: +Owned Local NVIDIA NIM, compatible endpoints without a native attachment, retained Pi local selections, retained Hermes local selections without an attachment, and retained legacy shared local routes use `inference.local` through the OpenShell gateway: ```text Supported agent runtime (sandbox) ──▶ inference.local ──▶ OpenShell gateway ──▶ Provider endpoint ``` -NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use restricted providers attached to the sandbox. For example, NVIDIA Endpoints uses this path: +NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code hosted compatible selections and qualified Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use restricted providers attached to the sandbox. For example, NVIDIA Endpoints uses this path: ```text Supported agent runtime (sandbox) ──▶ integrate.api.nvidia.com ──▶ NVIDIA Endpoints diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 6034a3b4f95..f5e5939d110 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1226,8 +1226,8 @@ During that lease, another `launch` still verifies these conditions: - The recorded inference selection matches its provider path, and every configured route serves a valid response to a bounded request for the recorded model. - NVIDIA Endpoints, supported fixed hosted selections, and new Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections verify their recorded sandbox-attached provider and request the model at its native endpoint without a shared-route models probe. - Owned Local NVIDIA NIM, retained legacy and other custom shared routes use `inference.local`. + NVIDIA Endpoints, supported fixed hosted selections, and new Ollama, standalone vLLM, llama.cpp, Deep Agents Code hosted compatible, and qualifying host-local compatible selections verify their recorded sandbox-attached provider and request the model at its native endpoint without a shared-route models probe. + Owned Local NVIDIA NIM, retained legacy routes, and other custom routes without an attachment use `inference.local`. Their models probe must return HTTP 2xx, except that a retained legacy `openrouter-api` shared route can return HTTP 404 for `GET /v1/models` when the bounded model request succeeds. A failed model request rejects readiness before the agent starts. @@ -1301,7 +1301,7 @@ A recognized broken shared-route result lets `connect` attempt DNS or route repa When the initial shared-route probe cannot return a trusted result, `connect` fails closed before health-driven repair and before opening SSH. It prints a bounded, redacted last-probe detail and points you to `$$nemoclaw doctor`. -For NVIDIA Endpoints, supported fixed hosted selections, new Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections, and new OpenClaw Ollama, standalone vLLM, and llama.cpp selections, `connect` does not inspect or change the unrelated shared route. +For NVIDIA Endpoints, supported fixed hosted selections, new Deep Agents Code hosted compatible selections and qualified Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and host-local compatible selections, and new OpenClaw Ollama, standalone vLLM, and llama.cpp selections, `connect` does not inspect or change the unrelated shared route. It verifies the recorded sandbox-attached provider and sends a native inference request for the selected model. The request must return a structurally valid successful inference result before `connect` opens SSH. @@ -1729,7 +1729,7 @@ $$nemoclaw my-assistant start When that deadline expires, `start` keeps the existing container, exits non-zero, and prints the `NEMOCLAW_CONNECT_TIMEOUT` value to use on the next run. -After the gateway and forward checks pass, `start` sends one inference request through the sandbox's selected provider path using its recorded provider and model. NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections, use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, other custom shared routes, Pi local selections, and retained shared local selections use `inference.local`. A reachable provider path can still reject an inference request or return an invalid result, so the command exits non-zero in either case. It prints the probe result, including the HTTP status when the path returned one, and points you to the sandbox doctor command. Ordinary providers request 16 reply tokens; Gemini requests 256 reply tokens so reasoning has room before visible content. The request allows up to 90 seconds, and the host wrapper allows 95 seconds for bounded process startup and cleanup. Each run consumes provider tokens on a hosted path. When the sandbox records no provider or no model, `start` skips the request and exits `0`. For a shared gateway route, `doctor` still classifies an HTTP `401` or `403` response as reachable. The native NVIDIA doctor probe instead requires a 2xx response from the sandbox-attached provider. Correct the provider credential when `start` reports either status. +After the gateway and forward checks pass, `start` sends one inference request through the sandbox's selected provider path using its recorded provider and model. NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code hosted compatible selections and qualified Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections, use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, compatible endpoints without a native attachment, retained Pi and Hermes local selections without an attachment, and retained shared local routes use `inference.local`. A reachable provider path can still reject an inference request or return an invalid result, so the command exits non-zero in either case. It prints the probe result, including the HTTP status when the path returned one, and points you to the sandbox doctor command. Ordinary providers request 16 reply tokens; Gemini requests 256 reply tokens so reasoning has room before visible content. The request allows up to 90 seconds, and the host wrapper allows 95 seconds for bounded process startup and cleanup. Each run consumes provider tokens on a hosted path. When the sandbox records no provider or no model, `start` skips the request and exits `0`. For a shared gateway route, `doctor` still classifies an HTTP `401` or `403` response as reachable. The native NVIDIA doctor probe instead requires a 2xx response from the sandbox-attached provider. Correct the provider credential when `start` reports either status. ### `$$nemoclaw status` @@ -1781,7 +1781,7 @@ $$nemoclaw my-assistant status --json $$nemoclaw sandbox status my-assistant --json ``` -The command probes the selected provider path from inside the sandbox, and when that probe reports the path reachable it sends an inference request over the same path. NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections, use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, other custom shared routes, Pi local selections, and retained shared local selections use `inference.local`. That inference request is the authoritative inference health check, and both checks exercise the path that agent traffic uses. The main `Inference` line reports one of these states: +The command probes the selected provider path from inside the sandbox, and when that probe reports the path reachable it sends an inference request over the same path. NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code hosted compatible selections and qualified Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections, use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, compatible endpoints without a native attachment, retained Pi and Hermes local selections without an attachment, and retained shared local routes use `inference.local`. That inference request is the authoritative inference health check, and both checks exercise the path that agent traffic uses. The main `Inference` line reports one of these states: | State | Meaning | | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -1936,7 +1936,7 @@ The `Registered gateway binding` result depends on the sandbox registry: - For a registered sandbox with an invalid binding, the check fails, and `doctor` does not select, probe, or recover a gateway from that binding. - For an unregistered sandbox name, `doctor` uses the gateway selected by `NEMOCLAW_GATEWAY_PORT` for its other gateway checks and omits the `Registered gateway binding` check. -For inference health, `doctor` treats the selected provider path inside the sandbox as authoritative. Providers that still use the shared route probe the shared gateway route at `https://inference.local/v1/models`; responses from `200` through `499`, including `401` and `403`, pass that reachability check. NVIDIA Endpoints instead probes its sandbox-attached provider at `https://integrate.api.nvidia.com/v1/models` and requires a 2xx response. For the shared route, HTTP `500` through `599`, interim `100` through `199`, transport failures with status `000`, invalid status values, and an unavailable authoritative probe fail the check. For native NVIDIA, any non-2xx response or unavailable probe fails it. Fixed hosted providers and qualified Deep Agents Code, Hermes, and OpenClaw local selections verify their recorded attachment and make an inference request to their native endpoint. +For inference health, `doctor` treats the selected provider path inside the sandbox as authoritative. Providers that still use the shared route probe the shared gateway route at `https://inference.local/v1/models`; responses from `200` through `499`, including `401` and `403`, pass that reachability check. NVIDIA Endpoints instead probes its sandbox-attached provider at `https://integrate.api.nvidia.com/v1/models` and requires a 2xx response. For the shared route, HTTP `500` through `599`, interim `100` through `199`, transport failures with status `000`, invalid status values, and an unavailable authoritative probe fail the check. For native NVIDIA, any non-2xx response or unavailable probe fails it. Fixed hosted providers, new Deep Agents Code hosted compatible selections, and qualified Deep Agents Code, Hermes, and OpenClaw local selections verify their recorded attachment and make an inference request to their native endpoint. A healthy shared route cannot satisfy that check. Host-side upstream probes use the same authenticated model-invocation checks as status and remain diagnostic only, so their failure does not fail `doctor` when the authoritative in-sandbox path is reachable. For gateway runtimes, `doctor` also reports an informational `Serving process: not checked` result because its fresh sandbox probes do not attest the long-running gateway process. This result does not fail the readiness check. Terminal runtimes omit it because they have no long-running gateway process. The `Inference` `Route` check warns when either the provider or model is unknown. After the gateway is healthy, run `$$nemoclaw status` to refresh the route information. Warnings do not make the command fail. Failed checks, including a failed or unavailable authoritative inference route, exit non-zero so scripts can use `doctor` as a readiness gate. Use `--json` for machine-readable output. For a `compatible-endpoint` route that uses `openai-completions`, the JSON report includes an informational `Inference` check labeled `Reasoning effort`. The check reports `low`, `medium`, `high`, or `endpoint-default` and never includes credentials. Because the check has `info` status, it does not change the command's exit status. @@ -2141,7 +2141,7 @@ The sandbox must be running for `nemohermes my-assistant gateway-token --quiet` -`gateway-token` is not applicable to Deep Agents sandboxes because there is no OpenClaw gateway token. NVIDIA Endpoints, supported fixed hosted providers, and new Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, retained legacy and other custom shared routes use `inference.local`. +`gateway-token` is not applicable to Deep Agents sandboxes because there is no OpenClaw gateway token. NVIDIA Endpoints, supported fixed hosted providers, and new Ollama, standalone vLLM, llama.cpp, hosted compatible, and qualifying host-local compatible selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, retained legacy routes, and other custom routes without an attachment use `inference.local`. @@ -3341,8 +3341,8 @@ After the rebuild succeeds, retrieve the replacement token with `nemohermes my-a After restore, the command starts Deep Agents with the complete source native home, including `config.toml`, hooks, skills, memories, histories, packages, and unrelated native entries. Before changing the sandbox, rebuild verifies that the recorded provider path is still reachable and that the target provider, model, reasoning settings, web search selection, base image, current agent MCP source, and current live OpenShell policy can be captured. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. -Deep Agents also uses attached providers for qualified local endpoints. -Local NVIDIA NIM, other agents' local selections, and other custom endpoints retain the shared `inference.local` route. If those checks fail after backup, NemoClaw keeps the existing sandbox intact. Use rebuild after a failed Deep Agents version check, after enabling Tavily Search, or as the apply step for legacy MCP migration. +Deep Agents also uses attached providers for hosted compatible and qualified local endpoints. +Local NVIDIA NIM, other agents' shared local selections, and custom endpoints without an attachment retain the shared `inference.local` route. If those checks fail after backup, NemoClaw keeps the existing sandbox intact. Use rebuild after a failed Deep Agents version check, after enabling Tavily Search, or as the apply step for legacy MCP migration. Legacy Deep Agents Code local sandboxes without an attachment stop before rebuild. Local NVIDIA NIM retains its shared rebuild path. Back up the sandbox data before running `nemo-deepagents onboard --fresh --name --recreate-sandbox`. diff --git a/docs/reference/enterprise-readiness.mdx b/docs/reference/enterprise-readiness.mdx index 3a6e953924e..50563513a0b 100644 --- a/docs/reference/enterprise-readiness.mdx +++ b/docs/reference/enterprise-readiness.mdx @@ -82,7 +82,7 @@ Each row links to deeper documentation and, when a concrete fix is in progress, | Audit and session records | Supported with caveats | OpenClaw stores per-session JSONL event logs, and Hermes stores its own runtime state. Deep Agents Code has no equivalent session-log export, but it can emit bounded-content traces when you explicitly enable observability. Export is manual per sandbox. | | Resource quotas | Supported with caveats | The entrypoint applies best-effort process and file-descriptor limits (`ulimit -u 512`, `ulimit -n 65536`). Set hard limits through the container runtime for fail-closed enforcement. Refer to [Process Controls](../security/security-controls/process-controls). | | Cost and spend controls | Platform or partner-owned | Deny-by-default egress and routed inference reduce exfiltration and stray endpoints, but NemoClaw does not enforce per-token spend budgets. Set spend limits with your inference provider and monitor unattended agents. | -| Credential isolation | Supported | Inference credentials stay in OpenShell provider state and never enter the sandbox. NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, other custom shared routes, Pi local selections, and retained shared local selections use `inference.local`. Hermes live qualification for the new local lifecycle remains pending. Refer to [How Inference Routing Works](../inference/about-inference-routing) for qualification and existing-sandbox boundaries. CLI output redaction adds defense in depth, and OpenClaw sandboxes also run a memory secret scanner. Refer to [Credential Storage](../security/credential-storage) and [Security Best Practices](../security/best-practices). | +| Credential isolation | Supported | Inference credentials stay in OpenShell provider state and never enter the sandbox. NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code hosted compatible selections and qualified Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, compatible endpoints without a native attachment, retained Pi and Hermes local selections without an attachment, and retained shared local routes use `inference.local`. Hermes live qualification for the new local lifecycle remains pending. Refer to [How Inference Routing Works](../inference/about-inference-routing) for qualification and existing-sandbox boundaries. CLI output redaction adds defense in depth, and OpenClaw sandboxes also run a memory secret scanner. Refer to [Credential Storage](../security/credential-storage) and [Security Best Practices](../security/best-practices). | | Upgrades and lifecycle | Supported with caveats | Upgrade NemoClaw, then run `$$nemoclaw rebuild` to recreate the sandbox with the current image while backing up and restoring its complete native home. Do not update the agent runtime in place inside the sandbox because the agent is image-pinned. Version attachment flexibility remains tracked in [#2217](https://github.com/NVIDIA/NemoClaw/issues/2217). | | State persistence | Supported | OpenShell preserves the native home/workspace across routine lifecycle operations. NemoClaw transfers the complete native tree during rebuild and recreation. Refer to [Persist Sandbox State](../manage-sandboxes/state-and-backups/create-and-restore-snapshots). | | Remote and cloud deployment | Supported with caveats | Provision a tested Linux host, connect over SSH, run the installer, and run `$$nemoclaw onboard`. For OpenClaw or Hermes, keep the agent dashboard bound to loopback and access it through SSH port forwarding. Automatic recovery after a host reboot is not guaranteed; follow the manual recovery flow in [Deploy to a Headless Server](../deployment/deploy-to-headless-server). | @@ -132,7 +132,7 @@ Use the following phrasing to describe NemoClaw accurately in customer and field - Describe NemoClaw as an open-source reference stack for running agents more safely inside OpenShell, in active development, rather than a finished enterprise control plane. - State that egress is deny-by-default and that an operator approves new endpoints, so the agent cannot reach arbitrary hosts. -- Explain that inference credentials remain in OpenShell provider state, so the sandbox never holds provider keys. NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, other custom shared routes, Pi local selections, and retained shared local selections use `inference.local`. Hermes live qualification for the new local lifecycle remains pending. Refer to [How Inference Routing Works](../inference/about-inference-routing) for qualification and existing-sandbox boundaries. +- Explain that inference credentials remain in OpenShell provider state, so the sandbox never holds provider keys. NVIDIA Endpoints, supported fixed hosted providers, and new Deep Agents Code hosted compatible selections and qualified Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use native endpoints through sandbox-attached OpenShell providers. Owned Local NVIDIA NIM, compatible endpoints without a native attachment, retained Pi and Hermes local selections without an attachment, and retained shared local routes use `inference.local`. Hermes live qualification for the new local lifecycle remains pending. Refer to [How Inference Routing Works](../inference/about-inference-routing) for qualification and existing-sandbox boundaries. - Frame centralized fleet management, operator RBAC, enterprise SSO, and multi-tenant isolation as roadmap-only or out of scope today, and avoid presenting them as current capabilities. - Position cost controls and spend limits as provider-owned, and recommend setting provider-side limits for unattended agents. - When a customer hits a known limitation, point to the workaround in this guide and the tracked issue rather than promising a fix date. diff --git a/docs/reference/network-policies.mdx b/docs/reference/network-policies.mdx index 9bb23c7812d..4bde28d0a28 100644 --- a/docs/reference/network-policies.mdx +++ b/docs/reference/network-policies.mdx @@ -239,13 +239,14 @@ Refer to [How Inference Routing Works](../inference/about-inference-routing) for -New Deep Agents Code sandboxes use native providers for qualified Ollama, standalone vLLM, llama.cpp, and host-local compatible endpoints. -The attached profile permits only `GET /v1/models` and `POST /v1/chat/completions` on the recorded host and port, using the listed Python and curl binaries. +New Deep Agents Code sandboxes use attached providers for hosted compatible endpoints. +Qualified Ollama, standalone vLLM, llama.cpp, and host-local compatible endpoints also use native providers. +The local attached profile permits only `GET /v1/models` and `POST /v1/chat/completions` on the recorded host and port, using the listed Python and curl binaries. These local HTTP endpoints do not use TLS termination on port `443`. OpenShell replaces the sandbox's credential placeholder at the allowed endpoint. For a credentialless upstream, the protected local proxy removes the sentinel authorization header before forwarding the request. -Nonqualifying custom endpoints and owned Local NVIDIA NIM retain the shared `inference.local` route. +Custom endpoints without a native attachment and owned Local NVIDIA NIM retain the shared `inference.local` route. Existing DCode local sandboxes without native authority require explicit recreation; NemoClaw does not migrate them automatically. Before recreation, download the state you need to an independently managed location and verify that the download is complete. Follow [Transfer State Manually](../manage-sandboxes/state-and-backups/transfer-state-manually) before deleting the existing sandbox. diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index 434c76ff542..3f3ffc7857b 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -65,7 +65,7 @@ Pick the matching onboarding entry point for each agent. |-------|--------|---------|-------| | OpenClaw | Tested | Yes | Default agent runtime. Onboard with `nemoclaw onboard` (no `--agent` flag required). | | Hermes | Tested | No | First-class agent with dedicated CLI (`nemohermes`), Dockerfile, manifest, docs, and Hermes E2E coverage in `.github/workflows/e2e.yaml` (`hermes-e2e`, `hermes-*`, and Hermes rebuild/switch jobs). Onboard with `nemohermes onboard` or `nemoclaw onboard --agent hermes`. Unlocks the Hermes Provider inference route. Known structural gaps: model-provider compatibility registry is empty (backfilled after failures, see `nemoclaw-blueprint/model-specific-setup/hermes/README.md`); no Hermes-specific unit tests in `nemoclaw/src/`; macOS and WSL CI suites do not differentiate agents. Suitable for evaluation and the documented onboarding paths; production parity with OpenClaw is not yet asserted. | -| LangChain Deep Agents Code | Tested | No | Terminal-oriented coding agent runtime with no in-sandbox gateway or dashboard, built on the Deep Agents SDK. Its manifest is `agents/langchain-deepagents-code/manifest.yaml`, and its binary is `dcode`. Onboard with `nemo-deepagents onboard` or `nemoclaw onboard --agent langchain-deepagents-code`, and follow [the quickstart](/user-guide/deepagents/get-started/quickstart). NemoClaw runs it as a managed agent runtime: unmanaged sandbox, Model Context Protocol (MCP) server, and shell overrides are rejected, and credential-bearing proxy URLs are dropped from persisted shell environment values. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use sandbox-attached providers for qualified host-local endpoints. Owned Local NVIDIA NIM and other custom or local selections retain the shared `inference.local` route. Live runtime acceptance, broader launch material, and terminal-agent diagnostics are tracked at open issue #4861. | +| LangChain Deep Agents Code | Tested | No | Terminal-oriented coding agent runtime with no in-sandbox gateway or dashboard, built on the Deep Agents SDK. Its manifest is `agents/langchain-deepagents-code/manifest.yaml`, and its binary is `dcode`. Onboard with `nemo-deepagents onboard` or `nemoclaw onboard --agent langchain-deepagents-code`, and follow [the quickstart](/user-guide/deepagents/get-started/quickstart). NemoClaw runs it as a managed agent runtime: unmanaged sandbox, Model Context Protocol (MCP) server, and shell overrides are rejected, and credential-bearing proxy URLs are dropped from persisted shell environment values. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Ollama, standalone vLLM, llama.cpp, hosted compatible, and qualifying host-local compatible selections use sandbox-attached providers. Owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared local selections use `inference.local`. Live runtime acceptance, broader launch material, and terminal-agent diagnostics are tracked at open issue #4861. | {/* agent-status:end */} ## Platforms @@ -135,7 +135,7 @@ Each row below is a launch-facing capability claim that NemoClaw makes in docs, |------------|--------|-------| | Guided onboarding | Tested | Single-command interactive wizard (`$$nemoclaw onboard`) that walks the user through inference provider selection, credential setup, and sandbox creation or update. It opens the agent dashboard when the selected runtime provides one. Non-interactive mode is supported with `--non-interactive` and `NEMOCLAW_*` environment variables for CI and scripted installs. | | Sandboxed execution | Tested with limitations | Landlock, seccomp, network namespace isolation, no-new-privileges, privilege dropping, and process limits (nproc 512 at `scripts/lib/sandbox-rlimits.sh:8`) are tested and on by default. OpenShell owns capability enforcement for managed entrypoints, exec processes, and connect shells; those paths do not call NemoClaw capability handling. Direct container runtimes that explicitly override the image user to root retain the existing compatibility drop and required `setpriv` transition. Direct container launches must configure capabilities at the container runtime. | -| Managed inference | Tested | Provider-routed model calls through OpenShell, transparent to the agent. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Deep Agents Code selections of Ollama, standalone vLLM, llama.cpp, and qualifying local compatible endpoints use sandbox-attached providers for their qualified host-local endpoints. Hermes implements the same attachment path, with live qualification for its new local lifecycle pending. New OpenClaw selections of Ollama, standalone vLLM, and llama.cpp also use sandbox-attached providers, and their `status`, `connect`, and `start` health checks send requests to the recorded provider endpoint. Earlier OpenClaw sandboxes without a sandbox-attached provider, Pi, owned Local NVIDIA NIM, and other custom or local selections retain the shared `inference.local` route. OpenShell keeps provider credentials outside the sandbox. Supports every entry in the Providers table. | +| Managed inference | Tested | Provider-routed model calls through OpenShell, transparent to the agent. NVIDIA Endpoints and supported fixed hosted providers use native endpoints through sandbox-attached OpenShell providers. New Deep Agents Code selections of Ollama, standalone vLLM, llama.cpp, hosted compatible endpoints, and qualified host-local compatible endpoints use sandbox-attached providers. Hermes uses attached providers for qualified host-local Ollama, standalone vLLM, llama.cpp, and compatible selections; live qualification for its new local lifecycle remains pending. New OpenClaw selections of Ollama, standalone vLLM, and llama.cpp also use attached providers, and their `status`, `connect`, and `start` health checks send requests to the recorded provider endpoint. Earlier OpenClaw sandboxes without a sandbox-attached provider, Pi, owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared local selections use `inference.local`. OpenShell keeps provider credentials outside the sandbox. Supports every entry in the Providers table. | | Declarative network policy | Tested | YAML-defined egress with policy presets. Presets include `slack`, `discord`, `telegram`, `weather`, `openclaw-pricing`, `huggingface`, `npm`, `pypi`, `brew`, and others. Hot-reloadable at runtime with `$$nemoclaw policy add`. | | State persistence and rebuild transfer | Tested | OpenShell preserves the native agent home during routine lifecycle operations. Before destructive replacement, NemoClaw archives and integrity-checks the complete native home/workspace, then restores it into the replacement sandbox. | | Agent skills | Tested | Packaged agent skills are discoverable by Cursor, Claude Code, and other coding assistants under `.agents/skills/`. Skills also install into the sandbox with `$$nemoclaw skill install`. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 0259643b726..fd782634317 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2068,7 +2068,7 @@ Verify that the inference provider endpoint is reachable from the host. Check th $$nemoclaw status ``` -The main `Inference` line checks the provider path from inside the sandbox and sends an inference request over the path the agent uses. For shared-route providers, including owned Local NVIDIA NIM, `status` probes `https://inference.local/v1/models` before it sends the request through `https://inference.local`. For NVIDIA Endpoints, fixed hosted selections, and new Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections with a recorded native attachment, it verifies that attachment and sends the request to the selected native endpoint; it does not probe `inference.local`. When an inference request returns a transient gateway status, `status` makes up to three request attempts before it reports a failure. Shared-route retries repeat the route probe and request, while native providers retry only the native request. It prints the next attempt and two-second delay to stderr before each retry. Each inference request attempt has a 90-second request timeout. If the main line shows `unauthorized`, `unhealthy`, `unreachable`, or `not probed`, inspect the labeled diagnostic lines to identify the failing hop. An `unauthorized` line means the route answered but rejected the request, so refresh the provider credential rather than the route. +The main `Inference` line checks the provider path from inside the sandbox and sends an inference request over the path the agent uses. For shared-route providers, including owned Local NVIDIA NIM, `status` probes `https://inference.local/v1/models` before it sends the request through `https://inference.local`. For NVIDIA Endpoints, fixed hosted selections, and new Deep Agents Code hosted compatible selections and qualified Deep Agents Code and Hermes Ollama, standalone vLLM, llama.cpp, and host-local compatible selections, plus new OpenClaw Ollama, standalone vLLM, and llama.cpp selections with a recorded native attachment, it verifies that attachment and sends the request to the selected native endpoint; it does not probe `inference.local`. When an inference request returns a transient gateway status, `status` makes up to three request attempts before it reports a failure. Shared-route retries repeat the route probe and request, while native providers retry only the native request. It prints the next attempt and two-second delay to stderr before each retry. Each inference request attempt has a 90-second request timeout. If the main line shows `unauthorized`, `unhealthy`, `unreachable`, or `not probed`, inspect the labeled diagnostic lines to identify the failing hop. An `unauthorized` line means the route answered but rejected the request, so refresh the provider credential rather than the route. Hermes live qualification for the new local lifecycle remains pending. diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index f2d3aaa8a9a..281e327e3a2 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -114,7 +114,7 @@ NemoClaw allows a credential source only in the process that is intended to hold | Execution path | Rule | |---|---| -| Host-brokered provider | Discover and use host credentials only through OpenShell-managed providers. NVIDIA Endpoints, supported fixed hosted providers, and new qualifying Deep Agents Code and Hermes local selections except owned Local NVIDIA NIM use sandbox-attached native providers. Owned Local NIM and other routes retain `inference.local`. The sandbox receives no raw host credential. | +| Host-brokered provider | Discover and use host credentials only through OpenShell-managed providers. NVIDIA Endpoints, supported fixed hosted providers, new Deep Agents Code hosted compatible endpoints, qualified Deep Agents Code and Hermes local selections, and new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use sandbox-attached providers. Owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared routes use `inference.local`. The sandbox receives no raw host credential. | | Sandbox-direct provider | Keep only credential sources that are deliberately provisioned and reachable inside the sandbox. | | Ambient metadata provider | Disable SDK discovery when the metadata endpoint cannot exist across the sandbox boundary. Do not make an internal or link-local endpoint reachable merely to satisfy an automatic probe. | | New agent or provider | Declare the enabled credential sources and add agent-specific tests proving that unavailable sources are not probed. | @@ -159,8 +159,8 @@ If direct Hermes Bedrock support is added later, it must apply the same unavaila #### Deep Agents Managed Inference Credential Discovery NemoClaw configures the managed `dcode` runtime with a generated OpenAI-compatible route. -New NVIDIA Endpoints, supported fixed hosted providers, Ollama, standalone vLLM, llama.cpp, and qualifying local compatible selections use sandbox-attached native providers. -Owned Local NVIDIA NIM and other routes retain `inference.local`. +New NVIDIA Endpoints, supported fixed hosted providers, hosted compatible endpoints, Ollama, standalone vLLM, llama.cpp, and qualifying host-local compatible selections use sandbox-attached providers. +Owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared routes use `inference.local`. The sandbox receives a scoped credential reference for native providers or a placeholder key for shared routes. OpenShell keeps the upstream provider credential outside the sandbox. The managed launchers reject credential-shaped environment values and upstream auth files before Deep Agents Code starts, so provider discovery stays on the host-side OpenShell boundary. @@ -335,7 +335,7 @@ The agent never receives the provider API key. | Aspect | Detail | |---|---| -| Default | NVIDIA Endpoints, supported fixed hosted providers, and new qualifying Deep Agents Code and Hermes local selections except owned Local NVIDIA NIM use sandbox-attached native providers. Owned Local NIM and other routes retain `inference.local`. OpenShell owns the credential and upstream endpoint in both cases. | +| Default | NVIDIA Endpoints, supported fixed hosted providers, new Deep Agents Code hosted compatible endpoints, qualified Deep Agents Code and Hermes local selections, and new OpenClaw Ollama, standalone vLLM, and llama.cpp selections use sandbox-attached providers. Owned Local NVIDIA NIM, compatible endpoints without an attachment, and retained shared routes use `inference.local`. OpenShell owns the credential and upstream endpoint in both cases. | | What you can change | You select a supported provider. NemoClaw selects and enforces that provider's managed path. | | Risk if bypassed | Adding an unowned provider endpoint to network policy bypasses NemoClaw's managed provider boundary and could let an agent use a stolen or hardcoded key. The NVIDIA Endpoints path is not a bypass: its checked-in OpenShell provider profile restricts the endpoint, methods, paths, and binaries, then rewrites a placeholder credential at the egress boundary. | | Recommendation | Do not add inference provider hosts to network policy for NemoClaw model traffic. Use the provider path created by onboarding. The `claude-code` preset is a separate opt-in exception for running the Claude Code CLI with its own credentials, not a way to configure NemoClaw inference. | diff --git a/test/agents/deepagents/langchain-deepagents-code-config.test.ts b/test/agents/deepagents/langchain-deepagents-code-config.test.ts index 857c7f0d4a1..89e650fddeb 100644 --- a/test/agents/deepagents/langchain-deepagents-code-config.test.ts +++ b/test/agents/deepagents/langchain-deepagents-code-config.test.ts @@ -227,6 +227,17 @@ describe("LangChain Deep Agents Code config generator", () => { expect(config).not.toContain("[models.providers.openrouter]"); }); + it("keeps the compatible credential when its endpoint equals the native NVIDIA URL", () => { + const config = runGenerator({ + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", + NEMOCLAW_UPSTREAM_ENDPOINT_URL: "https://integrate.api.nvidia.com/v1", + NEMOCLAW_INFERENCE_BASE_URL: "https://integrate.api.nvidia.com/v1", + }); + + expect(config).toContain('api_key_env = "COMPATIBLE_API_KEY"'); + expect(config).not.toContain('api_key_env = "NVIDIA_INFERENCE_API_KEY"'); + }); + it("rejects upstream endpoint URLs with control characters before writing config", () => { const result = runGeneratorProcess({ NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", diff --git a/test/agents/deepagents/langchain-deepagents-code-image.test.ts b/test/agents/deepagents/langchain-deepagents-code-image.test.ts index 0f504b7165f..b22db039e23 100644 --- a/test/agents/deepagents/langchain-deepagents-code-image.test.ts +++ b/test/agents/deepagents/langchain-deepagents-code-image.test.ts @@ -1030,6 +1030,10 @@ describe("LangChain Deep Agents Code image contracts", () => { "https://integrate\\.api\\.nvidia\\.com/v1", "references_attached_provider_placeholder_key", 'api_key_env[[:space:]]*=[[:space:]]*"NVIDIA_INFERENCE_API_KEY"', + "references_native_compatible_route", + 'HOSTED_ENDPOINT="${NEMOCLAW_ENDPOINT_URL:-https://inference-api.nvidia.com/v1}"', + "references_compatible_provider_placeholder_key", + 'selected_openai_provider_scalar api_key_env)" = "COMPATIBLE_API_KEY"', "configured_inference_route_contract", "classify_headless_output", '"schema_version", "command", "data"', @@ -1167,12 +1171,50 @@ describe("LangChain Deep Agents Code image contracts", () => { 'api_key_env = "NVIDIA_INFERENCE_API_KEY"', ].join("\n"), ], + [ + "native-compatible", + [ + "[models.providers.openai]", + 'base_url = "https://inference-api.nvidia.com/v1"', + 'api_key_env = "COMPATIBLE_API_KEY"', + ].join("\n"), + ], ])("selects the %s Deep Agents Code inference route contract", (expected, config) => { expect(runHeadlessCheckHelper("inference-route-contract", { CONFIG: config })).toBe( `${expected}\n`, ); }); + it.each(["https://custom.example.test/v1", "https://integrate.api.nvidia.com/v1"])( + "accepts selected compatible endpoint %s and rejects a different endpoint", + (endpoint) => { + const config = (baseUrl: string) => + [ + "[models.providers.openai]", + `base_url = "${baseUrl}"`, + 'api_key_env = "COMPATIBLE_API_KEY"', + ].join("\n"); + expect( + runHeadlessCheckHelper("inference-route-contract", { + CONFIG: config(endpoint), + NEMOCLAW_ENDPOINT_URL: endpoint, + }), + ).toBe("native-compatible\n"); + expect(() => + runHeadlessCheckHelper("inference-route-contract", { + CONFIG: config("https://other.example.test/v1"), + NEMOCLAW_ENDPOINT_URL: endpoint, + }), + ).toThrow(); + expect(() => + runHeadlessCheckHelper("inference-route-contract", { + CONFIG: config(endpoint).replace("COMPATIBLE_API_KEY", "DEEPAGENTS_CODE_OPENAI_API_KEY"), + NEMOCLAW_ENDPOINT_URL: endpoint, + }), + ).toThrow(); + }, + ); + it.each([ [ "native endpoint with managed placeholder", @@ -1189,6 +1231,28 @@ describe("LangChain Deep Agents Code image contracts", () => { 'api_key_env = "NVIDIA_INFERENCE_API_KEY"', ].join("\n"), ], + [ + "compatible endpoint with managed placeholder", + [ + "[models.providers.openai]", + 'base_url = "https://inference-api.nvidia.com/v1"', + 'api_key_env = "DEEPAGENTS_CODE_OPENAI_API_KEY"', + ].join("\n"), + ], + [ + "commented compatible key", + '[models.providers.openai]\nbase_url = "https://inference-api.nvidia.com/v1"\n' + + '# api_key_env = "COMPATIBLE_API_KEY"\napi_key_env = "DEEPAGENTS_CODE_OPENAI_API_KEY"', + ], + ...["[models.providers.other]", "[[other]]"].map((header) => [ + `compatible key in ${header}`, + [ + "[models.providers.openai]", + 'base_url = "https://inference-api.nvidia.com/v1"', + header, + 'api_key_env = "COMPATIBLE_API_KEY"', + ].join("\n"), + ]), ])("rejects an inconsistent Deep Agents Code route contract: %s", (_case, config) => { expect(() => runHeadlessCheckHelper("inference-route-contract", { CONFIG: config })).toThrow(); }); diff --git a/test/agents/deepagents/langchain-deepagents-code-managed-model-params.test.ts b/test/agents/deepagents/langchain-deepagents-code-managed-model-params.test.ts index e4a1811956a..17b3e80b994 100644 --- a/test/agents/deepagents/langchain-deepagents-code-managed-model-params.test.ts +++ b/test/agents/deepagents/langchain-deepagents-code-managed-model-params.test.ts @@ -132,11 +132,18 @@ for upstream in ("ollama-local", "vllm-local", "llama-cpp-local", "compatible-en else: raise AssertionError("invalid native-local reference accepted") -# Ambient local credentials cannot replace the managed hosted-route token. +# Ambient local credentials cannot replace a hosted compatible reference. os.environ[name] = "openshell:resolve:env:v42_" + name -for endpoint in ("https://inference.local/v1", "https://host.openshell.internal:8000/v1", "http://172.32.2.3:8000/v1"): +write_owned(${JSON.stringify(routeFile)}, "https://inference.local/v1") +assert config._get_provider_kwargs("openai")["api_key"] == "nemoclaw-managed-inference" +for endpoint in ("https://host.openshell.internal:8000/v1", "http://172.32.2.3:8000/v1"): write_owned(${JSON.stringify(routeFile)}, endpoint) - assert config._get_provider_kwargs("openai")["api_key"] == "nemoclaw-managed-inference" + try: + config._get_provider_kwargs("openai") + except RuntimeError: + pass + else: + raise AssertionError("local token replaced a hosted compatible reference") write_owned(${JSON.stringify(routeFile)}, "http://host.openshell.internal:8000/v1") write_owned(${JSON.stringify(providerFile)}, "nvidia-prod") assert config._get_provider_kwargs("openai")["api_key"] == "nemoclaw-managed-inference" @@ -204,6 +211,66 @@ print("native-managed-constructor-placeholder-ok") expect(output).toContain("native-managed-constructor-placeholder-ok"); }); + it("passes the issued compatible placeholder to the hosted DCode constructor (#13164)", () => { + const tempDir = createPackageFixture(); + patchFixture(tempDir); + const routeFile = path.join(tempDir, "managed-inference-base-url"); + const providerFile = managedUpstreamProviderPath(tempDir); + const validation = ` +import os +from pathlib import Path +from deepagents_code import config + +def write_owned(file, value): + path = Path(file) + path.chmod(0o644) + path.write_text(value + "\\n") + path.chmod(0o444) + +write_owned(${JSON.stringify(providerFile)}, "compatible-endpoint") +name = "COMPATIBLE_API_KEY" +os.environ["NEMOCLAW_LOCAL_INFERENCE_TOKEN"] = "openshell:resolve:env:v42_NEMOCLAW_LOCAL_INFERENCE_TOKEN" +for endpoint in ("https://inference-api.nvidia.com/v1", "https://integrate.api.nvidia.com/v1", "https://compatible.example/v1"): + write_owned(${JSON.stringify(routeFile)}, endpoint) + for reference in ( + "openshell:resolve:env:" + name, + "openshell:resolve:env:v12_" + name, + "openshell:resolve:env:s" + "a" * 64 + "_" + name, + ): + os.environ[name] = reference + for adapter in ("openai", "openrouter"): + resolved = config._get_provider_kwargs(adapter) + assert resolved["api_key"] == reference + assert resolved["base_url"] == endpoint + + for invalid in ("", "raw-secret", "nemoclaw-managed-inference", "openshell:resolve:env:v12_OTHER_KEY"): + os.environ[name] = invalid + try: + config._get_provider_kwargs("openai") + except RuntimeError as error: + assert not invalid or invalid not in str(error) + else: + raise AssertionError("hosted compatible route accepted an invalid reference") + del os.environ[name] + try: + config._get_provider_kwargs("openai") + except RuntimeError: + pass + else: + raise AssertionError("hosted compatible route accepted a missing reference") + +for endpoint in ("https://inference.local/v1", "http://localhost:8000/v1"): + write_owned(${JSON.stringify(routeFile)}, endpoint) + assert config._get_provider_kwargs("openai")["api_key"] == "nemoclaw-managed-inference" +print("native-compatible-constructor-placeholder-ok") +`; + const output = execFileSync("python3", ["-c", validation], { + env: { PATH: process.env.PATH, PYTHONPATH: tempDir }, + encoding: "utf8", + }); + expect(output).toContain("native-compatible-constructor-placeholder-ok"); + }); + it("uses only the native OpenRouter resolver handle for Deep Code requests (#13045)", () => { const tempDir = createPackageFixture(); patchFixture(tempDir); diff --git a/test/e2e/e2e-cloud-experimental/checks/03-deepagents-code-nemotron-ultra-profile.sh b/test/e2e/e2e-cloud-experimental/checks/03-deepagents-code-nemotron-ultra-profile.sh index 72cdfc66b2e..345098b43dd 100755 --- a/test/e2e/e2e-cloud-experimental/checks/03-deepagents-code-nemotron-ultra-profile.sh +++ b/test/e2e/e2e-cloud-experimental/checks/03-deepagents-code-nemotron-ultra-profile.sh @@ -20,7 +20,10 @@ set -euo pipefail SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-}}" +REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)}" PREFIX="03-deepagents-code-nemotron-ultra-profile" +HOSTED_ENDPOINT="${NEMOCLAW_ENDPOINT_URL:-https://inference-api.nvidia.com/v1}" +HOSTED_ENDPOINT="$(node --no-warnings "${REPO}/test/e2e/e2e-cloud-experimental/normalize-hosted-endpoint.ts" "$HOSTED_ENDPOINT")" fail() { printf '%s: FAIL: %s\n' "$PREFIX" "$1" >&2 @@ -40,7 +43,10 @@ profile_contract_source() { import asyncio import hashlib import importlib.metadata +import os +import re import socket +import sys from pathlib import Path import tomllib @@ -125,8 +131,8 @@ assert configured_model_id in MANAGED_MODEL_IDS, default_model provider = config["models"]["providers"]["openai"] assert provider["models"] == [configured_model_id] -assert provider["api_key_env"] == "DEEPAGENTS_CODE_OPENAI_API_KEY" -assert provider["base_url"] == "https://inference.local/v1" +assert provider["api_key_env"] == "COMPATIBLE_API_KEY" +assert provider["base_url"] == sys.argv[1] assert provider["enabled"] is True assert provider["params"] == { "use_responses_api": False, @@ -162,10 +168,15 @@ assert set(_NEMOCLAW_NEMOTRON_ULTRA_MODEL_IDS) == set(MANAGED_MODEL_IDS) MANAGED_BASE_URL = managed_inference_base_url() assert MANAGED_BASE_URL == provider["base_url"], MANAGED_BASE_URL -# The resolver supplies only the fixed synthetic credential. The provider -# credential never reaches the constructor or appears in the returned settings. +COMPATIBLE_REFERENCE = os.environ.get("COMPATIBLE_API_KEY", "") +assert re.fullmatch( + r"openshell:resolve:env:(?:(?:v[0-9]{1,20}|s[a-f0-9]{64})_)?COMPATIBLE_API_KEY", + COMPATIBLE_REFERENCE, +), "compatible provider reference is missing or invalid" +# The resolver passes the issued reference to OpenShell. It never receives the +# host's raw provider credential. OPENAI_CONTRACT = { - "api_key": "nemoclaw-managed-inference", + "api_key": COMPATIBLE_REFERENCE, "base_url": MANAGED_BASE_URL, "use_responses_api": False, } @@ -179,7 +190,7 @@ MANAGED_ULTRA_EXTRA_BODY = { **EXPECTED_EXTRA_BODY, } OPENROUTER_CONTRACT = { - "api_key": "nemoclaw-managed-inference", + "api_key": COMPATIBLE_REFERENCE, "base_url": MANAGED_BASE_URL, } # Deliberate near miss: a different Nemotron generation must not be shaped. @@ -448,7 +459,7 @@ sandbox_exec "test -x /opt/venv/bin/python3" >/dev/null || fail "/opt/venv/bin/p profile_source="$(profile_contract_source)" profile_output="$( openshell sandbox exec --name "$SANDBOX_NAME" -- \ - /opt/venv/bin/python3 -I -c "$profile_source" 2>&1 + /opt/venv/bin/python3 -I -c "$profile_source" "$HOSTED_ENDPOINT" 2>&1 )" || fail "Nemotron Ultra harness profile contract failed: $profile_output" printf '%s\n' "$profile_output" | grep -Fq "NEMOCLAW_NEMOTRON_ULTRA_PROFILE_OK:" || fail "profile verification marker is missing" pass "configured ChatOpenAI resolves the complete Nemotron Ultra profile without inference" diff --git a/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh b/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh index 53340aa02a2..01e81d60628 100755 --- a/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh +++ b/test/e2e/e2e-cloud-experimental/checks/04-deepagents-code-fresh-reonboard.sh @@ -16,6 +16,7 @@ REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)}" CLI="${NEMOCLAW_CLI_BIN:-${REPO}/bin/nemoclaw.js}" PREFIX="04-deepagents-code-fresh-reonboard" HOSTED_ENDPOINT="${NEMOCLAW_ENDPOINT_URL:-https://inference-api.nvidia.com/v1}" +HOSTED_ENDPOINT="$(node --no-warnings "${REPO}/test/e2e/e2e-cloud-experimental/normalize-hosted-endpoint.ts" "$HOSTED_ENDPOINT")" CREDENTIAL_CANARY="nemoclaw-dcode-config-get-canary" REJECTED_MUTATION_MODEL="nemoclaw-rejected-config-mutation" PERSONAL_LOGIN_PROFILE="/sandbox/.bash_profile" @@ -102,7 +103,7 @@ assert_identity() { [ "$route" = "inference" ] || fail "$phase identity route is '${route:-missing}'" [ "$provider" = "compatible-endpoint" ] || fail "$phase identity provider is '${provider:-missing}'" [ "$observed_model" = "openai:${model}" ] || fail "$phase identity model is '${observed_model:-missing}'" - [ "$endpoint" = "https://inference.local/v1" ] || fail "$phase identity endpoint is '${endpoint:-missing}'" + [ "$endpoint" = "$HOSTED_ENDPOINT" ] || fail "$phase identity endpoint is '${endpoint:-missing}'" } is_positive_integer() { @@ -183,14 +184,14 @@ import sys import tomllib path = Path("/sandbox/.deepagents/config.toml") -model, credential_canary = sys.argv[1:] +model, credential_canary, endpoint = sys.argv[1:] text = path.read_text(encoding="utf-8") config = tomllib.loads(text) provider = config["models"]["providers"]["openai"] assert config["models"]["default"] == f"openai:{model}" assert provider["models"] == [model] -assert provider["api_key_env"] == "DEEPAGENTS_CODE_OPENAI_API_KEY" -assert provider["base_url"] == "https://inference.local/v1" +assert provider["api_key_env"] == "COMPATIBLE_API_KEY" +assert provider["base_url"] == endpoint assert config["update"] == {"check": True, "auto_update": True} assert config["ui"] == {"show_scrollbar": True, "show_url_open_toast": False, "theme": "discard"} assert config["threads"] == { @@ -414,7 +415,7 @@ pass "fresh re-onboard restores the complete native config byte-for-byte" verify_source="$(verify_config_source)" verify_output="$( openshell sandbox exec --name "$SANDBOX_NAME" -- \ - /opt/venv/bin/python3 -I -c "$verify_source" "$model_a" "$CREDENTIAL_CANARY" 2>&1 + /opt/venv/bin/python3 -I -c "$verify_source" "$model_a" "$CREDENTIAL_CANARY" "$HOSTED_ENDPOINT" 2>&1 )" || fail "live DCode config does not preserve the complete native-state boundary" printf '%s\n' "$verify_output" | grep -Fq "NEMOCLAW_DCODE_FRESH_CONFIG_VERIFIED" || fail "fresh config verification marker is missing" pass "config keeps declared and unrecognized native state" diff --git a/test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh b/test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh index b79e4ff0240..7dc89ed6621 100755 --- a/test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh +++ b/test/e2e/e2e-cloud-experimental/checks/07-deepagents-code-headless-inference.sh @@ -31,8 +31,11 @@ set -euo pipefail SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-onboard}}" +REPO="${REPO:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)}" PREFIX="07-deepagents-code-headless-inference" HEADLESS_TIMEOUT="${DEEPAGENTS_HEADLESS_TIMEOUT:-120}" +HOSTED_ENDPOINT="${NEMOCLAW_ENDPOINT_URL:-https://inference-api.nvidia.com/v1}" +HOSTED_ENDPOINT="$(node --no-warnings "${REPO}/test/e2e/e2e-cloud-experimental/normalize-hosted-endpoint.ts" "$HOSTED_ENDPOINT")" SKILL_CLI_TIMEOUT_SECONDS=180 ok() { printf '%s\n' "${PREFIX}: OK ($*)"; } @@ -382,20 +385,66 @@ references_attached_provider_placeholder_key() { grep -Eq 'api_key_env[[:space:]]*=[[:space:]]*"NVIDIA_INFERENCE_API_KEY"' } +selected_openai_provider_scalar() { + local key="$1" + awk -v key="$key" ' + /^\[\[?[^]]+\]\]?[[:space:]]*$/ { + selected = ($0 == "[models.providers.openai]") + next + } + selected && $0 ~ "^[[:space:]]*" key "[[:space:]]*=" { + pattern = "^[[:space:]]*" key "[[:space:]]*=[[:space:]]*\"[^\"]*\"[[:space:]]*$" + if ($0 !~ pattern) { + invalid = 1 + next + } + value = $0 + sub("^[[:space:]]*" key "[[:space:]]*=[[:space:]]*\"", "", value) + sub(/"[[:space:]]*$/, "", value) + count++ + } + END { + if (count == 1 && !invalid) print value + } + ' +} + +references_native_compatible_route() { + local configured_endpoint + configured_endpoint="$(selected_openai_provider_scalar base_url)" + [ "$configured_endpoint" = "$HOSTED_ENDPOINT" ] +} + +references_compatible_provider_placeholder_key() { + [ "$(selected_openai_provider_scalar api_key_env)" = "COMPATIBLE_API_KEY" ] +} + configured_inference_route_contract() { local config config="$(cat)" if printf '%s\n' "$config" | references_native_nvidia_route \ && printf '%s\n' "$config" | references_attached_provider_placeholder_key \ && ! printf '%s\n' "$config" | references_managed_inference_route \ - && ! printf '%s\n' "$config" | references_managed_placeholder_key; then + && ! printf '%s\n' "$config" | references_managed_placeholder_key \ + && ! printf '%s\n' "$config" | references_native_compatible_route \ + && ! printf '%s\n' "$config" | references_compatible_provider_placeholder_key; then printf '%s\n' native-nvidia return 0 fi + if printf '%s\n' "$config" | references_native_compatible_route \ + && printf '%s\n' "$config" | references_compatible_provider_placeholder_key \ + && ! printf '%s\n' "$config" | references_managed_inference_route \ + && ! printf '%s\n' "$config" | references_managed_placeholder_key \ + && ! printf '%s\n' "$config" | references_attached_provider_placeholder_key; then + printf '%s\n' native-compatible + return 0 + fi if printf '%s\n' "$config" | references_managed_inference_route \ && printf '%s\n' "$config" | references_managed_placeholder_key \ && ! printf '%s\n' "$config" | references_native_nvidia_route \ - && ! printf '%s\n' "$config" | references_attached_provider_placeholder_key; then + && ! printf '%s\n' "$config" | references_attached_provider_placeholder_key \ + && ! printf '%s\n' "$config" | references_native_compatible_route \ + && ! printf '%s\n' "$config" | references_compatible_provider_placeholder_key; then printf '%s\n' managed return 0 fi @@ -612,6 +661,9 @@ main() { if [ "$route_contract" = "native-nvidia" ]; then pass "config.toml routes through the native NVIDIA endpoint" pass "config.toml uses the attached-provider placeholder API key" + elif [ "$route_contract" = "native-compatible" ]; then + pass "config.toml routes through the attached compatible endpoint" + pass "config.toml uses the compatible-provider placeholder API key" elif [ "$route_contract" = "managed" ]; then pass "config.toml routes through the managed inference.local endpoint" pass "config.toml uses the managed Deep Agents Code placeholder API key" diff --git a/test/e2e/e2e-cloud-experimental/normalize-hosted-endpoint.ts b/test/e2e/e2e-cloud-experimental/normalize-hosted-endpoint.ts new file mode 100644 index 00000000000..2bc2d48bfe0 --- /dev/null +++ b/test/e2e/e2e-cloud-experimental/normalize-hosted-endpoint.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { unsafeEndpointUrlViolation } from "../../../src/lib/core/endpoint-url-safety.ts"; +import { normalizeProviderBaseUrl } from "../../../src/lib/core/provider-endpoint.ts"; + +const endpoint = process.argv[2]; +const violation = unsafeEndpointUrlViolation(endpoint); +if (!endpoint?.trim() || violation) { + console.error(`Hosted endpoint ${violation?.reason ?? "is required."}`); + process.exit(1); +} +process.stdout.write(normalizeProviderBaseUrl(endpoint, "openai")); diff --git a/test/e2e/fixtures/phases/lifecycle-dcode-invalid-credential.ts b/test/e2e/fixtures/phases/lifecycle-dcode-invalid-credential.ts index dedcd3f79ca..c1d3f359f17 100644 --- a/test/e2e/fixtures/phases/lifecycle-dcode-invalid-credential.ts +++ b/test/e2e/fixtures/phases/lifecycle-dcode-invalid-credential.ts @@ -3,6 +3,10 @@ import { randomUUID } from "node:crypto"; +import { DCODE_MANAGED_EXEC_LAUNCHER } from "../../../../src/lib/actions/sandbox/connect-inference-route-probe.ts"; +import { nativeHostedAuthHeaderScript } from "../../../../src/lib/inference/native-provider/contract.ts"; +import { hostedNativeProviderForAttachment } from "../../../../src/lib/inference/native-provider/hosted-attachment.ts"; +import { recordedNativeProviderAttachment } from "../../../../src/lib/inference/native-provider/recorded-selection.ts"; import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { assertExitZero, outputContainsReadySandbox, resultText } from "../clients/command.ts"; import type { HostCliClient } from "../clients/host.ts"; @@ -40,6 +44,7 @@ export interface DcodeInvalidCredentialRebuildOptions { gatewayName: string; providerName: string; credentialEnv: string; + endpoint: string; model: string; validCredential: string; } @@ -74,10 +79,10 @@ export function dcodeInvalidCredentialRebuildOptionsFromRegistryEntry( if (!validCredential) { throw new Error("DCode invalid-credential lifecycle requires the original provider credential"); } - const providerName = requiredString(entry, "provider"); - if (providerName !== HOSTED_INFERENCE_PROVIDER_NAME) { + const logicalProvider = requiredString(entry, "provider"); + if (logicalProvider !== HOSTED_INFERENCE_PROVIDER_NAME) { throw new Error( - `DCode invalid-credential lifecycle requires provider '${HOSTED_INFERENCE_PROVIDER_NAME}', got '${providerName}'`, + `DCode invalid-credential lifecycle requires provider '${HOSTED_INFERENCE_PROVIDER_NAME}', got '${logicalProvider}'`, ); } const credentialEnv = @@ -89,10 +94,29 @@ export function dcodeInvalidCredentialRebuildOptionsFromRegistryEntry( `DCode invalid-credential lifecycle requires credential env '${HOSTED_INFERENCE_CREDENTIAL_ENV}'`, ); } + const attachment = recordedNativeProviderAttachment({ + provider: logicalProvider, + endpointUrl: typeof entry.endpointUrl === "string" ? entry.endpointUrl : undefined, + credentialEnv, + preferredInferenceApi: + typeof entry.preferredInferenceApi === "string" ? entry.preferredInferenceApi : undefined, + nativeHostedProviderAttachment: entry.nativeHostedProviderAttachment, + }); + const nativeProvider = attachment && hostedNativeProviderForAttachment(attachment); + if ( + !nativeProvider?.custom || + nativeProvider.api !== "openai-completions" || + nativeProvider.credentialEnv !== credentialEnv + ) { + throw new Error( + "DCode invalid-credential lifecycle requires an attached native completion provider", + ); + } return { gatewayName: requiredString(entry, "gatewayName"), - providerName, + providerName: nativeProvider.providerName, credentialEnv, + endpoint: `${nativeProvider.endpoint.replace(/\/+$/u, "")}/chat/completions`, model: requiredString(entry, "model"), validCredential, }; @@ -105,6 +129,7 @@ export function isDcodeInvalidCredentialRebuildOptions( "gatewayName" in options && "providerName" in options && "credentialEnv" in options && + "endpoint" in options && "model" in options && "validCredential" in options ); @@ -205,21 +230,19 @@ async function probeRoute( return await deps.sandbox.exec( sandboxName, [ - "curl", - "-sS", - "-o", - "/dev/null", - "-w", - "%{http_code}", - "--connect-timeout", - "5", - "--max-time", - "15", - "-H", - "Content-Type: application/json", - "--data-binary", + DCODE_MANAGED_EXEC_LAUNCHER, + "/bin/sh", + "-c", + [ + nativeHostedAuthHeaderScript({ + credentialEnv: options.credentialEnv, + api: "openai-completions", + }), + 'exec curl -q -sS -o /dev/null -w "%{http_code}" --connect-timeout 5 --max-time 15 -H "Content-Type: application/json" -H "$AUTH_HEADER" --data-binary "$1" "$2"', + ].join("; "), + "dcode-native-inference-probe", payload, - "https://inference.local/v1/chat/completions", + options.endpoint, ], { artifactName: `lifecycle-dcode-route-${phase}-${attempt}`, diff --git a/test/e2e/live/cloud-experimental-checks.ts b/test/e2e/live/cloud-experimental-checks.ts index fb382a796c5..255af58f41f 100644 --- a/test/e2e/live/cloud-experimental-checks.ts +++ b/test/e2e/live/cloud-experimental-checks.ts @@ -12,6 +12,7 @@ import { requireDcodeBaseImageReference, } from "../fixtures/dcode-base-image.ts"; import type { E2ETargetFixtures } from "../fixtures/e2e-test.ts"; +import { DEFAULT_HOSTED_INFERENCE_BASE_URL } from "../fixtures/hosted-inference.ts"; import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; @@ -93,6 +94,7 @@ export function buildCloudExperimentalCommandEnv( ...buildAvailabilityProbeEnv(base), CLOUD_EXPERIMENTAL_MODEL: base.NEMOCLAW_MODEL, COMPATIBLE_API_KEY: apiKey, + NEMOCLAW_ENDPOINT_URL: base.NEMOCLAW_ENDPOINT_URL || DEFAULT_HOSTED_INFERENCE_BASE_URL, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_E2E_CLOUD_API_KEY_ENV: "COMPATIBLE_API_KEY", NEMOCLAW_NON_INTERACTIVE: "1", diff --git a/test/e2e/support/cloud-hosted-endpoint.test.ts b/test/e2e/support/cloud-hosted-endpoint.test.ts new file mode 100644 index 00000000000..56621e9ddc0 --- /dev/null +++ b/test/e2e/support/cloud-hosted-endpoint.test.ts @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { runHeadlessCheckHelper } from "../../helpers/langchain-deepagents-code-headless.ts"; + +const repo = process.cwd(); +const checks = [ + ["03-deepagents-code-nemotron-ultra-profile.sh", "fail"], + ["04-deepagents-code-fresh-reonboard.sh", "fail"], + ["07-deepagents-code-headless-inference.sh", "ok"], +] as const; + +describe.each(checks)("%s hosted endpoint", (file, firstFunction) => { + const script = readFileSync( + path.join(repo, "test/e2e/e2e-cloud-experimental/checks", file), + "utf8", + ); + // Execute initialization only; the remaining script owns live sandbox operations. + const initialization = script.slice(0, script.indexOf(`\n${firstFunction}() {`)); + function run(endpoint?: string) { + return spawnSync("/bin/bash", ["-c", `${initialization}\nprintf '%s' "$HOSTED_ENDPOINT"`], { + encoding: "utf8", + env: { + PATH: `${path.dirname(process.execPath)}:/usr/bin:/bin`, + REPO: repo, + ...(endpoint === undefined ? {} : { NEMOCLAW_ENDPOINT_URL: endpoint }), + }, + }); + } + + it.each([ + [undefined, "https://inference-api.nvidia.com/v1"], + ["https://custom.example/v1", "https://custom.example/v1"], + ["https://custom.example/v1///", "https://custom.example/v1"], + ["https://CUSTOM.example:443/v1/chat/completions/", "https://custom.example/v1"], + [" https://custom.example/v1/models ", "https://custom.example/v1"], + ])("uses the onboarding endpoint for input %s", (input, expected) => { + const result = run(input); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe(expected); + }); + + it.each([ + "https://user:secret@custom.example/v1", + "https://custom.example/v1?key=secret", + "https://custom.example/v1#secret", + "https://custom.example/v1\n", + "not-a-url", + ])("rejects unsafe endpoint %s before running checks", (input) => { + const result = run(input); + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).not.toContain(input); + }); +}); + +it.each([ + "https://custom.example.test/v1/", + "https://CUSTOM.example.test:443/v1/chat/completions/", +])("accepts canonical config for %s and rejects different routes or keys", (endpoint) => { + const config = (baseUrl: string, key = "COMPATIBLE_API_KEY") => + `[models.providers.openai]\nbase_url = "${baseUrl}"\napi_key_env = "${key}"\n`; + expect( + runHeadlessCheckHelper("inference-route-contract", { + CONFIG: config("https://custom.example.test/v1"), + NEMOCLAW_ENDPOINT_URL: endpoint, + }), + ).toBe("native-compatible\n"); + expect(() => + runHeadlessCheckHelper("inference-route-contract", { + CONFIG: config("https://other.example.test/v1"), + NEMOCLAW_ENDPOINT_URL: endpoint, + }), + ).toThrow(); + expect(() => + runHeadlessCheckHelper("inference-route-contract", { + CONFIG: config("https://custom.example.test/v1", "DEEPAGENTS_CODE_OPENAI_API_KEY"), + NEMOCLAW_ENDPOINT_URL: endpoint, + }), + ).toThrow(); +}); diff --git a/test/e2e/support/e2e-phase-lifecycle.test.ts b/test/e2e/support/e2e-phase-lifecycle.test.ts index 4025cb9a1d2..9cea0f98833 100644 --- a/test/e2e/support/e2e-phase-lifecycle.test.ts +++ b/test/e2e/support/e2e-phase-lifecycle.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, expectTypeOf, it } from "vitest"; +import { customNativeProvider } from "../../../src/lib/inference/native-provider/custom.ts"; import { type CommandRunner, GatewayClient, @@ -554,11 +555,30 @@ describe("LifecyclePhaseFixture profile dispatch", () => { describe("LifecyclePhaseFixture DCode invalid-credential rebuild", () => { const sandboxName = "e2e-dcode-cloud"; const validCredential = "valid-fixture-credential"; + const nativeProvider = customNativeProvider({ + provider: "compatible-endpoint", + endpointUrl: "https://inference-api.nvidia.com/v1", + api: "openai-completions", + credentialEnv: "COMPATIBLE_API_KEY", + allowedIps: ["8.8.8.8"], + }); + const attachment = { + schemaVersion: 1, + profileId: nativeProvider.profileId, + providerName: nativeProvider.providerName, + providerId: "provider-fixture-id", + endpointUrl: nativeProvider.endpoint, + allowedIps: nativeProvider.allowedIps, + custom: nativeProvider.custom, + }; const options = dcodeInvalidCredentialRebuildOptionsFromRegistryEntry( { agent: "langchain-deepagents-code", gatewayName: "nemoclaw", provider: "compatible-endpoint", + endpointUrl: nativeProvider.endpoint, + preferredInferenceApi: "openai-completions", + nativeHostedProviderAttachment: attachment, model: "nvidia/nvidia/nemotron-3-ultra", }, validCredential, @@ -683,6 +703,12 @@ describe("LifecyclePhaseFixture DCode invalid-credential rebuild", () => { call.command === "openshell" && call.args.slice(0, 2).join(" ") === "provider update", ); expect(providerUpdates).toHaveLength(2); + expect(providerUpdates[0].args).toContain(nativeProvider.providerName); + const baselineProbe = runner.calls.find( + (call) => call.options?.artifactName === "lifecycle-dcode-route-baseline-1", + ); + expect(baselineProbe?.args).toContain(options.endpoint); + expect(baselineProbe?.args.join(" ")).not.toContain("inference.local"); const invalidCredential = providerUpdates[0].options?.env?.COMPATIBLE_API_KEY; expect(invalidCredential).toMatch(/^nvapi-e2e-invalid-/); expect(providerUpdates[0].args).not.toContain(invalidCredential); @@ -836,8 +862,9 @@ describe("LifecyclePhaseFixture DCode invalid-credential rebuild", () => { it("derives only the expected DCode compatible-endpoint binding", () => { expect(options).toEqual({ gatewayName: "nemoclaw", - providerName: "compatible-endpoint", + providerName: nativeProvider.providerName, credentialEnv: "COMPATIBLE_API_KEY", + endpoint: "https://inference-api.nvidia.com/v1/chat/completions", model: "nvidia/nvidia/nemotron-3-ultra", validCredential, }); @@ -852,5 +879,28 @@ describe("LifecyclePhaseFixture DCode invalid-credential rebuild", () => { validCredential, ), ).toThrow(/registry agent/); + expect(() => + dcodeInvalidCredentialRebuildOptionsFromRegistryEntry( + { + agent: "langchain-deepagents-code", + gatewayName: "nemoclaw", + provider: "compatible-endpoint", + model: "nvidia/model", + }, + validCredential, + ), + ).toThrow(/attached native completion provider/); + expect(() => + dcodeInvalidCredentialRebuildOptionsFromRegistryEntry( + { + agent: "langchain-deepagents-code", + gatewayName: "nemoclaw", + provider: "compatible-endpoint", + model: "nvidia/model", + nativeHostedProviderAttachment: { ...attachment, providerName: "wrong-provider" }, + }, + validCredential, + ), + ).toThrow(/native hosted provider attachment/); }); }); diff --git a/test/e2e/support/platform-parity-cloud-experimental.test.ts b/test/e2e/support/platform-parity-cloud-experimental.test.ts index e9a4bafebbd..034b6c02d84 100644 --- a/test/e2e/support/platform-parity-cloud-experimental.test.ts +++ b/test/e2e/support/platform-parity-cloud-experimental.test.ts @@ -1075,6 +1075,7 @@ assert_status_mode disabled AWS_SECRET_ACCESS_KEY: "do-not-copy", GITHUB_TOKEN: "do-not-copy", [DCODE_BASE_IMAGE_ENV]: baseImageReference, + NEMOCLAW_ENDPOINT_URL: "https://custom.example.test/v1", NEMOCLAW_MODEL: "model-a", RANDOM_RUNNER_SECRET: "do-not-copy", }); @@ -1082,6 +1083,7 @@ assert_status_mode disabled expect(env).toMatchObject({ COMPATIBLE_API_KEY: "secret-key", CLOUD_EXPERIMENTAL_MODEL: "model-a", + NEMOCLAW_ENDPOINT_URL: "https://custom.example.test/v1", NEMOCLAW_SANDBOX_NAME: "deepagents-sandbox", SANDBOX_NAME: "deepagents-sandbox", OPENSHELL_GATEWAY: "nemoclaw", diff --git a/test/helpers/langchain-deepagents-code-headless.ts b/test/helpers/langchain-deepagents-code-headless.ts index d6c763446c0..1cd6d64ad07 100644 --- a/test/helpers/langchain-deepagents-code-headless.ts +++ b/test/helpers/langchain-deepagents-code-headless.ts @@ -56,6 +56,7 @@ type HeadlessCheckEnvironment = Partial< | "DCODE_EXIT" | "DEEPAGENTS_HEADLESS_TIMEOUT" | "HEADLESS_OUTPUT" + | "NEMOCLAW_ENDPOINT_URL" | "PROC_ROOT" | "TOKEN", string @@ -152,7 +153,9 @@ export function runHeadlessCheckHelper( DCODE_EXIT: env.DCODE_EXIT ?? "", DEEPAGENTS_HEADLESS_TIMEOUT: env.DEEPAGENTS_HEADLESS_TIMEOUT ?? "", HEADLESS_OUTPUT: env.HEADLESS_OUTPUT ?? "", - PATH: "/usr/bin:/bin", + NEMOCLAW_ENDPOINT_URL: env.NEMOCLAW_ENDPOINT_URL ?? "", + PATH: `${path.dirname(process.execPath)}:/usr/bin:/bin`, + REPO: repoRoot, PROC_ROOT: env.PROC_ROOT ?? "", TOKEN: env.TOKEN ?? "", }, @@ -166,10 +169,15 @@ export function runHeadlessCheckSnippet( ): string { const source = fs .readFileSync(sourcePath, "utf8") - .replace("${BASH_SOURCE[0]}", "${BASH_SOURCE[0]-}"); + .replaceAll("${BASH_SOURCE[0]}", "${BASH_SOURCE[0]-}"); return execFileSync("/bin/bash", ["-s"], { encoding: "utf8", - env: { ...process.env, ...env }, + env: { + ...process.env, + REPO: repoRoot, + ...env, + PATH: `${env.PATH ?? process.env.PATH ?? "/usr/bin:/bin"}:${path.dirname(process.execPath)}`, + }, input: `${source}\n${snippet}\n`, }); } From e7edcc49629bd025039b04925de55c9332f499b3 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas <34834085+prekshivyas@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:36:33 -0700 Subject: [PATCH 14/22] fix(e2e): admit approved OpenShell templates in GPU controller (#13486) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Outcome The protected GPU controller can project independently approved OpenShell installer and runtime templates from the candidate commit. Candidates covered by main's existing trust records retain their existing path. ## Reason The controller rejected the approved OpenShell upgrade in #12603 before GPU qualification started. It validated candidate templates only against main's registry, although the installer check already supported commit-bound candidate admission. Historical failure: [protected GPU job in run 39206820005](https://github.com/NVIDIA/NemoClaw/actions/runs/39206820005/job/117922700270). ### Related issues Refs #12603. ## Changes - Reuse the existing independent maintainer approval check for new candidate trust during prepare, verify, and restore. - Compare the admitted registry and installer consumers with immutable candidate Git blobs before changing the trusted checkout. - Pass the admitted releases to both installer and feature-gate validation; retain artifact identity, revocation, SDK archive, and projection restrictions. - Give only the four admission steps a read-only GitHub token. Verify approval before installing the reviewed SDK or building the controller, then recheck before live qualification. - Preserve rejection without mutation, partial-write rollback, and ownership checks. Document restoration failure when approval is revoked. ## Verification - `node node_modules/vitest/vitest.mjs run --project e2e-support` with the protected workspace, projection, workflow, candidate trust, and upgrade admission test files — 241 tests passed. - Workspace regressions rerun after fixture restructuring — 22 tests passed. - Revision `95cad4745bec8bf5d0d1b0d6a69d37d77858aaaa`: the focused protected-runtime workflow guard suite passed 111 tests, including rejection of SDK installation before approval verification. - `npm run docs` — 125 routes checked; Fern reported zero errors and two warnings. - Normal pre-commit, commit-msg, and pre-push hooks — passed, including publication validation and CLI/plugin/JS-config TypeScript checks. - Exact-head manual PR E2E passed: `jobs=cloud-onboard,full-e2e,security-posture` ([run 39302613910](https://github.com/NVIDIA/NemoClaw/actions/runs/39302613910)) and `jobs=managed-image-protected-runtime` ([run 39305366189](https://github.com/NVIDIA/NemoClaw/actions/runs/39305366189)). See the [E2E evidence comment](https://github.com/NVIDIA/NemoClaw/pull/13486#issuecomment-6276655640) for selectors, tested commit, and receipts. - No live E2E assertions or budgets changed. No secrets, API keys, or credentials added. ## Review notes This repairs candidate admission only. The full protected GPU suite passed on this PR head with the candidate OpenShell `0.0.116` inputs recorded in its artifact. That result does not qualify the separate OpenShell 0.1.3 upgrade in #12603. Its trusted harness still needs the `inference set` and Docker bridge migrations before that upgrade can be qualified. The workflow change must land on main before #12603's trusted controller can use it. This PR does not authorize merging #12603 or claim 0.1.3 GPU platform qualification. Independent code, security, and documentation review covered commit `18b4a45c22aa066f6ad6d7e1a44499f14200614b`. Exact-head CI and CodeRabbit review completed on `95cad4745bec8bf5d0d1b0d6a69d37d77858aaaa`; the self-hosted llama.cpp GPU check passed on its second attempt. The Advisor reported three P1 findings, all rejected in the [maintainer disposition](https://github.com/NVIDIA/NemoClaw/pull/13486#issuecomment-6275486695); its blocker gate remains red. Independent human review and maintainer acceptance remain required. If approval is revoked or GitHub admission becomes unavailable after preparation, verification and restoration fail without changing projected files. The isolated checkout remains for diagnosis; stale approval is not reused. --- Signed-off-by: Prekshi Vyas Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> ## Summary by CodeRabbit * **Security** * Protected E2E runtime operations now check candidate approval before projecting OpenShell sources when the candidate is not already trusted. Approval checks apply to preparation, verification, and restoration. * OpenShell verification now runs before SDK download and remains in place after installation. The workflow has read-only access to pull request details for approval checks. * **Documentation** * Clarified approval requirements, token scope, verification order, failure outcomes, and recovery guidance. Approval does not permit new SDK dependency graphs on protected E2E hosts, qualify Pi, or pass candidate catalogs to the Jetson operator. --------- Signed-off-by: Prekshi Vyas Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Co-authored-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .github/workflows/e2e.yaml | 26 +- test/e2e/README.md | 22 +- ...d-image-protected-runtime-workflow.test.ts | 50 ++-- .../protected-openshell-workspace.test.ts | 246 ++++++++++-------- test/support/protected-openshell-workspace.ts | 152 +++++++++++ ...ge-protected-runtime-workflow-boundary.mts | 40 ++- tools/e2e/protected-openshell-projection.mts | 6 +- tools/e2e/protected-openshell-workspace.mts | 100 ++++++- 8 files changed, 491 insertions(+), 151 deletions(-) create mode 100644 test/support/protected-openshell-workspace.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d4486e3a3e7..36de4ae9d62 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -5213,6 +5213,7 @@ jobs: timeout-minutes: 75 permissions: contents: read + pull-requests: read env: CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime @@ -5361,10 +5362,22 @@ jobs: - id: openshell-projection name: Bind reviewed candidate OpenShell runtime shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} run: | set -euo pipefail node tools/e2e/protected-openshell-workspace.mts prepare > "$E2E_ARTIFACT_DIR/openshell-projection.json" + - name: Verify reviewed candidate OpenShell runtime before consumption + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} + run: | + set -euo pipefail + node tools/e2e/protected-openshell-workspace.mts verify > "$E2E_ARTIFACT_DIR/openshell-projection-initially-verified.json" + - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -5500,6 +5513,15 @@ jobs: shell: bash run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + - name: Verify reviewed candidate OpenShell runtime + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} + run: | + set -euo pipefail + node tools/e2e/protected-openshell-workspace.mts verify > "$E2E_ARTIFACT_DIR/openshell-projection-verified.json" + - name: Run all-agent GPU, local inference, rollback, and cleanup qualification env: NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} @@ -5510,7 +5532,6 @@ jobs: echo "::error::Protected NIM qualification must execute trusted workflow code" >&2 exit 1 } - node tools/e2e/protected-openshell-workspace.mts verify > "$E2E_ARTIFACT_DIR/openshell-projection-verified.json" export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" OPENSHELL_BIN="$(command -v openshell)" export OPENSHELL_BIN @@ -5546,6 +5567,9 @@ jobs: - name: Restore trusted OpenShell sources if: ${{ always() && steps.openshell-projection.outcome == 'success' }} shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} run: | set -euo pipefail node tools/e2e/protected-openshell-workspace.mts restore > "$E2E_ARTIFACT_DIR/openshell-projection-restored.json" diff --git a/test/e2e/README.md b/test/e2e/README.md index 2a7907ef04e..66396c53683 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -350,11 +350,10 @@ The API credential is limited to trusted preparation and removed before npm runs If CI starts before the approved artifact exists, wait for the admission workflow's SDK job, then rerun CI. Changed or revoked approval cannot reuse the older artifact. -This does not yet extend the protected runtime's source projection, permit a new -SDK graph on protected E2E hosts, qualify Pi, or pass candidate catalogs to the Jetson operator. A data-admission -comment is not permission to execute arbitrary candidate code on a protected -host. Those boundaries require their own supported implementation and validation; -passing the hash check is not full E2E qualification. +The protected runtime controller uses candidate approval for source projection as described under [Push and Manual PR E2E](#push-and-manual-pr-e2e). +Approval does not permit a new SDK dependency graph on protected E2E hosts, qualify Pi, or pass candidate catalogs to the Jetson operator. +Those boundaries require their own supported implementation and validation. +Passing the hash check is not full E2E qualification. Unchanged runs pass the selected base revision and complete cohort receipt to every stock-onboarding consumer. Changed-input runs pass the authenticated candidate catalog separately to those consumers. @@ -2274,9 +2273,20 @@ Leave `targets` empty. Keep `include_staging_brev_launchable=false`. The candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. The GPU job reads OpenShell sources from the candidate commit as data, then validates their pins and operational templates. +When the candidate needs trust absent from the controller's registry, preparation, both verifications, and restoration recheck the [candidate approval](#candidate-openshell-release-admission). +The approval must match the open PR, candidate commit, and base commit. +The approved registry and installer sources must match the candidate's committed files. +Candidates covered by the controller's registry need no additional approval. +The job exposes its GitHub token as `GITHUB_TOKEN` only to the four trusted source-projection steps, with `contents: read` and `pull-requests: read` permissions. +The token exists in those step environments and is absent from the live qualification environment. It projects only the reviewed installer, supervisor and feature-check modules, plus version literals, into the trusted controller checkout. -It builds that controller after projection and verifies the projected sources before qualification. +It verifies the projected sources before installing the SDK or building the controller, then rechecks them before qualification. The job restores the trusted sources after qualification and retains source-digest records with its artifacts. +An admission rejection changes no files. +A failed verification blocks qualification. +A failed restoration fails the job. +If admission fails after preparation, the projected checkout remains for diagnosis. +Inspect the job log and projection artifacts before recovering the runner or dispatching another run. Those records identify tested inputs; they do not prove live E2E success. The job also selects the candidate OpenShell SDK from the controller's reviewed current or replacement archives. It projects the SDK identity into the trusted manifest and lockfile, then installs the verified archive before building the CLI. diff --git a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts index b46f0ffcec7..6e4335392d3 100644 --- a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts +++ b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts @@ -118,27 +118,47 @@ describe("protected managed-image runtime workflow", () => { expect(validateManagedImageProtectedRuntimeWorkflow(value)).not.toEqual([]); }); - it("rejects missing source verification", () => { + it.each([ + ["Verify reviewed candidate OpenShell runtime before consumption", "run", "echo bypass"], + ["Verify reviewed candidate OpenShell runtime before consumption", "if", "false"], + ["Verify reviewed candidate OpenShell runtime before consumption", "continue-on-error", true], + ["Verify reviewed candidate OpenShell runtime before consumption", "env", {}], + ["Verify reviewed candidate OpenShell runtime", "run", "echo bypass"], + ["Verify reviewed candidate OpenShell runtime", "if", "false"], + ["Verify reviewed candidate OpenShell runtime", "continue-on-error", true], + ["Verify reviewed candidate OpenShell runtime", "env", {}], + [ + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + "env", + { GITHUB_TOKEN: "${{ github.token }}" }, + ], + ])("rejects admission step %s with altered %s", (name, field, replacement) => { + const value = workflow(); + namedStep(value, String(name))[String(field)] = replacement; + expect(validateManagedImageProtectedRuntimeWorkflow(value)).not.toEqual([]); + }); + + it("rejects consuming projected sources before verifying approval", () => { const value = workflow(); - const step = namedStep( + const job = runtimeJob(value); + const steps = job.steps as Array>; + const verify = namedStep( value, - "Run all-agent GPU, local inference, rollback, and cleanup qualification", + "Verify reviewed candidate OpenShell runtime before consumption", ); - const command = - 'node tools/e2e/protected-openshell-workspace.mts verify > "$E2E_ARTIFACT_DIR/openshell-projection-verified.json"'; - step.run = String(step.run).replace(command, ""); + const install = namedStep(value, "Install reviewed OpenShell SDK from trusted controller"); + const withoutVerify = steps.filter((step) => step !== verify); + const index = withoutVerify.indexOf(install); + job.steps = [...withoutVerify.slice(0, index + 1), verify, ...withoutVerify.slice(index + 1)]; expect(validateManagedImageProtectedRuntimeWorkflow(value)).not.toEqual([]); }); it("rejects source verification after qualification", () => { const value = workflow(); - const step = namedStep( - value, - "Run all-agent GPU, local inference, rollback, and cleanup qualification", - ); - const command = - 'node tools/e2e/protected-openshell-workspace.mts verify > "$E2E_ARTIFACT_DIR/openshell-projection-verified.json"'; - step.run = `${String(step.run).replace(command, "")}\n${command}\n`; + const job = runtimeJob(value); + const steps = job.steps as Array>; + const verify = namedStep(value, "Verify reviewed candidate OpenShell runtime"); + job.steps = [...steps.filter((step) => step !== verify), verify]; expect(validateManagedImageProtectedRuntimeWorkflow(value)).not.toEqual([]); }); @@ -793,12 +813,12 @@ describe("protected managed-image runtime workflow", () => { ); }); - it("rejects protected runtime permissions beyond same-run artifact access", () => { + it("rejects protected runtime permissions beyond source and approval reads", () => { const value = workflow(); runtimeJob(value).permissions = { actions: "read", contents: "read" }; expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( - "managed-image-protected-runtime permissions must be exactly contents: read", + "managed-image-protected-runtime permissions must be exactly contents: read and pull-requests: read", ); }); diff --git a/test/e2e/support/protected-openshell-workspace.test.ts b/test/e2e/support/protected-openshell-workspace.test.ts index 47fe8de860f..fb28df81816 100644 --- a/test/e2e/support/protected-openshell-workspace.test.ts +++ b/test/e2e/support/protected-openshell-workspace.test.ts @@ -4,10 +4,15 @@ import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import * as fs from "node:fs"; -import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { identity } from "../../helpers/candidate-openshell-trust"; +import { + CANDIDATE_OPENSHELL_SOURCES, + CANDIDATE_OPENSHELL_TRUST_PATH, +} from "../../../tools/e2e/candidate-openshell-trust.mts"; +import { fixture, approvedFixture, git, roots } from "../../support/protected-openshell-workspace"; import { manageProtectedOpenShellWorkspace } from "../../../tools/e2e/protected-openshell-workspace.mts"; vi.mock("node:fs", async (importOriginal) => { @@ -20,81 +25,107 @@ vi.mock("node:fs", async (importOriginal) => { }; }); -const sources = [ - "nemoclaw-blueprint/blueprint.yaml", - "scripts/install-openshell.sh", - "scripts/brev-launchable-ci-cpu.sh", - "src/lib/onboard/docker-driver-gateway-runtime.ts", - "src/lib/onboard/openshell-feature-gate.ts", - "src/lib/onboard/openshell-version.ts", - "package.json", - "package-lock.json", - "ci/reviewed-npm-audit.json", -]; -const roots: string[] = []; afterEach(() => { vi.restoreAllMocks(); vi.mocked(fs.openSync).mockReset(); for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); -function git(root: string, ...args: string[]) { - return execFileSync( - "git", +describe("protected OpenShell workspace", () => { + it("projects independently approved installer and feature gate templates and rechecks each phase", async () => { + const { input, request } = approvedFixture(); + await expect(manageProtectedOpenShellWorkspace("prepare", input)).resolves.toMatchObject({ + operation: "prepare", + }); + expect( + request.mock.calls.filter(([path]) => path === "/repos/NVIDIA/NemoClaw/pulls/123"), + ).toHaveLength(2); + request.mockClear(); + await expect(manageProtectedOpenShellWorkspace("verify", input)).resolves.toMatchObject({ + operation: "verify", + }); + expect( + request.mock.calls.filter(([path]) => path === "/repos/NVIDIA/NemoClaw/pulls/123"), + ).toHaveLength(2); + request.mockClear(); + await expect(manageProtectedOpenShellWorkspace("restore", input)).resolves.toMatchObject({ + operation: "restore", + }); + expect( + request.mock.calls.filter(([path]) => path === "/repos/NVIDIA/NemoClaw/pulls/123"), + ).toHaveLength(2); + expect(git(input.trustedRoot, "status", "--porcelain")).toBe(""); + }); + + it.each<[string, (value: ReturnType) => void, RegExp]>([ [ - "-c", - "core.hooksPath=/dev/null", - "-c", - "commit.gpgsign=false", - "-c", - "user.name=Fixture", - "-c", - "user.email=fixture@example.invalid", - "-C", - root, - ...args, + "missing", + ({ approval }) => { + approval.approvals.splice(0); + }, + /approval|decision/i, ], - { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, - ).trim(); -} -function fixture() { - const root = fs.mkdtempSync(join(tmpdir(), "protected-openshell-workspace-")); - roots.push(root); - const trustedRoot = join(root, "trusted"); - const candidateRoot = join(root, "candidate"); - for (const target of [trustedRoot, candidateRoot]) { - fs.mkdirSync(target); - git(target, "init", "--quiet"); - for (const path of sources) { - const file = join(target, path); - fs.mkdirSync(dirname(file), { recursive: true }); - fs.copyFileSync(new URL(`../../../${path}`, import.meta.url), file); - fs.chmodSync(file, path.endsWith(".sh") ? 0o755 : 0o644); - } - fs.writeFileSync(join(target, "controller.txt"), "trusted fixture\n"); - } - fs.writeFileSync( - join(trustedRoot, "nemoclaw-blueprint/blueprint.yaml"), - 'name: fixture\nmin_openshell_version: "0.0.1"\nmax_openshell_version: "0.0.1"\n', - ); - fs.writeFileSync( - join(trustedRoot, "src/lib/onboard/openshell-version.ts"), - 'export const SUPPORTED_OPENSHELL_FALLBACK_VERSION = "0.0.1";\n', - ); - git(trustedRoot, "add", "."); - git(trustedRoot, "commit", "--quiet", "-m", "fixture"); - git(candidateRoot, "add", "."); - git(candidateRoot, "commit", "--quiet", "-m", "fixture"); - return { - trustedRoot, - candidateRoot, - workflowSha: git(trustedRoot, "rev-parse", "HEAD"), - candidateSha: git(candidateRoot, "rev-parse", "HEAD"), - }; -} + [ + "stale", + ({ approval }) => { + approval.approvals[0]!.body = `NemoClaw-Upgrade-E2E: approve ${identity.candidateSha} ${identity.baseSha}`; + }, + /approval|decision/i, + ], + [ + "registry", + ({ approval }) => { + approval.files[CANDIDATE_OPENSHELL_TRUST_PATH] += "\n"; + }, + /Committed candidate registry differs/, + ], + [ + "source", + ({ approval }) => { + approval.files[CANDIDATE_OPENSHELL_SOURCES.blueprintSource] += "\n"; + }, + /Committed candidate source differs/, + ], + ])("rejects %s candidate approval before mutation", async (_scenario, mutate, expected) => { + const value = approvedFixture(); + mutate(value); + await expect(manageProtectedOpenShellWorkspace("prepare", value.input)).rejects.toThrow( + expected, + ); + expect(git(value.input.trustedRoot, "status", "--porcelain")).toBe(""); + }); -describe("protected OpenShell workspace", () => { - it("binds the workflow CLI to dispatch commits and emits records for each phase", () => { + it("rejects an unapproved feature gate template before mutation", async () => { + const { input, approval } = approvedFixture(); + fs.appendFileSync( + join(input.candidateRoot, "src/lib/onboard/openshell-feature-gate.ts"), + "\nunreviewed_operation();\n", + ); + git(input.candidateRoot, "add", "."); + git(input.candidateRoot, "commit", "--quiet", "-m", "unapproved feature"); + input.candidateSha = git(input.candidateRoot, "rev-parse", "HEAD"); + approval.approvals[0]!.body = `NemoClaw-Upgrade-E2E: approve ${input.candidateSha} ${identity.baseSha}`; + await expect(manageProtectedOpenShellWorkspace("prepare", input)).rejects.toThrow( + /feature gate operational template/, + ); + expect(git(input.trustedRoot, "status", "--porcelain")).toBe(""); + }); + + it("rejects revoked approval before verification and leaves projected files untouched", async () => { + const { input, approval } = approvedFixture(); + await manageProtectedOpenShellWorkspace("prepare", input); + const before = git(input.trustedRoot, "diff", "--no-ext-diff"); + approval.approvals[0]!.body = `NemoClaw-Upgrade-E2E: revoke ${input.candidateSha} ${identity.baseSha}`; + await expect(manageProtectedOpenShellWorkspace("verify", input)).rejects.toThrow( + "Candidate upgrade needs an independent maintainer's current commit-bound E2E approval", + ); + await expect(manageProtectedOpenShellWorkspace("restore", input)).rejects.toThrow( + "Candidate upgrade needs an independent maintainer's current commit-bound E2E approval", + ); + expect(git(input.trustedRoot, "diff", "--no-ext-diff")).toBe(before); + }); + + it("binds the workflow CLI to dispatch commits and emits records for each phase", async () => { const input = fixture(); const command = fileURLToPath( new URL("../../../tools/e2e/protected-openshell-workspace.mts", import.meta.url), @@ -122,16 +153,16 @@ describe("protected OpenShell workspace", () => { expect(git(input.trustedRoot, "status", "--porcelain")).toBe(""); }); - it("prepares candidate pins, verifies their bytes, and restores the trusted checkout", () => { + it("prepares candidate pins, verifies their bytes, and restores the trusted checkout", async () => { const input = fixture(); - const prepared = manageProtectedOpenShellWorkspace("prepare", input); + const prepared = await manageProtectedOpenShellWorkspace("prepare", input); expect(prepared).toMatchObject({ workflowSha: input.workflowSha, candidateSha: input.candidateSha, operation: "prepare", }); expect(prepared.files).toHaveLength(7); - expect(manageProtectedOpenShellWorkspace("verify", input)).toEqual({ + expect(await manageProtectedOpenShellWorkspace("verify", input)).toEqual({ ...prepared, operation: "verify", }); @@ -142,33 +173,35 @@ describe("protected OpenShell workspace", () => { .digest("hex"), })); expect(actualDigests).toEqual(prepared.files); - manageProtectedOpenShellWorkspace("restore", input); + await manageProtectedOpenShellWorkspace("restore", input); expect(git(input.trustedRoot, "status", "--porcelain")).toBe(""); - expect(() => manageProtectedOpenShellWorkspace("restore", input)).not.toThrow(); + await expect(manageProtectedOpenShellWorkspace("restore", input)).resolves.toBeDefined(); }); it.each(["workflowSha", "candidateSha"] as const)( "rejects a mismatched %s before changing files", - (key) => { + async (key) => { const input = fixture(); input[key] = "a".repeat(40); - expect(() => manageProtectedOpenShellWorkspace("prepare", input)).toThrow(/dispatch commit/); + await expect(manageProtectedOpenShellWorkspace("prepare", input)).rejects.toThrow( + /dispatch commit/, + ); expect(git(input.trustedRoot, "status", "--porcelain")).toBe(""); }, ); - it("uses committed candidate blobs instead of mutable worktree content", () => { + it("uses committed candidate blobs instead of mutable worktree content", async () => { const input = fixture(); fs.appendFileSync( join(input.candidateRoot, "scripts/install-openshell.sh"), "\nunreviewed_operation\n", ); - const result = manageProtectedOpenShellWorkspace("prepare", input); + const result = await manageProtectedOpenShellWorkspace("prepare", input); expect(result.releaseVersion).toMatch(/^\d+\.\d+\.\d+$/); - expect(() => manageProtectedOpenShellWorkspace("verify", input)).not.toThrow(); + await expect(manageProtectedOpenShellWorkspace("verify", input)).resolves.toBeDefined(); }); - it("rejects committed unreviewed candidate operations", () => { + it("rejects committed unreviewed candidate operations", async () => { const input = fixture(); fs.appendFileSync( join(input.candidateRoot, "scripts/install-openshell.sh"), @@ -177,41 +210,50 @@ describe("protected OpenShell workspace", () => { git(input.candidateRoot, "add", "."); git(input.candidateRoot, "commit", "--quiet", "-m", "unreviewed"); input.candidateSha = git(input.candidateRoot, "rev-parse", "HEAD"); - expect(() => manageProtectedOpenShellWorkspace("prepare", input)).toThrow(/template/); + await expect(manageProtectedOpenShellWorkspace("prepare", input)).rejects.toThrow(/template/); expect(git(input.trustedRoot, "status", "--porcelain")).toBe(""); }); it.each(["prepare", "verify", "restore"] as const)( "rejects unrelated controller edits during %s", - (operation) => { + async (operation) => { const input = fixture(); fs.writeFileSync(join(input.trustedRoot, "controller.txt"), "modified"); - expect(() => manageProtectedOpenShellWorkspace(operation, input)).toThrow(/Unrelated/); + await expect(manageProtectedOpenShellWorkspace(operation, input)).rejects.toThrow( + /Unrelated/, + ); expect(fs.readFileSync(join(input.trustedRoot, "controller.txt"), "utf8")).toBe("modified"); }, ); - it.each(["file", "parent"])("rejects a symlinked destination %s without following it", (kind) => { - const input = fixture(); - const path = join( - input.trustedRoot, - kind === "file" ? "scripts/install-openshell.sh" : "scripts", - ); - const moved = `${path}.retained`; - fs.renameSync(path, moved); - fs.symlinkSync(moved, path); - expect(() => manageProtectedOpenShellWorkspace("prepare", input)).toThrow(/Unsafe/); - expect(fs.readlinkSync(path)).toBe(moved); - }); + it.each(["file", "parent"])( + "rejects a symlinked destination %s without following it", + async (kind) => { + const input = fixture(); + const path = join( + input.trustedRoot, + kind === "file" ? "scripts/install-openshell.sh" : "scripts", + ); + const moved = `${path}.retained`; + fs.renameSync(path, moved); + fs.symlinkSync(moved, path); + await expect(manageProtectedOpenShellWorkspace("prepare", input)).rejects.toThrow(/Unsafe/); + expect(fs.readlinkSync(path)).toBe(moved); + }, + ); - it("rejects tampering before verification or restoration", () => { + it("rejects tampering before verification or restoration", async () => { const input = fixture(); - manageProtectedOpenShellWorkspace("prepare", input); + await manageProtectedOpenShellWorkspace("prepare", input); const path = join(input.trustedRoot, "src/lib/onboard/openshell-version.ts"); fs.appendFileSync(path, "\nunreviewed_operation();\n"); const before = git(input.trustedRoot, "diff", "--no-ext-diff"); - expect(() => manageProtectedOpenShellWorkspace("verify", input)).toThrow(/Unexpected verify/); - expect(() => manageProtectedOpenShellWorkspace("restore", input)).toThrow(/Unexpected restore/); + await expect(manageProtectedOpenShellWorkspace("verify", input)).rejects.toThrow( + /Unexpected verify/, + ); + await expect(manageProtectedOpenShellWorkspace("restore", input)).rejects.toThrow( + /Unexpected restore/, + ); expect(git(input.trustedRoot, "diff", "--no-ext-diff")).toBe(before); }); @@ -227,7 +269,7 @@ describe("protected OpenShell workspace", () => { fs.symlinkSync(retained, path); return original(file, flags, mode); }); - expect(() => manageProtectedOpenShellWorkspace("prepare", input)).toThrow(/Unsafe/); + await expect(manageProtectedOpenShellWorkspace("prepare", input)).rejects.toThrow(/Unsafe/); expect(fs.readlinkSync(path)).toBe(retained); expect(fs.statSync(retained)).toMatchObject({ ino: before.ino, @@ -236,13 +278,13 @@ describe("protected OpenShell workspace", () => { }); }); - it("rejects a hard-linked destination and closes its opened descriptor", () => { + it("rejects a hard-linked destination and closes its opened descriptor", async () => { const input = fixture(); const path = join(input.trustedRoot, "scripts/install-openshell.sh"); fs.linkSync(path, `${path}.retained`); vi.mocked(fs.openSync).mockClear(); vi.mocked(fs.closeSync).mockClear(); - expect(() => manageProtectedOpenShellWorkspace("prepare", input)).toThrow(/Unsafe/); + await expect(manageProtectedOpenShellWorkspace("prepare", input)).rejects.toThrow(/Unsafe/); const opened = vi.mocked(fs.openSync).mock.results.map((result) => result.value); expect(opened.length).toBeGreaterThan(0); expect(vi.mocked(fs.closeSync).mock.calls.map(([fd]) => fd)).toEqual(opened); @@ -257,7 +299,7 @@ describe("protected OpenShell workspace", () => { .mockImplementationOnce(() => { throw new Error("fixture replacement failure"); }); - expect(() => manageProtectedOpenShellWorkspace("prepare", input)).toThrow( + await expect(manageProtectedOpenShellWorkspace("prepare", input)).rejects.toThrow( /fixture replacement failure/, ); expect(git(input.trustedRoot, "status", "--porcelain")).toBe(""); diff --git a/test/support/protected-openshell-workspace.ts b/test/support/protected-openshell-workspace.ts new file mode 100644 index 00000000000..6d3d3d09bd2 --- /dev/null +++ b/test/support/protected-openshell-workspace.ts @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import * as fs from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { vi } from "vitest"; +import { fixture as approvalFixture, identity } from "../helpers/candidate-openshell-trust"; +import { + CANDIDATE_OPENSHELL_SOURCES, + CANDIDATE_OPENSHELL_TRUST_PATH, +} from "../../tools/e2e/candidate-openshell-trust.mts"; +import { + readOpenShellTrustRegistry, + validateInstallerSources, + validateOpenShellFeatureGateSource, +} from "../../scripts/checks/extract-installer-pins.mts"; + +const sources = [ + "nemoclaw-blueprint/blueprint.yaml", + "scripts/install-openshell.sh", + "scripts/brev-launchable-ci-cpu.sh", + "src/lib/onboard/docker-driver-gateway-runtime.ts", + "src/lib/onboard/openshell-feature-gate.ts", + "src/lib/onboard/openshell-version.ts", + "package.json", + "package-lock.json", + "ci/reviewed-npm-audit.json", +]; +export const roots: string[] = []; + +export function git(root: string, ...args: string[]) { + return execFileSync( + "git", + [ + "-c", + "core.hooksPath=/dev/null", + "-c", + "commit.gpgsign=false", + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-C", + root, + ...args, + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ).trim(); +} +export function fixture() { + const root = fs.mkdtempSync(join(tmpdir(), "protected-openshell-workspace-")); + roots.push(root); + const trustedRoot = join(root, "trusted"); + const candidateRoot = join(root, "candidate"); + for (const target of [trustedRoot, candidateRoot]) { + fs.mkdirSync(target); + git(target, "init", "--quiet"); + for (const path of sources) { + const file = join(target, path); + fs.mkdirSync(dirname(file), { recursive: true }); + fs.copyFileSync(new URL(`../../${path}`, import.meta.url), file); + fs.chmodSync(file, path.endsWith(".sh") ? 0o755 : 0o644); + } + fs.writeFileSync(join(target, "controller.txt"), "trusted fixture\n"); + } + fs.writeFileSync( + join(trustedRoot, "nemoclaw-blueprint/blueprint.yaml"), + 'name: fixture\nmin_openshell_version: "0.0.1"\nmax_openshell_version: "0.0.1"\n', + ); + fs.writeFileSync( + join(trustedRoot, "src/lib/onboard/openshell-version.ts"), + 'export const SUPPORTED_OPENSHELL_FALLBACK_VERSION = "0.0.1";\n', + ); + git(trustedRoot, "add", "."); + git(trustedRoot, "commit", "--quiet", "-m", "fixture"); + git(candidateRoot, "add", "."); + git(candidateRoot, "commit", "--quiet", "-m", "fixture"); + return { + trustedRoot, + candidateRoot, + workflowSha: git(trustedRoot, "rev-parse", "HEAD"), + candidateSha: git(candidateRoot, "rev-parse", "HEAD"), + }; +} + +function templateDigest(check: () => unknown): string { + try { + check(); + } catch (error) { + const digest = /actual_sha256=([a-f0-9]{64})/.exec(String(error))?.[1]; + if (digest) return digest; + throw error; + } + throw new Error("Fixture must introduce an untrusted template"); +} + +export function approvedFixture() { + const input = fixture(); + const approval = approvalFixture(); + const candidate = Object.fromEntries( + Object.entries(CANDIDATE_OPENSHELL_SOURCES).map(([key, path]) => [ + key, + fs.readFileSync(join(input.candidateRoot, path), "utf8"), + ]), + ) as Record; + const selected = validateInstallerSources(candidate); + const release = structuredClone(selected.installerReleases[0]!); + candidate.installerSource += "\n# Reviewed installer fixture change.\n"; + release.installerTemplateSha256 = [ + ...release.installerTemplateSha256, + templateDigest(() => validateInstallerSources(candidate)), + ]; + const featurePath = "src/lib/onboard/openshell-feature-gate.ts"; + const featureSource = + fs.readFileSync(join(input.candidateRoot, featurePath), "utf8") + + "\n// Reviewed feature gate fixture change.\n"; + release.featureGateTemplateSha256 = [ + ...(release.featureGateTemplateSha256 ?? []), + templateDigest(() => validateOpenShellFeatureGateSource(featureSource, release.version)), + ]; + for (const [key, path] of Object.entries(CANDIDATE_OPENSHELL_SOURCES)) + approval.files[path] = candidate[key as keyof typeof candidate]; + const registry = readOpenShellTrustRegistry(); + approval.files[CANDIDATE_OPENSHELL_TRUST_PATH] = JSON.stringify({ + ...registry, + releases: [...registry.releases.filter((prior) => prior.version !== release.version), release], + }); + for (const [path, source] of Object.entries(approval.files)) + fs.writeFileSync(join(input.candidateRoot, path), source); + fs.writeFileSync(join(input.candidateRoot, featurePath), featureSource); + git(input.candidateRoot, "add", "."); + git(input.candidateRoot, "commit", "--quiet", "-m", "approved fixture"); + input.candidateSha = git(input.candidateRoot, "rev-parse", "HEAD"); + approval.approvals[0]!.body = `NemoClaw-Upgrade-E2E: approve ${input.candidateSha} ${identity.baseSha}`; + const request = vi.fn(async (path: string) => { + const result = await approval.request(path); + if (path === "/repos/NVIDIA/NemoClaw/pulls/123") { + const pr = result as { head: { sha: string } }; + return { ...pr, head: { ...pr.head, sha: input.candidateSha } }; + } + return result; + }); + return { + input: Object.assign(input, { + admission: { prNumber: identity.prNumber, baseSha: identity.baseSha, request }, + }), + approval, + request, + }; +} diff --git a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts index 99e7dd88ba4..ce5321963dd 100644 --- a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts +++ b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts @@ -112,8 +112,8 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR errors.push(`${JOB_ID} must run on the protected amd64 GPU runner`); } if (job["timeout-minutes"] !== 75) errors.push(`${JOB_ID} must keep the 75 minute timeout`); - if (!isDeepStrictEqual(job.permissions, { contents: "read" })) { - errors.push(`${JOB_ID} permissions must be exactly contents: read`); + if (!isDeepStrictEqual(job.permissions, { contents: "read", "pull-requests": "read" })) { + errors.push(`${JOB_ID} permissions must be exactly contents: read and pull-requests: read`); } if (job["continue-on-error"] !== undefined) { errors.push(`${JOB_ID} must not weaken failures with continue-on-error`); @@ -265,19 +265,43 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR ); if (projection?.id !== "openshell-projection") errors.push(`${JOB_ID} must identify the OpenShell projection`); + const verifyBeforeUse = requireStep( + errors, + workflowSteps, + "Verify reviewed candidate OpenShell runtime before consumption", + ); + const verify = requireStep(errors, workflowSteps, "Verify reviewed candidate OpenShell runtime"); const restore = requireStep(errors, workflowSteps, "Restore trusted OpenShell sources"); + const admissionEnv = { + GITHUB_TOKEN: "${{ github.token }}", + UPGRADE_PR_NUMBER: "${{ inputs.pr_number }}", + }; + if ( + jobEnv.GITHUB_TOKEN !== undefined || + jobEnv.GH_TOKEN !== undefined || + workflowSteps.some( + (step) => + ![projection, verifyBeforeUse, verify, restore].includes(step) && + (record(step.env).GITHUB_TOKEN !== undefined || record(step.env).GH_TOKEN !== undefined), + ) + ) + errors.push(`${JOB_ID} must expose GitHub credentials only to candidate admission operations`); + if (verifyBeforeUse?.if !== undefined || verify?.if !== undefined) + errors.push(`${JOB_ID} must not skip OpenShell verification`); if (restore?.if !== "${{ always() && steps.openshell-projection.outcome == 'success' }}") { errors.push(`${JOB_ID} must restore sources after a successful OpenShell projection`); } for (const [step, operation, output] of [ [projection, "prepare", "openshell-projection.json"], + [verifyBeforeUse, "verify", "openshell-projection-initially-verified.json"], + [verify, "verify", "openshell-projection-verified.json"], [restore, "restore", "openshell-projection-restored.json"], ] as const) { const expected = `set -euo pipefail\nnode tools/e2e/protected-openshell-workspace.mts ${operation} > "$E2E_ARTIFACT_DIR/${output}"\n`; if ( step?.run !== expected || step?.shell !== "bash" || - step?.env !== undefined || + !isDeepStrictEqual(step?.env, admissionEnv) || step?.["continue-on-error"] !== undefined ) { errors.push(`${JOB_ID} must use the reviewed OpenShell ${operation} operation`); @@ -476,19 +500,11 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR errors.push(`${JOB_ID} must expose NVIDIA_API_KEY only to trusted qualification code`); } requireFragments(errors, qualification, [ - 'node tools/e2e/protected-openshell-workspace.mts verify > "$E2E_ARTIFACT_DIR/openshell-projection-verified.json"', '[[ "$(git rev-parse --verify HEAD)" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA" ]]', 'OPENSHELL_BIN="$(command -v openshell)"\nexport OPENSHELL_BIN', "tools/e2e/live-vitest-invocation.mts run", `--test-path ${LIVE_TEST_PATH}`, ]); - const qualificationRun = text(qualification?.run); - const verifyPosition = qualificationRun.indexOf( - "node tools/e2e/protected-openshell-workspace.mts verify", - ); - if (verifyPosition < 0 || verifyPosition > qualificationRun.indexOf("OPENSHELL_BIN=")) { - errors.push(`${JOB_ID} must verify projected sources before runtime qualification`); - } if (text(qualification?.run).includes(".candidate-runtime")) { errors.push(`${JOB_ID} trusted qualification must not execute candidate checkout paths`); } @@ -522,6 +538,7 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR "Prepare E2E workspace", "Validate protected runtime activation contract", "Bind reviewed candidate OpenShell runtime", + "Verify reviewed candidate OpenShell runtime before consumption", "Download reviewed OpenShell SDK archive", "Install reviewed OpenShell SDK from trusted controller", "Build trusted CLI with reviewed OpenShell runtime", @@ -530,6 +547,7 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR "Start isolated protected runtime registry", "Build exact all-agent protected runtime images", "Install OpenShell CLI", + "Verify reviewed candidate OpenShell runtime", "Run all-agent GPU, local inference, rollback, and cleanup qualification", "Remove isolated protected runtime registry", "Restore trusted OpenShell sources", diff --git a/tools/e2e/protected-openshell-projection.mts b/tools/e2e/protected-openshell-projection.mts index 0b6531983c2..cce2ae4f841 100644 --- a/tools/e2e/protected-openshell-projection.mts +++ b/tools/e2e/protected-openshell-projection.mts @@ -4,6 +4,7 @@ import { validateInstallerSources, validateOpenShellFeatureGateSource, + type OpenShellReleaseTrust, } from "../../scripts/checks/extract-installer-pins.mts"; import { isDeepStrictEqual } from "node:util"; import { parseAuditConfig } from "../../scripts/audit-reviewed-npm-graph.mts"; @@ -28,9 +29,10 @@ function replaceOne(source: string, pattern: RegExp, replacement: string, label: export function projectProtectedOpenShellSources( candidate: ProtectedOpenShellCandidateSources, trusted: { blueprintSource: string; versionSource: string }, + releases?: readonly OpenShellReleaseTrust[], ): { releaseVersion: string; files: Record } { - const { releaseVersion } = validateInstallerSources(candidate); - validateOpenShellFeatureGateSource(candidate.featureGateSource, releaseVersion); + const { releaseVersion } = validateInstallerSources(candidate, releases); + validateOpenShellFeatureGateSource(candidate.featureGateSource, releaseVersion, releases); const qualified = [ ...candidate.supervisorRuntimeSource.matchAll( /^const QUALIFIED_STABLE_OPENSHELL_VERSION = "([0-9]+\.[0-9]+\.[0-9]+)";$/gm, diff --git a/tools/e2e/protected-openshell-workspace.mts b/tools/e2e/protected-openshell-workspace.mts index 39d222b4982..2adfd126429 100644 --- a/tools/e2e/protected-openshell-workspace.mts +++ b/tools/e2e/protected-openshell-workspace.mts @@ -21,11 +21,22 @@ import { projectProtectedOpenShellSdk, } from "./protected-openshell-projection.mts"; +import { + readOpenShellTrustRegistry, + resolveOpenShellTrustRegistry, +} from "../../scripts/checks/extract-installer-pins.mts"; +import { parseOpenShellTrustRegistry } from "../../scripts/checks/openshell-trust-data.mts"; +import { + CANDIDATE_OPENSHELL_SOURCES, + CANDIDATE_OPENSHELL_TRUST_PATH, + registryCoveredByTrusted, + reviewedCandidateOpenShellTrust, +} from "./candidate-openshell-trust.mts"; +import { type UpgradeRequest } from "./candidate-upgrade-admission.mts"; +import { githubRequest } from "./base-image-publication.mts"; + const SOURCE_PATHS = { - blueprintSource: "nemoclaw-blueprint/blueprint.yaml", - installerSource: "scripts/install-openshell.sh", - brevInstallerSource: "scripts/brev-launchable-ci-cpu.sh", - supervisorRuntimeSource: "src/lib/onboard/docker-driver-gateway-runtime.ts", + ...CANDIDATE_OPENSHELL_SOURCES, featureGateSource: "src/lib/onboard/openshell-feature-gate.ts", packageSource: "package.json", lockSource: "package-lock.json", @@ -38,6 +49,7 @@ export interface ProtectedOpenShellWorkspace { candidateRoot: string; workflowSha: string; candidateSha: string; + admission?: { prNumber: number; baseSha: string; request: UpgradeRequest }; } function git(root: string, args: string[]): string { @@ -119,8 +131,53 @@ function replaceSource(path: string, source: string, mode: number): void { } } +/** Recheck approval for each phase; local receipts cannot authorize new executable templates. */ +async function candidateReleases( + input: ProtectedOpenShellWorkspace, + candidate: Record, +) { + const registryPath = CANDIDATE_OPENSHELL_TRUST_PATH; + const present = git(input.candidateRoot, [ + "ls-tree", + "-z", + input.candidateSha, + "--", + registryPath, + ]); + const registrySource = present + ? committedSource(input.candidateRoot, input.candidateSha, registryPath) + : undefined; + const registry = + registrySource === undefined + ? { schemaVersion: 1 as const, releases: [], revokedTemplateSha256: [] } + : parseOpenShellTrustRegistry(registrySource); + const trusted = readOpenShellTrustRegistry(); + if (registryCoveredByTrusted(registry, trusted)) return resolveOpenShellTrustRegistry(trusted); + if (!input.admission) + throw new Error("Candidate OpenShell trust requires live upgrade admission"); + const approved = await reviewedCandidateOpenShellTrust( + { + prNumber: input.admission.prNumber, + baseSha: input.admission.baseSha, + candidateSha: input.candidateSha, + }, + input.admission.request, + ); + if (registrySource !== approved.registrySource) + throw new Error("Committed candidate registry differs from the approved commit"); + for (const key of Object.keys( + CANDIDATE_OPENSHELL_SOURCES, + ) as (keyof typeof CANDIDATE_OPENSHELL_SOURCES)[]) { + if (candidate[key] !== approved.sources[key]) + throw new Error( + `Committed candidate source differs from the approved commit: ${CANDIDATE_OPENSHELL_SOURCES[key]}`, + ); + } + return approved.releases; +} + /** Apply only reviewed modules and release literals to an isolated trusted workflow checkout. */ -export function manageProtectedOpenShellWorkspace( +export async function manageProtectedOpenShellWorkspace( operation: "prepare" | "verify" | "restore", input: ProtectedOpenShellWorkspace, ) { @@ -134,14 +191,19 @@ export function manageProtectedOpenShellWorkspace( committedSource(input.candidateRoot, input.candidateSha, path), ]), ) as Record; - const projected = projectProtectedOpenShellSources(candidate, { - blueprintSource: committedSource( - input.trustedRoot, - input.workflowSha, - SOURCE_PATHS.blueprintSource, - ), - versionSource: committedSource(input.trustedRoot, input.workflowSha, VERSION_PATH), - }); + const releases = await candidateReleases(input, candidate); + const projected = projectProtectedOpenShellSources( + candidate, + { + blueprintSource: committedSource( + input.trustedRoot, + input.workflowSha, + SOURCE_PATHS.blueprintSource, + ), + versionSource: committedSource(input.trustedRoot, input.workflowSha, VERSION_PATH), + }, + releases, + ); const sdk = projectProtectedOpenShellSdk( candidate, { @@ -231,11 +293,21 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) } console.log( JSON.stringify( - manageProtectedOpenShellWorkspace(operation as "prepare" | "verify" | "restore", { + await manageProtectedOpenShellWorkspace(operation as "prepare" | "verify" | "restore", { trustedRoot: process.env.GITHUB_WORKSPACE!, candidateRoot: process.env.NEMOCLAW_E2E_TESTED_ROOT!, workflowSha: process.env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA!, candidateSha: process.env.CHECKOUT_SHA!, + admission: { + prNumber: Number(process.env.UPGRADE_PR_NUMBER), + baseSha: process.env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA ?? "", + request: (url) => { + const token = process.env.GITHUB_TOKEN; + if (!token) + throw new Error("Candidate upgrade admission requires authenticated GitHub access"); + return githubRequest(url, token); + }, + }, }), null, 2, From 88900837e9705c105470c6c3e33835e4b2a9a523 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter Date: Mon, 19 Oct 2026 14:19:59 -0700 Subject: [PATCH 15/22] test(dcode): set repo root for copied headless check (#13498) ## Outcome The Deep Agents Code proxy runtime contract test finds its shared endpoint normalizer when it runs a copy of cloud check 07 from a temporary directory. This fixes the deterministic CLI shard failure that appeared after #13363 merged. ## Reason The test copies the check script to a temporary path but did not set `REPO`. The check therefore derived the repository root from that temporary path and tried to load the normalizer outside the checkout. The [failed CLI shard](https://github.com/NVIDIA/NemoClaw/actions/runs/39308061441/job/118250984620) reproduced the missing module. The aggregate CLI and gate failures were consequences of that shard. ### Related issues Refs #13363 and #13164. ## Changes - Set `REPO` to the test checkout root in the copied-script fixture, matching the shared headless test helper. Runtime code is unchanged. ## Verification - Before the fix, the affected proxy runtime suite reproduced the missing-module failure in five tests. - `npx vitest run --project integration test/agents/deepagents/langchain-deepagents-code-headless-runtime.test.ts test/agents/deepagents/langchain-deepagents-code-proxy-runtime-contract.test.ts` passed after the fix: 2 files, 9 tests. - Credential-isolated `npm run validate:pr` passed, including repository checks, builds, and pre-push checks. The signed commit and push hooks passed. - GitHub marks commit `1e4774b2fb7b6bce547ae645206254993b666189` Verified. The diff contains no secrets or credentials. --- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> ## Summary by CodeRabbit * **Tests** * Updated runtime contract checks to use the current working directory while continuing to respect inherited environment settings and caller-provided overrides. Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Co-authored-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../langchain-deepagents-code-proxy-runtime-contract.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/agents/deepagents/langchain-deepagents-code-proxy-runtime-contract.test.ts b/test/agents/deepagents/langchain-deepagents-code-proxy-runtime-contract.test.ts index cb4246fd505..d494f7ec856 100644 --- a/test/agents/deepagents/langchain-deepagents-code-proxy-runtime-contract.test.ts +++ b/test/agents/deepagents/langchain-deepagents-code-proxy-runtime-contract.test.ts @@ -32,7 +32,7 @@ function runHeadlessCheckHelper( ): string { return execFileSync("bash", ["-c", `source "$1"; ${snippet}`, "bash", sourcePath], { encoding: "utf8", - env: { ...process.env, ...env }, + env: { ...process.env, REPO: process.cwd(), ...env }, }); } From 5375ea759e9a9a9843fb2943986597bc48188d1a Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:21:47 -0700 Subject: [PATCH 16/22] chore(pi): refresh qualification receipts for staging repair --- ci/pi-agent-qualification-v1-linux-amd64.json | 8 ++++---- ci/pi-agent-qualification-v1-linux-arm64.json | 8 ++++---- src/lib/agent/candidate-authority.ts | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/pi-agent-qualification-v1-linux-amd64.json b/ci/pi-agent-qualification-v1-linux-amd64.json index 8252283bdb4..1d65e04e9f3 100644 --- a/ci/pi-agent-qualification-v1-linux-amd64.json +++ b/ci/pi-agent-qualification-v1-linux-amd64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/amd64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:a789c4b50b25e02a990389c7d0b2a8dcf8b801036eb4cda43ee6a7574ab409ce", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:a789c4b50b25e02a990389c7d0b2a8dcf8b801036eb4cda43ee6a7574ab409ce", + "digest": "sha256:de4c0bbdb1512e3a9a61f467d8da8a08aa337390628f35031e95330e16d98b17", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:de4c0bbdb1512e3a9a61f467d8da8a08aa337390628f35031e95330e16d98b17", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "0a9ca6d8178eb27b0ede47094fa76c5674ff60a6", + "revision": "f421e01e62b4d03a1410157dcce11c3a1aaf3a7c", "release": "v0.1.0", - "cohort": "ghrun-39286562149-1" + "cohort": "ghrun-39308521500-3" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/ci/pi-agent-qualification-v1-linux-arm64.json b/ci/pi-agent-qualification-v1-linux-arm64.json index eb1d8764659..e04a42bbe21 100644 --- a/ci/pi-agent-qualification-v1-linux-arm64.json +++ b/ci/pi-agent-qualification-v1-linux-arm64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/arm64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:75bbc5644e9119dc9824e4ad30637ca27032e352d10584e8a624cc1a9c9f5378", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:75bbc5644e9119dc9824e4ad30637ca27032e352d10584e8a624cc1a9c9f5378", + "digest": "sha256:2be599dc736b782af348788ae0bce6d2c01a7327977113315afdfb90b5c50a28", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:2be599dc736b782af348788ae0bce6d2c01a7327977113315afdfb90b5c50a28", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "0a9ca6d8178eb27b0ede47094fa76c5674ff60a6", + "revision": "f421e01e62b4d03a1410157dcce11c3a1aaf3a7c", "release": "v0.1.0", - "cohort": "ghrun-39286562149-1" + "cohort": "ghrun-39308521500-3" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/src/lib/agent/candidate-authority.ts b/src/lib/agent/candidate-authority.ts index dfd5caaa71c..aa0183df307 100644 --- a/src/lib/agent/candidate-authority.ts +++ b/src/lib/agent/candidate-authority.ts @@ -13,8 +13,8 @@ export const CANDIDATE_QUALIFICATION_RECEIPT_DIGESTS: Readonly< Record > = Object.freeze({ pi: Object.freeze([ - "0a76f266e2831b50b86635f72cf62697932e2f5822a7b291131809a17eb3a87b", - "b785ac087a120b75c06614b419c5c35ee869eaecb2709919af3ef5d7437cc6eb", + "dd32f76c82140b590943c04071afd3759d537956ce7d5064afa6009b4e716bbd", + "f135ae7a02e6f92077198616237e6db5df8e9edeb1d00b4faa3cf8df23aad605", ]), }); From d087dbcad90ee639016ba0d259556f024340acb5 Mon Sep 17 00:00:00 2001 From: "Deepak Jain (DJ)" Date: Mon, 19 Oct 2026 16:04:20 -0700 Subject: [PATCH 17/22] fix(onboard): wait for routed gateway before native restart (#13481) ## Outcome Fresh Model Router onboarding waits for OpenClaw to accept connections before requesting its native gateway restart. A gateway that is still starting can finish within the existing startup window instead of immediately failing step 7/8 with `ECONNREFUSED`. ## Reason Sandbox creation can finish before the OpenClaw listener starts. Routed setup previously configured and restarted the gateway without waiting; the restart command failed before its own post-restart readiness check could run. Existing setup tests assumed the restart succeeded and missed this ordering gap. ### Related issues Fixes #13465. ## Changes Reuse the bounded native startup check before routed setup restarts the selected gateway. Require positive readiness, revalidate sandbox identity after waiting, and retain native restart failure reporting. Custom-image pairing and ordinary-provider setup keep their existing paths. Regression tests cover initial connection refusal followed by readiness, the selected gateway and port, deadline exhaustion, late readiness, inconclusive transport, and identity changes. ## Verification - `npx vitest run --project cli src/lib/onboard/openclaw-setup.test.ts src/lib/onboard/machine/final-flow-phases.test.ts src/lib/onboard/machine/final-flow-phases.runtime.test.ts src/lib/onboard/machine/final-flow-composition.test.ts src/lib/onboard/machine/finalization-deps.test.ts --maxWorkers 2`: 141 tests passed across five files. The initial regression failed before the fix because setup performed zero readiness probes. - `npx vitest run --project integration test/automation/pull-requests/growth-guardrails.test.ts`: seven growth guardrails passed. - `NODE_OPTIONS=--max-old-space-size=8192 npm run typecheck:cli` and `npm run build:cli`: passed. Changed-file formatting and lint passed. - `src/lib/actions/sandbox/gateway-restart.test.ts`: 10 passed; 26 failed before reaching their test action because this environment denies the host lifecycle lock write. - Not run: Ubuntu 26.04 fresh CPU/GPU Model Router onboarding, resume, and an NVIDIA Endpoints control. The available Docker runtime has less than the required 8 GiB memory. - Normal pre-commit, commit-message, and pre-push hooks passed, including repository checks, source-shape and growth guards, and secret scanning. - The diff contains no secrets, API keys, or credentials. ## Review notes Decision: Accept this focused bug fix despite the pending triage label. Reason: the reported race remains on current `main`. Placement: readiness coordination stays in NemoClaw onboarding; OpenClaw owns gateway lifecycle. Accountable maintainer: @deepujain, who supplied this direction for the implementation. Validation plan: the deterministic regression and failure cases above, with the Ubuntu journeys still requiring platform validation. Self-review of NVIDIA/NemoClaw commit 748fd984564ece7cd1d894ba94eb0f95412b119c covers the sensitive files `src/lib/onboard.ts`, `src/lib/onboard/openclaw-setup.ts`, and `src/lib/onboard/openclaw-setup.test.ts`. Source inspection and focused tests found no change to pairing, credentials, or native restart ownership. Independent maintainer review remains pending. --- Signed-off-by: Deepak Jain ## Summary by CodeRabbit * **Bug Fixes** * Routed OpenClaw setup now waits for the selected gateway to report ready before restarting it. Setup stops if startup times out, readiness cannot be confirmed, or the sandbox changes while startup is being checked, helping prevent restarts against an unavailable or incorrect gateway. Signed-off-by: Deepak Jain --- src/lib/onboard.ts | 4 +- src/lib/onboard/openclaw-setup.test.ts | 164 ++++++++++++++++++++++++- src/lib/onboard/openclaw-setup.ts | 17 ++- 3 files changed, 176 insertions(+), 9 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 749bcd33f16..60d96a88b26 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2401,15 +2401,15 @@ const setupMessagingChannels = messagingChannelSetup.createSetupMessagingChannel prompt, }); const configSyncDeps = { getProviderSelectionConfig, sandboxCommandExecutor: sandboxExec }; -const syncNemoClawConfigInSandbox = createNemoClawConfigSync(configSyncDeps); const configureOpenclawSandbox = openclawSetup.createConfigureOpenclawSandbox({ - syncNemoClawConfigInSandbox, + syncNemoClawConfigInSandbox: createNemoClawConfigSync(configSyncDeps), }); const setupOpenclaw = openclawSetup.createOpenclawSetup({ step, agentProductName, configureOpenclawSandbox, restartNativeGateway: restartNativeGatewayForInitialSetup, + waitForGatewayStartup: finalizationHandlerDeps.waitForStartedOpenclawGatewayProcess, shouldRestartNativeGateway: isRoutedInferenceProvider, }); const { diff --git a/src/lib/onboard/openclaw-setup.test.ts b/src/lib/onboard/openclaw-setup.test.ts index 3210e1c23f1..bae90d08f1d 100644 --- a/src/lib/onboard/openclaw-setup.test.ts +++ b/src/lib/onboard/openclaw-setup.test.ts @@ -2,6 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + isSandboxGatewayHttpReachableForStatus, + waitForStartedNativeGatewayProcess, +} from "../actions/sandbox/process-recovery"; const configMocks = vi.hoisted(() => ({ readSandboxConfig: vi.fn(), @@ -143,10 +147,19 @@ describe("OpenClaw sandbox setup", () => { configureOpenclawSandbox, initializeOpenclawInferenceRoute: vi.fn(async () => undefined), restartNativeGateway, + waitForGatewayStartup: vi.fn(async () => true), shouldRestartNativeGateway: (provider) => provider === "nvidia-router", }); - await setup("spark-box", "model", "nvidia-router", revalidateSandboxIdentity); + await setup( + "spark-box", + "model", + "nvidia-router", + revalidateSandboxIdentity, + null, + false, + "nemoclaw-19090", + ); expect(configureOpenclawSandbox).toHaveBeenCalledExactlyOnceWith( "spark-box", @@ -154,10 +167,146 @@ describe("OpenClaw sandbox setup", () => { "nvidia-router", revalidateSandboxIdentity, ); - expect(restartNativeGateway).toHaveBeenCalledExactlyOnceWith("spark-box"); + expect(restartNativeGateway).toHaveBeenCalledExactlyOnceWith("spark-box", "nemoclaw-19090"); expect(configureOpenclawSandbox).toHaveBeenCalledBefore(restartNativeGateway); }); + it("waits through connection refusal before restarting the routed gateway (#13465)", async () => { + let now = 0; + const runBuffered = vi + .fn() + .mockResolvedValueOnce({ + outcome: { kind: "completed", exitCode: 0 }, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSTOPPED", + stderr: "", + }) + .mockResolvedValueOnce({ + outcome: { kind: "completed", exitCode: 0 }, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING", + stderr: "", + }); + const probe = vi.fn((name, gateway, options) => + isSandboxGatewayHttpReachableForStatus(name, gateway, { + ...options, + commandExecutor: { runBuffered } as never, + getHealthProbeUrl: () => "http://127.0.0.1:18790/health", + }), + ); + const waitForStartedOpenclawGatewayProcess = vi.fn((name: string, gateway: string) => + waitForStartedNativeGatewayProcess(name, "openclaw", gateway, { + probe, + now: () => now, + delay: async (ms) => { + now += ms; + }, + environment: {}, + }), + ); + const restartNativeGateway = vi.fn(async () => ({ ok: true as const })); + const configureOpenclawSandbox = vi.fn(async () => undefined); + const setup = createOpenclawSetup({ + step: vi.fn(), + agentProductName: () => "OpenClaw", + configureOpenclawSandbox, + restartNativeGateway, + shouldRestartNativeGateway: () => true, + waitForGatewayStartup: waitForStartedOpenclawGatewayProcess, + }); + + await setup("spark-box", "model", "nvidia-router", vi.fn(), null, false, "nemoclaw-19090"); + + expect(probe).toHaveBeenCalledTimes(2); + expect(runBuffered).toHaveBeenLastCalledWith( + expect.objectContaining({ + target: { kind: "named", gatewayName: "nemoclaw-19090" }, + command: expect.arrayContaining([expect.stringContaining("http://127.0.0.1:18790/health")]), + }), + ); + expect(probe).toHaveBeenLastCalledWith("spark-box", "nemoclaw-19090", { + startup: { timeoutMs: expect.any(Number) }, + }); + expect(configureOpenclawSandbox).toHaveBeenCalledBefore(waitForStartedOpenclawGatewayProcess); + expect(waitForStartedOpenclawGatewayProcess).toHaveBeenCalledBefore(restartNativeGateway); + expect(restartNativeGateway).toHaveBeenCalledExactlyOnceWith("spark-box", "nemoclaw-19090"); + }); + + it.each([ + { name: "deadline exhaustion", observation: false, elapsed: 0 }, + { name: "inconclusive transport", observation: null, elapsed: 0 }, + { name: "readiness after the deadline", observation: true, elapsed: 30_000 }, + ])("withholds routed restart and success on $name (#13465)", async ({ observation, elapsed }) => { + let now = 0; + const probe = vi.fn(async () => { + now += elapsed; + return observation; + }); + const restartNativeGateway = vi.fn(async () => ({ ok: true as const })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + try { + const setup = createOpenclawSetup({ + step: vi.fn(), + agentProductName: () => "OpenClaw", + configureOpenclawSandbox: vi.fn(async () => undefined), + shouldRestartNativeGateway: () => true, + restartNativeGateway, + waitForGatewayStartup: (name, gateway) => + waitForStartedNativeGatewayProcess(name, "openclaw", gateway, { + probe, + now: () => now, + delay: async (ms) => { + now += ms; + }, + environment: {}, + }), + }); + await expect( + setup("spark-box", "model", "nvidia-router", vi.fn(), null, false, "nemoclaw-19090"), + ).rejects.toThrow("startup did not settle before routed gateway restart"); + expect(now).toBeLessThanOrEqual(30_000); + expect(probe).toHaveBeenCalled(); + expect(restartNativeGateway).not.toHaveBeenCalled(); + expect(log.mock.calls.flat().join("\n")).not.toContain("gateway launched"); + } finally { + log.mockRestore(); + } + }); + + it("refuses routed restart when identity changes during startup settlement (#13465)", async () => { + const revalidateSandboxIdentity = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementationOnce(() => { + throw new Error("sandbox identity changed"); + }); + const restartNativeGateway = vi.fn(); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + try { + const setup = createOpenclawSetup({ + step: vi.fn(), + agentProductName: () => "OpenClaw", + configureOpenclawSandbox: vi.fn(async () => undefined), + shouldRestartNativeGateway: () => true, + restartNativeGateway, + waitForGatewayStartup: vi.fn(async () => true), + }); + await expect( + setup( + "spark-box", + "model", + "nvidia-router", + revalidateSandboxIdentity, + null, + false, + "nemoclaw-19090", + ), + ).rejects.toThrow("sandbox identity changed"); + expect(restartNativeGateway).not.toHaveBeenCalled(); + expect(log.mock.calls.flat().join("\n")).not.toContain("gateway launched"); + } finally { + log.mockRestore(); + } + }); + it("leaves ordinary providers on their initial native gateway", async () => { const restartNativeGateway = vi.fn(async () => ({ ok: true as const })); const setup = createOpenclawSetup({ @@ -166,6 +315,7 @@ describe("OpenClaw sandbox setup", () => { configureOpenclawSandbox: vi.fn(async () => undefined), initializeOpenclawInferenceRoute: vi.fn(async () => undefined), restartNativeGateway, + waitForGatewayStartup: vi.fn(async () => true), shouldRestartNativeGateway: (provider) => provider === "nvidia-router", }); @@ -191,6 +341,7 @@ describe("OpenClaw sandbox setup", () => { }), initializeOpenclawInferenceRoute, restartNativeGateway: vi.fn(async () => ({ ok: true as const })), + waitForGatewayStartup: vi.fn(async () => true), shouldRestartNativeGateway: () => false, }); @@ -224,6 +375,7 @@ describe("OpenClaw sandbox setup", () => { configureOpenclawSandbox: vi.fn(async () => undefined), initializeOpenclawInferenceRoute, restartNativeGateway: vi.fn(async () => ({ ok: true as const })), + waitForGatewayStartup: vi.fn(async () => true), shouldRestartNativeGateway: () => false, }); @@ -254,6 +406,7 @@ describe("OpenClaw sandbox setup", () => { }, initializeOpenclawInferenceRoute: vi.fn(async () => undefined), restartNativeGateway: vi.fn(), + waitForGatewayStartup: vi.fn(async () => true), shouldRestartNativeGateway: () => false, }); @@ -280,12 +433,13 @@ describe("OpenClaw sandbox setup", () => { failureLayer: "native agent command", detail: "restart rejected", })), + waitForGatewayStartup: vi.fn(async () => true), shouldRestartNativeGateway: () => true, }); - await expect(setup("spark-box", "model", "nvidia-router")).rejects.toThrow( - /native gateway restart failed.*restart rejected/, - ); + await expect( + setup("spark-box", "model", "nvidia-router", undefined, null, false, "nemoclaw-19090"), + ).rejects.toThrow(/native gateway restart failed.*restart rejected/); expect(log.mock.calls.flat().join("\n")).not.toContain("gateway launched"); } finally { log.mockRestore(); diff --git a/src/lib/onboard/openclaw-setup.ts b/src/lib/onboard/openclaw-setup.ts index 02420c497aa..57739dfdad0 100644 --- a/src/lib/onboard/openclaw-setup.ts +++ b/src/lib/onboard/openclaw-setup.ts @@ -89,7 +89,11 @@ export interface OpenclawSetupDeps { step(n: number, total: number, msg: string): void; agentProductName(): string; shouldRestartNativeGateway(provider: string): boolean; - restartNativeGateway(sandboxName: string): Promise< + waitForGatewayStartup(sandboxName: string, gatewayName: string): Promise; + restartNativeGateway( + sandboxName: string, + gatewayName?: string, + ): Promise< | { ok: true } | { ok: false; @@ -138,8 +142,17 @@ export function createOpenclawSetup(deps: OpenclawSetupDeps) { revalidateSandboxIdentity, ); } else if (deps.shouldRestartNativeGateway(provider)) { + if (!gatewayName) { + throw new Error("Routed OpenClaw setup requires an explicit gateway name."); + } + revalidateSandboxIdentity?.(`wait for native OpenClaw gateway in sandbox '${sandboxName}'`); + if ((await deps.waitForGatewayStartup(sandboxName, gatewayName)) !== true) { + throw new Error( + `OpenClaw startup did not settle before routed gateway restart for sandbox '${sandboxName}'.`, + ); + } revalidateSandboxIdentity?.(`restart native OpenClaw gateway in sandbox '${sandboxName}'`); - const restart = await deps.restartNativeGateway(sandboxName); + const restart = await deps.restartNativeGateway(sandboxName, gatewayName); if (!restart.ok) { throw new Error( `OpenClaw native gateway restart failed during setup (${restart.failureLayer}): ${restart.detail}`, From 29a4a58a7a33ad03e10d388ca4427f3e3ea23db7 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas <34834085+prekshivyas@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:32:12 -0700 Subject: [PATCH 18/22] feat(inference): enable hosted Super VL image inputs (#13371) ## Outcome NVIDIA Endpoints offers `nvidia/nemotron-3.5-super-vl-120b-a12b` in the bundled model picker. Selecting it enables text and image input for OpenClaw, including unattended Brev onboarding. ## Reason The hosted Super VL model is available with image input, but unattended onboarding recognizes only the local NVFP4 checkpoint identity. The hosted model therefore retains text-only configuration. Adding the hosted model to the picker alone does not set its input capabilities. Model source: [NVIDIA API Catalog](https://build.nvidia.com/nvidia/nemotron-3.5-super-vl-120b-a12b). ## Changes - Append hosted Super VL to the NVIDIA fallback catalog consumed by Brev's generated picker, preserving existing menu choices and the shared default model. - Default its OpenClaw input capability to text and image while preserving explicit operator overrides. - Keep OpenRouter's existing fallback choices. Brev extracts literal model objects from each provider array, so these entries remain explicit. - Cover model selection, the real setup-flow handoff in interactive and unattended modes, and propagation through the managed startup profile, environment mapper, and generated OpenClaw configuration. - Document the hosted model's automatic input selection and explicit-override precedence in the capability guide and OpenClaw command reference. ## Verification - Focused Vitest run across eight inference and onboarding test files: **296 passed**. - Targeted onboarding-selection and Brev diagnostic-cleanup regression checks: **2 passed**. - Setup-flow suites after the review repair: **35 passed**, including both new hosted-model cases. - `npm run docs`: passed with zero errors. Fern reports an unauthenticated redirect-check warning and an unchanged theme-contrast warning. Generated reference text appears only in the OpenClaw variant. - CLI type-checking with an 8 GiB Node heap: passed on the repaired commit through the installed pre-push hook. - Brev `sync-nemoclaw-catalog.mjs` at `e7a1503ae904ba2a4c7bb32d5419a97e053ccb64`, against this candidate: generated picker includes Super VL; existing order, default, and OpenRouter choices preserved. - Installed pre-commit, commit-message, and pre-push hooks: passed. All three published commits are GitHub Verified. - Required source CI passed on `dc809fa0d92204077bf1b31200e7608a2f9cd5d8`: [run 38966651411](https://github.com/NVIDIA/NemoClaw/actions/runs/38966651411), including all 12 CLI test shards. - CodeRabbit completed with no actionable findings. All nine Advisor source finding ledgers are clear, but [the Advisor blocker gate](https://github.com/NVIDIA/NemoClaw/actions/runs/38969037988/job/117204184211) failed on one unresolved manual-only `network-policy` E2E recommendation. This review gate remains unresolved. - Supplemental managed-image qualification is tracked separately in [run 38966651487](https://github.com/NVIDIA/NemoClaw/actions/runs/38966651487); it is not evidence of hosted Super VL inference. - Reviewed diff: no secrets, API keys, credentials, or unrelated changes. ## Review notes Self-reviewed commit `dc809fa0d92204077bf1b31200e7608a2f9cd5d8` in `NVIDIA/NemoClaw`, covering changed paths under `src/lib/inference/` and `src/lib/onboard/`. Independent maintainer review and scope acceptance remain pending. Validation ran in a container without host credentials, SSH agent, or Docker socket; installed publication hooks use the same boundary. Brev rollout requires selecting an exact tagged NemoClaw release containing this change, then rebuilding its image and regenerating the model catalog. No production image was promoted. Live hosted inference and a deployed Brev browser session were not tested; local Spark qualification is separate evidence. Before production rollout, qualify a hosted text request and image request through the deployed Brev/OpenClaw flow. --- Signed-off-by: Prekshi Vyas ## Summary by CodeRabbit * **New Features** * Added NVIDIA Nemotron 3.5 Super VL 120B to the cloud model picker and featured model options. * During setup, the hosted model defaults to text and image inputs. Valid explicit input settings are preserved, and setup proceeds without an additional prompt. * Kept Nemotron 3.5 Super VL 120B out of OpenRouter fallback options, which continue to include Nemotron 3 Ultra, Nemotron 3 Super, and MiniMax M3. --- .../configure-model-capabilities.mdx | 7 +-- docs/reference/commands.mdx | 2 +- src/lib/inference/config.test.ts | 10 +++++ src/lib/inference/config.ts | 5 ++- src/lib/inference/model-prompts.test.ts | 9 ++++ .../inference/nvidia-featured-models.test.ts | 1 + .../inference-input-capability.test.ts | 17 +++++++ src/lib/onboard/inference-input-capability.ts | 5 +++ .../managed-startup-profile-builder.test.ts | 45 +++++++++++++++++++ .../setup-nim-flow-hosted-models.test.ts | 41 +++++++++++++++++ 10 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 src/lib/onboard/setup-nim-flow-hosted-models.test.ts diff --git a/docs/inference/configure-model-capabilities.mdx b/docs/inference/configure-model-capabilities.mdx index dd812886565..44778ff5535 100644 --- a/docs/inference/configure-model-capabilities.mdx +++ b/docs/inference/configure-model-capabilities.mdx @@ -70,10 +70,11 @@ export NEMOCLAW_INFERENCE_INPUTS=text,image nemoclaw onboard ``` -During interactive onboarding, NemoClaw prompts for **Text only** or **Text + Image** when the discovered model name appears multimodal and `NEMOCLAW_INFERENCE_INPUTS` is not already valid. -When `NEMOCLAW_INFERENCE_INPUTS` is unset, non-interactive onboarding enables `text,image` only for a registered managed-vLLM model whose catalog declares multimodal input. +For hosted NVIDIA Endpoints model `nvidia/nemotron-3.5-super-vl-120b-a12b`, both interactive and non-interactive onboarding select `text,image` automatically unless you supply an explicit valid override. +For other models, interactive onboarding prompts for **Text only** or **Text + Image** when the discovered model name appears multimodal and `NEMOCLAW_INFERENCE_INPUTS` is not already valid. +When `NEMOCLAW_INFERENCE_INPUTS` is unset, non-interactive onboarding also enables `text,image` for a registered managed-vLLM model whose catalog declares multimodal input. This includes NVIDIA Nemotron 3.5 Super VL on one DGX Station. -Other models default to `text`; a model name that merely looks multimodal does not enable image input. +Other models default to `text` during non-interactive onboarding; a model name that merely looks multimodal does not enable image input. ## Apply Capability Changes diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f5e5939d110..31e2e92d97a 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4361,7 +4361,7 @@ The following variables let you tune onboarding without editing the Dockerfile o | `NEMOCLAW_INFERENCE_INPUTS` | comma-separated list of `text` and/or `image` | OpenClaw only. - Declares model input modalities. Unsupported or duplicate values are rejected. An explicit valid value takes precedence; unattended onboarding defaults to `text,image` for registered multimodal managed-vLLM models such as Super VL. | | + Declares model input modalities. Unsupported or duplicate values are rejected. An explicit valid value takes precedence. Hosted `nvidia/nemotron-3.5-super-vl-120b-a12b` defaults to `text,image` in both interactive and unattended onboarding. Unattended onboarding also selects `text,image` for registered multimodal managed-vLLM models such as Super VL. | | `NEMOCLAW_OPENCLAW_OTEL` | `1` to enable | Enables OpenClaw conversation diagnostics export through the `diagnostics-otel` plugin. Disabled by default. | | `NEMOCLAW_OPENCLAW_OTEL_ENDPOINT` | OTLP/HTTP URL | Sets the OpenTelemetry collector endpoint for OpenClaw diagnostics. Defaults to diff --git a/src/lib/inference/config.test.ts b/src/lib/inference/config.test.ts index 9e3fb362d6f..ff687bed394 100644 --- a/src/lib/inference/config.test.ts +++ b/src/lib/inference/config.test.ts @@ -22,6 +22,7 @@ import { LLAMA_CPP_LOCAL_CREDENTIAL_ENV, MANAGED_PROVIDER_ID, OLLAMA_LOCAL_CREDENTIAL_ENV, + OPENROUTER_CLOUD_MODEL_OPTIONS, planInferenceRouteReconcile, resolveAgentDefaultCloudModel, resolveAgentInferenceApi, @@ -103,6 +104,7 @@ describe("inference selection config", () => { expect(CLOUD_MODEL_OPTIONS).toEqual([ { id: "nvidia/nemotron-3-ultra-550b-a55b", label: "Nemotron 3 Ultra 550B" }, { id: "nvidia/nemotron-3-super-120b-a12b", label: "Nemotron 3 Super 120B" }, + { id: "nvidia/nemotron-3.5-super-vl-120b-a12b", label: "Nemotron 3.5 Super VL 120B" }, ]); expect(CLOUD_MODEL_OPTIONS.map((option: { id: string }) => option.id)).not.toContain( "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", @@ -128,6 +130,14 @@ describe("inference selection config", () => { ); }); + it("keeps the NVIDIA hosted Super VL addition out of the OpenRouter fallback", () => { + expect(OPENROUTER_CLOUD_MODEL_OPTIONS.map((option) => option.id)).toEqual([ + "nvidia/nemotron-3-ultra-550b-a55b", + "nvidia/nemotron-3-super-120b-a12b", + "minimaxai/minimax-m3", + ]); + }); + it("aligns Hermes Provider defaults with the Hermes Agent Nous catalog", () => { expect(DEFAULT_HERMES_PROVIDER_MODEL).toBe("moonshotai/kimi-k2.6"); expect(HERMES_PROVIDER_MODEL_OPTIONS.slice(0, 10)).toEqual([ diff --git a/src/lib/inference/config.ts b/src/lib/inference/config.ts index e65e11ddc48..46a518ab15f 100644 --- a/src/lib/inference/config.ts +++ b/src/lib/inference/config.ts @@ -106,9 +106,12 @@ export const DEFAULT_HERMES_PROVIDER_MODEL = HERMES_PROVIDER_MODEL_OPTIONS[0]; export const CLOUD_MODEL_OPTIONS = [ { id: "nvidia/nemotron-3-ultra-550b-a55b", label: "Nemotron 3 Ultra 550B" }, { id: "nvidia/nemotron-3-super-120b-a12b", label: "Nemotron 3 Super 120B" }, + { id: "nvidia/nemotron-3.5-super-vl-120b-a12b", label: "Nemotron 3.5 Super VL 120B" }, ]; +// Brev extracts literal entries from each provider array; keep these entries explicit. export const OPENROUTER_CLOUD_MODEL_OPTIONS = [ - ...CLOUD_MODEL_OPTIONS, + { id: "nvidia/nemotron-3-ultra-550b-a55b", label: "Nemotron 3 Ultra 550B" }, + { id: "nvidia/nemotron-3-super-120b-a12b", label: "Nemotron 3 Super 120B" }, { id: "minimaxai/minimax-m3", label: "Minimax M3" }, ]; export const DEFAULT_ROUTE_PROFILE = "inference-local"; diff --git a/src/lib/inference/model-prompts.test.ts b/src/lib/inference/model-prompts.test.ts index 648b891723d..9b6ee5a5bf7 100644 --- a/src/lib/inference/model-prompts.test.ts +++ b/src/lib/inference/model-prompts.test.ts @@ -57,6 +57,15 @@ describe("model prompt helpers", () => { expect(result).toBe("live/default"); }); + it("selects hosted Super VL after the existing cloud model choices", async () => { + const result = await promptCloudModel({ + promptFn: promptSequence(["3"]), + writeLine: vi.fn(), + }); + + expect(result).toBe("nvidia/nemotron-3.5-super-vl-120b-a12b"); + }); + it("keeps a separate safe custom model as the Other prefill (#5827)", async () => { const promptFn = promptSequence(["2", ""]); const validateNvidiaEndpointModelFn = vi.fn((model: string) => ({ diff --git a/src/lib/inference/nvidia-featured-models.test.ts b/src/lib/inference/nvidia-featured-models.test.ts index ec01a84dade..2b18469475d 100644 --- a/src/lib/inference/nvidia-featured-models.test.ts +++ b/src/lib/inference/nvidia-featured-models.test.ts @@ -163,6 +163,7 @@ describe("NVIDIA featured model catalog", () => { expect(models.map((model) => model.id)).toEqual([ "nvidia/nemotron-3-ultra-550b-a55b", "nvidia/nemotron-3-super-120b-a12b", + "nvidia/nemotron-3.5-super-vl-120b-a12b", ]); expect(warnings).toEqual([ " Warning: failed to load NVIDIA's featured model catalog; falling back to the bundled list (service unavailable; HTTP 503).", diff --git a/src/lib/onboard/inference-input-capability.test.ts b/src/lib/onboard/inference-input-capability.test.ts index 97025f8530b..96fbbcc7226 100644 --- a/src/lib/onboard/inference-input-capability.test.ts +++ b/src/lib/onboard/inference-input-capability.test.ts @@ -5,6 +5,21 @@ import { describe, expect, it, vi } from "vitest"; import { maybePromptForInferenceInputCapability } from "./inference-input-capability"; describe("catalog inference inputs for unattended onboarding", () => { + it.each([true, false])( + "defaults hosted Super VL to text and image with unattended=%s", + async (unattended) => { + const env: NodeJS.ProcessEnv = {}; + const prompt = vi.fn(); + await maybePromptForInferenceInputCapability("nvidia/nemotron-3.5-super-vl-120b-a12b", { + env, + isNonInteractive: () => unattended, + prompt, + }); + expect(env.NEMOCLAW_INFERENCE_INPUTS).toBe("text,image"); + expect(prompt).not.toHaveBeenCalled(); + }, + ); + it.each([ "nemotron-3.5-super-vl-120b", "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", @@ -27,6 +42,8 @@ describe("catalog inference inputs for unattended onboarding", () => { { model: null, inputs: undefined }, { model: "nemotron-3.5-super-vl-120b", inputs: "text" }, { model: "nemotron-3.5-super-vl-120b", inputs: "image,text" }, + { model: "nvidia/nemotron-3.5-super-vl-120b-a12b", inputs: "text" }, + { model: "nvidia/nemotron-3.5-super-vl-120b-a12b", inputs: "image,text" }, ])( "preserves explicit inputs or unknown capability: $model / $inputs", async ({ model, inputs }) => { diff --git a/src/lib/onboard/inference-input-capability.ts b/src/lib/onboard/inference-input-capability.ts index 3aa0314739f..859a03c707a 100644 --- a/src/lib/onboard/inference-input-capability.ts +++ b/src/lib/onboard/inference-input-capability.ts @@ -31,6 +31,11 @@ export async function maybePromptForInferenceInputCapability( ): Promise { const env = deps.env || process.env; if (isValidInferenceInputsOverride(env.NEMOCLAW_INFERENCE_INPUTS)) return; + // The hosted model has a different identity from the local NVFP4 checkpoint. + if (model === "nvidia/nemotron-3.5-super-vl-120b-a12b") { + env.NEMOCLAW_INFERENCE_INPUTS = "text,image"; + return; + } if (deps.isNonInteractive()) { // Unattended onboarding cannot answer the vision prompt. Use declared // catalog capabilities, never a model-name hint, for its default. diff --git a/src/lib/onboard/managed-startup-profile-builder.test.ts b/src/lib/onboard/managed-startup-profile-builder.test.ts index ba16fcbefe3..08c7ad8e37b 100644 --- a/src/lib/onboard/managed-startup-profile-builder.test.ts +++ b/src/lib/onboard/managed-startup-profile-builder.test.ts @@ -6,7 +6,10 @@ import { createHash } from "node:crypto"; import { describe, expect, it } from "vitest"; +import { buildConfig as buildOpenClawConfig } from "../../../scripts/generate-openclaw-config.mts"; import { PEM } from "./__test-helpers__/corporate-ca-fixtures"; +import { maybePromptForInferenceInputCapability } from "./inference-input-capability"; +import { mapManagedStartupProfileToAgentEnvironment } from "./managed-startup/agent-environment"; import { decodeManagedStartupProfile } from "./managed-startup/profile"; import { assertManagedStartupProfileBuilderInventoryCoverage, @@ -376,6 +379,48 @@ describe("buildManagedStartupProfile", () => { ); }); + it.each([undefined, "text"])( + "carries hosted Super VL inputs through startup with override %s", + async (override) => { + const model = "nvidia/nemotron-3.5-super-vl-120b-a12b"; + const environment: NodeJS.ProcessEnv = { NEMOCLAW_INFERENCE_INPUTS: override }; + await maybePromptForInferenceInputCapability(model, { + env: environment, + isNonInteractive: () => true, + prompt: async () => { + throw new Error("Unattended onboarding must not prompt"); + }, + }); + const transport = buildManagedStartupProfile( + openClawInput({ + environment, + inference: { + routeProvider: "inference", + upstreamProvider: "nvidia-prod", + model, + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + primaryModelRef: `inference/${model}`, + compatibility: {}, + }, + }), + ); + const mapped = mapManagedStartupProfileToAgentEnvironment(transport.profile); + const config = buildOpenClawConfig({ + ...mapped.configurationEnvironment, + ...mapped.runtimeEnvironment, + }); + expect(config).toMatchObject({ + models: { + providers: { + inference: { models: [{ id: model, input: override ? ["text"] : ["image", "text"] }] }, + }, + }, + }); + }, + ); + it("builds OpenClaw with every stock behavior knob and canonical transport", () => { const plan = messagingPlan("openclaw"); const extraAgents = { diff --git a/src/lib/onboard/setup-nim-flow-hosted-models.test.ts b/src/lib/onboard/setup-nim-flow-hosted-models.test.ts new file mode 100644 index 00000000000..267d0112fad --- /dev/null +++ b/src/lib/onboard/setup-nim-flow-hosted-models.test.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { makeDeps } from "./__test-helpers__/setup-nim-flow"; +import { createSetupNim } from "./setup-nim-flow"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("hosted model setup flow", () => { + it.each([false, true])( + "passes hosted Super VL to input capability selection with unattended mode %s", + async (nonInteractive) => { + const model = "nvidia/nemotron-3.5-super-vl-120b-a12b"; + const maybePromptForInferenceInputCapability = vi.fn(async () => {}); + const setupNim = createSetupNim( + makeDeps({ + isNonInteractive: () => nonInteractive, + getNonInteractiveProvider: () => "build", + maybePromptForInferenceInputCapability, + handleRemoteProviderSelection: async ({ selected }, state) => { + expect(selected.key).toBe("build"); + state.model = model; + state.provider = "nvidia-prod"; + state.endpointUrl = "https://integrate.api.nvidia.com/v1"; + state.credentialEnv = "NVIDIA_INFERENCE_API_KEY"; + return "selected"; + }, + }), + ); + + const result = await setupNim(null); + + expect(result.model).toBe(model); + expect(maybePromptForInferenceInputCapability).toHaveBeenCalledExactlyOnceWith(model); + }, + ); +}); From c2bfcac9dcedd0fb47186261c11da6d563ecd995 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter Date: Mon, 19 Oct 2026 16:36:31 -0700 Subject: [PATCH 19/22] test(onboard): honor configured gateway in dashboard fixture (#13500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Outcome The dashboard port test passes when NemoClaw uses its default gateway port and when `NEMOCLAW_GATEWAY_PORT=18080` or `9123`. The production allocator is unchanged. ## Reason The test wrote its current sandbox to the default gateway registry with port 8080. Under a gateway override, the allocator correctly treated that row as a sibling and exhausted its two observed ports. ## Changes - Build the current sandbox's registry path, gateway name, and gateway port from the configured gateway port. - Keep the sibling gateway distinct from the configured gateway, including when the configured port is 9123. Keep the forward observations unchanged. ## Verification - `env -u NEMOCLAW_GATEWAY_PORT npx vitest run --project cli src/lib/onboard/dashboard-port.test.ts` — 57 tests passed. - `NEMOCLAW_GATEWAY_PORT=18080 npx vitest run --project cli src/lib/onboard/dashboard-port.test.ts` — 57 tests passed. - `NEMOCLAW_GATEWAY_PORT=9123 npx vitest run --project cli src/lib/onboard/dashboard-port.test.ts` — 57 tests passed. - `npm run validate:pr` — passed against canonical main `88900837e9705c105470c6c3e33835e4b2a9a523`. - Diff review — one test file changed; no secrets, API keys, or credentials. ## Review notes `src/lib/onboard/dashboard-port.test.ts` matches the contributor sensitive-path policy. I reviewed the diff and gateway-port test runs on commit `cb63fa94ec0afd9e4ecf417319359f17c49cda65` in `NVIDIA/NemoClaw`. No independent pre-publication review exists. This draft awaits review. --- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> ## Summary by CodeRabbit * **Tests** * Expanded dashboard port registry coverage to verify that gateways are identified by their respective ports, including when multiple gateways use separate ports. These checks provide broader validation of gateway occupancy reporting across different test configurations. No user-facing behavior changes are included in this update. --------- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Co-authored-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- src/lib/onboard/dashboard-port.test.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/lib/onboard/dashboard-port.test.ts b/src/lib/onboard/dashboard-port.test.ts index f62b7e1a414..2e50e705909 100644 --- a/src/lib/onboard/dashboard-port.test.ts +++ b/src/lib/onboard/dashboard-port.test.ts @@ -10,6 +10,8 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { GATEWAY_PORT } from "../core/ports"; +import { nemoclawStateRoot } from "../state/state-root"; import type { OpenShellForwardAdapter, OpenShellForwardObservation, @@ -31,6 +33,7 @@ import { withDashboardPortReservationLock, withDashboardPortReservationScope, } from "./dashboard-port"; +import { resolveGatewayName } from "./gateway-binding/identity"; describe("dashboard-port override intent", () => { it.each([ @@ -873,18 +876,21 @@ describe("getRegistryOccupiedDashboardPorts", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dashboard-host-index-")); try { vi.stubEnv("HOME", home); - const defaultRoot = path.join(home, ".nemoclaw"); - const siblingRoot = path.join(defaultRoot, "gateways", "9123"); + const currentRoot = nemoclawStateRoot(home, GATEWAY_PORT); + const siblingPort = GATEWAY_PORT === 9123 ? 9124 : 9123; + const siblingRoot = nemoclawStateRoot(home, siblingPort); + assert.notEqual(currentRoot, siblingRoot); + fs.mkdirSync(currentRoot, { recursive: true }); fs.mkdirSync(siblingRoot, { recursive: true }); fs.writeFileSync( - path.join(defaultRoot, "sandboxes.json"), + path.join(currentRoot, "sandboxes.json"), JSON.stringify({ defaultSandbox: "instance-b", sandboxes: { "instance-b": { name: "instance-b", - gatewayName: "nemoclaw", - gatewayPort: 8080, + gatewayName: resolveGatewayName(GATEWAY_PORT), + gatewayPort: GATEWAY_PORT, dashboardPort: 18790, }, }, @@ -897,8 +903,8 @@ describe("getRegistryOccupiedDashboardPorts", () => { sandboxes: { "instance-a": { name: "instance-a", - gatewayName: "nemoclaw-9123", - gatewayPort: 9123, + gatewayName: resolveGatewayName(siblingPort), + gatewayPort: siblingPort, dashboardPort: 18789, }, }, @@ -906,7 +912,7 @@ describe("getRegistryOccupiedDashboardPorts", () => { ); const occupied = getRegistryOccupiedDashboardPorts("instance-b"); - assert.equal(occupied.get("18789"), "instance-a (gateway 9123)"); + assert.equal(occupied.get("18789"), `instance-a (gateway ${siblingPort})`); assert.equal( findAvailableDashboardPortFromObservations( "instance-b", From 5ac01ac17a7a80ddc52cc3609b91c4671fb9f2b6 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Mon, 19 Oct 2026 17:05:28 -0700 Subject: [PATCH 20/22] fix(pi): qualify the repaired settings generator Signed-off-by: Prekshi Vyas --- ci/pi-agent-qualification-v1-linux-amd64.json | 8 ++++---- ci/pi-agent-qualification-v1-linux-arm64.json | 8 ++++---- src/lib/agent/candidate-authority.ts | 4 ++-- .../runtime/pi-candidate-runtime-artifacts.test.ts | 1 - 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ci/pi-agent-qualification-v1-linux-amd64.json b/ci/pi-agent-qualification-v1-linux-amd64.json index 1d65e04e9f3..7c7d396a465 100644 --- a/ci/pi-agent-qualification-v1-linux-amd64.json +++ b/ci/pi-agent-qualification-v1-linux-amd64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/amd64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:de4c0bbdb1512e3a9a61f467d8da8a08aa337390628f35031e95330e16d98b17", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:de4c0bbdb1512e3a9a61f467d8da8a08aa337390628f35031e95330e16d98b17", + "digest": "sha256:df061c97ba5d1200155016a1e9805dcfc6f06e4b2f7e4520ab38dc60ec3dc8e8", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:df061c97ba5d1200155016a1e9805dcfc6f06e4b2f7e4520ab38dc60ec3dc8e8", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "f421e01e62b4d03a1410157dcce11c3a1aaf3a7c", + "revision": "f4414ccdfdc2e18a84d3f7199d639ae653dcd58d", "release": "v0.1.0", - "cohort": "ghrun-39308521500-3" + "cohort": "ghrun-39343622961-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/ci/pi-agent-qualification-v1-linux-arm64.json b/ci/pi-agent-qualification-v1-linux-arm64.json index e04a42bbe21..a8b3e5ccb59 100644 --- a/ci/pi-agent-qualification-v1-linux-arm64.json +++ b/ci/pi-agent-qualification-v1-linux-arm64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/arm64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:2be599dc736b782af348788ae0bce6d2c01a7327977113315afdfb90b5c50a28", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:2be599dc736b782af348788ae0bce6d2c01a7327977113315afdfb90b5c50a28", + "digest": "sha256:13a6aff892d06e891b8983498b7b5370f5d8be2671ccf18e4455c7f1c6e987e1", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:13a6aff892d06e891b8983498b7b5370f5d8be2671ccf18e4455c7f1c6e987e1", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "f421e01e62b4d03a1410157dcce11c3a1aaf3a7c", + "revision": "f4414ccdfdc2e18a84d3f7199d639ae653dcd58d", "release": "v0.1.0", - "cohort": "ghrun-39308521500-3" + "cohort": "ghrun-39343622961-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/src/lib/agent/candidate-authority.ts b/src/lib/agent/candidate-authority.ts index aa0183df307..36fb94af8ba 100644 --- a/src/lib/agent/candidate-authority.ts +++ b/src/lib/agent/candidate-authority.ts @@ -13,8 +13,8 @@ export const CANDIDATE_QUALIFICATION_RECEIPT_DIGESTS: Readonly< Record > = Object.freeze({ pi: Object.freeze([ - "dd32f76c82140b590943c04071afd3759d537956ce7d5064afa6009b4e716bbd", - "f135ae7a02e6f92077198616237e6db5df8e9edeb1d00b4faa3cf8df23aad605", + "bb61a1632b9050e6c83e264483acd39792f776cbd301a427c71784c2ee114727", + "d4d0281932aa1dc9b78030079768cfcbb21e7b9261f476c061b70c7d01b56ad3", ]), }); diff --git a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts index adaa985871c..b71aa8198d4 100644 --- a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts +++ b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts @@ -533,7 +533,6 @@ syncBuiltinESMExports();\n`, defaultModel: "selected-model", }); expect(fs.statSync(target).mode & 0o777).toBe(0o600); - expect(fs.readFileSync(settingsPath)).toEqual(fs.readFileSync(target)); } finally { fs.rmSync(home, { recursive: true, force: true }); } From 318a57168cd83840fffbfbaecf2ffb7a2fb618d6 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Mon, 19 Oct 2026 17:38:10 -0700 Subject: [PATCH 21/22] fix(pi): preserve catalog links during rollback Signed-off-by: Prekshi Vyas --- agents/pi/generate-config.ts | 20 +- .../pi-candidate-runtime-artifacts.test.ts | 172 ++++++++++++++---- 2 files changed, 148 insertions(+), 44 deletions(-) diff --git a/agents/pi/generate-config.ts b/agents/pi/generate-config.ts index 330aca3d370..d51a686ae83 100644 --- a/agents/pi/generate-config.ts +++ b/agents/pi/generate-config.ts @@ -197,8 +197,8 @@ function buildConfig(settings: Settings): ManagedPiConfig { }; } -function resolveSettingsDestination(settingsPath: string): string { - let destination = settingsPath; +function resolveFileDestination(filePath: string): string { + let destination = filePath; for (;;) { try { return realpathSync.native(destination); @@ -237,7 +237,7 @@ function syncManagedDefaults(configDir: string, model: string, nativeLocal: bool if (!nativeLocal) return; existing = {}; } - const destination = resolveSettingsDestination(settingsPath); + const destination = resolveFileDestination(settingsPath); const stagedPath = join(dirname(destination), `.settings.json.${randomUUID()}.tmp`); let staged = false; let descriptor: number | null = null; @@ -284,24 +284,26 @@ function main(): void { const configPath = join(configDir, "models.json"); const config = buildConfig(settings); + // Snapshot, write, and roll back the same target without removing a user's link. + const configDestination = resolveFileDestination(configPath); let previousCatalog: Buffer | null; try { - previousCatalog = readFileSync(configPath); + previousCatalog = readFileSync(configDestination); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; previousCatalog = null; } try { - writeFileSync(configPath, config.text, { mode: 0o600 }); - chmodSync(configPath, 0o600); + writeFileSync(configDestination, config.text, { mode: 0o600 }); + chmodSync(configDestination, 0o600); syncManagedDefaults(configDir, config.model, config.nativeLocal); } catch (error) { try { if (previousCatalog === null) { - rmSync(configPath, { force: true }); + rmSync(configDestination, { force: true }); } else { - writeFileSync(configPath, previousCatalog, { mode: 0o600 }); - chmodSync(configPath, 0o600); + writeFileSync(configDestination, previousCatalog, { mode: 0o600 }); + chmodSync(configDestination, 0o600); } } catch (rollbackError) { throw new AggregateError([error, rollbackError], "Pi catalog rollback failed."); diff --git a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts index b71aa8198d4..cf8d073f820 100644 --- a/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts +++ b/test/agents/openclaw/runtime/pi-candidate-runtime-artifacts.test.ts @@ -261,43 +261,145 @@ describe("Pi managed model catalog generation", () => { }, ); - it("preserves the previous Pi catalog when settings sync fails (#13362)", () => { - const selected = { - NEMOCLAW_MODEL: "previous-model", - NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", - NEMOCLAW_INFERENCE_BASE_URL: "http://host.openshell.internal:11434/v1", - }; - const { home, status, stderr } = generate(selected); - try { - expect(status, stderr).toBe(0); - const configDir = path.join(home, ".pi/agent"); - const modelsPath = path.join(configDir, "models.json"); - const settingsPath = path.join(configDir, "settings.json"); - const previousCatalog = fs.readFileSync(modelsPath); - fs.writeFileSync(settingsPath, "{invalid json"); + it.each([ + { name: "regular", prepare: (_home: string, _catalog: string) => undefined, linked: false }, + { + name: "symlinked", + prepare: (home: string, catalog: string) => { + const target = path.join(home, "user-models.json"); + fs.renameSync(catalog, target); + fs.symlinkSync(target, catalog); + }, + linked: true, + }, + ])( + "preserves the previous $name Pi catalog when settings sync fails (#13362)", + ({ prepare, linked }) => { + const selected = { + NEMOCLAW_MODEL: "previous-model", + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", + NEMOCLAW_INFERENCE_BASE_URL: "http://host.openshell.internal:11434/v1", + }; + const { home, status, stderr } = generate(selected); + try { + expect(status, stderr).toBe(0); + const configDir = path.join(home, ".pi/agent"); + const modelsPath = path.join(configDir, "models.json"); + const settingsPath = path.join(configDir, "settings.json"); + prepare(home, modelsPath); + const previousCatalog = fs.readFileSync(modelsPath); + fs.writeFileSync(settingsPath, "{invalid json"); - const failed = spawnSync( - process.execPath, - [path.join(root, "agents/pi/generate-config.ts")], - { - cwd: root, - encoding: "utf8", - env: { - PATH: process.env.PATH ?? "", - HOME: home, - ...selected, - NEMOCLAW_MODEL: "next-model", + const failed = spawnSync( + process.execPath, + [path.join(root, "agents/pi/generate-config.ts")], + { + cwd: root, + encoding: "utf8", + env: { + PATH: process.env.PATH ?? "", + HOME: home, + ...selected, + NEMOCLAW_MODEL: "next-model", + }, }, - }, - ); - expect(failed.status).not.toBe(0); - expect(fs.readFileSync(modelsPath)).toEqual(previousCatalog); - expect(fs.statSync(modelsPath).mode & 0o777).toBe(0o600); - expect(fs.readFileSync(settingsPath, "utf8")).toBe("{invalid json"); - } finally { - fs.rmSync(home, { recursive: true, force: true }); - } - }); + ); + expect(failed.status).not.toBe(0); + expect(fs.readFileSync(modelsPath)).toEqual(previousCatalog); + expect(fs.statSync(modelsPath).mode & 0o777).toBe(0o600); + expect(fs.readFileSync(settingsPath, "utf8")).toBe("{invalid json"); + expect(fs.lstatSync(modelsPath).isSymbolicLink()).toBe(linked); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }, + ); + + it.each([ + { + name: "absolute", + prepare: (home: string, catalog: string) => { + const target = path.join(home, "user-models.json"); + fs.symlinkSync(target, catalog); + return target; + }, + }, + { + name: "relative", + prepare: (home: string, catalog: string) => { + fs.symlinkSync("../../user-models.json", catalog); + return path.join(home, "user-models.json"); + }, + }, + { + name: "chained", + prepare: (home: string, catalog: string) => { + const intermediate = path.join(home, "catalog-link.json"); + fs.symlinkSync("user-models.json", intermediate); + fs.symlinkSync(intermediate, catalog); + return path.join(home, "user-models.json"); + }, + }, + { + name: "missing-parent", + prepare: (home: string, catalog: string) => { + const target = path.join(home, "missing-directory/user-models.json"); + fs.symlinkSync(target, catalog); + return target; + }, + }, + ])( + "preserves the $name catalog link through failed startup and retry (#13362)", + ({ prepare }) => { + const selected = { + NEMOCLAW_MODEL: "failed-model", + NEMOCLAW_UPSTREAM_PROVIDER: "compatible-endpoint", + NEMOCLAW_INFERENCE_BASE_URL: "http://host.openshell.internal:11434/v1", + }; + let target = ""; + let originalLink = ""; + const { home, status } = generate(selected, (home) => { + const configDir = path.join(home, ".pi/agent"); + fs.mkdirSync(configDir, { recursive: true }); + const modelsPath = path.join(configDir, "models.json"); + target = prepare(home, modelsPath); + originalLink = fs.readlinkSync(modelsPath); + fs.writeFileSync(path.join(configDir, "settings.json"), "{invalid json"); + }); + try { + expect(status).not.toBe(0); + const modelsPath = path.join(home, ".pi/agent/models.json"); + const settingsPath = path.join(home, ".pi/agent/settings.json"); + expect(fs.readlinkSync(modelsPath)).toBe(originalLink); + expect(fs.existsSync(target)).toBe(false); + expect(fs.readFileSync(settingsPath, "utf8")).toBe("{invalid json"); + + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(settingsPath, "{}"); + const retry = spawnSync( + process.execPath, + [path.join(root, "agents/pi/generate-config.ts")], + { + cwd: root, + encoding: "utf8", + env: { + PATH: process.env.PATH ?? "", + HOME: home, + ...selected, + NEMOCLAW_MODEL: "retry-model", + }, + }, + ); + expect(retry.status, retry.stderr).toBe(0); + expect(JSON.parse(fs.readFileSync(target, "utf8")).defaultModel).toBe("retry-model"); + expect(JSON.parse(fs.readFileSync(settingsPath, "utf8")).defaultModel).toBe("retry-model"); + expect(fs.statSync(target).mode & 0o777).toBe(0o600); + expect(fs.readlinkSync(modelsPath)).toBe(originalLink); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }, + ); it("preserves both Pi files when settings permission update fails (#13362)", () => { const selected = { From cb5fc19bebdeea5a29e3561d5f18c2631263abe8 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Mon, 19 Oct 2026 18:03:05 -0700 Subject: [PATCH 22/22] fix(pi): qualify catalog rollback symlink repair Signed-off-by: Prekshi Vyas --- ci/pi-agent-qualification-v1-linux-amd64.json | 8 ++++---- ci/pi-agent-qualification-v1-linux-arm64.json | 8 ++++---- src/lib/agent/candidate-authority.ts | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/pi-agent-qualification-v1-linux-amd64.json b/ci/pi-agent-qualification-v1-linux-amd64.json index 7c7d396a465..a92e5c8e99e 100644 --- a/ci/pi-agent-qualification-v1-linux-amd64.json +++ b/ci/pi-agent-qualification-v1-linux-amd64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/amd64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:df061c97ba5d1200155016a1e9805dcfc6f06e4b2f7e4520ab38dc60ec3dc8e8", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:df061c97ba5d1200155016a1e9805dcfc6f06e4b2f7e4520ab38dc60ec3dc8e8", + "digest": "sha256:1b7fc149d9d4733f7696ee28a2d410616efc86b17e00f62f7358ba5f7ae8945c", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:1b7fc149d9d4733f7696ee28a2d410616efc86b17e00f62f7358ba5f7ae8945c", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "f4414ccdfdc2e18a84d3f7199d639ae653dcd58d", + "revision": "318a57168cd83840fffbfbaecf2ffb7a2fb618d6", "release": "v0.1.0", - "cohort": "ghrun-39343622961-1" + "cohort": "ghrun-39349087946-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/ci/pi-agent-qualification-v1-linux-arm64.json b/ci/pi-agent-qualification-v1-linux-arm64.json index a8b3e5ccb59..fae776b2782 100644 --- a/ci/pi-agent-qualification-v1-linux-arm64.json +++ b/ci/pi-agent-qualification-v1-linux-arm64.json @@ -3,13 +3,13 @@ "agent": "pi", "platform": "linux/arm64", "image": "ghcr.io/nvidia/nemoclaw/pi-sandbox", - "digest": "sha256:13a6aff892d06e891b8983498b7b5370f5d8be2671ccf18e4455c7f1c6e987e1", - "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:13a6aff892d06e891b8983498b7b5370f5d8be2671ccf18e4455c7f1c6e987e1", + "digest": "sha256:9d4c2d87efd9b85c765922bc5d59c91427aac0a3dd24832e24b11a643350f1d3", + "reference": "ghcr.io/nvidia/nemoclaw/pi-sandbox@sha256:9d4c2d87efd9b85c765922bc5d59c91427aac0a3dd24832e24b11a643350f1d3", "source": { "repository": "NVIDIA/NemoClaw", - "revision": "f4414ccdfdc2e18a84d3f7199d639ae653dcd58d", + "revision": "318a57168cd83840fffbfbaecf2ffb7a2fb618d6", "release": "v0.1.0", - "cohort": "ghrun-39343622961-1" + "cohort": "ghrun-39349087946-1" }, "startupProfileContractVersion": 1, "capabilityContractVersion": 1 diff --git a/src/lib/agent/candidate-authority.ts b/src/lib/agent/candidate-authority.ts index 36fb94af8ba..36211744847 100644 --- a/src/lib/agent/candidate-authority.ts +++ b/src/lib/agent/candidate-authority.ts @@ -13,8 +13,8 @@ export const CANDIDATE_QUALIFICATION_RECEIPT_DIGESTS: Readonly< Record > = Object.freeze({ pi: Object.freeze([ - "bb61a1632b9050e6c83e264483acd39792f776cbd301a427c71784c2ee114727", - "d4d0281932aa1dc9b78030079768cfcbb21e7b9261f476c061b70c7d01b56ad3", + "01d2019bac5227bcc0fd41ecef8b018857e7b68f81d8d6b2d39fbcf8fea0ce64", + "e011809172224f6e09649afbeb9c91d4a06500fdd17a8ea0698d4f55f8ca18da", ]), });