diff --git a/src/main/lib/agentTap.test.ts b/src/main/lib/agentTap.test.ts index 3dff5a122..2432a8119 100644 --- a/src/main/lib/agentTap.test.ts +++ b/src/main/lib/agentTap.test.ts @@ -39,6 +39,7 @@ const { ALLOWED_EVENTS, ALLOWED_FIELD_NAMES, REASONS, + STARTUP_STEPS, PRODUCT_EVENTS, PRODUCT_FIELDS, _resetProductEventBudgetForTest @@ -58,7 +59,12 @@ const AGENT_EVENTS = [ 'node_fetch_failed', 'health_check_failed', 'agent_exited', - 'agent_error' + 'agent_error', + 'startup_step', + 'startup_stalled', + 'startup_failed', + 'document_host_degraded', + 'agent_orphans_stopped' ] describe('agentTap', () => { @@ -114,7 +120,32 @@ describe('agentTap', () => { it('exposes exactly the agent field allowlist', () => { expect([...ALLOWED_FIELD_NAMES].sort()).toEqual( - ['agent_version', 'code', 'duration_ms', 'node_version', 'reason'].sort() + [ + 'agent_version', + 'code', + 'duration_ms', + 'node_version', + 'reason', + 'step', + 'outcome', + 'attempt', + 'count', + 'uptime_ms', + 'startup_ms', + 'ready', + 'crash', + 'missing', + 'python_arch', + 'python_version', + 'crdt', + 'cli', + 'comfyui', + 'comfyui_found_by', + 'comfyui_match', + 'shell', + 'sandbox', + 'assets' + ].sort() ) }) @@ -189,7 +220,11 @@ describe('agentTap', () => { 'checksum_mismatch', 'permission_denied', 'disabled', - 'unknown' + 'unknown', + 'data_dir_locked', + 'address_in_use', + 'not_set_up', + 'config_invalid' ].sort() ) }) @@ -411,6 +446,17 @@ describe('agentTap', () => { }) }) + describe('startup step cap', () => { + it('holds four full launches an hour', () => { + vi.useFakeTimers() + vi.setSystemTime(0) + const tap = createAgentTap(baseOpts) + const line = '[agent-event] startup_step step=listen outcome=ok duration_ms=1\n' + tap.ingest(line.repeat(241), 'stdout') + expect(captured).toHaveLength(240) + }) + }) + describe('rate window', () => { it('keeps the cap until a full hour has passed', () => { vi.useFakeTimers() @@ -915,3 +961,135 @@ describe('agentTap consent gating', () => { ]) }) }) + +// The host's startup records, byte-identical to cloud's +// services/agent/packaging/testdata/startup_events.txt, and the step ids to +// its startup_step_ids.txt. +const STARTUP_FIXTURE = fs + .readFileSync(path.join(__dirname, 'fixtures', 'agent-startup-events.txt'), 'utf8') + .trimEnd() + .split('\n') +const STARTUP_STEP_IDS = fs + .readFileSync(path.join(__dirname, 'fixtures', 'agent-startup-step-ids.txt'), 'utf8') + .trim() + .split('\n') + +describe('agentTap startup records', () => { + let captured: Array<{ event: string; ctx: Record }> + + beforeEach(() => { + captured = [] + vi.spyOn(telemetry, 'emit').mockImplementation((event, ctx) => { + captured.push({ event, ctx: ctx as Record }) + }) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + // As ComfyUI prints a record it logs: ANSI level tag, CRLF on Windows. + function ingest(lines: string[]): void { + createAgentTap({ installationId: 'inst-1' }).ingest( + lines.map((line) => `\u001b[32m[INFO]\u001b[0m ${line}\r\n`).join(''), + 'stderr' + ) + } + + function fields(line: string): Record { + ingest([line]) + expect(captured, line).toHaveLength(1) + const ctx = { ...captured[0]?.ctx } + for (const key of ['installation_id', 'variant', 'release', 'core_beta_flags']) delete ctx[key] + captured = [] + return ctx + } + + it('knows every step id the agent reports, plus init and other', () => { + expect([...STARTUP_STEPS].sort()).toEqual([...STARTUP_STEP_IDS, 'init', 'other'].sort()) + }) + + it('forwards every fixture record with its typed fields', () => { + ingest(STARTUP_FIXTURE) + expect(captured.map((c) => c.event)).toEqual( + STARTUP_FIXTURE.map((line) => `comfy.desktop.comfyui.agent.${line.split(' ')[1]}`) + ) + expect(captured[1]?.ctx).toMatchObject({ + step: 'cli_provision', + outcome: 'degraded', + duration_ms: 30500, + reason: 'network_error' + }) + expect(captured[4]?.ctx).toMatchObject({ step: 'object_info', reason: 'timeout' }) + expect(captured[6]?.ctx).toMatchObject({ duration_ms: 41000, node_version: 'v22.11.0' }) + }) + + it('types the host lifecycle fields', () => { + expect( + fields( + '[agent-event] agent_starting agent_version=0.1.0 attempt=2 python_arch=amd64 python_version=3.12.4' + ) + ).toEqual({ + agent_version: '0.1.0', + attempt: 2, + python_arch: 'amd64', + python_version: '3.12.4' + }) + expect( + fields( + '[agent-event] agent_started duration_ms=4100 agent_version=0.1.0 startup_ms=3900 crdt=on cli=provisioned comfyui=found comfyui_found_by=explicit comfyui_match=mismatch shell=off sandbox=appcontainer assets=enabled' + ) + ).toEqual({ + duration_ms: 4100, + agent_version: '0.1.0', + startup_ms: 3900, + crdt: 'on', + cli: 'provisioned', + comfyui: 'found', + comfyui_found_by: 'explicit', + comfyui_match: 'mismatch', + shell: 'off', + sandbox: 'appcontainer', + assets: 'enabled' + }) + expect( + fields('[agent-event] agent_exited code=2 uptime_ms=812 ready=false crash=panic') + ).toEqual({ + code: 2, + uptime_ms: 812, + ready: false, + crash: 'panic' + }) + expect(fields('[agent-event] agent_orphans_stopped count=2')).toEqual({ count: 2 }) + expect(fields('[agent-event] package_missing missing=comfy_cli_and_node')).toEqual({ + missing: 'comfy_cli_and_node' + }) + expect(fields('[agent-event] startup_failed step=init reason=config_invalid')).toEqual({ + step: 'init', + reason: 'config_invalid' + }) + }) + + it('reports a state value it does not know as unknown, keeping the event', () => { + expect(fields('[agent-event] startup_step step=new_step outcome=great duration_ms=5')).toEqual({ + step: 'unknown', + outcome: 'unknown', + duration_ms: 5 + }) + }) + + it('drops an event whose count is negative or whose version is not one', () => { + ingest([ + '[agent-event] agent_exited uptime_ms=-1', + '[agent-event] agent_orphans_stopped count=-3', + '[agent-event] agent_starting python_version=three' + ]) + expect(captured).toEqual([]) + }) + + it('never lets a line set the platform or arch Desktop attaches itself', () => { + expect(fields('[agent-event] agent_starting attempt=1 platform=x arch=y')).toEqual({ + attempt: 1 + }) + }) +}) diff --git a/src/main/lib/agentTap.ts b/src/main/lib/agentTap.ts index 2b5ac8f39..b0aaac439 100644 --- a/src/main/lib/agentTap.ts +++ b/src/main/lib/agentTap.ts @@ -21,7 +21,14 @@ export const ALLOWED_EVENTS: ReadonlySet = new Set([ 'node_fetch_failed', 'health_check_failed', 'agent_exited', - 'agent_error' + 'agent_error', + // The agent's startup, built by its v1 host from the agent's own log + // (cloud services/agent/packaging/comfy_agent/host.py AgentRecords). + 'startup_step', + 'startup_stalled', + 'startup_failed', + 'document_host_degraded', + 'agent_orphans_stopped' ]) // Absent from ALLOWED_EVENTS, so no line can forge it. @@ -41,7 +48,34 @@ export const REASONS: ReadonlySet = new Set([ 'checksum_mismatch', 'permission_denied', 'disabled', - 'unknown' + 'unknown', + 'data_dir_locked', + 'address_in_use', + 'not_set_up', + 'config_invalid' +]) + +// The agent's startup step ids: byte-identical to cloud's +// services/agent/packaging/testdata/startup_step_ids.txt, plus "init" (a +// failure before the first step) and "other" (a step this list does not know). +export const STARTUP_STEPS: ReadonlySet = new Set([ + 'init', + 'open_data_dir', + 'cli_provision', + 'find_comfyui', + 'code_search_canary', + 'find_node', + 'sandbox_config', + 'egress_proxy', + 'sandbox_setup', + 'sandbox_grants', + 'object_info', + 'knowledge', + 'cli_skills', + 'detect_environment', + 'document_host', + 'listen', + 'other' ]) // A Set, not an object literal, so untrusted keys like `constructor` can't hit the prototype. @@ -50,7 +84,56 @@ export const ALLOWED_FIELD_NAMES: ReadonlySet = new Set([ 'duration_ms', 'agent_version', 'node_version', - 'reason' + 'reason', + // Typed in LIFECYCLE_ENUMS / LIFECYCLE_COUNTS / fieldValue. + 'step', + 'outcome', + 'attempt', + 'count', + 'uptime_ms', + 'startup_ms', + 'ready', + 'crash', + 'missing', + 'python_arch', + 'python_version', + 'crdt', + 'cli', + 'comfyui', + 'comfyui_found_by', + 'comfyui_match', + 'shell', + 'sandbox', + 'assets' +]) + +// Closed sets for the lifecycle fields that name a state. A value outside its +// set still reports the event, as "unknown", like `reason`. +const LIFECYCLE_ENUMS: ReadonlyMap> = new Map( + Object.entries({ + step: STARTUP_STEPS, + outcome: ['ok', 'degraded', 'failed'], + crash: ['panic', 'fatal_error', 'none'], + missing: ['comfy_cli', 'node', 'comfy_cli_and_node'], + python_arch: ['amd64', 'arm64', 'other'], + crdt: ['on', 'disabled', 'failed', 'other'], + cli: ['provisioned', 'override', 'unavailable', 'other'], + comfyui: ['found', 'not_found'], + comfyui_found_by: ['explicit', 'server', 'known_location', 'none', 'other'], + comfyui_match: ['match', 'mismatch', 'unknown', 'other'], + shell: ['on', 'off'], + sandbox: ['none', 'seatbelt', 'appcontainer', 'other'], + assets: ['enabled', 'disabled', 'unknown', 'other'] + }).map(([name, values]): [string, ReadonlySet] => [name, new Set(values)]) +) + +// Lifecycle fields that are non-negative counts (unlike `code`, an exit code). +const LIFECYCLE_COUNTS: ReadonlySet = new Set([ + 'duration_ms', + 'attempt', + 'count', + 'uptime_ms', + 'startup_ms' ]) const VERSION = /^v?\d{1,6}(?:\.\d{1,6}){1,3}(?:[-+.]?[0-9A-Za-z][0-9A-Za-z.+-]{0,39})?$/ @@ -175,11 +258,14 @@ type AgentTapOptions = { function fieldValue(key: string, rawValue: string): TelemetryValue | undefined { if (key === 'reason') return REASONS.has(rawValue) ? rawValue : 'unknown' - if (key === 'agent_version' || key === 'node_version') { + const values = LIFECYCLE_ENUMS.get(key) + if (values) return values.has(rawValue) ? rawValue : 'unknown' + if (key === 'ready') return bool(rawValue) + if (key === 'agent_version' || key === 'node_version' || key === 'python_version') { return VERSION.test(rawValue) ? rawValue : undefined } const value = /^-?\d+$/.test(rawValue) ? Number(rawValue) : NaN - if (!Number.isSafeInteger(value) || (key === 'duration_ms' && value < 0)) return undefined + if (!Number.isSafeInteger(value) || (LIFECYCLE_COUNTS.has(key) && value < 0)) return undefined return value } @@ -261,6 +347,9 @@ export function parseAgentEventLine(line: string): AgentEvent | null { // Per-event budget on top of telemetry's own per-minute limit. const PER_EVENT_HOURLY_CAP = 60 +// One start writes a startup_step per step (15) on each of up to 4 attempts, +// so the default cap would hold one launch an hour. +const LIFECYCLE_HOURLY_CAPS: Readonly> = { startup_step: 240 } const RATE_WINDOW_MS = 60 * 60_000 export function createAgentTap(opts: AgentTapOptions): { @@ -313,7 +402,7 @@ export function createAgentTap(opts: AgentTapOptions): { const { event, fields } = parsed === UNKNOWN_EVENT ? { event: UNKNOWN_EVENTS_DROPPED, fields: { count: 1 } } : parsed const bucket = rateBucket(event) - if (bucket.count >= PER_EVENT_HOURLY_CAP) return + if (bucket.count >= (LIFECYCLE_HOURLY_CAPS[event] ?? PER_EVENT_HOURLY_CAP)) return bucket.count++ try { // Base context merged last so parsed fields can never override it. diff --git a/src/main/lib/fixtures/agent-startup-events.txt b/src/main/lib/fixtures/agent-startup-events.txt new file mode 100644 index 000000000..d4bc6425c --- /dev/null +++ b/src/main/lib/fixtures/agent-startup-events.txt @@ -0,0 +1,10 @@ +[agent-event] startup_step step=find_comfyui outcome=ok duration_ms=12 +[agent-event] startup_step step=cli_provision outcome=degraded duration_ms=30500 reason=network_error +[agent-event] startup_step step=object_info outcome=failed duration_ms=120000 reason=timeout +[agent-event] startup_stalled step=sandbox_setup duration_ms=95000 +[agent-event] startup_failed step=object_info reason=timeout +[agent-event] node_fetch_started +[agent-event] node_fetched duration_ms=41000 node_version=v22.11.0 +[agent-event] node_fetch_failed duration_ms=360000 reason=timeout +[agent-event] node_found node_version=v20.18.1 +[agent-event] document_host_degraded reason=crashed diff --git a/src/main/lib/fixtures/agent-startup-step-ids.txt b/src/main/lib/fixtures/agent-startup-step-ids.txt new file mode 100644 index 000000000..64336ee3f --- /dev/null +++ b/src/main/lib/fixtures/agent-startup-step-ids.txt @@ -0,0 +1,15 @@ +open_data_dir +cli_provision +find_comfyui +code_search_canary +find_node +sandbox_config +egress_proxy +sandbox_setup +sandbox_grants +object_info +knowledge +cli_skills +detect_environment +document_host +listen diff --git a/src/shared/datadogMirroredEvents.test.ts b/src/shared/datadogMirroredEvents.test.ts index 5eb934e8c..12f4cdd1c 100644 --- a/src/shared/datadogMirroredEvents.test.ts +++ b/src/shared/datadogMirroredEvents.test.ts @@ -36,6 +36,19 @@ describe('isDatadogMirroredEvent', () => { expect(isDatadogMirroredEvent(name)).toBe(mirrored) }) + // Local Agent failures page; its startup timing and ready summary do not. + it.each([ + ['comfy.desktop.comfyui.agent.startup_failed', true], + ['comfy.desktop.comfyui.agent.startup_stalled', true], + ['comfy.desktop.comfyui.agent.health_check_failed', true], + ['comfy.desktop.comfyui.agent.agent_error', true], + ['comfy.desktop.comfyui.agent.document_host_degraded', true], + ['comfy.desktop.comfyui.agent.startup_step', false], + ['comfy.desktop.comfyui.agent.agent_started', false] + ])('mirrors %s: %s', (name, mirrored) => { + expect(isDatadogMirroredEvent(name)).toBe(mirrored) + }) + it('returns false for unknown event names', () => { expect(isDatadogMirroredEvent('comfy.desktop.not.a.real.event')).toBe(false) }) diff --git a/src/shared/datadogMirroredEvents.ts b/src/shared/datadogMirroredEvents.ts index 9701023da..334eddf9c 100644 --- a/src/shared/datadogMirroredEvents.ts +++ b/src/shared/datadogMirroredEvents.ts @@ -122,7 +122,17 @@ export const DATADOG_MIRRORED_EVENT_NAMES: ReadonlySet = new Set([ 'comfy.desktop.comfyui.assets.scanner.walk_failed', 'comfy.desktop.comfyui.assets.scanner.metadata_failed', 'comfy.desktop.comfyui.assets.seeder.batch_insert_failed', - 'comfy.desktop.comfyui.assets.seeder.scan_failed' + 'comfy.desktop.comfyui.assets.seeder.scan_failed', + // Local Agent failures from agentTap: a start that died (and the step it + // died on), a step past its expected time, a failed health check, the host + // giving up or failing to spawn the agent, and the workflow-editing host + // failing after start. Monitors group on `step` / `reason` per release. + // Startup timing (startup_step, agent_started) stays PostHog-only. + 'comfy.desktop.comfyui.agent.startup_failed', + 'comfy.desktop.comfyui.agent.startup_stalled', + 'comfy.desktop.comfyui.agent.health_check_failed', + 'comfy.desktop.comfyui.agent.agent_error', + 'comfy.desktop.comfyui.agent.document_host_degraded' ]) export function isDatadogMirroredEvent(eventName: string): boolean {