From e66982ff4cde61f9f3ccc11f79988a7a778547c6 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:31:06 -0400 Subject: [PATCH 1/2] fix(worker): terminate provider billing retry loops --- .../__tests__/opencode-server.test.ts | 76 +++++++++++++++++++ .../lib/harnesses/opencode-server/harness.ts | 64 +++++++++++++++- .../provider-error-recovery.test.ts | 34 ++++++++- .../provider-error-recovery.ts | 19 ++++- 4 files changed, 190 insertions(+), 3 deletions(-) diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts index 62e0a91b..8c72abcc 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts @@ -1913,6 +1913,82 @@ describe('OpenCodeServerHarness', () => { } }); + it('terminates an OpenCode retry loop for provider billing suspension', async () => { + const { client, harness } = createHarness(); + const taskEvents: TaskEvent[] = []; + const persistedEnvelopes: AcpPersistedEnvelope[] = []; + + harness.subscribe((event) => taskEvents.push(event)); + harness.subscribeRuntimePersistedEnvelope((envelope) => + persistedEnvelopes.push(envelope), + ); + + try { + await connectHarness(harness, client); + + expect( + harness.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { + text: 'Start work.', + visibleInTranscript: true, + }, + }), + ).toBe(true); + + await vi.waitFor(() => { + expect(client.promptAsync).toHaveBeenCalledTimes(1); + }); + + expect( + harness.sendCommand({ + commandName: TaskCommandName.SendMessage, + data: { + text: 'Queued follow-up.', + visibleInTranscript: true, + }, + }), + ).toBe(true); + + await client.emit({ + type: 'session.status', + properties: { + sessionID: 'ses_1', + status: { + type: 'retry', + attempt: 1, + message: + 'Your account org-redacted is suspended due to insufficient balance, please recharge your account.', + next: Date.now() + 2_000, + }, + }, + }); + + expect(client.abort).toHaveBeenCalledWith({ + sessionId: 'ses_1', + signal: expect.any(AbortSignal), + }); + expect( + taskEvents.some( + (event) => event.eventName === TaskEventName.TaskAborted, + ), + ).toBe(true); + expect(harness.getQueuedMessages()).toEqual([]); + expect(client.promptAsync).toHaveBeenCalledTimes(1); + expect( + persistedEnvelopes.some( + (envelope) => + envelope.eventType === ACP_ENVELOPE_EVENT_TYPES.AssistantMessage && + String(envelope.payload.text ?? '').includes( + 'suspended due to insufficient balance', + ), + ), + ).toBe(true); + } finally { + harness.dispose(); + } + }); + it('retries a cyber policy refusal with safer framing without aborting the task', async () => { const { client, harness } = createHarness(); const taskEvents: TaskEvent[] = []; diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts index a5651570..5c01ebec 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts @@ -77,6 +77,7 @@ import { import { formatOpenCodeProviderErrorRetryNoticeText, getOpenCodeProviderErrorRecovery, + isOpenCodeTerminalProviderError, type OpenCodeProviderErrorRecovery, } from './provider-error-recovery'; import type { @@ -3507,7 +3508,32 @@ export class OpenCodeServerHarness const status = asRecord(properties?.status); const statusType = asString(status?.type); - if (statusType === 'busy' || statusType === 'retry') { + if (statusType === 'retry') { + const sessionId = + asString(properties?.sessionID) ?? + asString(properties?.sessionId) ?? + this.sessionId; + const message = asString(status?.message); + + if ( + sessionId && + message && + isOpenCodeTerminalProviderError({ message }) + ) { + await this.terminateOpenCodeProviderRetry(sessionId, message); + return; + } + + // Retry transitions prove the session is alive, but not that the turn's + // loop advanced. Keep the stall watchdog armed during transient backoff. + this.ignoreNextProviderRecoverySessionIdle = false; + this.inFlight = true; + this.stallWatchdogs.noteActivity(); + this.stallWatchdogs.ensureTurnStallArmed(); + return; + } + + if (statusType === 'busy') { // If OpenCode omitted the paired session.idle event, do not let a stale // guard consume the retry's eventual idle transition. this.ignoreNextProviderRecoverySessionIdle = false; @@ -3529,6 +3555,42 @@ export class OpenCodeServerHarness } } + private async terminateOpenCodeProviderRetry( + sessionId: string, + message: string, + ): Promise { + this.logger.error( + `OpenCode reported a terminal provider error as retryable sessionId=${sessionId}: ${message}`, + ); + + // Stop OpenCode's internal retry loop. It reports the intentional abort as + // MessageAbortedError, which is redundant with the provider error below. + this.armReplayAbortErrorSuppression(); + try { + await this.client.abort({ + sessionId, + signal: this.eventAbortController.signal, + }); + } catch (error) { + this.logger.warn( + `Failed to abort OpenCode after terminal provider retry status sessionId=${sessionId}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + + await this.handleSessionError({ + type: 'session.error', + properties: { + sessionID: sessionId, + error: { + name: 'APIError', + data: { message, isRetryable: false }, + }, + }, + }); + } + private async handleSessionIdle( payload: OpenCodeEventPayload, ): Promise { diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts index 43af95e0..4d522592 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { getOpenCodeProviderErrorRecovery } from './provider-error-recovery'; +import { + getOpenCodeProviderErrorRecovery, + isOpenCodeTerminalProviderError, +} from './provider-error-recovery'; describe('getOpenCodeProviderErrorRecovery', () => { it('detects an UnknownError-wrapped cyber policy refusal', () => { @@ -59,4 +62,33 @@ describe('getOpenCodeProviderErrorRecovery', () => { }), ).toBeNull(); }); + + it.each([ + 'Your account org-redacted is suspended due to insufficient balance, please recharge your account or check your plan and billing details', + JSON.stringify({ + error: { + code: 'insufficient_balance', + message: 'There is not enough credit to run this request.', + }, + }), + ])( + 'classifies provider billing and suspension errors as terminal', + (error) => { + expect(isOpenCodeTerminalProviderError(error)).toBe(true); + expect(getOpenCodeProviderErrorRecovery(error)).toBeNull(); + }, + ); + + it('classifies payment-required responses as terminal', () => { + expect( + isOpenCodeTerminalProviderError({ + name: 'APIError', + data: { + message: 'Payment required', + statusCode: 402, + isRetryable: true, + }, + }), + ).toBe(true); + }); }); diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts index 1268f0d4..18718607 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts @@ -28,19 +28,23 @@ const POLICY_CODES = new Set([ ]); const TERMINAL_CODES = new Set([ + 'account_suspended', 'authentication_error', 'billing_error', 'context_length_exceeded', + 'credit_balance_too_low', 'forbidden', + 'insufficient_balance', 'insufficient_quota', 'invalid_api_key', 'invalid_model', 'model_not_found', + 'payment_required', 'permission_denied', 'unauthorized', ]); -const TERMINAL_STATUS_CODES = new Set([400, 401, 403, 404, 422]); +const TERMINAL_STATUS_CODES = new Set([400, 401, 402, 403, 404, 422]); function collectProviderErrorValues(error: unknown): unknown[] { const pending: Array<{ value: unknown; depth: number }> = [ @@ -146,6 +150,9 @@ function isExplicitlyTerminal(values: unknown[]): boolean { lower.includes('model is not available') || lower.includes('model not found') || lower.includes('maximum context length') || + lower.includes('insufficient balance') || + lower.includes('please recharge') || + (lower.includes('account') && lower.includes('is suspended')) || lower.includes('insufficient quota') ) { return true; @@ -156,6 +163,16 @@ function isExplicitlyTerminal(values: unknown[]): boolean { return false; } +/** + * OpenCode exposes errors it has decided to retry through session.status + * before it emits a terminal session.error. Providers occasionally mark + * billing or account failures as retryable, so the harness must be able to + * override that decision before OpenCode enters an unbounded backoff loop. + */ +export function isOpenCodeTerminalProviderError(error: unknown): boolean { + return isExplicitlyTerminal(collectProviderErrorValues(error)); +} + /** * Provider session errors are recoverable by default because they normally * invalidate one model turn, not the OpenCode session or Roomote task. Keep a From a085b5ed0b86190be37a42f6b6de000549f99e65 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 18 Jul 2026 03:33:17 +0000 Subject: [PATCH 2/2] fix(worker): match message-only payment required retries Classify bare "Payment required" retry status messages as terminal so OpenCode does not keep backoff when only the status message is available. --- .../__tests__/opencode-server.test.ts | 74 +++++++++++++++++++ .../provider-error-recovery.test.ts | 10 +++ .../provider-error-recovery.ts | 1 + 3 files changed, 85 insertions(+) diff --git a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts index 8c72abcc..4d9f114c 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/__tests__/opencode-server.test.ts @@ -1989,6 +1989,80 @@ describe('OpenCodeServerHarness', () => { } }); + it('terminates an OpenCode retry loop for message-only payment required status', async () => { + const { client, harness } = createHarness(); + const taskEvents: TaskEvent[] = []; + const persistedEnvelopes: AcpPersistedEnvelope[] = []; + + harness.subscribe((event) => taskEvents.push(event)); + harness.subscribeRuntimePersistedEnvelope((envelope) => + persistedEnvelopes.push(envelope), + ); + + try { + await connectHarness(harness, client); + + expect( + harness.sendCommand({ + commandName: TaskCommandName.StartNewTask, + data: { + text: 'Start work.', + visibleInTranscript: true, + }, + }), + ).toBe(true); + + await vi.waitFor(() => { + expect(client.promptAsync).toHaveBeenCalledTimes(1); + }); + + expect( + harness.sendCommand({ + commandName: TaskCommandName.SendMessage, + data: { + text: 'Queued follow-up.', + visibleInTranscript: true, + }, + }), + ).toBe(true); + + // Retry status only carries message; no statusCode/code fields. + await client.emit({ + type: 'session.status', + properties: { + sessionID: 'ses_1', + status: { + type: 'retry', + attempt: 1, + message: 'Payment required', + next: Date.now() + 2_000, + }, + }, + }); + + expect(client.abort).toHaveBeenCalledWith({ + sessionId: 'ses_1', + signal: expect.any(AbortSignal), + }); + expect( + taskEvents.some( + (event) => event.eventName === TaskEventName.TaskAborted, + ), + ).toBe(true); + expect(harness.getQueuedMessages()).toEqual([]); + expect(client.promptAsync).toHaveBeenCalledTimes(1); + expect( + persistedEnvelopes.some( + (envelope) => + envelope.eventType === ACP_ENVELOPE_EVENT_TYPES.AssistantMessage && + String(envelope.payload.text ?? '').includes('Payment required'), + ), + ).toBe(true); + } finally { + harness.dispose(); + } + }); + it('retries a cyber policy refusal with safer framing without aborting the task', async () => { const { client, harness } = createHarness(); const taskEvents: TaskEvent[] = []; diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts index 4d522592..efe3ca83 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.test.ts @@ -91,4 +91,14 @@ describe('getOpenCodeProviderErrorRecovery', () => { }), ).toBe(true); }); + + it('classifies message-only payment-required retry status as terminal', () => { + // handleSessionStatus only has the retry status message, not statusCode/code. + expect( + isOpenCodeTerminalProviderError({ message: 'Payment required' }), + ).toBe(true); + expect( + getOpenCodeProviderErrorRecovery({ message: 'Payment required' }), + ).toBeNull(); + }); }); diff --git a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts index 18718607..dcb5a388 100644 --- a/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts +++ b/apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts @@ -153,6 +153,7 @@ function isExplicitlyTerminal(values: unknown[]): boolean { lower.includes('insufficient balance') || lower.includes('please recharge') || (lower.includes('account') && lower.includes('is suspended')) || + lower.includes('payment required') || lower.includes('insufficient quota') ) { return true;