[Fix] Tasks keep retrying when provider billing fails

apps/worker/src/sandbox-server/lib/harnesses/opencode-server/harness.ts

@@ -77,6 +77,7 @@ import {
 import {
   formatOpenCodeProviderErrorRetryNoticeText,
   getOpenCodeProviderErrorRecovery,
+  isOpenCodeTerminalProviderError,
   type OpenCodeProviderErrorRecovery,
 } from './provider-error-recovery';
 import type {
@@ -3507,7 +3508,32 @@ export class OpenCodeServerHarness
     const status = asRecord(properties?.status);
     const statusType = asString(status?.type);
 
-    if (statusType === 'busy' || statusType === 'retry') {
+    if (statusType === 'retry') {
+      const sessionId =
+        asString(properties?.sessionID) ??
+        asString(properties?.sessionId) ??
+        this.sessionId;
+      const message = asString(status?.message);
+
+      if (
+        sessionId &&
+        message &&
+        isOpenCodeTerminalProviderError({ message })
+      ) {
+        await this.terminateOpenCodeProviderRetry(sessionId, message);
+        return;
+      }
+
+      // Retry transitions prove the session is alive, but not that the turn's
+      // loop advanced. Keep the stall watchdog armed during transient backoff.
+      this.ignoreNextProviderRecoverySessionIdle = false;
+      this.inFlight = true;
+      this.stallWatchdogs.noteActivity();
+      this.stallWatchdogs.ensureTurnStallArmed();
+      return;
+    }
+
+    if (statusType === 'busy') {
       // If OpenCode omitted the paired session.idle event, do not let a stale
       // guard consume the retry's eventual idle transition.
       this.ignoreNextProviderRecoverySessionIdle = false;
@@ -3529,6 +3555,42 @@ export class OpenCodeServerHarness
     }
   }
 
+  private async terminateOpenCodeProviderRetry(
+    sessionId: string,
+    message: string,
+  ): Promise<void> {
+    this.logger.error(
+      `OpenCode reported a terminal provider error as retryable sessionId=${sessionId}: ${message}`,
+    );
+
+    // Stop OpenCode's internal retry loop. It reports the intentional abort as
+    // MessageAbortedError, which is redundant with the provider error below.
+    this.armReplayAbortErrorSuppression();
+    try {
+      await this.client.abort({
+        sessionId,
+        signal: this.eventAbortController.signal,
+      });
+    } catch (error) {
+      this.logger.warn(
+        `Failed to abort OpenCode after terminal provider retry status sessionId=${sessionId}: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+    }
+
+    await this.handleSessionError({
+      type: 'session.error',
+      properties: {
+        sessionID: sessionId,
+        error: {
+          name: 'APIError',
+          data: { message, isRetryable: false },
+        },
+      },
+    });
+  }
+
   private async handleSessionIdle(
     payload: OpenCodeEventPayload,
   ): Promise<void> {

apps/worker/src/sandbox-server/lib/harnesses/opencode-server/provider-error-recovery.ts

@@ -28,19 +28,23 @@ const POLICY_CODES = new Set([
 ]);
 
 const TERMINAL_CODES = new Set([
+  'account_suspended',
   'authentication_error',
   'billing_error',
   'context_length_exceeded',
+  'credit_balance_too_low',
   'forbidden',
+  'insufficient_balance',
   'insufficient_quota',
   'invalid_api_key',
   'invalid_model',
   'model_not_found',
+  'payment_required',
   'permission_denied',
   'unauthorized',
 ]);
 
-const TERMINAL_STATUS_CODES = new Set([400, 401, 403, 404, 422]);
+const TERMINAL_STATUS_CODES = new Set([400, 401, 402, 403, 404, 422]);
 
 function collectProviderErrorValues(error: unknown): unknown[] {
   const pending: Array<{ value: unknown; depth: number }> = [
@@ -146,6 +150,10 @@ function isExplicitlyTerminal(values: unknown[]): boolean {
         lower.includes('model is not available') ||
         lower.includes('model not found') ||
         lower.includes('maximum context length') ||
+        lower.includes('insufficient balance') ||
+        lower.includes('please recharge') ||
+        (lower.includes('account') && lower.includes('is suspended')) ||
+        lower.includes('payment required') ||
         lower.includes('insufficient quota')
       ) {
         return true;
@@ -156,6 +164,16 @@ function isExplicitlyTerminal(values: unknown[]): boolean {
   return false;
 }
 
+/**
+ * OpenCode exposes errors it has decided to retry through session.status
+ * before it emits a terminal session.error. Providers occasionally mark
+ * billing or account failures as retryable, so the harness must be able to
+ * override that decision before OpenCode enters an unbounded backoff loop.
+ */
+export function isOpenCodeTerminalProviderError(error: unknown): boolean {
+  return isExplicitlyTerminal(collectProviderErrorValues(error));
+}
+
 /**
  * Provider session errors are recoverable by default because they normally
  * invalidate one model turn, not the OpenCode session or Roomote task. Keep a