From 2b6e298f1285eddaac052e762a2dd119c4475e3b Mon Sep 17 00:00:00 2001 From: Evgeny Formanenko Date: Thu, 1 Oct 2026 11:58:53 +0300 Subject: [PATCH] fix(rpc): retry a bare HTTP 500 and pre-check failures of traced mined blocks A 500 whose body is not a JSON-RPC response comes from a proxy in front of the node, and the same call succeeds when repeated. A 500 with a JSON-RPC envelope stays fatal: Bitcoin Core and others answer ordinary errors that way. A proxy can also label its error page or an empty body as JSON, so an error body that does not parse is kept as text for the status check instead of failing as a parse error. A transaction in a block passed its pre-checks when it was mined, so a trace of that block failing them means the serving node has wrong state for it. These errors are retried for replay traces whatever code or proxy cause they come under, including one error for a whole batch of a single method, and counted as `wrong_state`, so a node that never recovers shows up in the metrics as the dump stalls. Closes #585 Co-Authored-By: Claude Opus 5.5 (1M context) --- ...0-and-trace-precheck_2026-10-01-08-25.json | 10 + ...0-and-trace-precheck_2026-10-01-08-25.json | 10 + evm/evm-rpc/src/rpc-client.ts | 40 ++- evm/evm-rpc/test/rpc.transient-errors.test.ts | 307 +++++++++++++++++- .../src/client.connection-error.test.ts | 219 ++++++++++++- util/rpc-client/src/client.ts | 33 ++ util/rpc-client/src/transport/http.ts | 32 +- 7 files changed, 639 insertions(+), 12 deletions(-) create mode 100644 common/changes/@subsquid/evm-rpc/fix-rpc-retry-bare-500-and-trace-precheck_2026-10-01-08-25.json create mode 100644 common/changes/@subsquid/rpc-client/fix-rpc-retry-bare-500-and-trace-precheck_2026-10-01-08-25.json diff --git a/common/changes/@subsquid/evm-rpc/fix-rpc-retry-bare-500-and-trace-precheck_2026-10-01-08-25.json b/common/changes/@subsquid/evm-rpc/fix-rpc-retry-bare-500-and-trace-precheck_2026-10-01-08-25.json new file mode 100644 index 000000000..e2e1c3d7b --- /dev/null +++ b/common/changes/@subsquid/evm-rpc/fix-rpc-retry-bare-500-and-trace-precheck_2026-10-01-08-25.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/evm-rpc", + "comment": "Retry transaction pre-check failures (insufficient funds, nonce too low or high, intrinsic gas too low) returned while tracing a mined block, under any code or proxy cause", + "type": "patch" + } + ], + "packageName": "@subsquid/evm-rpc" +} diff --git a/common/changes/@subsquid/rpc-client/fix-rpc-retry-bare-500-and-trace-precheck_2026-10-01-08-25.json b/common/changes/@subsquid/rpc-client/fix-rpc-retry-bare-500-and-trace-precheck_2026-10-01-08-25.json new file mode 100644 index 000000000..f5beb8ae0 --- /dev/null +++ b/common/changes/@subsquid/rpc-client/fix-rpc-retry-bare-500-and-trace-precheck_2026-10-01-08-25.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/rpc-client", + "comment": "Retry HTTP 500 whose body is not a JSON-RPC response, which comes from a proxy in front of the node rather than from the node. A failed response labeled as JSON whose body does not parse now surfaces as an HTTP error with the status, not as a parse error", + "type": "patch" + } + ], + "packageName": "@subsquid/rpc-client" +} diff --git a/evm/evm-rpc/src/rpc-client.ts b/evm/evm-rpc/src/rpc-client.ts index 20ef6a36c..5fc5d0747 100644 --- a/evm/evm-rpc/src/rpc-client.ts +++ b/evm/evm-rpc/src/rpc-client.ts @@ -6,7 +6,7 @@ export interface EvmRpcClientOptions extends RpcClientOptions { * Whether internal server errors should be treated as retryable. * * This includes: - * - HTTP 500 (internal server error) + * - HTTP 500 carrying a JSON-RPC error (a 500 with any other body is always retried) * - RPC -32000 (catch-all) * - RPC -32603 (internal error) */ @@ -29,6 +29,9 @@ export class EvmRpcClient extends RpcClient { if (this.isRpcRateLimitError(err)) { return true } + if (this.isTraceStateError(err)) { + return true + } if (this.isRpcTransientError(err)) { return true } @@ -51,12 +54,15 @@ export class EvmRpcClient extends RpcClient { getRetryKind(err: Error): string { if (err instanceof RpcError) { if (this.isRpcRateLimitError(err)) return 'rate_limit' + if (this.isTraceStateError(err)) return 'wrong_state' if (this.isRpcTransientError(err)) return 'transient' let kind = super.getRetryKind(err) if (kind != 'other') return kind if (this.isRpcInternalError(err)) return 'internal' } if (err instanceof HttpError && err.response.status === 500) { + // a 500 without a JSON-RPC answer, retried by RpcClient as a proxy failure + if (super.isConnectionError(err)) return super.getRetryKind(err) return this.retryInternalServerErrors ? 'internal' : 'transient' } // Thrown here when the endpoint had no usable answer yet: a null or an @@ -91,9 +97,41 @@ export class EvmRpcClient extends RpcClient { isRpcTransientError(err: RpcError): boolean { return isTransientError(err.code, err.message, err.data) } + + /** + * A transaction of a mined block failed its pre-checks while being traced. + * It passed them when it was mined, so the node that served the trace has + * wrong state for the block. A proxy reports this as an execution exception, + * which is final for a call that runs a new transaction, but not here. + */ + isTraceStateError(err: RpcError): boolean { + // attached by RpcClient to an error received in answer to a call + let method = (err as {rpcMethod?: unknown}).rpcMethod + let tracesMinedTx = typeof method == 'string' && MINED_TX_TRACES.has(method) + if (!tracesMinedTx) return false + + let messages = [err.message, ...getErrorCauses(err.data).map(cause => cause.message)] + return messages.some(message => message != null && TX_PRECHECK_FAILURE.test(message)) + } } +// Calls that re-execute transactions already in a block. `debug_traceCall`, +// `trace_call` and the like run a new transaction, whose pre-check failure is final. +const MINED_TX_TRACES = new Set([ + 'debug_traceBlockByHash', + 'debug_traceBlockByNumber', + 'debug_traceTransaction', + 'trace_block', + 'trace_replayBlockTransactions', + 'trace_replayTransaction', + 'trace_transaction', +]) + + +const TX_PRECHECK_FAILURE = /insufficient funds for gas|nonce too (low|high)|intrinsic gas too low/i + + const TRANSIENT_ERRORS = [ // proxy and aggregator summaries of failed upstream attempts /upstreams? (not synced|transport errors?|validation mismatch|timeouts?|missing data)/i, diff --git a/evm/evm-rpc/test/rpc.transient-errors.test.ts b/evm/evm-rpc/test/rpc.transient-errors.test.ts index 3ca1b25b6..0ebf4577a 100644 --- a/evm/evm-rpc/test/rpc.transient-errors.test.ts +++ b/evm/evm-rpc/test/rpc.transient-errors.test.ts @@ -175,8 +175,20 @@ describe('EvmRpcClient.isConnectionError', () => { expect(client.isConnectionError(httpError(500, [transient, {...transient, id: 3}]))).toBe(true) }) - it('leaves a bare HTTP 500 to retryInternalServerErrors', () => { - let err = httpError(500, 'Internal Server Error\n') + it('retries HTTP 500 whose body is not a JSON-RPC answer', () => { + let err = httpError(500, 'Uncaught exception') + expect(client.isConnectionError(err)).toBe(true) + expect(clientRetrying500.isConnectionError(err)).toBe(true) + }) + + it('retries HTTP 500 with a proxy error object, which is not a JSON-RPC answer', () => { + let err = httpError(500, {error: {message: 'Internal Server Error'}}) + expect(client.isConnectionError(err)).toBe(true) + expect(client.getRetryKind(err)).toBe('http') + }) + + it('leaves HTTP 500 with a permanent JSON-RPC error to retryInternalServerErrors', () => { + let err = httpError(500, {jsonrpc: '2.0', id: 1, error: {code: -32000, message: 'execution reverted'}}) expect(client.isConnectionError(err)).toBe(false) expect(clientRetrying500.isConnectionError(err)).toBe(true) }) @@ -194,15 +206,302 @@ describe('EvmRpcClient.getRetryKind', () => { ], [new RpcError({code: -32000, message: 'execution timeout'}), 'timeout'], [new RetryError('server returned unexpected result: null is not an object'), 'no_result'], - [httpError(500, {error: {code: 19, message: 'Temporary internal error. Please retry'}}), 'transient'], + [ + httpError(500, { + jsonrpc: '2.0', + id: 1, + error: {code: 19, message: 'Temporary internal error. Please retry'}, + }), + 'transient', + ], + [httpError(500, {error: {code: 19, message: 'Temporary internal error. Please retry'}}), 'http'], + [httpError(500, 'Uncaught exception'), 'http'], [httpError(503, ''), 'http'], + [traceError('debug_traceBlockByHash', -32000, 'insufficient funds for gas * price + value'), 'wrong_state'], ])('classifies %s as %s', (err, kind) => { expect(client.getRetryKind(err)).toBe(kind) }) it('classifies internal errors retried by retryInternalServerErrors as internal', () => { expect(clientRetrying500.getRetryKind(new RpcError({code: -32603, message: 'Internal error'}))).toBe('internal') - expect(clientRetrying500.getRetryKind(httpError(500, 'Internal Server Error\n'))).toBe('internal') + expect( + clientRetrying500.getRetryKind( + httpError(500, {jsonrpc: '2.0', id: 1, error: {code: -32000, message: 'execution reverted'}}), + ), + ).toBe('internal') + expect(clientRetrying500.getRetryKind(httpError(500, 'Uncaught exception'))).toBe('http') + }) +}) + +const PRECHECK_FAILURE = 'tracing failed: insufficient funds for gas * price + value: address 0x01 have 0 want 100' + +// An error in answer to a call, as RpcClient hands it over. +function traceError(method: string, code: number, message: string, data?: unknown): RpcError { + return Object.assign(new RpcError({code, message, data}), {rpcMethod: method}) +} + +// As a proxy passes on the node's -32000. +function proxiedTraceError(method: string, message = PRECHECK_FAILURE): RpcError { + return traceError(method, -32003, message, { + code: 'ErrEndpointExecutionException', + message, + details: {originalCode: -32000}, + }) +} + +describe('EvmRpcClient on a pre-check failure while tracing a mined block', () => { + let client = new EvmRpcClient({url: 'http://localhost:1', log: null}) + + it.each([ + 'debug_traceBlockByHash', + 'debug_traceBlockByNumber', + 'debug_traceTransaction', + 'trace_block', + 'trace_replayBlockTransactions', + 'trace_replayTransaction', + 'trace_transaction', + ])('retries %s', (method) => { + expect(client.isConnectionError(proxiedTraceError(method))).toBe(true) + }) + + it.each([ + [-32000, 'insufficient funds for gas * price + value: address 0x01 have 0 want 100'], + [-32000, 'nonce too low: address 0x01, tx: 5 state: 6'], + [-32000, 'nonce too high: address 0x01, tx: 7 state: 6'], + [-32000, 'intrinsic gas too low: have 0, want 21000'], + [-32603, 'insufficient funds for gas * price + value'], + [-32000, 'err: insufficient funds for gas * price + value: address 0x01 have 0 want 100 (supplied gas 21000)'], + [-32000, 'Insufficient funds for gas * price + value'], + [1, 'nonce too low'], + ])('retries %i %s straight from the node', (code, message) => { + expect(client.isConnectionError(traceError('debug_traceBlockByHash', code, message))).toBe(true) + }) + + it('finds the failure in a cause under a generic proxy message', () => { + let err = traceError('debug_traceBlockByHash', -32603, 'all upstream attempts failed', { + code: 'ErrUpstreamsExhausted', + cause: [{code: 'ErrEndpointExecutionException', message: PRECHECK_FAILURE}], + }) + expect(client.isConnectionError(err)).toBe(true) + }) + + it('finds the failure in a cause whose own message is generic', () => { + let err = traceError('trace_block', -32603, 'gave up retrying on network-level after 1s', { + code: 'ErrFailsafeRetryExceeded', + cause: { + code: 'ErrUpstreamsExhausted', + cause: [ + { + code: 'ErrUpstreamRequest', + cause: {code: 'ErrEndpointExecutionException', message: 'nonce too high'}, + }, + ], + }, + }) + expect(client.isConnectionError(err)).toBe(true) + }) + + it.each([ + 'eth_call', + 'eth_estimateGas', + 'eth_sendRawTransaction', + 'debug_traceCall', + 'trace_call', + 'trace_callMany', + ])('does not retry it for %s, which runs a new transaction', (method) => { + expect(client.isConnectionError(proxiedTraceError(method))).toBe(false) + }) + + it('does not retry it when the method is unknown', () => { + let err = new RpcError({code: -32003, message: PRECHECK_FAILURE, data: {code: 'ErrEndpointExecutionException'}}) + expect(client.isConnectionError(err)).toBe(false) + }) + + it.each([ + 'execution reverted', + 'insufficient funds for transfer', + 'out of gas', + 'tracing failed: max fee per gas less than block base fee', + ])('does not retry other execution errors of a trace: %s', (message) => { + expect(client.isConnectionError(proxiedTraceError('debug_traceBlockByHash', message))).toBe(false) + }) +}) + +type Call = {id: number; method: string} +type Reply = {status: number; contentType: string; body: unknown} + +const PRECHECK_ERROR = { + code: -32003, + message: PRECHECK_FAILURE, + data: {code: 'ErrEndpointExecutionException', message: PRECHECK_FAILURE}, +} + +function json(body: unknown, status = 200): Reply { + return {status, contentType: 'application/json', body} +} + +// an error for each call, as the node answers +function perCallErrors(calls: Call | Call[]): Reply { + let answer = (call: Call) => ({jsonrpc: '2.0', id: call.id, error: PRECHECK_ERROR}) + return json(Array.isArray(calls) ? calls.map(answer) : answer(calls)) +} + +// one error for the whole batch, as a proxy may answer +function wholeBatchError(): Reply { + return json({jsonrpc: '2.0', id: null, error: PRECHECK_ERROR}) +} + +describe('EvmRpcClient through the transport', () => { + let server: http.Server + let url: string + let replies: ((calls: Call | Call[]) => Reply)[] + + beforeEach(async () => { + replies = [] + server = http.createServer((req, res) => { + let body = '' + req.on('data', (chunk) => { + body += chunk + }) + req.on('end', () => { + let calls = JSON.parse(body) + let answer = (call: Call) => ({jsonrpc: '2.0', id: call.id, result: []}) + let reply = replies.shift()?.(calls) ?? json(Array.isArray(calls) ? calls.map(answer) : answer(calls)) + res.writeHead(reply.status, {'content-type': reply.contentType}) + res.end(typeof reply.body == 'string' ? reply.body : JSON.stringify(reply.body)) + }) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + url = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + }) + + afterEach(async () => { + // a test that fails before closing its client leaves a keep-alive socket open + server.closeAllConnections() + await new Promise((resolve) => server.close(() => resolve())) + }) + + function client(retryAttempts = 1): EvmRpcClient { + return new EvmRpcClient({url, log: null, retryAttempts, retrySchedule: [0]}) + } + + function traceBatch(...methods: string[]) { + return methods.map((method, i) => ({method, params: [`0x0${i + 1}`, {tracer: 'callTracer'}]})) + } + + describe('on a pre-check failure', () => { + it('retries a trace call', async () => { + replies = [perCallErrors] + let rpc = client() + + await expect(rpc.call('debug_traceBlockByHash', ['0x01', {tracer: 'callTracer'}])).resolves.toEqual([]) + expect(rpc.getMetrics().retriedErrors).toEqual({wrong_state: 1}) + rpc.close() + }) + + it('retries a batch of trace calls that fail one by one', async () => { + replies = [perCallErrors] + let rpc = client() + + await expect(rpc.batchCall(traceBatch('trace_block', 'trace_block'))).resolves.toEqual([[], []]) + expect(rpc.getMetrics().retriedErrors).toEqual({wrong_state: 1}) + rpc.close() + }) + + it('retries a batch of trace calls that fails as a whole', async () => { + replies = [wholeBatchError] + let rpc = client() + + let batch = traceBatch('debug_traceBlockByHash', 'debug_traceBlockByHash') + await expect(rpc.batchCall(batch)).resolves.toEqual([[], []]) + expect(rpc.getMetrics().retriedErrors).toEqual({wrong_state: 1}) + rpc.close() + }) + + it('keeps retrying while the node serves wrong state', async () => { + replies = [perCallErrors, perCallErrors, perCallErrors] + let rpc = client(3) + + await expect(rpc.call('debug_traceTransaction', ['0x01'])).resolves.toEqual([]) + expect(rpc.getMetrics().retriedErrors).toEqual({wrong_state: 3}) + rpc.close() + }) + + it('fails once the retries are spent', async () => { + replies = [perCallErrors, perCallErrors] + let rpc = client(1) + + await expect(rpc.call('debug_traceTransaction', ['0x01'])).rejects.toBeInstanceOf(RpcError) + rpc.close() + }) + + it('fails a batch of several methods that fails as a whole', async () => { + replies = [wholeBatchError] + let rpc = client() + + let batch = traceBatch('debug_traceBlockByHash', 'eth_getBlockByHash') + await expect(rpc.batchCall(batch)).rejects.toBeInstanceOf(RpcError) + expect(rpc.getMetrics().retriedErrors).toEqual({}) + rpc.close() + }) + + it('fails eth_call at once', async () => { + replies = [perCallErrors] + let rpc = client() + + await expect(rpc.call('eth_call', [{to: '0x01'}, 'latest'])).rejects.toBeInstanceOf(RpcError) + expect(rpc.getMetrics().retriedErrors).toEqual({}) + rpc.close() + }) + + it('fails a batch of eth_call that fails as a whole at once', async () => { + replies = [wholeBatchError] + let rpc = client() + + await expect(rpc.batchCall(traceBatch('eth_call', 'eth_call'))).rejects.toBeInstanceOf(RpcError) + expect(rpc.getMetrics().retriedErrors).toEqual({}) + rpc.close() + }) + }) + + describe('on HTTP 500', () => { + it('retries the bare 500 of a proxy', async () => { + replies = [() => ({status: 500, contentType: 'text/plain;charset=UTF-8', body: 'Uncaught exception'})] + let rpc = client() + + await expect(rpc.call('eth_blockNumber')).resolves.toEqual([]) + expect(rpc.getMetrics().retriedErrors).toEqual({http: 1}) + rpc.close() + }) + + it('retries a 500 with a proxy error object', async () => { + replies = [() => json({error: {message: 'Internal Server Error'}}, 500)] + let rpc = client() + + await expect(rpc.call('eth_blockNumber')).resolves.toEqual([]) + expect(rpc.getMetrics().retriedErrors).toEqual({http: 1}) + rpc.close() + }) + + it('retries a 500 whose JSON-RPC error is transient', async () => { + let error = {code: 19, message: 'Temporary internal error. Please retry'} + replies = [(call) => json({jsonrpc: '2.0', id: (call as Call).id, error}, 500)] + let rpc = client() + + await expect(rpc.call('eth_blockNumber')).resolves.toEqual([]) + expect(rpc.getMetrics().retriedErrors).toEqual({transient: 1}) + rpc.close() + }) + + it('fails on a 500 whose JSON-RPC error is permanent', async () => { + let error = {code: -32601, message: 'The method x does not exist/is not available'} + replies = [(call) => json({jsonrpc: '2.0', id: (call as Call).id, error}, 500)] + let rpc = client() + + await expect(rpc.call('x')).rejects.toBeInstanceOf(HttpError) + expect(rpc.getMetrics().retriedErrors).toEqual({}) + rpc.close() + }) }) }) diff --git a/util/rpc-client/src/client.connection-error.test.ts b/util/rpc-client/src/client.connection-error.test.ts index ca12accfb..c41fcc68f 100644 --- a/util/rpc-client/src/client.connection-error.test.ts +++ b/util/rpc-client/src/client.connection-error.test.ts @@ -3,10 +3,10 @@ import type {AddressInfo} from 'node:net' import {HttpError, HttpResponse} from '@subsquid/http-client' import {afterEach, beforeEach, describe, expect, it} from 'vitest' import {RpcClient} from './client' -import {RetryError, RpcError} from './errors' +import {RetryError, RpcError, RpcProtocolError} from './errors' -function httpError(status: number): HttpError { - return new HttpError(new HttpResponse(1, 'http://localhost', status, new Headers() as any, '', false)) +function httpError(status: number, body: unknown = ''): HttpError { + return new HttpError(new HttpResponse(1, 'http://localhost', status, new Headers() as any, body, false)) } describe('RpcClient.isConnectionError', () => { @@ -16,9 +16,59 @@ describe('RpcClient.isConnectionError', () => { expect(client.isConnectionError(httpError(status))).toBe(true) }) - it.each([400, 401, 403, 404, 500])('does not retry HTTP %i', (status) => { + it.each([400, 401, 403, 404])('does not retry HTTP %i', (status) => { expect(client.isConnectionError(httpError(status))).toBe(false) }) + + // a proxy in front of the node failed + it.each([ + ['a text body', 'Uncaught exception'], + ['an HTML page', '

500 Internal Server Error

'], + ['an empty body', undefined], + ['bytes', Buffer.from('Internal Server Error')], + ['a JSON body that is not JSON-RPC', {message: 'Internal server error'}], + ['a JSON body with an error string', {error: 'upstream failed'}], + ['an error object with no JSON-RPC envelope', {error: {message: 'Internal Server Error'}}], + ['an error object with a code but no envelope', {error: {code: 500, message: 'upstream failed'}}], + ['an envelope with neither result nor error', {jsonrpc: '2.0', id: 1}], + ['an empty JSON array', []], + ['a JSON array of proxy errors', [{error: {message: 'Internal Server Error'}}]], + ['a JSON string', 'Internal Server Error'], + ['JSON null sent as text', 'null'], + ])('retries HTTP 500 with %s', (_, body) => { + expect(client.isConnectionError(httpError(500, body))).toBe(true) + }) + + // the node answered the request + it.each([ + [ + 'a JSON-RPC 1.0 error, as Bitcoin Core sends', + {result: null, error: {code: -8, message: 'Block height out of range'}, id: 1}, + ], + ['a JSON-RPC 2.0 error', {jsonrpc: '2.0', id: 1, error: {code: -32000, message: 'execution reverted'}}], + [ + 'a JSON-RPC 2.0 error with a null id', + {jsonrpc: '2.0', id: null, error: {code: -32700, message: 'Parse error'}}, + ], + ['a JSON-RPC result', {jsonrpc: '2.0', id: 1, result: '0x1'}], + [ + 'a batch answer', + [ + {jsonrpc: '2.0', id: 1, result: '0x1'}, + {jsonrpc: '2.0', id: 2, error: {code: -32000, message: 'x'}}, + ], + ], + ['a JSON-RPC error sent as text', '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"x"}}'], + ['a JSON-RPC error sent as bytes', Buffer.from('{"result":null,"error":{"code":-8,"message":"x"},"id":1}')], + ])('does not retry HTTP 500 with %s', (_, body) => { + expect(client.isConnectionError(httpError(500, body))).toBe(false) + }) + + it('judges only a 500 by its body', () => { + let jsonRpcError = {jsonrpc: '2.0', id: 1, error: {code: -32000, message: 'x'}} + expect(client.isConnectionError(httpError(400, 'Uncaught exception'))).toBe(false) + expect(client.isConnectionError(httpError(502, jsonRpcError))).toBe(true) + }) }) describe('RpcClient.getRetryKind', () => { @@ -31,6 +81,7 @@ describe('RpcClient.getRetryKind', () => { [httpError(524), 'timeout'], [new RpcError({code: -32000, message: 'execution timeout'}), 'timeout'], [httpError(502), 'http'], + [httpError(500, 'Uncaught exception'), 'http'], [httpError(521), 'http'], [new RetryError(), 'retry'], ])('classifies %s as %s', (err, kind) => { @@ -78,6 +129,15 @@ describe('RpcClient retried error metrics', () => { client.close() }) + it('retries a bare HTTP 500 from a proxy', async () => { + statuses = [500] + let client = new RpcClient({url, log: null, retryAttempts: 1, retrySchedule: [0]}) + + await expect(client.call('eth_blockNumber')).resolves.toBe('0x1') + expect(client.getMetrics().retriedErrors).toEqual({http: 1}) + client.close() + }) + it('does not count the error that fails the request', async () => { statuses = [429, 502] let client = new RpcClient({url, log: null, retryAttempts: 1, retrySchedule: [0]}) @@ -102,3 +162,154 @@ describe('RpcClient retried error metrics', () => { client.close() }) }) + +interface Reply { + status: number + contentType: string + body: string +} + +describe('RpcClient through the HTTP transport', () => { + let server: http.Server + let url: string + let replies: Reply[] + + beforeEach(async () => { + replies = [] + server = http.createServer((req, res) => { + let body = '' + req.on('data', (chunk) => { + body += chunk + }) + req.on('end', () => { + let calls = JSON.parse(body) + let answer = (call: {id: number}) => ({jsonrpc: '2.0', id: call.id, result: '0x1'}) + let reply = replies.shift() ?? { + status: 200, + contentType: 'application/json', + body: JSON.stringify(Array.isArray(calls) ? calls.map(answer) : answer(calls)), + } + res.writeHead(reply.status, {'content-type': reply.contentType}) + res.end(reply.body) + }) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + url = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + }) + + afterEach(async () => { + // a test that fails before closing its client leaves a keep-alive socket open + server.closeAllConnections() + await new Promise((resolve) => server.close(() => resolve())) + }) + + function client(retryAttempts = 1): RpcClient { + return new RpcClient({url, log: null, retryAttempts, retrySchedule: [0]}) + } + + const JSON_RPC_ERROR = JSON.stringify({ + result: null, + error: {code: -8, message: 'Block height out of range'}, + id: 1, + }) + + // a proxy in front of the node failed + it.each<[string, Reply]>([ + ['plain text', {status: 500, contentType: 'text/plain;charset=UTF-8', body: 'Uncaught exception'}], + ['an HTML page', {status: 500, contentType: 'text/html', body: '

500 Internal Server Error

'}], + ['an empty body labeled as JSON', {status: 500, contentType: 'application/json', body: ''}], + [ + 'a malformed body labeled as JSON', + {status: 500, contentType: 'application/json', body: '{"error": "upstream'}, + ], + [ + 'a proxy error object', + {status: 500, contentType: 'application/json', body: '{"error":{"message":"Internal Server Error"}}'}, + ], + ['an untyped body', {status: 500, contentType: '', body: 'Internal Server Error'}], + ])('retries a 500 with %s', async (_, reply) => { + replies = [reply] + let rpc = client() + + await expect(rpc.call('eth_blockNumber')).resolves.toBe('0x1') + expect(rpc.getMetrics().retriedErrors).toEqual({http: 1}) + rpc.close() + }) + + // the node answered the request + it.each<[string, Reply]>([ + ['a JSON-RPC error', {status: 500, contentType: 'application/json', body: JSON_RPC_ERROR}], + ['a JSON-RPC error labeled as text', {status: 500, contentType: 'text/plain', body: JSON_RPC_ERROR}], + ])('fails on a 500 with %s', async (_, reply) => { + replies = [reply] + let rpc = client() + + await expect(rpc.call('getblockhash', [1e9])).rejects.toBeInstanceOf(HttpError) + expect(rpc.getMetrics().retriedErrors).toEqual({}) + rpc.close() + }) + + it('retries a gateway error whose JSON-labeled body does not parse', async () => { + replies = [{status: 502, contentType: 'application/json', body: ''}] + let rpc = client() + + await expect(rpc.call('eth_blockNumber')).resolves.toBe('0x1') + rpc.close() + }) + + it('fails a 400 whose JSON-labeled body does not parse with its status', async () => { + replies = [{status: 400, contentType: 'application/json', body: 'bad request'}] + let rpc = client() + + let err = await rpc.call('eth_blockNumber').catch((e) => e) + expect(err).toBeInstanceOf(HttpError) + expect(err.response.status).toBe(400) + expect(err.response.body).toBe('bad request') + rpc.close() + }) + + it('still rejects malformed JSON in a successful response', async () => { + replies = [{status: 200, contentType: 'application/json', body: '{"jsonrpc":'}] + let rpc = client() + + await expect(rpc.call('eth_blockNumber')).rejects.toBeInstanceOf(RpcProtocolError) + rpc.close() + }) + + describe('a whole-batch error', () => { + const ENVELOPE: Reply = { + status: 200, + contentType: 'application/json', + body: JSON.stringify({jsonrpc: '2.0', id: null, error: {code: -32003, message: 'request rejected'}}), + } + + it('carries the method of a batch of one method', async () => { + replies = [ENVELOPE] + let rpc = client(0) + let batch = [ + {method: 'debug_traceBlockByHash', params: ['0x01']}, + {method: 'debug_traceBlockByHash', params: ['0x02']}, + ] + + let err = await rpc.batchCall(batch).catch((e) => e) + expect(err).toBeInstanceOf(RpcError) + expect(err.code).toBe(-32003) + expect(err.rpcMethod).toBe('debug_traceBlockByHash') + rpc.close() + }) + + it('carries no method for a batch of several', async () => { + replies = [ENVELOPE] + let rpc = client(0) + let batch = [ + {method: 'debug_traceBlockByHash', params: ['0x01']}, + {method: 'eth_getBlockByHash', params: ['0x01', false]}, + ] + + let err = await rpc.batchCall(batch).catch((e) => e) + expect(err).toBeInstanceOf(RpcError) + expect(err.rpcMethod).toBeUndefined() + rpc.close() + }) + }) +}) diff --git a/util/rpc-client/src/client.ts b/util/rpc-client/src/client.ts index 94623bce7..fe71d5492 100644 --- a/util/rpc-client/src/client.ts +++ b/util/rpc-client/src/client.ts @@ -559,6 +559,10 @@ export class RpcClient { case 523: case 524: return true + // Bitcoin Core and others answer ordinary JSON-RPC errors with a 500. + // Any other body comes from a proxy in front of the node failing. + case 500: + return !isJsonRpcResponseBody(err.response.body) default: return false } @@ -650,3 +654,32 @@ function isRequestTimedOutError(err: unknown): boolean { return err instanceof RpcError && /request.*timed out/i.test(err.message) } + +function isJsonRpcResponseBody(body: unknown): boolean { + let json = parseJson(body) + let responses = Array.isArray(json) ? json : [json] + return responses.some(isJsonRpcResponse) +} + + +// JSON-RPC 1.0 responses, Bitcoin Core's among them, have no `jsonrpc` member, +// only `id`. A bare `error` object, with neither, comes from a proxy. +function isJsonRpcResponse(value: unknown): boolean { + if (value == null || typeof value != 'object') return false + + let hasEnvelope = 'jsonrpc' in value || 'id' in value + let hasOutcome = 'result' in value || 'error' in value + return hasEnvelope && hasOutcome +} + + +// JSON sent under a non-JSON content type arrives as text or bytes. +function parseJson(body: unknown): unknown { + if (typeof body != 'string' && !Buffer.isBuffer(body)) return body + try { + return JSON.parse(body.toString()) + } catch { + return undefined + } +} + diff --git a/util/rpc-client/src/transport/http.ts b/util/rpc-client/src/transport/http.ts index e4654b741..865a1c3cd 100644 --- a/util/rpc-client/src/transport/http.ts +++ b/util/rpc-client/src/transport/http.ts @@ -1,8 +1,9 @@ import {FetchRequest, FetchResponse, HttpAgent, HttpClient, HttpClientOptions} from '@subsquid/http-client' import {Logger} from '@subsquid/logger' +import {addErrorContext} from '@subsquid/util-internal' import {fixUnsafeIntegers} from '@subsquid/util-internal-json-fix-unsafe-integers' import {RpcError, RpcProtocolError} from '../errors' -import {Connection, RpcRequest, RpcResponse} from '../interfaces' +import {Connection, RpcErrorInfo, RpcRequest, RpcResponse} from '../interfaces' import {redactRpcUrlsInError} from '../redact' @@ -10,7 +11,7 @@ class RpcHttpClient extends HttpClient { fixUnsafeIntegers = false protected async handleResponseBody(req: FetchRequest, res: FetchResponse): Promise { - if (!res.ok) return super.handleResponseBody(req, res) + if (!res.ok) return this.handleErrorResponseBody(req, res) let json = await res.text() try { if (this.fixUnsafeIntegers) { @@ -21,6 +22,20 @@ class RpcHttpClient extends HttpClient { throw new RpcProtocolError(1008, `server returned invalid JSON: ${err.message}`) } } + + // A proxy's error page can come labeled as JSON. Kept as text when it does not + // parse, so that the status decides whether to retry, not a parse error. + private async handleErrorResponseBody(req: FetchRequest, res: FetchResponse): Promise { + let contentType = (res.headers.get('content-type') || '').split(';')[0] + if (contentType != 'application/json') return super.handleResponseBody(req, res) + + let text = await res.text() + try { + return JSON.parse(text) + } catch { + return text + } + } } @@ -100,7 +115,7 @@ export class HttpConnection implements Connection { // the misleading "should be an array" protocol error. let error = (res as unknown as RpcResponse | null)?.error if (error) { - throw new RpcError(error) + throw wholeBatchError(batch, error) } throw new RpcProtocolError(1008, `Response for a batch request should be an array`) } @@ -131,3 +146,14 @@ export class HttpConnection implements Connection { return res } } + + +// Carries the method, as an error in answer to a single call does, when the +// batch has only one: whether to retry can depend on it. +function wholeBatchError(batch: RpcRequest[], info: RpcErrorInfo): RpcError { + let err = new RpcError(info) + + let method = batch[0].method + let sameMethod = batch.every(call => call.method == method) + return sameMethod ? addErrorContext(err, {rpcMethod: method}) : err +}