diff --git a/src/integrationTestHelpers/anvilHarness.ts b/src/integrationTestHelpers/anvilHarness.ts index 5f5aa423..9193600e 100644 --- a/src/integrationTestHelpers/anvilHarness.ts +++ b/src/integrationTestHelpers/anvilHarness.ts @@ -46,7 +46,7 @@ import { refreshForkTime, setBalanceOnL1, } from './anvilHarnessHelpers'; - +import { type RpcCachingProxy, startRpcCachingProxy } from './rpcCachingProxy'; import type { CustomTimingParams, PrivateKeyAccountWithPrivateKey } from '../testHelpers'; export type AnvilTestStack = { @@ -75,6 +75,7 @@ let l1ContainerName: string | undefined; let l2ContainerName: string | undefined; let cleanupHookRegistered = false; let teardownStarted = false; +let l1RpcCachingProxy: RpcCachingProxy | undefined; export async function setupAnvilTestStack(): Promise { if (envPromise) { @@ -84,7 +85,9 @@ export async function setupAnvilTestStack(): Promise { teardownStarted = false; if (!cleanupHookRegistered) { - process.once('exit', () => teardownAnvilTestStack()); + process.once('exit', () => { + void teardownAnvilTestStack(); + }); cleanupHookRegistered = true; } @@ -118,6 +121,14 @@ export async function setupAnvilTestStack(): Promise { }, }; + const cacheFilePath = join(process.cwd(), '.cache', 'anvil-rpc-cache.json'); + + l1RpcCachingProxy = await startRpcCachingProxy(anvilForkUrl, cacheFilePath, { + forkBlockNumber: testConstants.DEFAULT_SEPOLIA_FORK_BLOCK_NUMBER, + }); + + const l1RpcUrlWithCaching = l1RpcCachingProxy.proxyUrl; + const harnessDeployer = createAccount(); const blockAdvancerAccount = createAccount(); @@ -130,7 +141,7 @@ export async function setupAnvilTestStack(): Promise { networkName: dockerNetworkName, l1RpcPort, anvilImage, - anvilForkUrl, + anvilForkUrl: l1RpcUrlWithCaching, anvilForkBlockNumber: testConstants.DEFAULT_SEPOLIA_FORK_BLOCK_NUMBER, chainId: sepolia.id, }); @@ -367,7 +378,7 @@ export async function setupAnvilTestStack(): Promise { return initializedEnv; })().catch((error) => { - teardownAnvilTestStack(); + void teardownAnvilTestStack(); throw error; }); @@ -388,6 +399,14 @@ export function teardownAnvilTestStack() { } teardownStarted = true; + if (l1RpcCachingProxy) { + for (const line of l1RpcCachingProxy.getSummaryLines()) { + console.log(line); + } + + l1RpcCachingProxy.close(); + } + cleanupCurrentHarnessResources({ l2ContainerName: l2ContainerName, l1ContainerName: l1ContainerName, @@ -399,6 +418,7 @@ export function teardownAnvilTestStack() { l1ContainerName = undefined; dockerNetworkName = undefined; runtimeDir = undefined; + l1RpcCachingProxy = undefined; envPromise = undefined; initializedEnv = undefined; } diff --git a/src/integrationTestHelpers/anvilHarnessHelpers.ts b/src/integrationTestHelpers/anvilHarnessHelpers.ts index 055d752a..ebfb2b6e 100644 --- a/src/integrationTestHelpers/anvilHarnessHelpers.ts +++ b/src/integrationTestHelpers/anvilHarnessHelpers.ts @@ -180,7 +180,6 @@ export async function fundL2Deployer(params: { return; } - const previousBalance = currentBalance; const { maxSubmissionCost, l2MaxFeePerGas } = await getRequiredRetryableFunding( l1Client, l2Client, @@ -211,9 +210,10 @@ export async function fundL2Deployer(params: { await l1Client.waitForTransactionReceipt({ hash: txHash }); const startedAt = Date.now(); + while (Date.now() - startedAt < 60_000) { currentBalance = await l2Client.getBalance({ address: deployer.address }); - if (currentBalance > previousBalance) { + if (currentBalance >= fundAmount) { return currentBalance; } @@ -266,7 +266,7 @@ export async function setBalanceOnL1(params: { const publicClient = createPublicClient({ transport: http(params.rpcUrl) }); await publicClient.request({ method: 'anvil_setBalance' as never, - params: [params.address, `0x${params.balance.toString(16)}`], + params: [params.address, `0x${params.balance.toString(16)}`] as never, }); } diff --git a/src/integrationTestHelpers/dockerHelpers.ts b/src/integrationTestHelpers/dockerHelpers.ts index 6e2231d7..34ab674c 100644 --- a/src/integrationTestHelpers/dockerHelpers.ts +++ b/src/integrationTestHelpers/dockerHelpers.ts @@ -208,6 +208,8 @@ export function startSourceL1AnvilContainer(params: { params.containerName, '--network', params.networkName, + '--add-host', + 'host.docker.internal:host-gateway', '--entrypoint', 'anvil', '-p', diff --git a/src/integrationTestHelpers/globalSetup.mjs b/src/integrationTestHelpers/globalSetup.mjs index 595f814b..cd6a1830 100644 --- a/src/integrationTestHelpers/globalSetup.mjs +++ b/src/integrationTestHelpers/globalSetup.mjs @@ -1,3 +1,9 @@ -import { setupAnvilTestStack } from './anvilHarness.ts'; +import { afterAll } from 'vitest'; + +import { setupAnvilTestStack, teardownAnvilTestStack } from './anvilHarness.ts'; await setupAnvilTestStack(); + +afterAll(() => { + teardownAnvilTestStack(); +}); diff --git a/src/integrationTestHelpers/rpcCachingProxy.ts b/src/integrationTestHelpers/rpcCachingProxy.ts new file mode 100644 index 00000000..2cf34d37 --- /dev/null +++ b/src/integrationTestHelpers/rpcCachingProxy.ts @@ -0,0 +1,237 @@ +import { createServer, IncomingHttpHeaders } from 'node:http'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { buffer } from 'node:stream/consumers'; + +type RpcRequest = { + id?: unknown; + jsonrpc?: string; + method?: string; + params?: unknown; +}; + +type RpcResponse = { + error?: unknown; + id?: unknown; + jsonrpc?: string; + result?: unknown; +}; + +type CacheEntry = { + error?: unknown; + jsonrpc: string; + result?: unknown; +}; + +type CacheData = Record; + +type CacheMetadata = { + forkBlockNumber: number; +}; + +type CacheFile = { + metadata: CacheMetadata; + entries: CacheData; +}; + +export type RpcCachingProxy = { + proxyUrl: string; + close: () => void; + getSummaryLines: () => string[]; +}; + +const CACHEABLE_METHODS = new Set([ + 'eth_chainId', + 'eth_gasPrice', + 'eth_getAccountInfo', + 'eth_getBalance', + 'eth_getBlockByNumber', + 'eth_getCode', + 'eth_getStorageAt', + 'eth_getTransactionCount', + 'eth_getTransactionReceipt', +]); + +function forwardHeaders(headers: IncomingHttpHeaders): Record { + const nextHeaders: Record = {}; + + for (const [key, value] of Object.entries(headers)) { + if (typeof value === 'undefined' || key === 'content-length' || key === 'host') { + continue; + } + + nextHeaders[key] = Array.isArray(value) ? value.join(', ') : value; + } + + return nextHeaders; +} + +function getCacheKey(request: RpcRequest): string | undefined { + if (typeof request.method !== 'string' || !CACHEABLE_METHODS.has(request.method)) { + return undefined; + } + + return JSON.stringify([request.method, request.params ?? []]); +} + +function getIdKey(id: unknown): string { + return JSON.stringify(id ?? null); +} + +function writeCacheFile(cacheFilePath: string, cacheFile: CacheFile) { + writeFileSync(cacheFilePath, JSON.stringify(cacheFile, null, 2)); +} + +function isCacheFile(value: unknown): value is CacheFile { + if (!value || typeof value !== 'object') { + return false; + } + + const maybeCacheFile = value as Partial; + return ( + !!maybeCacheFile.metadata && + typeof maybeCacheFile.metadata === 'object' && + typeof maybeCacheFile.metadata.forkBlockNumber === 'number' && + !!maybeCacheFile.entries && + typeof maybeCacheFile.entries === 'object' + ); +} + +function loadCacheData(params: { cacheFilePath: string; metadata: CacheMetadata }): { + cache: CacheData; + cacheInvalidated: boolean; +} { + const { cacheFilePath, metadata } = params; + + if (!existsSync(cacheFilePath)) { + writeCacheFile(cacheFilePath, { metadata, entries: {} }); + return { cache: {}, cacheInvalidated: false }; + } + + try { + const parsed = JSON.parse(readFileSync(cacheFilePath, 'utf8')) as unknown; + if (isCacheFile(parsed) && parsed.metadata.forkBlockNumber === metadata.forkBlockNumber) { + return { cache: parsed.entries, cacheInvalidated: false }; + } + } catch { + // Reset invalid cache files below. + } + + writeCacheFile(cacheFilePath, { metadata, entries: {} }); + return { cache: {}, cacheInvalidated: true }; +} + +export async function startRpcCachingProxy( + targetUrl: string, + cacheFilePath: string, + metadata: CacheMetadata, +): Promise { + mkdirSync(dirname(cacheFilePath), { recursive: true }); + + const { cache, cacheInvalidated } = loadCacheData({ cacheFilePath, metadata }); + + const stats = { + cacheInvalidated, + cacheHits: 0, + cacheMisses: 0, + requests: 0, + upstreamRequests: 0, + }; + + const server = createServer(async (request, response) => { + try { + const requestBody = await buffer(request); + const upstreamRequestBody = new Uint8Array(requestBody.byteLength); + upstreamRequestBody.set(requestBody); + + const parsed = JSON.parse(requestBody.toString('utf8')) as RpcRequest | RpcRequest[]; + const requests = Array.isArray(parsed) ? parsed : [parsed]; + + stats.requests += requests.length; + + const cacheKeys = requests.map((item) => getCacheKey(item)); + const cachedResponses = cacheKeys.map((cacheKey) => (cacheKey ? cache[cacheKey] : undefined)); + + if (cachedResponses.every((entry) => entry)) { + stats.cacheHits += requests.length; + response.setHeader('content-type', 'application/json'); + response.statusCode = 200; + response.end( + JSON.stringify( + Array.isArray(parsed) + ? requests.map((item, index) => ({ id: item.id ?? null, ...cachedResponses[index]! })) + : { id: requests[0].id ?? null, ...cachedResponses[0]! }, + ), + ); + return; + } + + stats.cacheMisses += cacheKeys.filter( + (cacheKey, index) => cacheKey && !cachedResponses[index], + ).length; + stats.upstreamRequests += 1; + + const upstreamResponse = await fetch(targetUrl, { + method: request.method, + headers: forwardHeaders(request.headers), + body: upstreamRequestBody, + }); + + const upstreamText = Buffer.from(await upstreamResponse.arrayBuffer()).toString('utf8'); + const upstreamResponses = [JSON.parse(upstreamText)].flat() as RpcResponse[]; + const upstreamById = new Map( + upstreamResponses.map((item) => [getIdKey(item.id), item] as const), + ); + + let cacheChanged = false; + + for (const [index, item] of requests.entries()) { + const cacheKey = cacheKeys[index]; + const upstreamItem = upstreamById.get(getIdKey(item.id)); + if (!cacheKey || !upstreamItem) { + continue; + } + + cache[cacheKey] = { + error: upstreamItem.error, + jsonrpc: upstreamItem.jsonrpc ?? '2.0', + result: upstreamItem.result, + }; + cacheChanged = true; + } + + if (cacheChanged) { + writeCacheFile(cacheFilePath, { metadata, entries: cache }); + } + + const contentType = upstreamResponse.headers.get('content-type'); + if (contentType) { + response.setHeader('content-type', contentType); + } + response.statusCode = upstreamResponse.status; + response.end(upstreamText); + } catch (error) { + response.statusCode = 502; + response.end( + JSON.stringify({ + error: error instanceof Error ? error.message : String(error), + }), + ); + } + }); + + server.listen(8449, '0.0.0.0'); + + return { + proxyUrl: `http://host.docker.internal:8449`, + getSummaryLines: () => [ + 'RPC proxy cache summary', + ` invalidated on startup: ${stats.cacheInvalidated ? 'yes' : 'no'}`, + ` requests: ${stats.requests}`, + ` cache hits: ${stats.cacheHits}`, + ` cache misses: ${stats.cacheMisses}`, + ` upstream HTTP requests: ${stats.upstreamRequests}`, + ], + close: () => server.close(), + }; +}