From 566d7f5052a711d9bff046b99af36fcc941b75e6 Mon Sep 17 00:00:00 2001 From: Facundo Date: Wed, 13 May 2026 13:31:12 -0300 Subject: [PATCH 01/17] refactor(p2p): merge FastTxCollection into TxCollection with sequential pipeline (#23245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Removes `FastTxCollection` as a separate class and absorbs all its logic directly into `TxCollection` - Replaces the old parallel file-store delay with a single sequential pipeline: node RPC → reqresp → file store, where each phase blocks on the previous (cancellation-aware) - File store collection is now driven by `IRequestTracker` — the same synchronization primitive used by node and reqresp paths. The tracker is the single source of truth for "is this tx still missing?" and "is this request still alive?" - `FileStoreTxCollection` simplified: dropped `start()`/`stop()`/persistent worker pool/`wakeSignal`. `startCollecting(requestTracker, context)` returns `Promise`, spins up its own per-call worker pool, and workers self-terminate when the tracker is cancelled (all-fetched / deadline / external) ## Collection flow inside `collectFast` 1. Start node RPC collection in the background 2. Wait `txCollectionFastNodesTimeoutBeforeReqRespMs` — interruptible by cancellation **or by node exhaustion** (so when no nodes are configured, reqresp starts immediately) 3. Start reqresp in the background (parallel with nodes) 4. Wait `txCollectionFileStoreFastDelayMs` — interruptible by cancellation or reqresp completion 5. Start file store collection in the background (its workers self-terminate) 6. `Promise.allSettled` on node + reqresp + file store `txCollectionFileStoreFastDelayMs` description updated to reflect it is now anchored to reqresp start, not collection start. ## File store / tracker integration - `FileStoreTxCollection.startCollecting` no longer takes `(txHashes, context, deadline)`; it takes `(requestTracker, context)` and reads the missing txs + deadline from the tracker - Workers check `requestTracker.isMissing(hash)` each scan — if the tx was found via another path (node/reqresp/gossipsub), the entry is dropped without an extra fetch - Workers race their backoff sleeps against `requestTracker.cancellationToken` — cancelling a request (deadline, `stopCollectingForBlocksUpTo/After`, or `stop()`) propagates to file store workers immediately - Removed `foundTxs`/`clearPending` plumbing on `FileStoreTxCollection` — the tracker handles both implicitly - `startCollecting` yields once after building its entry set, so a synchronous follow-up call (e.g. `markFetched` in tests, or the gossipsub-found path in production) lands before workers begin scanning ## Tests - `tx_collection.test.ts`: collapsed the `TestFastTxCollection` subclass; all accesses go directly through `TxCollection`. Added "starts reqresp immediately when no nodes are configured" covering the node-exhaustion shortcut - `file_store_tx_collection.test.ts`: rewritten for the new shape — no `start()`/`stop()`, lifecycle driven by the tracker (cancel to terminate workers). New "workers exit when tracker is cancelled" covers the per-call worker-pool teardown Closes https://linear.app/aztec-labs/issue/A-933/tx-collection-dont-retrieve-transactions-that-have-already-been via new synchronization with the request tracker. --- yarn-project/p2p/src/client/p2p_client.ts | 1 - .../p2p/src/services/tx_collection/config.ts | 4 +- .../tx_collection/fast_tx_collection.ts | 379 ----------------- .../file_store_tx_collection.test.ts | 166 +++----- .../tx_collection/file_store_tx_collection.ts | 157 +++---- .../p2p/src/services/tx_collection/index.ts | 3 +- .../tx_collection/tx_collection.test.ts | 87 ++-- .../services/tx_collection/tx_collection.ts | 395 +++++++++++++++--- 8 files changed, 491 insertions(+), 701 deletions(-) delete mode 100644 yarn-project/p2p/src/services/tx_collection/fast_tx_collection.ts diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index a91755a81b00..9e7c486033fc 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -269,7 +269,6 @@ export class P2PClient extends WithTracer implements P2P { throw new Error('Block stream not initialized'); } this.blockStream.start(); - await this.txCollection.start(); this.txFileStore?.start(); // Start slot monitor to call prepareForSlot when the slot changes diff --git a/yarn-project/p2p/src/services/tx_collection/config.ts b/yarn-project/p2p/src/services/tx_collection/config.ts index f8f5ceeea81f..68de3db09303 100644 --- a/yarn-project/p2p/src/services/tx_collection/config.ts +++ b/yarn-project/p2p/src/services/tx_collection/config.ts @@ -14,7 +14,7 @@ export type TxCollectionConfig = { txCollectionNodeRpcMaxBatchSize: number; /** A comma-separated list of file store URLs (s3://, gs://, file://, http://) for tx collection */ txCollectionFileStoreUrls: string[]; - /** Delay in ms before file store collection starts after fast collection is triggered */ + /** Delay in ms from reqresp start before file store collection begins */ txCollectionFileStoreFastDelayMs: number; /** Number of concurrent workers for fast file store collection */ txCollectionFileStoreFastWorkerCount: number; @@ -68,7 +68,7 @@ export const txCollectionConfigMappings: ConfigMappingsType }, txCollectionFileStoreFastDelayMs: { env: 'TX_COLLECTION_FILE_STORE_FAST_DELAY_MS', - description: 'Delay before file store collection starts after fast collection', + description: 'Delay in ms from reqresp start before file store collection begins', ...numberConfigHelper(2_000), }, txCollectionFileStoreFastWorkerCount: { diff --git a/yarn-project/p2p/src/services/tx_collection/fast_tx_collection.ts b/yarn-project/p2p/src/services/tx_collection/fast_tx_collection.ts deleted file mode 100644 index 7bcb1366342b..000000000000 --- a/yarn-project/p2p/src/services/tx_collection/fast_tx_collection.ts +++ /dev/null @@ -1,379 +0,0 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; -import { times } from '@aztec/foundation/collection'; -import { type Logger, createLogger } from '@aztec/foundation/log'; -import { sleep } from '@aztec/foundation/sleep'; -import { DateProvider, elapsed } from '@aztec/foundation/timer'; -import type { L2BlockInfo } from '@aztec/stdlib/block'; -import { type Tx, TxHash } from '@aztec/stdlib/tx'; - -import type { PeerId } from '@libp2p/interface'; - -import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js'; -import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js'; -import type { BlockTxsSource } from '../reqresp/index.js'; -import type { TxCollectionConfig } from './config.js'; -import { type IRequestTracker, RequestTracker } from './request_tracker.js'; -import type { FastCollectionRequest, FastCollectionRequestInput } from './tx_collection.js'; -import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js'; -import type { TxSource } from './tx_source.js'; - -/** - * Collect missing transactions for a block or proposal via reqresp. - * @param requestTracker - The missing transactions tracker - * @param blockTxsSource - The block or proposal containing the transactions - * @param pinnedPeer - Optional peer expected to have the transactions - * @returns The collected transactions - */ -export type IReqRespTxsCollector = ( - requestTracker: IRequestTracker, - blockTxsSource: BlockTxsSource, - pinnedPeer: PeerId | undefined, -) => Promise; - -export class FastTxCollection { - // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections - protected requests: Set = new Set(); - - constructor( - private readonly p2pService: BatchTxRequesterLibP2PService, - private nodes: TxSource[], - private txCollectionSink: TxCollectionSink, - private config: TxCollectionConfig, - private dateProvider: DateProvider = new DateProvider(), - private log: Logger = createLogger('p2p:tx_collection_service'), - protected reqRespTxsCollector?: IReqRespTxsCollector, - ) { - if (!this.reqRespTxsCollector) { - this.reqRespTxsCollector = (requestTracker, blockTxsSource, pinnedPeer) => - BatchTxRequester.collectAllTxs( - new BatchTxRequester( - requestTracker, - blockTxsSource, - pinnedPeer, - this.p2pService, - this.log, - this.dateProvider, - ).run(), - ); - } - } - - public async stop() { - this.requests.forEach(request => { - request.requestTracker.cancel(); - }); - await Promise.resolve(); - } - - public getFastCollectionRequests() { - return this.requests; - } - - public async collectFastFor( - input: FastCollectionRequestInput, - txHashes: TxHash[] | string[], - opts: { deadline: Date; pinnedPeer?: PeerId }, - ) { - const timeout = opts.deadline.getTime() - this.dateProvider.now(); - if (timeout <= 0) { - this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, { - deadline: opts.deadline.getTime(), - now: this.dateProvider.now(), - }); - return []; - } - - const blockInfo: L2BlockInfo = - input.type === 'proposal' - ? { ...input.blockProposal.toBlockInfo(), blockNumber: input.blockNumber } - : { ...input.block.toBlockInfo() }; - - const request: FastCollectionRequest = { - ...input, - blockInfo, - requestTracker: RequestTracker.create(txHashes, opts.deadline, this.dateProvider), - }; - - const [duration] = await elapsed(() => this.collectFast(request, { ...opts })); - - this.log.verbose( - `Collected ${request.requestTracker.collectedTxs.length} txs out of ${txHashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, - { - ...blockInfo, - duration, - requestType: input.type, - missingTxs: [...request.requestTracker.missingTxHashes], - }, - ); - return request.requestTracker.collectedTxs; - } - - protected async collectFast(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) { - this.requests.add(request); - const { blockInfo } = request; - - this.log.debug( - `Starting fast collection of ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, - { ...blockInfo, requestType: request.type, deadline: request.requestTracker.deadline }, - ); - - try { - // Start blasting all nodes for the txs. We give them a little time to respond before we start reqresp. - // We race against the cancellation token to exit as soon as all txs are collected, the deadline expires, - // or the request is externally cancelled. - const nodeCollectionPromise = this.collectFastFromNodes(request); - const waitBeforeReqResp = sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs); - await Promise.race([request.requestTracker.cancellationToken, waitBeforeReqResp]); - - // If we have collected all txs or the request was cancelled, we can stop here. - // Wait for node collection to settle so inner tasks finish before we return. - if (request.requestTracker.checkCancelled()) { - if (request.requestTracker.allFetched()) { - this.log.debug(`All txs collected for slot ${blockInfo.slotNumber} without reqresp`, blockInfo); - } - await nodeCollectionPromise; - return; - } - - // Start blasting reqresp for the remaining txs. Note that node collection keeps running in parallel. - // We stop when we have collected all txs, timed out, or both node collection and reqresp have given up. - // Inner tasks observe requestTracker.checkCancelled() and stop themselves, so this settles shortly after cancellation. - await Promise.allSettled([this.collectFastViaReqResp(request, opts), nodeCollectionPromise]); - } catch (err) { - this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, { - ...blockInfo, - missingTxs: request.requestTracker.missingTxHashes.values().map(txHash => txHash.toString()), - }); - } finally { - // Ensure no unresolved promises and remove the request from the set - request.requestTracker.cancel(); - this.requests.delete(request); - } - } - - /** - * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests - * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize - * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no - * tx is requested more than once at the same time to the same node. - */ - private async collectFastFromNodes(request: FastCollectionRequest): Promise { - if (this.nodes.length === 0) { - return; - } - - // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them. - const attemptsPerTx = [...request.requestTracker.missingTxHashes].map(txHash => ({ - txHash, - attempts: 0, - found: false, - })); - - // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected. - await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx))); - } - - private async collectFastFromNode( - request: FastCollectionRequest, - node: TxSource, - attemptsPerTx: { txHash: string; attempts: number; found: boolean }[], - ) { - const notFinished = () => !request.requestTracker.checkCancelled(); - - const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode; - const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize; - const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node - - const processBatch = async () => { - while (notFinished()) { - // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts, - // so we prioritize txs that have been requested less often. - const batch = []; - let index = 0; - while (batch.length < maxBatchSize) { - const txToRequest = attemptsPerTx[index++]; - if (!txToRequest) { - // No more txs to process - break; - } else if (!request.requestTracker.isMissing(txToRequest.txHash)) { - // Mark as found if it was found somewhere else, we'll then remove it from the array. - // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting. - txToRequest.found = true; - } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) { - // If the tx is not alredy being requested to this node, add it to the current batch and increase attempts. - // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that - // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting - // the same txs at the same time. - batch.push(txToRequest); - activeRequestsToThisNode.add(txToRequest.txHash); - txToRequest.attempts++; - } - } - - // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end. - attemptsPerTx.sort((a, b) => - a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found), - ); - const firstFoundTxIndex = attemptsPerTx.findIndex(tx => tx.found); - if (firstFoundTxIndex !== -1) { - attemptsPerTx.length = firstFoundTxIndex; - } - - // If we see no more txs to request, we can stop this "process" loop - if (batch.length === 0) { - return; - } - - const txHashes = batch.map(({ txHash }) => txHash); - // Collect this batch from the node - await this.txCollectionSink.collect( - async () => { - const result = await node.getTxsByHash(txHashes.map(TxHash.fromString)); - for (const tx of result.validTxs) { - request.requestTracker.markFetched(tx); - } - return result; - }, - txHashes, - { - description: `fast ${node.getInfo()}`, - node: node.getInfo(), - method: 'fast-node-rpc', - ...request.blockInfo, - }, - this.getAddContext(request), - ); - - // Clear from the active requests the txs we just requested - for (const requestedTx of batch) { - activeRequestsToThisNode.delete(requestedTx.txHash); - } - - // Sleep a bit until hitting the node again, but wake up immediately on cancellation - if (notFinished()) { - await Promise.race([ - sleep(this.config.txCollectionFastNodeIntervalMs), - request.requestTracker.cancellationToken, - ]); - } - } - }; - - // Kick off N parallel requests to the node, up to the maxParallelRequests limit - await Promise.all(times(maxParallelRequests, processBatch)); - } - - private async collectFastViaReqResp(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) { - const pinnedPeer = opts.pinnedPeer; - const blockInfo = request.blockInfo; - const slotNumber = blockInfo.slotNumber; - if (request.requestTracker.timeoutMs < 100) { - this.log.warn( - `Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, - { timeoutMs: request.requestTracker.timeoutMs, ...blockInfo }, - ); - return; - } - - this.log.debug( - `Starting fast reqresp for ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, - { ...blockInfo, timeoutMs: request.requestTracker.timeoutMs, pinnedPeer }, - ); - - try { - await this.txCollectionSink.collect( - async () => { - let blockTxsSource: BlockTxsSource; - if (request.type === 'proposal') { - blockTxsSource = request.blockProposal; - } else if (request.type === 'block') { - blockTxsSource = { - txHashes: request.block.body.txEffects.map(e => e.txHash), - archive: request.block.archive.root, - }; - } else { - throw new Error(`Unknown request type: ${(request as { type: string }).type}`); - } - - const result = await this.reqRespTxsCollector!(request.requestTracker, blockTxsSource, pinnedPeer); - return { validTxs: result, invalidTxHashes: [] }; - }, - Array.from(request.requestTracker.missingTxHashes), - { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo }, - this.getAddContext(request), - ); - } catch (err) { - this.log.error(`Error sending fast reqresp request for txs`, err, { - txs: [...request.requestTracker.missingTxHashes], - ...blockInfo, - }); - } - } - - /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ - private getAddContext(request: FastCollectionRequest): TxAddContext { - if (request.type === 'proposal') { - return { type: 'proposal', blockHeader: request.blockProposal.blockHeader }; - } else { - return { type: 'mined', block: request.block }; - } - } - - /** - * Handle txs by marking them as found for the requests that are waiting for them, and resolves the request if all its txs have been found. - * Called internally and from the main tx collection manager whenever the tx pool emits a tx-added event. - */ - public foundTxs(txs: Tx[]) { - for (const request of this.requests) { - for (const tx of txs) { - const txHash = tx.txHash.toString(); - // Remove the tx hash from the missing set, and add it to the found set. - if (request.requestTracker.markFetched(tx)) { - this.log.trace(`Found tx ${txHash} for fast collection request`, { - ...request.blockInfo, - txHash: tx.txHash.toString(), - type: request.type, - }); - if (request.requestTracker.allFetched()) { - this.log.trace(`All txs found for fast collection request`, { - ...request.blockInfo, - type: request.type, - }); - break; - } - } - } - } - } - - /** Returns the tx hashes that are still missing (from all requests). */ - public getMissingTxHashes(): TxHash[] { - return Array.from(this.requests.values()).flatMap(request => - Array.from(request.requestTracker.missingTxHashes).map(TxHash.fromString), - ); - } - - /** - * Stop collecting all txs for blocks less than or requal to the block number specified. - * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized. - */ - public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void { - for (const request of this.requests) { - if (request.blockInfo.blockNumber <= blockNumber) { - request.requestTracker.cancel(); - } - } - } - - /** - * Stop collecting all txs for blocks greater than the block number specified. - * To be called when there is a chain prune and previously mined txs are no longer relevant. - */ - public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void { - for (const request of this.requests) { - if (request.blockInfo.blockNumber > blockNumber) { - request.requestTracker.cancel(); - } - } - } -} diff --git a/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.test.ts b/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.test.ts index eacef127cfb2..f41512a1dbca 100644 --- a/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.test.ts +++ b/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.test.ts @@ -12,6 +12,7 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import type { TxPoolV2 } from '../../mem_pools/tx_pool_v2/interfaces.js'; import { type FileStoreCollectionConfig, FileStoreTxCollection } from './file_store_tx_collection.js'; import type { FileStoreTxSource } from './file_store_tx_source.js'; +import { type IRequestTracker, RequestTracker } from './request_tracker.js'; import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js'; describe('FileStoreTxCollection', () => { @@ -26,6 +27,11 @@ describe('FileStoreTxCollection', () => { let txs: Tx[]; let txHashes: TxHash[]; + let requestTracker: IRequestTracker; + + // Track in-flight startCollecting invocations so afterEach can shut them down cleanly. + let activeTrackers: IRequestTracker[]; + let activePromises: Promise[]; const makeFileStoreSource = (name: string) => { const source = mock(); @@ -49,6 +55,14 @@ describe('FileStoreTxCollection', () => { }); }; + /** Spawns a collection run and registers it for afterEach cleanup. */ + const startCollecting = (tracker: IRequestTracker, ctx: TxAddContext): Promise => { + activeTrackers.push(tracker); + const promise = fileStoreCollection.startCollecting(tracker, ctx); + activePromises.push(promise); + return promise; + }; + /** Waits for the sink to emit txs-added events for the expected number of txs. */ const waitForTxsAdded = (expectedCount: number) => { const { promise, resolve } = promiseWithResolvers(); @@ -102,33 +116,38 @@ describe('FileStoreTxCollection', () => { const block = await L2Block.random(BlockNumber(1)); context = { type: 'mined', block }; deadline = new Date(dateProvider.now() + 60 * 60 * 1000); + requestTracker = RequestTracker.create(txHashes, deadline, dateProvider); + + activeTrackers = []; + activePromises = []; }); afterEach(async () => { - await fileStoreCollection.stop(); + for (const t of activeTrackers) { + t.cancel(); + } + await Promise.allSettled(activePromises); jest.restoreAllMocks(); }); it('downloads txs when startCollecting is called', async () => { setFileStoreTxs(fileStoreSources[0], txs); - fileStoreCollection.start(); - const txsAddedPromise = waitForTxsAdded(txs.length); - fileStoreCollection.startCollecting(txHashes, context, deadline); + void startCollecting(requestTracker, context); await txsAddedPromise; expect(fileStoreSources[0].getTxsByHash).toHaveBeenCalled(); expect(txPool.addMinedTxs).toHaveBeenCalled(); }); - it('skips txs marked as found', async () => { + it('skips txs already marked fetched on the tracker', async () => { setFileStoreTxs(fileStoreSources[0], txs); - fileStoreCollection.start(); + // Mark first tx as found before queueing so it's never queued in the first place + requestTracker.markFetched(txs[0]); - fileStoreCollection.startCollecting(txHashes, context, deadline); - fileStoreCollection.foundTxs([txs[0]]); + void startCollecting(requestTracker, context); const txsAddedPromise = waitForTxsAdded(2); await txsAddedPromise; @@ -145,53 +164,25 @@ describe('FileStoreTxCollection', () => { // Pin random so we always start at source 0, ensuring we test the fallback to source 1 jest.spyOn(Math, 'random').mockReturnValue(0); - fileStoreCollection.start(); - + const tracker = RequestTracker.create([txHashes[0]], deadline, dateProvider); const txsAddedPromise = waitForTxsAdded(1); - fileStoreCollection.startCollecting([txHashes[0]], context, deadline); + void startCollecting(tracker, context); await txsAddedPromise; // Both stores should have been tried expect(fileStoreSources[0].getTxsByHash).toHaveBeenCalled(); expect(fileStoreSources[1].getTxsByHash).toHaveBeenCalled(); expect(txPool.addMinedTxs).toHaveBeenCalled(); - - jest.restoreAllMocks(); }); - it('does not start workers if no file store sources are configured', () => { + it('does not start workers if no file store sources are configured', async () => { const log = createLogger('test'); fileStoreCollection = new FileStoreTxCollection([], txCollectionSink, config, dateProvider, log); - fileStoreCollection.start(); - fileStoreCollection.startCollecting(txHashes, context, deadline); - - // With no sources, start() is a no-op (no workers spawned) and startCollecting() returns - // immediately, so no calls should have been made synchronously. - expect(fileStoreSources[0].getTxsByHash).not.toHaveBeenCalled(); - }); - - it('does not re-queue txs that are already pending', async () => { - setFileStoreTxs(fileStoreSources[0], txs); - setFileStoreTxs(fileStoreSources[1], txs); - - // Use single worker for deterministic behavior - const log = createLogger('test'); - config = { workerCount: 1, backoffBaseMs: 1000, backoffMaxMs: 5000 }; - fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); - - fileStoreCollection.start(); - - const txsAddedPromise = waitForTxsAdded(txs.length); - fileStoreCollection.startCollecting(txHashes, context, deadline); - fileStoreCollection.startCollecting(txHashes, context, deadline); // Duplicate call + // With no sources, startCollecting resolves immediately without making any calls. + await startCollecting(requestTracker, context); - await txsAddedPromise; - - // With 1 worker processing sequentially, each tx should be found on the first source. - // Duplicate startCollecting should not create extra entries. - const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); - expect(allCalls.length).toBe(txHashes.length); + expect(fileStoreSources[0].getTxsByHash).not.toHaveBeenCalled(); }); it('retries across sources when tx is not found initially', async () => { @@ -200,10 +191,9 @@ describe('FileStoreTxCollection', () => { config = { workerCount: 1, backoffBaseMs: 100, backoffMaxMs: 500 }; fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); - fileStoreCollection.start(); - // Initially both sources return empty - fileStoreCollection.startCollecting([txHashes[0]], context, deadline); + const tracker = RequestTracker.create([txHashes[0]], deadline, dateProvider); + void startCollecting(tracker, context); // Wait for first full cycle (2 sources = 2 calls) await waitForSourceCalls(fileStoreSources, 2); @@ -220,88 +210,54 @@ describe('FileStoreTxCollection', () => { expect(txPool.addMinedTxs).toHaveBeenCalled(); }); - it('expires entries past deadline', async () => { - const log = createLogger('test'); - config = { workerCount: 1, backoffBaseMs: 50, backoffMaxMs: 100 }; - fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); - - // Set a very short deadline - const shortDeadline = new Date(dateProvider.now() + 100); - - fileStoreCollection.start(); - fileStoreCollection.startCollecting([txHashes[0]], context, shortDeadline); - - // Wait for first full cycle (2 sources = 2 calls) - await waitForSourceCalls(fileStoreSources, 2); - - // Advance time past the deadline - dateProvider.setTime(dateProvider.now() + 200); - - // Clear mocks so we can distinguish new calls from old ones - jest.clearAllMocks(); - - // Add a new entry with a valid deadline and set up source to return it. - // This proves the worker is alive and the expired entry was cleaned up. - setFileStoreTxs(fileStoreSources[0], [txs[1]]); - const txsAddedPromise = waitForTxsAdded(1); - fileStoreCollection.startCollecting([txHashes[1]], context, deadline); - await txsAddedPromise; - - // Only txHashes[1] should have been requested after clearing mocks - const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); - const requestedHashes = allCalls.flat().flat(); - expect(requestedHashes).not.toContainEqual(txHashes[0]); - expect(requestedHashes).toContainEqual(txHashes[1]); - }); - - it('does not start collecting if deadline is in the past', () => { - const pastDeadline = new Date(dateProvider.now() - 1000); + it('does not start collecting if tracker is already cancelled', async () => { + requestTracker.cancel(); - fileStoreCollection.start(); - fileStoreCollection.startCollecting(txHashes, context, pastDeadline); + await startCollecting(requestTracker, context); - // startCollecting returns immediately without adding entries when deadline is past + // startCollecting returns immediately without spawning workers when tracker is cancelled expect(fileStoreSources[0].getTxsByHash).not.toHaveBeenCalled(); }); - it('foundTxs stops retry for found txs', async () => { + it('stops trying for txs marked fetched on the tracker after queuing', async () => { const log = createLogger('test'); config = { workerCount: 1, backoffBaseMs: 50, backoffMaxMs: 100 }; fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); setFileStoreTxs(fileStoreSources[0], [txs[1]]); - fileStoreCollection.start(); - fileStoreCollection.startCollecting(txHashes, context, deadline); + void startCollecting(requestTracker, context); - // Mark first tx as found - fileStoreCollection.foundTxs([txs[0]]); + // Externally mark tx[0] as found via the tracker (simulating node/reqresp/gossip finding it). + // startCollecting yields before spawning workers, so this runs before any source call is made. + requestTracker.markFetched(txs[0]); const txsAddedPromise = waitForTxsAdded(1); await txsAddedPromise; - // tx[0] should never have been attempted + // tx[0] should never have been attempted by the file store const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); const requestedHashes = allCalls.flat().flat(); expect(requestedHashes).not.toContainEqual(txHashes[0]); }); - it('clearPending removes all entries', async () => { - fileStoreCollection.start(); - fileStoreCollection.startCollecting(txHashes, context, deadline); - fileStoreCollection.clearPending(); + it('workers exit when tracker is cancelled', async () => { + // Long backoff so workers spend most of their time sleeping after a single attempt + const log = createLogger('test'); + config = { workerCount: 2, backoffBaseMs: 60_000, backoffMaxMs: 60_000 }; + fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); + + // Pre-set the tracker timer so a cancellation does not require real-time deadline expiry + const tracker = RequestTracker.create(txHashes, deadline, dateProvider); + const promise = startCollecting(tracker, context); - // Verify workers are alive but the cleared entries are gone by adding - // a new entry and confirming only it gets processed. - setFileStoreTxs(fileStoreSources[0], [txs[0]]); - const txsAddedPromise = waitForTxsAdded(1); - fileStoreCollection.startCollecting([txHashes[0]], context, deadline); - await txsAddedPromise; + // Let workers do at least one round of attempts + await waitForSourceCalls(fileStoreSources, 2); - // Only the newly added tx[0] should have been requested, not all 3 original txs - const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); - const requestedHashes = allCalls.flat().flat(); - expect(requestedHashes).not.toContainEqual(txHashes[1]); - expect(requestedHashes).not.toContainEqual(txHashes[2]); + tracker.cancel(); + + // The startCollecting promise resolves once all workers settle. Without this guarantee, the + // test would either hang or leak workers — both are caught by Jest's default timeout. + await promise; }); }); diff --git a/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.ts b/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.ts index 165ba3d9928a..abaf1b64ad6e 100644 --- a/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.ts +++ b/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.ts @@ -1,10 +1,11 @@ +import { times } from '@aztec/foundation/collection'; import { type Logger, createLogger } from '@aztec/foundation/log'; -import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise'; import { sleep } from '@aztec/foundation/sleep'; import { DateProvider } from '@aztec/foundation/timer'; -import { Tx, TxHash } from '@aztec/stdlib/tx'; +import { TxHash } from '@aztec/stdlib/tx'; import type { FileStoreTxSource } from './file_store_tx_source.js'; +import type { IRequestTracker } from './request_tracker.js'; import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js'; /** Configuration for a FileStoreTxCollection instance. */ @@ -16,8 +17,6 @@ export type FileStoreCollectionConfig = { type FileStoreTxEntry = { txHash: string; - context: TxAddContext; - deadline: Date; attempts: number; lastAttemptTime: number; nextSourceIndex: number; @@ -25,96 +24,60 @@ type FileStoreTxEntry = { /** * Collects txs from file stores as a fallback after P2P methods have been tried. - * Uses a shared worker pool that pulls entries with priority (fewest attempts first), - * retries with round-robin across sources, and applies exponential backoff between - * full cycles through all sources. + * Each call to startCollecting spins up its own worker pool which pulls entries with priority + * (fewest attempts first), retries with round-robin across sources, and applies exponential + * backoff between full cycles through all sources. Workers self-terminate when the request + * tracker is cancelled (deadline / all-fetched / external) or when there is nothing left to do. */ export class FileStoreTxCollection { - /** Map from tx hash string to entry for all pending downloads. */ - private entries = new Map(); - - /** Worker promises for the shared worker pool. */ - private workers: Promise[] = []; - - /** Whether the worker pool is running. */ - private running = false; - - /** Signal used to wake sleeping workers when new entries arrive or stop is called. */ - private wakeSignal: PromiseWithResolvers; - constructor( private readonly sources: FileStoreTxSource[], private readonly txCollectionSink: TxCollectionSink, private readonly config: FileStoreCollectionConfig, private readonly dateProvider: DateProvider = new DateProvider(), private readonly log: Logger = createLogger('p2p:file_store_tx_collection'), - ) { - this.wakeSignal = promiseWithResolvers(); - } - - /** Starts the shared worker pool. */ - public start(): void { - if (this.sources.length === 0) { - this.log.debug('No file store sources configured'); - return; - } - this.running = true; - for (let i = 0; i < this.config.workerCount; i++) { - this.workers.push(this.workerLoop()); - } - } - - /** Stops all workers and clears state. */ - public async stop(): Promise { - this.running = false; - this.wake(); - await Promise.all(this.workers); - this.workers = []; - this.entries.clear(); - } - - /** Adds entries to the shared map and wakes workers. */ - public startCollecting(txHashes: TxHash[], context: TxAddContext, deadline: Date): void { - if (this.sources.length === 0 || txHashes.length === 0) { - return; - } - if (+deadline <= this.dateProvider.now()) { + ) {} + + /** + * Spins up workers to download all txs still missing from the tracker, racing across the + * configured file store sources. Resolves once all workers settle. + */ + public async startCollecting(requestTracker: IRequestTracker, context: TxAddContext): Promise { + if (this.sources.length === 0 || requestTracker.checkCancelled()) { return; } - for (const txHash of txHashes) { - const hashStr = txHash.toString(); - if (!this.entries.has(hashStr)) { - this.entries.set(hashStr, { - txHash: hashStr, - context, - deadline, - attempts: 0, - lastAttemptTime: 0, - nextSourceIndex: Math.floor(Math.random() * this.sources.length), - }); - } + // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections + const entries: Set = new Set(); + for (const hashStr of requestTracker.missingTxHashes) { + entries.add({ + txHash: hashStr, + attempts: 0, + lastAttemptTime: 0, + nextSourceIndex: Math.floor(Math.random() * this.sources.length), + }); } - this.wake(); - } - /** Removes entries for txs that have been found elsewhere. */ - public foundTxs(txs: Tx[]): void { - for (const tx of txs) { - this.entries.delete(tx.getTxHash().toString()); + // Yield before spawning so the synchronous caller can finish any follow-up (eg. marking a tx + // as fetched on the tracker, or cancelling it) before workers begin scanning entries. + await Promise.resolve(); + if (requestTracker.checkCancelled()) { + return; } - } - /** Clears all pending entries. */ - public clearPending(): void { - this.entries.clear(); + await Promise.allSettled(times(this.config.workerCount, () => this.workerLoop(entries, requestTracker, context))); } - private async workerLoop(): Promise { - while (this.running) { - const action = this.getNextAction(); + private async workerLoop( + // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections + entries: Set, + requestTracker: IRequestTracker, + context: TxAddContext, + ): Promise { + while (!requestTracker.checkCancelled() && entries.size > 0) { + const action = this.getNextAction(entries, requestTracker); if (action.type === 'sleep') { - await action.promise; + await Promise.race([sleep(action.ms), requestTracker.cancellationToken]); continue; } @@ -133,10 +96,10 @@ export class FileStoreTxCollection { method: 'file-store', fileStore: source.getInfo(), }, - entry.context, + context, ); if (result.txs.length > 0) { - this.entries.delete(entry.txHash); + entries.delete(entry); } } catch (err) { this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, { err }); @@ -144,15 +107,20 @@ export class FileStoreTxCollection { } } - /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */ - private getNextAction(): { type: 'process'; entry: FileStoreTxEntry } | { type: 'sleep'; promise: Promise } { + /** Single-pass scan: removes stale entries, finds the best ready entry, or computes sleep time. */ + private getNextAction( + // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections + entries: Set, + requestTracker: IRequestTracker, + ): { type: 'process'; entry: FileStoreTxEntry } | { type: 'sleep'; ms: number } { const now = this.dateProvider.now(); let best: FileStoreTxEntry | undefined; let earliestReadyAt = Infinity; - for (const [key, entry] of this.entries) { - if (+entry.deadline <= now) { - this.entries.delete(key); + for (const entry of entries) { + // Drop entries whose tx was already found via another collection path. + if (!requestTracker.isMissing(entry.txHash)) { + entries.delete(entry); continue; } const backoffMs = this.getBackoffMs(entry); @@ -169,10 +137,9 @@ export class FileStoreTxCollection { if (best) { return { type: 'process', entry: best }; } - if (earliestReadyAt < Infinity) { - return { type: 'sleep', promise: this.sleepOrWake(earliestReadyAt - now) }; - } - return { type: 'sleep', promise: this.waitForWake() }; + // earliestReadyAt is finite whenever there are surviving entries; if entries became empty, + // the outer worker loop will exit on its next iteration via entries.size === 0. + return { type: 'sleep', ms: earliestReadyAt === Infinity ? 0 : earliestReadyAt - now }; } /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */ @@ -183,20 +150,4 @@ export class FileStoreTxCollection { } return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs); } - - /** Resolves the current wake signal and creates a new one. */ - private wake(): void { - this.wakeSignal.resolve(); - this.wakeSignal = promiseWithResolvers(); - } - - /** Waits until the wake signal is resolved. */ - private async waitForWake(): Promise { - await this.wakeSignal.promise; - } - - /** Sleeps for the given duration or until the wake signal is resolved. */ - private async sleepOrWake(ms: number): Promise { - await Promise.race([sleep(ms), this.wakeSignal.promise]); - } } diff --git a/yarn-project/p2p/src/services/tx_collection/index.ts b/yarn-project/p2p/src/services/tx_collection/index.ts index 4f151c32e27f..293ebdde7ab3 100644 --- a/yarn-project/p2p/src/services/tx_collection/index.ts +++ b/yarn-project/p2p/src/services/tx_collection/index.ts @@ -1,4 +1,3 @@ -export { TxCollection, type FastCollectionRequestInput } from './tx_collection.js'; -export { type IReqRespTxsCollector } from './fast_tx_collection.js'; +export { TxCollection, type FastCollectionRequestInput, type IReqRespTxsCollector } from './tx_collection.js'; export { type TxSource, createNodeRpcTxSources, NodeRpcTxSource } from './tx_source.js'; export { FileStoreTxSource, createFileStoreTxSources } from './file_store_tx_source.js'; diff --git a/yarn-project/p2p/src/services/tx_collection/tx_collection.test.ts b/yarn-project/p2p/src/services/tx_collection/tx_collection.test.ts index 750e09e34fb3..5cb61cbeedd9 100644 --- a/yarn-project/p2p/src/services/tx_collection/tx_collection.test.ts +++ b/yarn-project/p2p/src/services/tx_collection/tx_collection.test.ts @@ -16,9 +16,8 @@ import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interf import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js'; import type { BlockTxsSource } from '../reqresp/protocols/block_txs/block_txs_reqresp.js'; import { type TxCollectionConfig, txCollectionConfigMappings } from './config.js'; -import { FastTxCollection, type IReqRespTxsCollector } from './fast_tx_collection.js'; import type { FileStoreTxSource } from './file_store_tx_source.js'; -import { type FastCollectionRequest, TxCollection } from './tx_collection.js'; +import { type FastCollectionRequest, type IReqRespTxsCollector, TxCollection } from './tx_collection.js'; import type { TxSource } from './tx_source.js'; describe('TxCollection', () => { @@ -95,7 +94,7 @@ describe('TxCollection', () => { const setReqRespResponse = (promise: Promise) => { let lastArgs: Parameters | undefined; - txCollection.fastCollection.reqRespTxsCollector = jest.fn().mockImplementation((...x) => { + txCollection.reqRespTxsCollector = jest.fn().mockImplementation((...x) => { lastArgs = x; return promise; }); @@ -147,16 +146,16 @@ describe('TxCollection', () => { setReqRespTxs([]); }); - afterEach(async () => { - await txCollection.stop(); + afterEach(() => { + txCollection.stop(); }); - describe('fast collection', () => { + describe('fast tx collection', () => { it('collects txs from nodes only', async () => { setNodeTxs(nodes[0], txs); const collected = await txCollection.collectFastForBlock(block, txHashes, { deadline }); expect(nodes[0].getTxsByHash).toHaveBeenCalledWith(txHashes); - expect(txCollection.fastCollection.reqRespTxsCollector).not.toHaveBeenCalled(); + expect(txCollection.reqRespTxsCollector).not.toHaveBeenCalled(); expectTxsMinedInPool(txs); expect(collected).toEqual(txs); }); @@ -191,7 +190,7 @@ describe('TxCollection', () => { const collected = await txCollection.collectFastForBlock(block, txHashes, { deadline }); expect(nodes[0].getTxsByHash).toHaveBeenCalledWith(txHashes); expect(nodes[1].getTxsByHash).toHaveBeenCalledWith(txHashes); - expect(txCollection.fastCollection.reqRespTxsCollector).toHaveBeenCalledTimes(1); + expect(txCollection.reqRespTxsCollector).toHaveBeenCalledTimes(1); expectLastReqRespCollectorArgs(argsGetter); expectTxsMinedInPool([txs[0]]); expectTxsMinedInPool([txs[1]]); @@ -203,12 +202,26 @@ describe('TxCollection', () => { txCollection = new TestTxCollection(mockP2PService, [], constants, txPool, config, [], dateProvider); const argsGetter = setReqRespTxs(txs); const collected = await txCollection.collectFastForBlock(block, txHashes, { deadline }); - expect(txCollection.fastCollection.reqRespTxsCollector).toHaveBeenCalledTimes(1); + expect(txCollection.reqRespTxsCollector).toHaveBeenCalledTimes(1); expectLastReqRespCollectorArgs(argsGetter); expectTxsMinedInPool(txs); expect(collected).toEqual(txs); }); + it('starts reqresp immediately when no nodes are configured', async () => { + // Large initial wait — if reqresp were gated by it, the collection would take ~10s. + config = { ...config, txCollectionFastNodesTimeoutBeforeReqRespMs: 10_000 }; + txCollection = new TestTxCollection(mockP2PService, [], constants, txPool, config, [], dateProvider); + setReqRespTxs(txs); + + const startTime = dateProvider.now(); + const collected = await txCollection.collectFastForBlock(block, txHashes, { deadline }); + + expect(txCollection.reqRespTxsCollector).toHaveBeenCalledTimes(1); + expect(dateProvider.now() - startTime).toBeLessThan(1000); + expect(collected).toEqual(txs); + }); + it('keeps retrying txs not found until deadline', async () => { deadline = new Date(dateProvider.now() + 2000); setNodeTxs(nodes[0], [txs[0]]); @@ -219,7 +232,7 @@ describe('TxCollection', () => { expect(dateProvider.now()).toBeGreaterThanOrEqual(+deadline - 5); expect(nodes[0].getTxsByHash).toHaveBeenCalledWith(txHashes); expect(nodes[0].getTxsByHash).toHaveBeenCalledWith([txHashes[2]]); - expect(txCollection.fastCollection.reqRespTxsCollector).toHaveBeenCalledTimes(1); + expect(txCollection.reqRespTxsCollector).toHaveBeenCalledTimes(1); expectLastReqRespCollectorArgs(argsGetter); expectTxsMinedInPool([txs[0]]); expectTxsMinedInPool([txs[1]]); @@ -274,15 +287,15 @@ describe('TxCollection', () => { const collected = await txCollection.collectFastForBlock(block, txHashes, { deadline }); expect(collected).toEqual([]); expect(nodes[0].getTxsByHash).not.toHaveBeenCalled(); - expect(txCollection.fastCollection.reqRespTxsCollector).not.toHaveBeenCalled(); + expect(txCollection.reqRespTxsCollector).not.toHaveBeenCalled(); }); describe('cancellation signals', () => { /** Captures the FastCollectionRequest during collectFast, before it's removed in finally. */ const captureRequest = () => { let captured: FastCollectionRequest | undefined; - const origCollectFast = txCollection.fastCollection.collectFast.bind(txCollection.fastCollection); - jest.spyOn(txCollection.fastCollection, 'collectFast').mockImplementation((request, opts) => { + const origCollectFast = txCollection.collectFast.bind(txCollection); + jest.spyOn(txCollection, 'collectFast').mockImplementation((request, opts) => { captured = request; return origCollectFast(request, opts); }); @@ -319,7 +332,7 @@ describe('TxCollection', () => { setReqRespTxs([]); const collected = await txCollection.collectFastForBlock(block, txHashes, { deadline }); - expect(txCollection.fastCollection.reqRespTxsCollector).not.toHaveBeenCalled(); + expect(txCollection.reqRespTxsCollector).not.toHaveBeenCalled(); expect(collected).toEqual(txs); }); @@ -332,7 +345,7 @@ describe('TxCollection', () => { const collected = await txCollection.collectFastForBlock(block, txHashes, { deadline }); - expect(txCollection.fastCollection.reqRespTxsCollector).not.toHaveBeenCalled(); + expect(txCollection.reqRespTxsCollector).not.toHaveBeenCalled(); expect(dateProvider.now()).toBeGreaterThanOrEqual(+deadline - 5); expect(collected).toEqual([]); }); @@ -382,13 +395,13 @@ describe('TxCollection', () => { const request = getRequest(); expect(request).toBeDefined(); // Reqresp should not have started yet — we're still in the initial wait - expect(txCollection.fastCollection.reqRespTxsCollector).not.toHaveBeenCalled(); + expect(txCollection.reqRespTxsCollector).not.toHaveBeenCalled(); request.requestTracker.cancel(); await collectionPromise; // Should have exited without ever starting reqresp - expect(txCollection.fastCollection.reqRespTxsCollector).not.toHaveBeenCalled(); + expect(txCollection.reqRespTxsCollector).not.toHaveBeenCalled(); expect(dateProvider.now()).toBeLessThan(+deadline); }); @@ -406,7 +419,7 @@ describe('TxCollection', () => { const collectionPromise = txCollection.collectFastForBlock(block, txHashes, { deadline }); await sleep(200); - expect(txCollection.fastCollection.reqRespTxsCollector).toHaveBeenCalled(); + expect(txCollection.reqRespTxsCollector).toHaveBeenCalled(); getRequest().requestTracker.cancel(); collectorPromise.resolve([]); @@ -439,7 +452,7 @@ describe('TxCollection', () => { expect(request).toBeDefined(); expect(request.requestTracker.checkCancelled()).toBe(false); - await txCollection.stop(); + txCollection.stop(); expect(request.requestTracker.checkCancelled()).toBe(true); collectorPromise.resolve([]); @@ -489,13 +502,13 @@ describe('TxCollection', () => { const collectionPromise = txCollection.collectFastForBlock(block, txHashes, { deadline }); await sleep(100); - expect(txCollection.fastCollection.requests.size).toBe(1); + expect(txCollection.requests.size).toBe(1); txCollection.stopCollectingForBlocksUpTo(block.number); collectorPromise.resolve([]); await collectionPromise; - expect(txCollection.fastCollection.requests.size).toBe(0); + expect(txCollection.requests.size).toBe(0); }); }); }); @@ -529,17 +542,15 @@ describe('TxCollection', () => { it('collects txs from file store after configured delay', async () => { setFileStoreTxs(fileStoreSources[0], txs); - await txCollection.start(); - deadline = new Date(dateProvider.now() + 500); + // Long deadline so the collection ends when file store finds the txs (not when deadline fires) + deadline = new Date(dateProvider.now() + 5000); const collectionPromise = txCollection.collectFastForBlock(block, txHashes, { deadline }); - // File store should not have been called yet (delay hasn't elapsed) + // File store should not have been called yet (delays haven't elapsed) expect(fileStoreSources[0].getTxsByHash).not.toHaveBeenCalled(); - // Advance time past the configured file store delay - dateProvider.setTime(dateProvider.now() + 200); - // Allow the async sleep resolution and worker processing to complete - await sleep(200); + // Wait for: node wait (200ms default) + file store delay (100ms) + worker processing + await sleep(500); await collectionPromise; // File store should now have been called for each tx @@ -549,34 +560,28 @@ describe('TxCollection', () => { it('does not download txs from file store if found via P2P before delay expires', async () => { setFileStoreTxs(fileStoreSources[0], txs); - await txCollection.start(); - deadline = new Date(dateProvider.now() + 500); + // Long deadline so the collection ends when all txs are found (not when deadline fires) + deadline = new Date(dateProvider.now() + 5000); const collectionPromise = txCollection.collectFastForBlock(block, txHashes, { deadline }); - // Simulate all txs found via P2P before delay expires + // Simulate all txs found via P2P before delay expires — this cancels the tracker immediately txCollection.handleTxsAddedToPool({ txs, source: 'test' }); - // Now advance time past the delay - dateProvider.setTime(dateProvider.now() + 200); await sleep(100); await collectionPromise; - // File store should not have downloaded any txs because they were all found + // File store should not have downloaded any txs because they were all found before the delay const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); expect(allCalls.length).toBe(0); }); }); }); -class TestFastTxCollection extends FastTxCollection { +class TestTxCollection extends TxCollection { // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections declare requests: Set; - declare collectFast: (request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) => Promise; - declare reqRespTxsCollector?: IReqRespTxsCollector; -} - -class TestTxCollection extends TxCollection { - declare fastCollection: TestFastTxCollection; declare fileStoreFastCollection: TxCollection['fileStoreFastCollection']; declare handleTxsAddedToPool: TxPoolV2Events['txs-added']; + declare collectFast: (request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) => Promise; + declare reqRespTxsCollector?: IReqRespTxsCollector; } diff --git a/yarn-project/p2p/src/services/tx_collection/tx_collection.ts b/yarn-project/p2p/src/services/tx_collection/tx_collection.ts index 9a609fb408a3..30814392650c 100644 --- a/yarn-project/p2p/src/services/tx_collection/tx_collection.ts +++ b/yarn-project/p2p/src/services/tx_collection/tx_collection.ts @@ -1,7 +1,8 @@ import { BlockNumber } from '@aztec/foundation/branded-types'; +import { times } from '@aztec/foundation/collection'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; -import { DateProvider } from '@aztec/foundation/timer'; +import { DateProvider, elapsed } from '@aztec/foundation/timer'; import type { L2Block, L2BlockInfo } from '@aztec/stdlib/block'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import type { BlockProposal } from '@aztec/stdlib/p2p'; @@ -12,12 +13,13 @@ import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-clien import type { PeerId } from '@libp2p/interface'; import type { TxPoolV2, TxPoolV2Events } from '../../mem_pools/tx_pool_v2/interfaces.js'; +import { BatchTxRequester } from '../reqresp/batch-tx-requester/batch_tx_requester.js'; import type { BatchTxRequesterLibP2PService } from '../reqresp/batch-tx-requester/interface.js'; +import type { BlockTxsSource } from '../reqresp/index.js'; import type { TxCollectionConfig } from './config.js'; -import { FastTxCollection } from './fast_tx_collection.js'; import { FileStoreTxCollection } from './file_store_tx_collection.js'; import type { FileStoreTxSource } from './file_store_tx_source.js'; -import type { IRequestTracker } from './request_tracker.js'; +import { type IRequestTracker, RequestTracker } from './request_tracker.js'; import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js'; import type { TxSource } from './tx_source.js'; @@ -32,20 +34,36 @@ export type FastCollectionRequest = FastCollectionRequestInput & { blockInfo: L2BlockInfo; }; +/** + * Collect missing transactions for a block or proposal via reqresp. + * @param requestTracker - The missing transactions tracker + * @param blockTxsSource - The block or proposal containing the transactions + * @param pinnedPeer - Optional peer expected to have the transactions + * @returns The collected transactions + */ +export type IReqRespTxsCollector = ( + requestTracker: IRequestTracker, + blockTxsSource: BlockTxsSource, + pinnedPeer: PeerId | undefined, +) => Promise; + /** * Coordinates tx collection from remote RPC nodes, reqresp, and file store. * - * The fast collection methods quickly gather txs from RPC nodes and reqresp, usually for attesting - * to block proposals or preparing to prove an epoch. A delayed file-store fallback can also fetch - * txs if configured. Both paths send txs to the collection sink, which handles metrics and adds - * them to the tx pool. Whenever a tx is added to either the sink or the pool, this service is - * notified via events and stops collecting that tx across all in-flight requests. + * Runs a sequential pipeline: node RPC → reqresp → file store. Node collection starts immediately, + * reqresp starts after a configured delay, and file store (if configured) starts after a further + * delay. All paths send txs to the collection sink, which handles metrics and adds them to the + * tx pool. Whenever a tx is added to the sink or the pool, this service is notified and stops + * collecting that tx across all in-flight requests. */ export class TxCollection { - /** Fast collection methods */ - protected readonly fastCollection: FastTxCollection; + // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections + protected requests: Set = new Set(); - /** File store collection for fast (proposal/proving) path */ + /** The collector for txs via reqresp */ + protected reqRespTxsCollector?: IReqRespTxsCollector; + + /** File store collection for the fast (proposal/proving) path */ protected readonly fileStoreFastCollection: FileStoreTxCollection; /** Handles txs found by collection paths before adding to the pool */ @@ -57,12 +75,6 @@ export class TxCollection { /** Handler for the txs-added event from the tx collection sink */ protected readonly handleTxsFound: TxPoolV2Events['txs-added']; - /** Whether the service has been started. */ - private started = false; - - /** Whether file store sources are configured. */ - private readonly hasFileStoreSources: boolean; - constructor( private readonly p2pService: BatchTxRequesterLibP2PService, private readonly nodes: TxSource[], @@ -76,16 +88,18 @@ export class TxCollection { ) { this.txCollectionSink = new TxCollectionSink(this.txPool, telemetryClient, this.log); - this.fastCollection = new FastTxCollection( - this.p2pService, - this.nodes, - this.txCollectionSink, - this.config, - this.dateProvider, - this.log, - ); + this.reqRespTxsCollector = (requestTracker, blockTxsSource, pinnedPeer) => + BatchTxRequester.collectAllTxs( + new BatchTxRequester( + requestTracker, + blockTxsSource, + pinnedPeer, + this.p2pService, + this.log, + this.dateProvider, + ).run(), + ); - this.hasFileStoreSources = fileStoreSources.length > 0; this.fileStoreFastCollection = new FileStoreTxCollection( fileStoreSources, this.txCollectionSink, @@ -112,19 +126,11 @@ export class TxCollection { this.txPool.on('txs-added', this.handleTxsAddedToPool); } - /** Starts all collection loops. */ - public start(): Promise { - this.started = true; - this.fileStoreFastCollection.start(); - - // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup - return Promise.resolve(); - } - - /** Stops all activity. */ - public async stop() { - this.started = false; - await Promise.all([this.fastCollection.stop(), this.fileStoreFastCollection.stop()]); + /** Stops all activity. Cancels in-flight requests; file store workers self-terminate. */ + public stop() { + this.requests.forEach(request => { + request.requestTracker.cancel(); + }); this.txPool.removeListener('txs-added', this.handleTxsAddedToPool); this.txCollectionSink.removeListener('txs-added', this.handleTxsFound); @@ -145,48 +151,295 @@ export class TxCollection { } /** Collects the set of txs for the given proposal or block as fast as possible */ - public collectFastFor( + public async collectFastFor( input: FastCollectionRequestInput, txHashes: TxHash[] | string[], opts: { deadline: Date; pinnedPeer?: PeerId }, ) { + const timeout = opts.deadline.getTime() - this.dateProvider.now(); + if (timeout <= 0) { + this.log.warn(`Deadline for fast tx collection is in the past (${timeout}ms)`, { + deadline: opts.deadline.getTime(), + now: this.dateProvider.now(), + }); + return []; + } + const hashes = txHashes.map(h => (typeof h === 'string' ? TxHash.fromString(h) : h)); - // Delay file store collection to give P2P methods time to find txs first - if (this.hasFileStoreSources) { - const context = this.getAddContextForInput(input); - sleep(this.config.txCollectionFileStoreFastDelayMs) - .then(() => { - if (!this.started) { - return; - } + const blockInfo: L2BlockInfo = + input.type === 'proposal' + ? { ...input.blockProposal.toBlockInfo(), blockNumber: input.blockNumber } + : { ...input.block.toBlockInfo() }; + + const request: FastCollectionRequest = { + ...input, + blockInfo, + requestTracker: RequestTracker.create(hashes, opts.deadline, this.dateProvider), + }; + + const [duration] = await elapsed(() => this.collectFast(request, { pinnedPeer: opts.pinnedPeer })); + + this.log.verbose( + `Collected ${request.requestTracker.collectedTxs.length} txs out of ${hashes.length} for ${input.type} at slot ${blockInfo.slotNumber}`, + { + ...blockInfo, + duration, + requestType: input.type, + missingTxs: [...request.requestTracker.missingTxHashes], + }, + ); + return request.requestTracker.collectedTxs; + } + + protected async collectFast(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) { + this.requests.add(request); + const { blockInfo } = request; + + this.log.debug( + `Starting fast collection of ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, + { ...blockInfo, requestType: request.type, deadline: request.requestTracker.deadline }, + ); + + try { + // 1. Start node collection in the background. + // Note: this will be a noop if no nodes are configured. + const nodeCollectionPromise = this.collectFastFromNodes(request); + + // 2. Wait before starting reqresp, interruptible by cancellation or node exhaustion. + await Promise.race([ + request.requestTracker.cancellationToken, + sleep(this.config.txCollectionFastNodesTimeoutBeforeReqRespMs), + nodeCollectionPromise, // If node collection has finished (or if there are no nodes configured), we can exit early. + ]); + + // 3. Start reqresp in the background (runs in parallel with node collection). + // Note: this will be a noop if all TXs were already found. + const reqRespPromise = this.collectFastViaReqResp(request, opts); + + // 4. Wait before starting file store, interruptible by cancellation. + await Promise.race([ + request.requestTracker.cancellationToken, + sleep(this.config.txCollectionFileStoreFastDelayMs), + reqRespPromise, // If reqresp has finished, we can exit early. + ]); + + // 5. Start file store collection in the background. Self-terminates on tracker cancel / all-found. + // Note: this will be a noop if all TXs were already found. + const fileStorePromise = this.fileStoreFastCollection.startCollecting( + request.requestTracker, + this.getAddContext(request), + ); + + // 6. Wait for all paths to settle. + // NOTE: The request will automatically be cancelled after `opt.deadline` is reached. + await Promise.allSettled([reqRespPromise, nodeCollectionPromise, fileStorePromise]); + } catch (err) { + this.log.error(`Error collecting txs for ${request.type} for slot ${blockInfo.slotNumber}`, err, { + ...blockInfo, + missingTxs: request.requestTracker.missingTxHashes.values().map(txHash => txHash.toString()), + }); + } finally { + request.requestTracker.cancel(); + this.requests.delete(request); + } + } - // Only queue txs that are still missing after the delay. - const missingTxHashStrings = new Set(this.fastCollection.getMissingTxHashes().map(hash => hash.toString())); - const missingTxHashesToCollect = hashes.filter(hash => missingTxHashStrings.has(hash.toString())); - if (missingTxHashesToCollect.length > 0) { - this.fileStoreFastCollection.startCollecting(missingTxHashesToCollect, context, opts.deadline); + /** + * Starts collecting txs from all configured nodes. We send `txCollectionFastMaxParallelRequestsPerNode` requests + * in parallel to each node. We keep track of the number of attempts made to collect each tx, so we can prioritize + * the txs that have been requested less often whenever we need to send a new batch of requests. We ensure that no + * tx is requested more than once at the same time to the same node. + */ + private async collectFastFromNodes(request: FastCollectionRequest): Promise { + if (this.nodes.length === 0) { + return; + } + + // Keep a shared priority queue of all txs pending to be requested, sorted by the number of attempts made to collect them. + const attemptsPerTx = [...request.requestTracker.missingTxHashes].map(txHash => ({ + txHash, + attempts: 0, + found: false, + })); + + // Returns once we have finished all node loops. Each loop finishes when the deadline is hit, or all txs have been collected. + await Promise.allSettled(this.nodes.map(node => this.collectFastFromNode(request, node, attemptsPerTx))); + } + + private async collectFastFromNode( + request: FastCollectionRequest, + node: TxSource, + attemptsPerTx: { txHash: string; attempts: number; found: boolean }[], + ) { + const notFinished = () => !request.requestTracker.checkCancelled(); + + const maxParallelRequests = this.config.txCollectionFastMaxParallelRequestsPerNode; + const maxBatchSize = this.config.txCollectionNodeRpcMaxBatchSize; + const activeRequestsToThisNode = new Set(); // Track the txs being actively requested to this node + + const processBatch = async () => { + while (notFinished()) { + // Pull tx hashes from the attemptsPerTx array, which is sorted by attempts, + // so we prioritize txs that have been requested less often. + const batch = []; + let index = 0; + while (batch.length < maxBatchSize) { + const txToRequest = attemptsPerTx[index++]; + if (!txToRequest) { + // No more txs to process + break; + } else if (!request.requestTracker.isMissing(txToRequest.txHash)) { + // Mark as found if it was found somewhere else, we'll then remove it from the array. + // We don't delete it now since 'array.splice' is pretty expensive, so we do it after sorting. + txToRequest.found = true; + } else if (!activeRequestsToThisNode.has(txToRequest.txHash)) { + // If the tx is not already being requested to this node, add it to the current batch and increase attempts. + // Note that we increase the attempts *before* making the request, so the next `collectFastFromNode` that + // needs to grab txs to send, will pick txs that have been requested less often, instead of all requesting + // the same txs at the same time. + batch.push(txToRequest); + activeRequestsToThisNode.add(txToRequest.txHash); + txToRequest.attempts++; } - }) - .catch(err => this.log.error('Error in file store fast delay', err)); + } + + // After modifying the array by removing txs or updating attempts, re-sort it and trim the found txs from the end. + attemptsPerTx.sort((a, b) => + a.found === b.found ? a.attempts - b.attempts : Number(a.found) - Number(b.found), + ); + const firstFoundTxIndex = attemptsPerTx.findIndex(tx => tx.found); + if (firstFoundTxIndex !== -1) { + attemptsPerTx.length = firstFoundTxIndex; + } + + // If we see no more txs to request, we can stop this "process" loop + if (batch.length === 0) { + return; + } + + const txHashes = batch.map(({ txHash }) => txHash); + // Collect this batch from the node + await this.txCollectionSink.collect( + async () => { + const result = await node.getTxsByHash(txHashes.map(TxHash.fromString)); + for (const tx of result.validTxs) { + request.requestTracker.markFetched(tx); + } + return result; + }, + txHashes, + { + description: `fast ${node.getInfo()}`, + node: node.getInfo(), + method: 'fast-node-rpc', + ...request.blockInfo, + }, + this.getAddContext(request), + ); + + // Clear from the active requests the txs we just requested + for (const requestedTx of batch) { + activeRequestsToThisNode.delete(requestedTx.txHash); + } + + // Sleep a bit until hitting the node again, but wake up immediately on cancellation + if (notFinished()) { + await Promise.race([ + sleep(this.config.txCollectionFastNodeIntervalMs), + request.requestTracker.cancellationToken, + ]); + } + } + }; + + // Kick off N parallel requests to the node, up to the maxParallelRequests limit + await Promise.all(times(maxParallelRequests, processBatch)); + } + + private async collectFastViaReqResp(request: FastCollectionRequest, opts: { pinnedPeer?: PeerId }) { + const pinnedPeer = opts.pinnedPeer; + const blockInfo = request.blockInfo; + const slotNumber = blockInfo.slotNumber; + if (request.requestTracker.timeoutMs < 100) { + this.log.warn( + `Not initiating fast reqresp for txs for ${request.type} at slot ${blockInfo.slotNumber} due to timeout`, + { timeoutMs: request.requestTracker.timeoutMs, ...blockInfo }, + ); + return; + } + + if (request.requestTracker.checkCancelled()) { + this.log.debug(`No txs to collect via reqresp for ${request.type} at slot ${blockInfo.slotNumber}`, { + ...blockInfo, + }); + return; } - return this.fastCollection.collectFastFor(input, txHashes, opts); + this.log.debug( + `Starting fast reqresp for ${request.requestTracker.numberOfMissingTxs} txs for ${request.type} at slot ${blockInfo.slotNumber}`, + { ...blockInfo, timeoutMs: request.requestTracker.timeoutMs, pinnedPeer }, + ); + + try { + await this.txCollectionSink.collect( + async () => { + let blockTxsSource: BlockTxsSource; + if (request.type === 'proposal') { + blockTxsSource = request.blockProposal; + } else if (request.type === 'block') { + blockTxsSource = { + txHashes: request.block.body.txEffects.map(e => e.txHash), + archive: request.block.archive.root, + }; + } else { + throw new Error(`Unknown request type: ${(request as { type: string }).type}`); + } + + const result = await this.reqRespTxsCollector!(request.requestTracker, blockTxsSource, pinnedPeer); + return { validTxs: result, invalidTxHashes: [] }; + }, + Array.from(request.requestTracker.missingTxHashes), + { description: `reqresp for slot ${slotNumber}`, method: 'fast-req-resp', ...opts, ...request.blockInfo }, + this.getAddContext(request), + ); + } catch (err) { + this.log.error(`Error sending fast reqresp request for txs`, err, { + txs: [...request.requestTracker.missingTxHashes], + ...blockInfo, + }); + } } - /** Returns the TxAddContext for the given fast collection request input */ - private getAddContextForInput(input: FastCollectionRequestInput): TxAddContext { - if (input.type === 'proposal') { - return { type: 'proposal', blockHeader: input.blockProposal.blockHeader }; + /** Returns the TxAddContext for the given request, used by the sink to add txs to the pool correctly. */ + private getAddContext(request: FastCollectionRequest): TxAddContext { + if (request.type === 'proposal') { + return { type: 'proposal', blockHeader: request.blockProposal.blockHeader }; } else { - return { type: 'mined', block: input.block }; + return { type: 'mined', block: request.block }; } } - /** Mark the given txs as found. Stops collecting them. */ + /** Mark the given txs as found. Stops collecting them across all in-flight requests. */ private foundTxs(txs: Tx[]) { - this.fastCollection.foundTxs(txs); - this.fileStoreFastCollection.foundTxs(txs); + for (const request of this.requests) { + for (const tx of txs) { + if (request.requestTracker.markFetched(tx)) { + this.log.trace(`Found tx ${tx.txHash.toString()} for fast collection request`, { + ...request.blockInfo, + txHash: tx.txHash.toString(), + type: request.type, + }); + if (request.requestTracker.allFetched()) { + this.log.trace(`All txs found for fast collection request`, { + ...request.blockInfo, + type: request.type, + }); + break; + } + } + } + } } /** @@ -194,8 +447,11 @@ export class TxCollection { * To be called when we no longer care about gathering txs up to a certain block, eg when they become proven or finalized. */ public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void { - this.fastCollection.stopCollectingForBlocksUpTo(blockNumber); - this.fileStoreFastCollection.clearPending(); + for (const request of this.requests) { + if (request.blockInfo.blockNumber <= blockNumber) { + request.requestTracker.cancel(); + } + } } /** @@ -203,7 +459,10 @@ export class TxCollection { * To be called when there is a chain prune and previously mined txs are no longer relevant. */ public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void { - this.fastCollection.stopCollectingForBlocksAfter(blockNumber); - this.fileStoreFastCollection.clearPending(); + for (const request of this.requests) { + if (request.blockInfo.blockNumber > blockNumber) { + request.requestTracker.cancel(); + } + } } } From a7a0d8cf263bf244445d4930e8ea745f588fe1e2 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 13 May 2026 13:33:20 -0300 Subject: [PATCH 02/17] refactor(publisher): bundle-level simulate; drop per-action enqueue sims (#23165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Context `SequencerPublisher` simulates each enqueued L1 action individually at enqueue time, then sends them bundled through Multicall3. The `propose` checkpoint action is validated at enqueue and send time (the latter via a `preCheck` mechanism), but in isolation and relying on overrides. There is no simulation of the multicall payload before sending it, so a reverting tx is most likely not caught. This refactor: - Replaces the per-request `preCheck` mechanism with a **single bundle-level `eth_simulateV1`** of the assembled `aggregate3` payload, run right before send. If any entry reverts in sim it is dropped from the bundle, the reduced bundle is re-simulated to get an honest `gasUsed`, and the survivors are sent. Extracted to a `SequencerBundleSimulator`. - Drops the entire propose simulate at enqueue (`simulateProposeTx`, `validateCheckpointForSubmission`). The bundle simulate covers it. - Adds a new pre-broadcast `validateBlockHeader` call (calling `validateHeaderWithAttestations` with empty attestations + `ignoreSignatures: true`) that catches header-level bugs before we gossip the proposal to peers. Emits a new `header-validation-failed` event on failure. - Drops every per-action simulate at enqueue (governance signal **and** slashing votes/executes). Bundle simulate at send time is the single decision point for every per-action revert. `simulateAndEnqueueRequest` is deleted. We were enqueuing votes even if the simulation failed, after all. - Rewrites `sendRequestsAt` so it takes an L2 `SlotNumber`, derives the timestamp for the start of that slot, and sleeps until one L1 slot before that boundary, so we can land on the first L1 slot of the target L2 slot. - Centralises `SimulationOverridesPlan` construction into a single `buildCheckpointSimulationOverridesPlan` helper. The plan **always** pins both `pending` and `proven` chain tips (to the pipelined parent / invalidation target, or to the current snapshot when neither applies), so `STFLib.canPruneAtTime` cannot reintroduce a phantom prune during simulation. - Makes `SimulationOverridesBuilder.merge` undefined-safe: explicit `undefined` fields in an incoming plan no longer erase previously-set values. `withPendingTempCheckpointLogFields` now accepts a partial subset of fields. - Moves the payload-empty cache onto `GovernanceProposerContract` next to its concern. Only `isPayloadEmpty=false` is cached (a CREATE2 redeploy could go empty → populated). - Drops the old Multicall3 revert-recovery and per-request-resim machinery, since with `allowFailure: true` the top-level multicall is expected to land successfully. `Multicall3.forward` now throws `MulticallForwarderRevertedError` if the receipt reports a reverted status; the publisher does **not** rotate to a new publisher on that error (on-chain failure, not a send failure). Adds `Multicall3.hasCode` helper and a `simulateAggregate3` entrypoint used by the bundle simulator. - `L1TxUtils.sendTransaction` fails fast if `txTimeoutAt` has already elapsed when called. `SequencerPublisher.forwardWithPublisherRotation` re-checks the deadline at the head of each rotation iteration so it doesn't keep cycling through publishers after the L2 slot's submission window has closed. - Sequencer escape-hatch (`voteInSlotWithoutSyncing`) and full-escape-hatch (`voteOnSlotWithEscapeHatch`) vote-only paths now submit via `sendRequestsAt(slot)` rather than `sendRequests()`, so the bundle-simulate `block.timestamp` override matches the slot the EIP-712 vote signatures were generated for. The intended outcome is a publisher with one explicit re-validation point (the bundle simulate), measurable bundle gas (from the bundle simulate's `gasUsed`), and dead/duplicated state-override plumbing removed. ## Resulting simulations after this refactor The full list of simulation / gas-estimation steps that remain in a pipelined proposer slot, in execution order. ### Pre-build, in `Sequencer.doWork` 1. **`publisher.canProposeAt`** — rollup view call simulated with the centralised override plan. Cheap pre-check gate before any block-build work. 2. **`publisher.simulateInvalidateCheckpoint`** (conditional) — runs **only** if `syncedTo.pendingChainValidationStatus.valid === false` AND `!syncedTo.hasProposedCheckpoint`. Simulates the invalidate call against the rollup. Result becomes the `invalidateCheckpoint` package passed into `CheckpointProposalJob`. The previous code called this even when there's a proposed parent and discarded the result; this refactor adds the `!hasProposedCheckpoint` gate so we skip the wasted RPC. ### Per-slot, in `CheckpointProposalJob.proposeCheckpoint` 3. **CheckpointVoter votes** — `CheckpointVoter.enqueueVotes()` runs at the top of `execute()`, returning two promises that are awaited in parallel with block-build. It enqueues two kinds of votes via the publisher, **neither of which simulates at enqueue time** after this refactor: - **`enqueueGovernanceCastSignal`** — does an `isPayloadEmpty` pre-flight check (now on `GovernanceProposerContract`), then enqueues. No `eth_simulateV1`. - **`enqueueSlashingActions`** (one call per slashing action, type `vote-offenses` or `execute-slash`) — builds the request and enqueues. No `eth_simulateV1`. Real reverts on any of these are caught by the bundle simulate at send time, which drops the failing entry and proceeds with the survivors. 4. **`publisher.validateBlockHeader` (NEW: pre-broadcast)** — replaces the old `simulateProposeTx`-at-enqueue. Calls `validateHeaderWithAttestations` with empty attestations and `ignoreSignatures: true` so the rollup runs the header checks (archive match, slot match, timestamp, mana-min-fee, …) without needing real attestations. Runs **before** we gossip the proposal to peers. If it fails, abort the slot — log an error, emit `header-validation-failed`, don't broadcast, don't enqueue. 5. **`prepareProposeTx → validateBlobs estimateGas`** — kept as the blob-commitment **consistency check** (detects locally-built commitments not matching the blob sidecars). Returns `blobEvaluationGas`, which we stash on the propose `RequestWithExpiry` for use by the bundle gasLimit later. The simulate-step that previously paired with this (`simulateProposeTx`) is removed. ### Background pipeline, in `waitForAttestationsAndEnqueueSubmissionAsync` 6. **`publisher.simulateInvalidateCheckpoint` (conditional)** — runs **only** in the fallback path where attestation collection failed AND the pending chain turned out to be invalid. Triggered from `CheckpointProposalJob.enqueueInvalidation`. This is the second, late trigger for invalidation simulation — distinct from step 2's pre-build trigger. ### Send time, in `sendRequestsAt(targetSlot)` 7. **Bundle simulate (NEW)** — single `eth_simulateV1` of the assembled `aggregate3` payload, with `block.timestamp` overridden to the start of `targetSlot`, and state overrides = `[disableBlobCheck]` iff `propose` is in the bundle and `[]` otherwise. Per-entry result decoded from the returned `Result[]`. This is the **only** post-pipeline-sleep re-validation; it replaces the per-request `preCheck` mechanism entirely. 8. **Bundle re-simulate (NEW, conditional)** — runs **only** when step 7 dropped at least one entry. Re-runs the bundle simulate on the reduced payload to get an honest `gasUsed`, and applies the same per-entry decode so additional drops are caught. If the re-simulate falls back (node doesn't support `eth_simulateV1`), the publisher sends the **first-pass survivors only** with `MAX_L1_TX_LIMIT`; the entries that the first pass already proved would revert stay dropped and are reported as failed actions. ### Post-send No diagnostic-only simulate paths remain. `Multicall3.forward` throws `MulticallForwarderRevertedError` on a reverted receipt and re-throws on a send error; per-request revert resimulation has been removed. ## Known caveats - **`sendRequestsAt` early lead**: sleeps until `startOfTargetSlot - ethereumSlotDuration` to maximise inclusion in the first L1 block of the L2 slot. There is a known correctness risk: a tx mined in the L1 block immediately preceding the L2-slot boundary would revert via `ProposeLib.validateHeader`'s `slot == block.timestamp.slotFromTimestamp()` check. In practice the prior L1 block is usually already committed before this send wakes; if observed to be unreliable in production, tune the lead down, especially on tests. - **`validateBlockHeader` pre-broadcast coverage**: covers the `validateHeader` checks (archive, slot/timestamp, mana-min-fee, …) and the empty-attestation path of `validateHeaderWithAttestations`, but does NOT cover proposer-signature verification, inbox consumption (`Rollup__InvalidInHash`), or `header.inHash` match. Those still execute inside the full `propose` and are caught by the bundle simulate at send time. The cost of a rare miss is one wasted broadcast. - **Top-level `aggregate3` revert diagnostics removed**: the previous `Multicall3.forward` code decoded receipt-reverted reasons via `tryGetErrorFromRevertedTx` and did a per-request resim on send-throw. Both paths are gone. With `allowFailure: true` and `Multicall3.hasCode` covering the no-bytecode case, a reverted forwarder receipt is genuinely unexpected (OOG, forwarder bug). The throw of `MulticallForwarderRevertedError` is the only diagnostic surface — operators will need the transaction hash from the log to investigate. --- .test_patterns.yml | 12 + .../e2e_epochs/epochs_missed_l1_slot.test.ts | 6 +- .../epochs_proof_at_boundary.parallel.test.ts | 30 +- .../end-to-end/src/e2e_epochs/epochs_test.ts | 1 + .../e2e_l1_publisher/e2e_l1_publisher.test.ts | 134 +++- .../escape_hatch_vote_only.test.ts | 1 + .../contracts/chain_state_override.test.ts | 24 + .../src/contracts/chain_state_override.ts | 35 +- .../src/contracts/governance_proposer.ts | 30 + .../ethereum/src/contracts/multicall.test.ts | 82 +- .../ethereum/src/contracts/multicall.ts | 290 ++++--- .../ethereum/src/l1_tx_utils/l1_tx_utils.ts | 9 + .../publisher/sequencer-bundle-simulator.ts | 253 ++++++ .../src/publisher/sequencer-publisher.test.ts | 373 ++++----- .../src/publisher/sequencer-publisher.ts | 734 ++++++------------ .../src/sequencer/chain_state_overrides.ts | 213 ++--- .../sequencer/checkpoint_proposal_job.test.ts | 234 +++--- .../checkpoint_proposal_job.timing.test.ts | 4 +- .../src/sequencer/checkpoint_proposal_job.ts | 84 +- .../checkpoint_voter.ha.integration.test.ts | 7 +- .../sequencer-client/src/sequencer/events.ts | 23 +- .../src/sequencer/sequencer.test.ts | 100 +-- .../src/sequencer/sequencer.ts | 118 ++- 23 files changed, 1551 insertions(+), 1246 deletions(-) create mode 100644 yarn-project/sequencer-client/src/publisher/sequencer-bundle-simulator.ts diff --git a/.test_patterns.yml b/.test_patterns.yml index aaa4e6dadca2..10cd59cca8fd 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -172,6 +172,18 @@ tests: error_regex: "ContractFunctionExecutionError: The contract function" owners: - *mitch + # Under proposer pipelining each validator votes in its own slot and the votes + # don't aggregate into the same round, so the slashing quorum (3) is never + # reached within the 414s budget; the test consistently times out at the docker + # outer 600s (exit 124). The publisher refactor lands all vote-offenses tx's + # on L1 successfully — voteCount on the slasher proposer simply stays at 1 + # per round. This is a slashing-payload aggregation issue independent of + # publisher work; skip until the slashing team addresses it separately. + - regex: "e2e_p2p/valid_epoch_pruned_slash.test.ts" + skip: true + owners: + - *mitch + - *palla - regex: "archiver/src/archiver/archiver.test.ts" error_regex: "Received number of calls: 1" owners: diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_slot.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_slot.test.ts index 0c32eaab5353..eda99eef06c3 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_slot.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_slot.test.ts @@ -234,9 +234,13 @@ describe('e2e_epochs/epochs_missed_l1_slot', () => { await eth.setIntervalMining(L1_BLOCK_TIME); // Step 5: Wait for the next checkpoint to confirm block production resumed cleanly. + // We allow up to 3 L2 slots because the slot-N+1 propose for this checkpoint is dropped + // pre-send by bundleSimulate (the resumed L1 block lands in slot N, not slot N+1, so + // propose's validateHeader would revert), and the publisher retries one or two slots + // later once L1 timing realigns. const finalCheckpoint = CheckpointNumber(checkpointEvent.checkpointNumber + 1); logger.info(`Waiting for checkpoint ${finalCheckpoint}...`); - await test.waitUntilCheckpointNumber(finalCheckpoint, 60); + await test.waitUntilCheckpointNumber(finalCheckpoint, L2_SLOT_DURATION * 3); await monitor.run(); logger.info(`Checkpoint ${finalCheckpoint} published in slot ${monitor.l2SlotNumber}`); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_at_boundary.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_at_boundary.parallel.test.ts index 9ae7af5040f8..eaf403de188c 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_at_boundary.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_at_boundary.parallel.test.ts @@ -172,7 +172,8 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { // Tighter happy-path bound: the proof must land BEFORE the boundary slot's pipelined build kicks // off. With pipelining, the boundary slot's build starts at the start of the previous L2 slot // (i.e. boundaryTs - L2_SLOT_DURATION_IN_S). If the proof's L1 block is strictly earlier than - // that, the build at the boundary observes `tips.proven` already advanced and skips the override. + // that, the build at the boundary observes `tips.proven` already advanced so the proven pin is + // defensive only (no prune is due) and the boundary checkpoint publishes on the happy path. const assertProofMinedBeforeBoundaryBuild = async (proofReceipt: { blockNumber: bigint }, boundaryTs: bigint) => { const proofBlock = await test.l1Client.getBlock({ blockNumber: proofReceipt.blockNumber }); expect(proofBlock.timestamp).toBeLessThan(boundaryTs - BigInt(test.L2_SLOT_DURATION_IN_S)); @@ -201,8 +202,8 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { it('proof lands during slot build and checkpoint succeeds at boundary', async () => { // The proof for the unproven epoch lands AFTER the boundary slot's pipelined build starts but - // BEFORE the publisher's preCheck. The proven-override lets the boundary checkpoint build - // before the proof has landed; the preCheck succeeds because the proof arrives in time. + // BEFORE the publisher's preCheck. The proven pin lets the boundary checkpoint build before + // the proof has landed; the preCheck succeeds because the proof arrives in time. await setupTest({ aztecProofSubmissionEpochs: 1 }); const sequencers = nodes.map(node => node.getSequencer()!); @@ -238,17 +239,16 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { expect(boundaryPublished).toBeDefined(); const boundaryPreparing = events.preparing.filter(p => Number(p.targetSlot) === Number(boundarySlot)); - expect(boundaryPreparing.some(p => p.provenOverride !== undefined)).toBe(true); expect(boundaryPreparing.some(p => p.hadProposedParent)).toBe(true); expect(Number(test.monitor.checkpointNumber)).toBeGreaterThanOrEqual(Number(boundaryPublished!.checkpoint)); logger.warn(`Test passed. Final tip checkpoint=${test.monitor.checkpointNumber}`); }); - it('proof lands well before deadline and checkpoint succeeds without override', async () => { + it('proof lands well before deadline and checkpoint succeeds at boundary', async () => { // Sanity check: the prover runs on its natural schedule, so the proof lands well before the - // boundary epoch. By the time the boundary slot is built `tips.proven` is already advanced, - // `isPruneDueAtSlot` returns false, and the proven-override does not fire. + // boundary epoch. By the time the boundary slot is built `tips.proven` is already advanced + // and the proven pin is defensive only — but the boundary checkpoint must still publish. await setupTest({ aztecProofSubmissionEpochs: 1 }); const sequencers = nodes.map(node => node.getSequencer()!); @@ -272,15 +272,14 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { const boundaryPreparing = events.preparing.filter(p => Number(p.targetSlot) === Number(boundarySlot)); expect(boundaryPreparing.some(p => p.hadProposedParent)).toBe(true); - expect(boundaryPreparing.every(p => p.provenOverride === undefined)).toBe(true); expect(Number(test.monitor.checkpointNumber)).toBeGreaterThanOrEqual(Number(boundaryPublished!.checkpoint)); }); it('proof never lands so no checkpoint submission is attempted', async () => { - // The boundary slot's build applies the proven-override, but the publisher's preCheck rejects - // the propose tx because the proof never landed. After the prune fires on a later slot, a - // fresh propose advances the chain and a checkpoint is published in the new epoch. + // The boundary slot's build applies the proven pin, but the publisher's preCheck rejects the + // propose tx because the proof never landed. After the prune fires on a later slot, a fresh + // propose advances the chain and a checkpoint is published in the new epoch. await setupTest({ aztecProofSubmissionEpochs: 1 }); const sequencers = nodes.map(node => node.getSequencer()!); @@ -300,7 +299,6 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { const boundaryPreparing = events.preparing.filter(p => Number(p.targetSlot) === Number(boundarySlot)); expect(boundaryPreparing.some(p => p.hadProposedParent)).toBe(true); - expect(boundaryPreparing.some(p => p.provenOverride !== undefined)).toBe(true); // After the boundary fails, a subsequent slot's propose tx triggers the on-chain prune (since // the proof never landed and the deadline has expired) and resets `tips.pending`. The fresh @@ -314,7 +312,7 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { it('proof lands without a proposed parent and boundary checkpoint succeeds', async () => { // The slot before the boundary is paused so the boundary slot's build does not see a proposed - // parent. The proof still lands well before the deadline, so the proven-override never fires + // parent. The proof still lands well before the deadline, so the proven pin is defensive only // and the boundary checkpoint is published normally. await setupTest({ aztecProofSubmissionEpochs: 1 }); @@ -345,14 +343,13 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { const boundaryPreparing = events.preparing.filter(p => Number(p.targetSlot) === Number(boundarySlot)); expect(boundaryPreparing.length).toBeGreaterThan(0); expect(boundaryPreparing.every(p => !p.hadProposedParent)).toBe(true); - expect(boundaryPreparing.every(p => p.provenOverride === undefined)).toBe(true); expect(Number(test.monitor.checkpointNumber)).toBeGreaterThanOrEqual(Number(boundaryPublished!.checkpoint)); }); it('proof never lands without a proposed parent so no checkpoint submission is attempted', async () => { - // Same as the no-parent variant above but with the proof never landing. The proven-override - // fires (no parent + prune is due) but the publisher's preCheck rejects the propose, so no + // Same as the no-parent variant above but with the proof never landing. The proven pin fires + // (no parent + prune is due) but the publisher's preCheck rejects the propose, so no // checkpoint is published for the boundary slot. await setupTest({ aztecProofSubmissionEpochs: 1 }); @@ -378,7 +375,6 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { const boundaryPreparing = events.preparing.filter(p => Number(p.targetSlot) === Number(boundarySlot)); expect(boundaryPreparing.length).toBeGreaterThan(0); expect(boundaryPreparing.every(p => !p.hadProposedParent)).toBe(true); - expect(boundaryPreparing.some(p => p.provenOverride !== undefined)).toBe(true); // See the parent test for the reasoning: a subsequent slot's propose triggers the on-chain // prune in-tx, so the first post-boundary checkpoint lands within a couple of slots. diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts index d6d6905b6c61..d054b9aeb6c2 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts @@ -517,6 +517,7 @@ export class EpochsTestContext { 'proposer-rollup-check-failed', 'checkpoint-error', 'checkpoint-publish-failed', + 'header-validation-failed', 'pipelined-checkpoint-discarded', ...additionalFailEventKeys, ]; diff --git a/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts b/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts index 70503811095d..d9e62b7a5deb 100644 --- a/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts @@ -175,6 +175,23 @@ describe('L1Publisher integration', () => { } }; + // Warp the chain forward so that the current L2 slot matches `targetSlot`, and resync the + // dateProvider so `epochCache.getSlotNow()` (used by the bundle-level eth_simulateV1 and the + // L1 tx mine timestamp) also lands on `targetSlot`. The rollup contract rejects header slots + // that don't match block.timestamp, so the test must align both the chain and the date + // provider to the header's slot before calling sendRequests. + const progressToSlot = async (targetSlot: bigint) => { + const currentSlot = await rollup.getSlotNumber(); + if (BigInt(targetSlot) > BigInt(currentSlot)) { + await progressTimeBySlot(Number(BigInt(targetSlot) - BigInt(currentSlot))); + } + // Always resync the dateProvider so `epochCache.getSlotNow()` matches L1's block.timestamp. + // `sendRequests` derives its bundle-simulate timestamp from `getCurrentL2Slot()`, so if the + // dateProvider lags the chain the simulate runs at a stale slot and the rollup rejects the + // header with `HeaderLib__InvalidSlotNumber`. + await ethCheatCodes.syncDateProvider(); + }; + let port = 8545; // We increase the port for each test to avoid anvil conflicts const setup = async (deployL1ContractsArgs: Partial = {}) => { ({ rpcUrl, anvil } = await startAnvil({ port: port++ })); @@ -532,6 +549,8 @@ describe('L1Publisher integration', () => { CommitteeAttestationsAndSigners.empty(getSignatureContext()), Signature.empty(), ); + // Align chain time so the bundle simulate and the L1 send both run at the header's slot. + await progressToSlot(BigInt(checkpoint.header.slotNumber)); await publisher.sendRequests(); const logs = await l1Client.getLogs({ @@ -643,6 +662,8 @@ describe('L1Publisher integration', () => { new CommitteeAttestationsAndSigners(attestations, getSignatureContext()), signature, ); + // Align chain time so the bundle simulate and the L1 send both run at the header's slot. + await progressToSlot(BigInt(checkpoint.header.slotNumber)); const result = await publisher.sendRequests(); expect(result!.successfulActions).toEqual(['propose']); expect(result!.failedActions).toEqual([]); @@ -680,9 +701,23 @@ describe('L1Publisher integration', () => { expect(canPropose?.slot).toEqual(block.header.getSlot()); await publisher.validateBlockHeader(checkpoint.header); - await expect( - publisher.enqueueProposeCheckpoint(checkpoint, attestationsAndSigners, Signature.empty()), - ).rejects.toThrow(/ValidatorSelection__InvalidCommitteeCommitment/); + // Enqueue no longer simulates — the bundle simulate at send time drops the failing propose + // and sendRequests returns undefined (no surviving actions). The drop is reported via a + // warn log carrying the on-chain revert reason (raw hex selector since the propose request + // has no ABI attached). + const loggerWarnSpy = jest.spyOn((publisher as any).log, 'warn'); + await publisher.enqueueProposeCheckpoint(checkpoint, attestationsAndSigners, Signature.empty()); + await progressToSlot(BigInt(checkpoint.header.slotNumber)); + const result = await publisher.sendRequests(); + expect(result).toBeUndefined(); + // 0xca8d5954 == ValidatorSelection__InvalidCommitteeCommitment selector + expect(loggerWarnSpy).toHaveBeenCalledWith( + 'Bundle entry dropped: action reverted in sim', + expect.objectContaining({ + action: 'propose', + returnData: expect.stringMatching(/^0xca8d5954/), + }), + ); }); it('rejects flipped proposer signature', async () => { @@ -701,13 +736,25 @@ describe('L1Publisher integration', () => { validators.find(v => v.address.equals(proposer!))!, ); - await expect( - publisher.enqueueProposeCheckpoint( - checkpoint, - attestationsAndSigners, - flipSignature(attestationsAndSignersSignature), - ), - ).rejects.toThrow(/ECDSAInvalidSignatureS/); + // Enqueue no longer simulates — the bundle simulate at send time drops the failing propose + // and sendRequests returns undefined. + const loggerWarnSpy = jest.spyOn((publisher as any).log, 'warn'); + await publisher.enqueueProposeCheckpoint( + checkpoint, + attestationsAndSigners, + flipSignature(attestationsAndSignersSignature), + ); + await progressToSlot(BigInt(checkpoint.header.slotNumber)); + const result = await publisher.sendRequests(); + expect(result).toBeUndefined(); + // 0xd78bce0c == ECDSAInvalidSignatureS selector + expect(loggerWarnSpy).toHaveBeenCalledWith( + 'Bundle entry dropped: action reverted in sim', + expect.objectContaining({ + action: 'propose', + returnData: expect.stringMatching(/^0xd78bce0c/), + }), + ); }); it('rejects signature with invalid recovery value', async () => { @@ -732,8 +779,20 @@ describe('L1Publisher integration', () => { const wrongV = attestationsAndSignersSignature.v - 27; const wrongSig = new Signature(attestationsAndSignersSignature.r, attestationsAndSignersSignature.s, wrongV); - await expect(publisher.enqueueProposeCheckpoint(checkpoint, attestationsAndSigners, wrongSig)).rejects.toThrow( - /ECDSAInvalidSignature/, + // Enqueue no longer simulates — the bundle simulate at send time drops the failing propose + // and sendRequests returns undefined. + const loggerWarnSpy = jest.spyOn((publisher as any).log, 'warn'); + await publisher.enqueueProposeCheckpoint(checkpoint, attestationsAndSigners, wrongSig); + await progressToSlot(BigInt(checkpoint.header.slotNumber)); + const result = await publisher.sendRequests(); + expect(result).toBeUndefined(); + // 0xf645eedf == ECDSAInvalidSignature selector + expect(loggerWarnSpy).toHaveBeenCalledWith( + 'Bundle entry dropped: action reverted in sim', + expect.objectContaining({ + action: 'propose', + returnData: expect.stringMatching(/^0xf645eedf/), + }), ); }); @@ -810,9 +869,7 @@ describe('L1Publisher integration', () => { // Invalidate and propose logger.warn('Enqueuing requests to invalidate and propose the checkpoint'); publisher.enqueueInvalidateCheckpoint(invalidateRequest); - await publisher.enqueueProposeCheckpoint(checkpoint, attestationsAndSigners, attestationsAndSignersSignature, { - simulationOverridesPlan: invalidationSimulationOverridesPlan, - }); + await publisher.enqueueProposeCheckpoint(checkpoint, attestationsAndSigners, attestationsAndSignersSignature); const result = await publisher.sendRequests(); expect(result!.successfulActions).toEqual(['invalidate-by-insufficient-attestations', 'propose']); expect(result!.failedActions).toEqual([]); @@ -853,20 +910,24 @@ describe('L1Publisher integration', () => { const l1ToL2Messages = new Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).fill(new Fr(1n)); const { checkpoint } = await buildSingleCheckpoint({ l1ToL2Messages }); - // Expect the simulation to fail - const loggerErrorSpy = jest.spyOn((publisher as any).log, 'error'); - await expect( - publisher.enqueueProposeCheckpoint( - checkpoint, - CommitteeAttestationsAndSigners.empty(getSignatureContext()), - Signature.empty(), - ), - ).rejects.toThrow(/Rollup__InvalidInHash/); - expect(loggerErrorSpy).toHaveBeenNthCalledWith( - 2, - expect.stringMatching('Rollup__InvalidInHash'), - expect.anything(), - expect.objectContaining({ checkpointNumber: 1 }), + // Enqueue no longer simulates per action — the bundle simulate at send time drops the + // failing propose and reports the on-chain revert reason via a warn log. + const loggerWarnSpy = jest.spyOn((publisher as any).log, 'warn'); + await publisher.enqueueProposeCheckpoint( + checkpoint, + CommitteeAttestationsAndSigners.empty(getSignatureContext()), + Signature.empty(), + ); + await progressToSlot(BigInt(checkpoint.header.slotNumber)); + const result = await publisher.sendRequests(); + expect(result).toBeUndefined(); + // 0xcd6f4233 == Rollup__InvalidInHash selector + expect(loggerWarnSpy).toHaveBeenCalledWith( + 'Bundle entry dropped: action reverted in sim', + expect.objectContaining({ + action: 'propose', + returnData: expect.stringMatching(/^0xcd6f4233/), + }), ); }); }); @@ -1022,10 +1083,21 @@ describe('L1Publisher integration', () => { expect(BigInt(block2.slot)).toEqual(initialL2Slot + 1n); sendRequestsResult = undefined; await enqueueProposeL2Checkpoint(checkpoint2); + // Align chain time so the bundle simulate at send time runs at slot N+1 (matches the + // checkpoint2 header). Without this the bundle simulate (which uses getSlotNow()) sees + // the wrong slot and drops the propose entry. + await progressToSlot(BigInt(checkpoint2.header.slotNumber)); await sendRequests(); - // Wait for the new proposal to be sent to the pool - await retryUntil(() => ethCheatCodes.getTxPoolStatus().then(s => s.queued + s.pending > 1), 'tx queued', 20, 0.1); + // Wait for the new proposal to be sent to the pool. The progressToSlot warp above may have + // already mined the cancellation from the first proposal, so the pool may hold either the + // cancel-and-new-propose (two entries) or just the new propose (one entry). + await retryUntil( + () => ethCheatCodes.getTxPoolStatus().then(s => s.queued + s.pending >= 1), + 'tx queued', + 20, + 0.1, + ); // Mine a block await ethCheatCodes.mine(); diff --git a/yarn-project/end-to-end/src/e2e_sequencer/escape_hatch_vote_only.test.ts b/yarn-project/end-to-end/src/e2e_sequencer/escape_hatch_vote_only.test.ts index a333e9129db4..401c4159176b 100644 --- a/yarn-project/end-to-end/src/e2e_sequencer/escape_hatch_vote_only.test.ts +++ b/yarn-project/end-to-end/src/e2e_sequencer/escape_hatch_vote_only.test.ts @@ -161,6 +161,7 @@ describe('e2e_escape_hatch_vote_only', () => { 'checkpoint-publish-failed', 'proposer-rollup-check-failed', 'checkpoint-error', + 'header-validation-failed', ]; failEventTypes.forEach(eventType => { diff --git a/yarn-project/ethereum/src/contracts/chain_state_override.test.ts b/yarn-project/ethereum/src/contracts/chain_state_override.test.ts index 77d3b79f2459..f88c6574d9b2 100644 --- a/yarn-project/ethereum/src/contracts/chain_state_override.test.ts +++ b/yarn-project/ethereum/src/contracts/chain_state_override.test.ts @@ -66,6 +66,30 @@ describe('SimulationOverridesBuilder', () => { expect(plan?.chainTipsOverride).toEqual({ pending: CheckpointNumber(7), proven: CheckpointNumber(3) }); }); + it('merge does not erase prior chain tip values when the incoming half is undefined', () => { + const builder = new SimulationOverridesBuilder().withChainTips({ + pending: CheckpointNumber(7), + proven: CheckpointNumber(5), + }); + builder.merge({ chainTipsOverride: { pending: undefined, proven: CheckpointNumber(6) } }); + const plan = builder.build(); + expect(plan?.chainTipsOverride).toEqual({ pending: CheckpointNumber(7), proven: CheckpointNumber(6) }); + }); + + it('merge does not erase prior pending checkpoint state when the incoming field is undefined', () => { + const archive = Fr.random(); + const builder = new SimulationOverridesBuilder() + .withChainTips({ pending: CheckpointNumber(7) }) + .withPendingArchive(archive); + builder.merge({ + chainTipsOverride: { pending: CheckpointNumber(7) }, + pendingCheckpointState: { archive: undefined, slotNumber: SlotNumber(42) }, + }); + const plan = builder.build(); + expect(plan?.pendingCheckpointState?.archive).toEqual(archive); + expect(plan?.pendingCheckpointState?.slotNumber).toEqual(SlotNumber(42)); + }); + it('attaches temp checkpoint log fields under the configured pending checkpoint', () => { const headerHash = Fr.random(); const outHash = Fr.random(); diff --git a/yarn-project/ethereum/src/contracts/chain_state_override.ts b/yarn-project/ethereum/src/contracts/chain_state_override.ts index 6358f0cde0e0..8693981098d0 100644 --- a/yarn-project/ethereum/src/contracts/chain_state_override.ts +++ b/yarn-project/ethereum/src/contracts/chain_state_override.ts @@ -1,6 +1,7 @@ import { toHex as toPaddedHex } from '@aztec/foundation/bigint-buffer'; import type { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; import type { Buffer32 } from '@aztec/foundation/buffer'; +import { merge } from '@aztec/foundation/collection'; import type { Fr } from '@aztec/foundation/curves/bn254'; import type { StateOverride } from 'viem'; @@ -45,18 +46,22 @@ export class SimulationOverridesBuilder { return new SimulationOverridesBuilder().merge(plan); } - /** Merges another plan into this builder. Later values win on a per-half basis for chain tips. */ + /** + * Merges another plan into this builder. Later values win on a per-half basis for chain tips, + * but explicit `undefined` fields in the incoming plan are ignored so they cannot erase a + * previously-set value. + */ public merge(plan: SimulationOverridesPlan | undefined): this { if (!plan) { return this; } if (plan.chainTipsOverride) { - this.chainTipsOverride = { ...(this.chainTipsOverride ?? {}), ...plan.chainTipsOverride }; + this.chainTipsOverride = merge(this.chainTipsOverride ?? {}, plan.chainTipsOverride); + } + if (plan.pendingCheckpointState) { + this.pendingCheckpointState = merge(this.pendingCheckpointState ?? {}, plan.pendingCheckpointState); } - this.pendingCheckpointState = plan.pendingCheckpointState - ? { ...(this.pendingCheckpointState ?? {}), ...plan.pendingCheckpointState } - : this.pendingCheckpointState; this.disableBlobCheck = this.disableBlobCheck || (plan.disableBlobCheck ?? false); return this; @@ -87,15 +92,21 @@ export class SimulationOverridesBuilder { } /** - * Overrides the locally-derivable `tempCheckpointLogs` cell fields for the configured pending - * checkpoint. Callers populate these together because they all come from the same proposed - * checkpoint payload — there is no use case for setting them independently. + * Overrides one or more `tempCheckpointLogs` cell fields for the configured pending checkpoint. + * Fields are independent: any subset can be provided. The translator (`makeTempCheckpointLogOverride`) + * emits a stateDiff entry per field actually set, so unspecified fields stay at their on-chain + * values. + * + * `slotNumber` is load-bearing for `STFLib.canPruneAtTime`: when the simulation overrides `pending` + * to a checkpoint that has no on-chain `tempCheckpointLogs` entry yet, the missing slotNumber falls + * back to 0 and the contract treats the pending tip as belonging to epoch 0, triggering a phantom + * prune that silently undoes the `pending` override. */ public withPendingTempCheckpointLogFields(fields: { - headerHash: Fr; - outHash: Fr; - payloadDigest: Buffer32; - slotNumber: SlotNumber; + headerHash?: Fr; + outHash?: Fr; + payloadDigest?: Buffer32; + slotNumber?: SlotNumber; }): this { this.assertPendingCheckpointNumber(); this.pendingCheckpointState = { ...(this.pendingCheckpointState ?? {}), ...fields }; diff --git a/yarn-project/ethereum/src/contracts/governance_proposer.ts b/yarn-project/ethereum/src/contracts/governance_proposer.ts index 0210211bb28a..b9e169475854 100644 --- a/yarn-project/ethereum/src/contracts/governance_proposer.ts +++ b/yarn-project/ethereum/src/contracts/governance_proposer.ts @@ -20,6 +20,14 @@ import { ReadOnlyGovernanceContract, extractProposalIdFromLogs } from './governa export class GovernanceProposerContract implements IEmpireBase { private readonly proposer: GetContractReturnType; + /** + * Cache of bytecode-existence checks keyed by payload address. The check is stable for a + * contract's lifetime -- a contract either has code or it does not, and code cannot be removed + * after deployment (selfdestruct aside, which is not relevant here). Safe to memoize + * indefinitely for the lifetime of this instance. + */ + private readonly emptyPayloadCache: Map = new Map(); + constructor( public readonly client: ViemClient, address: Hex | EthAddress, @@ -133,6 +141,28 @@ export class GovernanceProposerContract implements IEmpireBase { return governance.hasActiveProposalWithPayload(payload); } + /** + * Returns true if the given payload address has no deployed bytecode. Used as a cheap + * pre-flight check before casting a governance signal — voting for a zero-code address + * is unrecoverable. + * + * We only cache the `false` result (address has bytecode). The `true` result is NOT + * cached because a CREATE2-redeployed address could go from empty to populated, and + * caching `true` would make us keep skipping a payload that later becomes valid. + */ + public async isPayloadEmpty(payload: EthAddress): Promise { + const key = payload.toString() as Hex; + if (this.emptyPayloadCache.get(key) === false) { + return false; + } + const code = await this.client.getCode({ address: key }); + const isEmpty = !code || code === '0x'; + if (!isEmpty) { + this.emptyPayloadCache.set(key, false); + } + return isEmpty; + } + public async submitRoundWinner( round: bigint, l1TxUtils: L1TxUtils, diff --git a/yarn-project/ethereum/src/contracts/multicall.test.ts b/yarn-project/ethereum/src/contracts/multicall.test.ts index 1804eaf1e307..c63077e1bb29 100644 --- a/yarn-project/ethereum/src/contracts/multicall.test.ts +++ b/yarn-project/ethereum/src/contracts/multicall.test.ts @@ -17,7 +17,6 @@ import { L1TxUtils, createL1TxUtils } from '../l1_tx_utils/index.js'; import type { Anvil } from '../test/start_anvil.js'; import { startAnvil } from '../test/start_anvil.js'; import type { ExtendedViemWalletClient } from '../types.js'; -import { FormattedViemError } from '../utils.js'; import { MULTI_CALL_3_ADDRESS, Multicall3, deployMulticall3 } from './multicall.js'; describe('Multicall3', () => { @@ -97,34 +96,65 @@ describe('Multicall3', () => { abi: GovernanceProposerAbi, }); - it('should be able to call multiple functions in a single transaction', async () => { + it('should not revert by default if a single call fails', async () => { await deployMulticall3(walletClient, logger); - const result = await Multicall3.forward( - [makeSuccessfulCall(), makeFailingCall()], - l1TxUtils, - undefined, - undefined, - deployed.l1ContractAddresses.rollupAddress.toString(), - logger, - { revertOnFailure: true }, - ); + const result = await Multicall3.forward([makeSuccessfulCall(), makeFailingCall()], l1TxUtils, undefined, undefined); expect(result).toBeDefined(); - expect(result).toBeInstanceOf(FormattedViemError); - const formattedError = result as FormattedViemError; - expect(formattedError.message).toContain('ValidatorSelection__InsufficientValidatorSetSize'); + expect(result.receipt.status).toBe('success'); }); - it('should not revert by default if a single call fails', async () => { - await deployMulticall3(walletClient, logger); - const result = await Multicall3.forward( - [makeSuccessfulCall(), makeFailingCall()], - l1TxUtils, - undefined, - undefined, - deployed.l1ContractAddresses.rollupAddress.toString(), - logger, - ); - expect(result).toBeDefined(); - expect('receipt' in result && result.receipt.status).toBe('success'); + describe('simulateAggregate3', () => { + beforeAll(async () => { + await deployMulticall3(walletClient, logger); + }); + + it('decodes per-entry results when all entries succeed', async () => { + const result = await Multicall3.simulateAggregate3([makeSuccessfulCall(), makeSuccessfulCall()], l1TxUtils); + expect(result.kind).toBe('decoded'); + if (result.kind !== 'decoded') { + return; + } + expect(result.entries).toHaveLength(2); + expect(result.entries[0].success).toBe(true); + expect(result.entries[1].success).toBe(true); + expect(result.gasUsed).toBeGreaterThan(0n); + }); + + it('marks reverted entries with a decoded revert reason', async () => { + const result = await Multicall3.simulateAggregate3([makeSuccessfulCall(), makeFailingCall()], l1TxUtils); + expect(result.kind).toBe('decoded'); + if (result.kind !== 'decoded') { + return; + } + expect(result.entries).toHaveLength(2); + expect(result.entries[0].success).toBe(true); + expect(result.entries[1].success).toBe(false); + expect(result.entries[1].revertReason).toContain('ValidatorSelection__InsufficientValidatorSetSize'); + }); + + it('honours fakeSenderBalance by overriding the sender balance for the simulate', async () => { + // Use a sender we have not funded so a real send would fail with insufficient funds. + const poorPrivateKey = '0x' + 'aa'.repeat(32); + const poorAccount = privateKeyToAccount(poorPrivateKey as `0x${string}`); + const poorClient = createExtendedL1Client([rpcUrl], poorAccount, foundry); + const poorL1TxUtils = createL1TxUtils(poorClient, { logger }); + + // Without fakeSenderBalance, the simulate would not fail on entry-level (call doesn't need + // value), but the eth_simulateV1 may still validate sender funds for gas. Either way, with + // fakeSenderBalance we explicitly cap balance high enough that no balance-related path can + // fail in the simulate. + const result = await Multicall3.simulateAggregate3([makeSuccessfulCall()], poorL1TxUtils, { + fakeSenderBalance: 10n ** 20n, + }); + expect(result.kind).toBe('decoded'); + if (result.kind !== 'decoded') { + return; + } + expect(result.entries[0].success).toBe(true); + }); + + it('reports hasCode() true after deployMulticall3', async () => { + expect(await Multicall3.hasCode(l1TxUtils)).toBe(true); + }); }); }); diff --git a/yarn-project/ethereum/src/contracts/multicall.ts b/yarn-project/ethereum/src/contracts/multicall.ts index 40e17970e5db..777b6a220ebe 100644 --- a/yarn-project/ethereum/src/contracts/multicall.ts +++ b/yarn-project/ethereum/src/contracts/multicall.ts @@ -1,16 +1,37 @@ -import { toHex as toPaddedHex } from '@aztec/foundation/bigint-buffer'; -import { TimeoutError } from '@aztec/foundation/error'; +import { EthAddress } from '@aztec/foundation/eth-address'; import type { Logger } from '@aztec/foundation/log'; -import { type Address, type EncodeFunctionDataParameters, type Hex, encodeFunctionData, multicall3Abi } from 'viem'; +import { + type Abi, + type Address, + type BlockOverrides, + type Hex, + type RequiredBy, + type StateOverride, + type TransactionReceipt, + decodeErrorResult, + decodeFunctionResult, + encodeFunctionData, + multicall3Abi, +} from 'viem'; import type { L1BlobInputs, L1TxConfig, L1TxRequest, L1TxUtils } from '../l1_tx_utils/index.js'; import type { ExtendedViemWalletClient } from '../types.js'; -import { FormattedViemError, formatViemError } from '../utils.js'; -import { RollupContract } from './rollup.js'; export const MULTI_CALL_3_ADDRESS = '0xcA11bde05977b3631167028862bE2a173976CA11' as const; +/** + * Thrown by `Multicall3.forward` when the forwarder transaction lands but the receipt reports a + * reverted status. This is not expected (aggregate3 uses allowFailure: true), so callers should + * treat it as a fatal on-chain failure rather than retrying on a different publisher. + */ +export class MulticallForwarderRevertedError extends Error { + constructor(public readonly receipt: TransactionReceipt) { + super(`Multicall3 forwarder tx reverted: ${receipt.transactionHash}`); + this.name = 'MulticallForwarderRevertedError'; + } +} + /** ABI fragment for aggregate3Value — not included in viem's multicall3Abi. */ export const aggregate3ValueAbi = [ { @@ -44,116 +65,177 @@ export const aggregate3ValueAbi = [ }, ] as const; +/** A single call to embed inside an aggregate3 simulation. The abi is used to decode revert reasons. */ +export type SimulateAggregate3Request = { + to: Address; + data: Hex; + /** Optional ABI used to decode the revert reason if this entry reverts. */ + abi?: Abi; +}; + +export type SimulateAggregate3EntryResult = { + success: boolean; + /** Decoded revert reason text when `success === false` and a request abi was provided. */ + revertReason?: string; + /** Raw return data hex. `'0x'` for successful entries with void return. */ + returnData: Hex; +}; + +/** + * Outcome of a bundle simulation. + * - `decoded`: eth_simulateV1 ran and produced a per-entry Result[]. Use `entries` for filtering. + * - `fallback`: the node does not support eth_simulateV1; `fallbackGasEstimate` was returned and no + * per-entry info is available. Caller should send the bundle as-is with a conservative gas cap. + */ +export type SimulateAggregate3Result = + | { kind: 'decoded'; entries: SimulateAggregate3EntryResult[]; gasUsed: bigint } + | { kind: 'fallback'; gasUsed: bigint }; + +export type SimulateAggregate3Options = { + blockOverrides?: BlockOverrides; + stateOverrides?: StateOverride; + /** + * If set, append a state override that fakes the sender's balance during the simulation so a + * low or zero balance does not cause the simulate to fail with insufficient funds. The fake + * balance is applied to `l1TxUtils.getSenderAddress()`. + */ + fakeSenderBalance?: bigint; + /** Gas cap to pass on the simulate call itself (defaults to viem's behavior). */ + gas?: bigint; + /** When eth_simulateV1 is unavailable, fall back to this gas estimate instead of throwing. */ + fallbackGasEstimate?: bigint; +}; + export class Multicall3 { - static async forward( + /** + * Returns true iff Multicall3 bytecode is deployed at MULTI_CALL_3_ADDRESS. An empty result from + * a non-existent contract would otherwise silently validate any bundle that uses Multicall3. + */ + static async hasCode(l1TxUtils: L1TxUtils): Promise { + const code = await l1TxUtils.getCode(EthAddress.fromString(MULTI_CALL_3_ADDRESS)); + return !!code && code !== '0x'; + } + + /** + * Simulates an aggregate3 call composed of the given requests via eth_simulateV1 and decodes the + * per-entry Result[]. Entries that revert are returned with a decoded revertReason (if the request + * provided an abi). + * + * Use this to pre-validate a bundle before sending it through `Multicall3.forward`. The caller can + * drop reverted entries from the bundle and re-simulate with the reduced list to get an accurate + * `gasUsed`. + */ + static async simulateAggregate3( + requests: SimulateAggregate3Request[], + l1TxUtils: L1TxUtils, + opts: SimulateAggregate3Options = {}, + ): Promise { + const calldata = encodeFunctionData({ + abi: multicall3Abi, + functionName: 'aggregate3', + args: [ + requests.map(r => ({ + target: r.to, + callData: r.data, + allowFailure: true, + })), + ], + }); + + const stateOverrides: StateOverride = [...(opts.stateOverrides ?? [])]; + if (opts.fakeSenderBalance !== undefined) { + stateOverrides.push({ + address: l1TxUtils.getSenderAddress().toString(), + balance: opts.fakeSenderBalance, + }); + } + + const simResult = await l1TxUtils.simulate( + { to: MULTI_CALL_3_ADDRESS, data: calldata, gas: opts.gas }, + opts.blockOverrides, + stateOverrides, + multicall3Abi, + { fallbackGasEstimate: opts.fallbackGasEstimate }, + ); + + if (simResult.result === '0x') { + return { kind: 'fallback', gasUsed: simResult.gasUsed }; + } + + const decoded = decodeFunctionResult({ + abi: multicall3Abi, + functionName: 'aggregate3', + data: simResult.result, + }) as readonly { success: boolean; returnData: `0x${string}` }[]; + + const entries: SimulateAggregate3EntryResult[] = decoded.map((entry, i) => { + if (entry.success) { + return { success: true, returnData: entry.returnData }; + } + let revertReason: string | undefined; + const abi = requests[i].abi; + if (abi && entry.returnData && entry.returnData !== '0x') { + try { + const decodedError = decodeErrorResult({ abi, data: entry.returnData }); + revertReason = `${decodedError.errorName}(${decodedError.args?.join(', ') ?? ''})`; + } catch { + // Decoding failed; leave revertReason undefined so the caller can log the raw returnData. + } + } + return { success: false, returnData: entry.returnData, revertReason }; + }); + + return { kind: 'decoded', entries, gasUsed: simResult.gasUsed }; + } + + /** + * Sends a batch of requests through aggregate3. Individual calls may fail (allowFailure: true), + * but the top-level multicall is expected to land successfully. Throws if the send fails or if + * the receipt reports a reverted status. + */ + static async forward( requests: L1TxRequest[], l1TxUtils: L1TxUtils, - gasConfig: L1TxConfig | undefined, + gasConfig: TOptGasLimitRequired extends true ? RequiredBy : L1TxConfig | undefined, blobConfig: L1BlobInputs | undefined, - rollupAddress: Hex, - logger: Logger, - opts: { revertOnFailure?: boolean } = {}, + opts: { gasLimitRequired?: TOptGasLimitRequired } = {}, ) { - requests = requests.filter(request => request.to !== null); - const args = requests.map(r => ({ - target: r.to!, - callData: r.data!, - allowFailure: !opts.revertOnFailure, - })); - const forwarderFunctionData: Required> = { + if (opts.gasLimitRequired && !gasConfig?.gasLimit) { + throw new Error('Multicall gasLimit is required when gasLimitRequired is true'); + } + + const args = requests + .filter(request => request.to !== null) + .map(r => ({ + target: r.to!, + callData: r.data!, + allowFailure: true, + })); + const encodedForwarderData = encodeFunctionData({ abi: multicall3Abi, functionName: 'aggregate3', args: [args], - }; - - const encodedForwarderData = encodeFunctionData(forwarderFunctionData); - try { - const { receipt, state } = await l1TxUtils.sendAndMonitorTransaction( - { - to: MULTI_CALL_3_ADDRESS, - data: encodedForwarderData, - abi: multicall3Abi, - }, - gasConfig, - blobConfig, - ); - - if (receipt.status === 'success') { - const stats = await l1TxUtils.getTransactionStats(receipt.transactionHash); - return { receipt, stats }; - } else { - logger.error('Forwarder transaction failed', undefined, { receipt }); - - const args = { - ...forwarderFunctionData, - address: MULTI_CALL_3_ADDRESS, - }; - - let errorMsg: string | undefined; - - if (blobConfig) { - const maxFeePerBlobGas = blobConfig.maxFeePerBlobGas ?? state.gasPrice.maxFeePerBlobGas; - if (maxFeePerBlobGas === undefined) { - errorMsg = 'maxFeePerBlobGas is required to get the error message'; - } else { - logger.debug('Trying to get error from reverted tx with blob config'); - errorMsg = await l1TxUtils.tryGetErrorFromRevertedTx( - encodedForwarderData, - args, - { - blobs: blobConfig.blobs, - kzg: blobConfig.kzg, - maxFeePerBlobGas, - }, - [ - { - address: rollupAddress, - stateDiff: [ - { - slot: toPaddedHex(RollupContract.checkBlobStorageSlot, true), - value: toPaddedHex(0n, true), - }, - ], - }, - ], - ); - } - } else { - logger.debug('Trying to get error from reverted tx without blob config'); - errorMsg = await l1TxUtils.tryGetErrorFromRevertedTx(encodedForwarderData, args, undefined, []); - } + }); - return { receipt, errorMsg }; - } - } catch (err) { - if (err instanceof TimeoutError) { - throw err; - } + const { receipt } = await l1TxUtils.sendAndMonitorTransaction( + { + to: MULTI_CALL_3_ADDRESS, + data: encodedForwarderData, + abi: multicall3Abi, + }, + gasConfig, + blobConfig, + ); - for (const request of requests) { - logger.debug('Simulating request', { request }); - const result = await l1TxUtils - .simulate(request, undefined, [ - { - address: rollupAddress, - stateDiff: [ - { slot: toPaddedHex(RollupContract.checkBlobStorageSlot, true), value: toPaddedHex(0n, true) }, - ], - }, - ]) - .catch(err => formatViemError(err, request.abi)); - if (result instanceof FormattedViemError) { - logger.error('Found error in simulation', result, { - to: request.to ?? 'null', - data: request.data, - }); - - return result; - } - } - logger.warn('Failed to get error from reverted tx', { err }); - throw err; + // This shouldn't happen. Any failure in individual calls is swallowed by forward since we set + // allowFailure to true for all calls, so a reverted status here would indicate a problem with + // the Multicall3 contract itself or the forwarder transaction (such as an out-of-gas). + if (receipt.status !== 'success') { + throw new MulticallForwarderRevertedError(receipt); } + + const stats = await l1TxUtils.getTransactionStats(receipt.transactionHash); + return { receipt, stats, multicallData: encodedForwarderData }; } /** Batch multiple value transfers into a single aggregate3Value call on Multicall3. */ diff --git a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts index 3ca1526cecdb..5acf9209114f 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts @@ -229,6 +229,15 @@ export class L1TxUtils extends ReadOnlyL1TxUtils { throw new InterruptError(`Transaction sending is interrupted`); } + // Fail fast before doing any work (gas estimation, balance check) if the caller's deadline + // has already passed. The same check is repeated after gas estimation in case it took long + // enough to push us past the deadline. + if (gasConfigOverrides?.txTimeoutAt && new Date() > gasConfigOverrides.txTimeoutAt) { + throw new TimeoutError( + `Transaction timed out before sending (now ${new Date().toISOString()} > timeoutAt ${gasConfigOverrides.txTimeoutAt.toISOString()})`, + ); + } + try { const gasConfig = merge(this.config, gasConfigOverrides); const account = this.getSenderAddress().toString(); diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-bundle-simulator.ts b/yarn-project/sequencer-client/src/publisher/sequencer-bundle-simulator.ts new file mode 100644 index 000000000000..5ae7c1647117 --- /dev/null +++ b/yarn-project/sequencer-client/src/publisher/sequencer-bundle-simulator.ts @@ -0,0 +1,253 @@ +import type { EpochCache } from '@aztec/epoch-cache'; +import { Multicall3, type RollupContract, buildSimulationOverridesStateOverride } from '@aztec/ethereum/contracts'; +import { type L1TxUtils, MAX_L1_TX_LIMIT } from '@aztec/ethereum/l1-tx-utils'; +import { formatViemError } from '@aztec/ethereum/utils'; +import type { SlotNumber } from '@aztec/foundation/branded-types'; +import { type Logger, createLogger } from '@aztec/foundation/log'; +import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; + +import type { Hex, StateOverride } from 'viem'; + +import type { RequestWithExpiry } from './sequencer-publisher.js'; + +/** A request that was dropped by bundle simulation, with the decoded revert reason. */ +export type DroppedRequest = { + request: RequestWithExpiry; + revertReason: string | undefined; + returnData: Hex | undefined; +}; + +/** + * Result of {@link SequencerBundleSimulator.simulate}. + * + * - `success`: simulation succeeded. `requests` is the filtered survivor list, `gasLimit` is + * the bumped gas limit derived from `gasUsed` (plus blob evaluation gas). `droppedRequests` + * lists the entries that were observed to revert in simulation. + * - `fallback`: the node does not support eth_simulateV1 (or the simulate call threw). The + * caller should send `requests` as-is with a safe gas limit (e.g. {@link MAX_L1_TX_LIMIT}). + * `droppedRequests` carries any entries that the first pass already proved reverted, so the + * caller does not re-include them when the second pass falls back. + * - `aborted`: the bundle cannot be sent. `droppedRequests` contains only entries that were + * actually observed to revert (so they can be reported as simulation failures); it is empty + * when the abort was caused by an empty input bundle. + */ +export type BundleSimulateResult = + | { kind: 'success'; requests: RequestWithExpiry[]; gasLimit: bigint; droppedRequests: DroppedRequest[] } + | { kind: 'fallback'; requests: RequestWithExpiry[]; droppedRequests: DroppedRequest[] } + | { kind: 'aborted'; reason: AbortReason; droppedRequests: DroppedRequest[] }; + +export type AbortReason = 'empty-bundle' | 'all-reverted' | 'second-pass-reverts'; + +type SimulatePassResult = + | { kind: 'decoded'; survivors: RequestWithExpiry[]; droppedRequests: DroppedRequest[]; gasUsed: bigint } + | { kind: 'fallback' }; + +/** + * Bundle-level simulator for the aggregate3 payload that `SequencerPublisher` is about to send. + * + * Runs `eth_simulateV1` against `Multicall3.aggregate3`, drops entries that revert, and returns + * a gasLimit for the survivors. When `eth_simulateV1` is unavailable, signals fallback to the + * caller so it can send the bundle as-is with a conservative gas limit. + */ +export class SequencerBundleSimulator { + private readonly log: Logger; + + constructor( + private readonly deps: { + getL1TxUtils: () => L1TxUtils; + rollupContract: RollupContract; + epochCache: EpochCache; + log?: Logger; + }, + ) { + this.log = deps.log ?? createLogger('sequencer:publisher:bundle-simulator'); + } + + /** + * Simulates the given bundle at the target slot's start timestamp and filters out entries + * that revert. + * + * - If all entries pass on the first pass, returns `success` with the gasLimit. + * - If some entries revert, re-simulates the survivors. If the second pass is clean, returns + * `success` with the survivors and dropped entries. If the second pass surfaces any revert, + * returns `aborted` — we refuse to send a bundle whose composition still has internal + * reverts after one round of filtering. + * - If eth_simulateV1 is unavailable, returns `fallback`. The caller is expected to send the + * bundle as-is with a safe gas limit. + * + * The simulation `block.timestamp` is always the target L2 slot's start timestamp, since + * propose's `validateHeader` and EIP-712 signature checks both derive a slot from + * `block.timestamp` and compare against the slot the validator signed for. + * + * Known limitation: on networks where L1 is mining behind cadence (missed L1 slots, anvil with + * overridden timestamps), the actual `block.timestamp` at send time can land in the prior L2 + * slot. In that case `propose` would revert silently inside the multicall. The simulator does + * not detect this case because it simulates AT the target timestamp — the prior implementation + * used `min(predictedNextL1Ts, targetTimestamp)` to surface this failure mode at simulate time. + */ + public async simulate(validRequests: RequestWithExpiry[], targetSlot: SlotNumber): Promise { + if (validRequests.length === 0) { + return { kind: 'aborted', reason: 'empty-bundle', droppedRequests: [] }; + } + // Pin the publisher we'll use across the whole simulate call so that the publisher's rotation + // can't change l1TxUtils mid-flight. + const l1TxUtils = this.deps.getL1TxUtils(); + + const proposeRequest = validRequests.find(r => r.action === 'propose'); + const simulateTimestamp = getTimestampForSlot(targetSlot, this.deps.epochCache.getL1Constants()); + const firstPassOverrides = await this.buildStateOverrides(!!proposeRequest); + + const firstPass = await this.simulateAndDecode(l1TxUtils, validRequests, simulateTimestamp, firstPassOverrides); + + if (firstPass.kind === 'fallback') { + this.log.warn('Bundle simulate fallback (eth_simulateV1 unavailable); caller will send bundle as-is', { + actions: validRequests.map(r => r.action), + }); + return { kind: 'fallback', requests: validRequests, droppedRequests: [] }; + } + + if (firstPass.survivors.length === 0) { + this.log.warn('All bundle entries dropped in simulation; aborting send', { + actions: validRequests.map(r => r.action), + }); + return { kind: 'aborted', reason: 'all-reverted', droppedRequests: firstPass.droppedRequests }; + } + + if (firstPass.droppedRequests.length === 0) { + return this.buildSuccessResult(l1TxUtils, firstPass.survivors, [], firstPass.gasUsed, proposeRequest); + } + + this.log.warn('Some bundle entries reverted; re-simulating reduced bundle', { + droppedActions: firstPass.droppedRequests.map(d => d.request.action), + remainingActions: firstPass.survivors.map(r => r.action), + }); + + // Rebuild overrides for the reduced bundle: if propose was dropped, we no longer need the blob-check override + const proposeSurvived = proposeRequest !== undefined && firstPass.survivors.includes(proposeRequest); + const secondPassOverrides = proposeSurvived ? firstPassOverrides : await this.buildStateOverrides(false); + const secondPass = await this.simulateAndDecode( + l1TxUtils, + firstPass.survivors, + simulateTimestamp, + secondPassOverrides, + ); + + if (secondPass.kind === 'fallback') { + this.log.warn( + 'Bundle simulate errored on second pass (eth_simulateV1 unavailable); sending first-pass survivors as-is', + { + actions: firstPass.survivors.map(r => r.action), + droppedActions: firstPass.droppedRequests.map(d => d.request.action), + }, + ); + return { kind: 'fallback', requests: firstPass.survivors, droppedRequests: firstPass.droppedRequests }; + } + + // We refuse to chase reverts through repeated trimming: anything other than a clean second pass aborts the whole send + if (secondPass.droppedRequests.length > 0) { + this.log.error('Re-simulate surfaced reverts; aborting send', { + secondPassDroppedActions: secondPass.droppedRequests.map(d => d.request.action), + }); + return { + kind: 'aborted', + reason: 'second-pass-reverts', + droppedRequests: [...firstPass.droppedRequests, ...secondPass.droppedRequests], + }; + } + + return this.buildSuccessResult( + l1TxUtils, + secondPass.survivors, + firstPass.droppedRequests, + secondPass.gasUsed, + proposeRequest, + ); + } + + private buildSuccessResult( + l1TxUtils: L1TxUtils, + survivors: RequestWithExpiry[], + droppedRequests: DroppedRequest[], + bundleGasUsed: bigint, + proposeRequest: RequestWithExpiry | undefined, + ): BundleSimulateResult { + const proposeSurvived = proposeRequest !== undefined && survivors.includes(proposeRequest); + const blobEvaluationGas = proposeSurvived ? (proposeRequest?.blobEvaluationGas ?? 0n) : 0n; + const gasLimit = this.computeGasLimit(l1TxUtils, bundleGasUsed, blobEvaluationGas); + this.log.debug('Bundle simulate complete', { + survivingRequests: survivors.length, + bundleGasUsed, + gasLimit, + actions: survivors.map(r => r.action), + }); + return { kind: 'success', requests: survivors, gasLimit, droppedRequests }; + } + + /** + * `gasLimit = bumpGasLimit(ceil(gasUsed * 64 / 63))`, plus blob evaluation gas if a propose + * survived, capped at the L1 block gas limit. + */ + private computeGasLimit(l1TxUtils: L1TxUtils, bundleGasUsed: bigint, blobEvaluationGas: bigint): bigint { + const gasUsedWithEip150 = (bundleGasUsed * 64n + 62n) / 63n; + const gasLimit = l1TxUtils.bumpGasLimit(gasUsedWithEip150) + blobEvaluationGas; + return gasLimit > MAX_L1_TX_LIMIT ? MAX_L1_TX_LIMIT : gasLimit; + } + + /** + * eth_simulateV1 cannot carry blob sidecar data, so disable the rollup's on-chain blob check + * when a propose is in the bundle. + */ + private buildStateOverrides(hasProposeAction: boolean): Promise { + return buildSimulationOverridesStateOverride( + this.deps.rollupContract, + hasProposeAction ? { disableBlobCheck: true } : undefined, + ); + } + + private async simulateAndDecode( + l1TxUtils: L1TxUtils, + requests: RequestWithExpiry[], + simulateTimestamp: bigint, + stateOverrides: StateOverride, + ): Promise { + let simResult: Awaited>; + try { + simResult = await Multicall3.simulateAggregate3( + requests.map(r => ({ to: r.request.to! as Hex, data: r.request.data! as Hex, abi: r.request.abi })), + l1TxUtils, + { + blockOverrides: { time: simulateTimestamp, gasLimit: MAX_L1_TX_LIMIT * 2n }, + stateOverrides, + gas: MAX_L1_TX_LIMIT, + fallbackGasEstimate: MAX_L1_TX_LIMIT, + }, + ); + } catch (err) { + this.log.warn('Bundle simulate threw; treating as fallback', { + err: formatViemError(err), + actions: requests.map(r => r.action), + }); + return { kind: 'fallback' }; + } + + if (simResult.kind === 'fallback') { + return { kind: 'fallback' }; + } + + const survivors: RequestWithExpiry[] = []; + const droppedRequests: DroppedRequest[] = []; + for (let i = 0; i < requests.length; i++) { + const entry = simResult.entries[i]; + if (entry.success) { + survivors.push(requests[i]); + continue; + } + droppedRequests.push({ + request: requests[i], + revertReason: entry.revertReason, + returnData: entry.returnData, + }); + } + return { kind: 'decoded', survivors, droppedRequests, gasUsed: simResult.gasUsed }; + } +} diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts index ce8479609636..1c5bdf1250fd 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts @@ -5,19 +5,17 @@ import type { L1ContractsConfig } from '@aztec/ethereum/config'; import { type GovernanceProposerContract, Multicall3, + MulticallForwarderRevertedError, type RollupContract, - type SimulationOverridesPlan, type SlashingProposerContract, } from '@aztec/ethereum/contracts'; import { - type GasPrice, type L1TxUtils, type L1TxUtilsConfig, + MAX_L1_TX_LIMIT, defaultL1TxUtilsConfig, } from '@aztec/ethereum/l1-tx-utils'; -import { FormattedViemError } from '@aztec/ethereum/utils'; -import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { Fr } from '@aztec/foundation/curves/bn254'; +import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { TimeoutError } from '@aztec/foundation/error'; import { EthAddress } from '@aztec/foundation/eth-address'; import { sleep } from '@aztec/foundation/sleep'; @@ -33,9 +31,12 @@ import { type MockProxy, mock } from 'jest-mock-extended'; import { type GetCodeReturnType, type GetTransactionReceiptReturnType, + type Hex, type PrivateKeyAccount, type TransactionReceipt, encodeFunctionData, + encodeFunctionResult, + multicall3Abi, toHex, } from 'viem'; import { privateKeyToAccount } from 'viem/accounts'; @@ -168,10 +169,13 @@ describe('SequencerPublisher', () => { (l1TxUtils as any).estimateGas.mockResolvedValue(GAS_GUESS); (l1TxUtils as any).simulate.mockResolvedValue({ gasUsed: 1_000_000n, result: '0x' }); (l1TxUtils as any).bumpGasLimit.mockImplementation((val: bigint) => val + (val * 20n) / 100n); + l1TxUtils.getSenderBalance.mockResolvedValue(10_000_000_000_000_000_000n); // 10 ETH, sufficient for all tests (l1TxUtils as any).client = { account: { address: '0x1234567890123456789012345678901234567890', }, + getGasPrice: () => Promise.resolve(1n), + getBlock: () => Promise.resolve({ timestamp: 0n }), }; const currentL2Slot = publisher.getCurrentL2Slot(); @@ -230,7 +234,8 @@ describe('SequencerPublisher', () => { forwardSpy.mockResolvedValue({ receipt: proposeTxReceipt, - errorMsg: undefined, + stats: undefined, + multicallData: '0x', }); await publisher.sendRequests(); @@ -274,8 +279,7 @@ describe('SequencerPublisher', () => { expect.objectContaining({ blobs: expect.any(Array), }), - mockRollupAddress, - expect.anything(), // the logger + { gasLimitRequired: true }, ); expect(forwardSpy.mock.calls[0][2]?.gasLimit).toBeGreaterThan(2_000_000n); @@ -291,7 +295,8 @@ describe('SequencerPublisher', () => { it('errors if forwarder tx fails', async () => { forwardSpy.mockRejectedValueOnce(new Error()).mockResolvedValueOnce({ receipt: proposeTxReceipt, - errorMsg: undefined, + stats: undefined, + multicallData: '0x', }); await publisher.enqueueProposeCheckpoint( @@ -312,7 +317,14 @@ describe('SequencerPublisher', () => { secondL1TxUtils = mock(); secondL1TxUtils.getBlockNumber.mockResolvedValue(1n); secondL1TxUtils.getSenderAddress.mockReturnValue(EthAddress.random()); - secondL1TxUtils.getSenderBalance.mockResolvedValue(1000n); + secondL1TxUtils.getSenderBalance.mockResolvedValue(10_000_000_000_000_000_000n); // 10 ETH + (secondL1TxUtils as any).client = { + account: { address: EthAddress.random().toString() }, + getGasPrice: () => Promise.resolve(1n), + }; + (secondL1TxUtils as any).bumpGasLimit = (val: bigint) => val + (val * 20n) / 100n; + (secondL1TxUtils as any).simulate = () => Promise.resolve({ gasUsed: 1_000_000n, result: '0x' }); + (secondL1TxUtils as any).getBlockNumber = () => Promise.resolve(1n); getNextPublisher = jest.fn(); @@ -352,7 +364,7 @@ describe('SequencerPublisher', () => { it('rotates to next publisher when forward throws and retries successfully', async () => { forwardSpy .mockRejectedValueOnce(new Error('RPC error')) - .mockResolvedValueOnce({ receipt: proposeTxReceipt, errorMsg: undefined }); + .mockResolvedValueOnce({ receipt: proposeTxReceipt, stats: undefined, multicallData: '0x' }); getNextPublisher.mockResolvedValueOnce(secondL1TxUtils); await rotatingPublisher.enqueueProposeCheckpoint( @@ -371,7 +383,6 @@ describe('SequencerPublisher', () => { expect.anything(), expect.anything(), expect.anything(), - expect.anything(), ); expect(forwardSpy).toHaveBeenNthCalledWith( 2, @@ -380,7 +391,6 @@ describe('SequencerPublisher', () => { expect.anything(), expect.anything(), expect.anything(), - expect.anything(), ); expect(getNextPublisher).toHaveBeenCalledWith([l1TxUtils.getSenderAddress()]); // Result is defined (rotation succeeded and tx was sent) @@ -424,152 +434,220 @@ describe('SequencerPublisher', () => { expect(result).toBeUndefined(); }); - it('does not rotate when forward returns a revert (on-chain failure)', async () => { - forwardSpy.mockResolvedValue({ receipt: { ...proposeTxReceipt, status: 'reverted' }, errorMsg: 'revert reason' }); - + it('does not enter the rotation loop when txTimeoutAt is already in the past', async () => { + const pastTimeout = new Date(Date.now() - 1000); await rotatingPublisher.enqueueProposeCheckpoint( new Checkpoint(l2Block.archive, header, [l2Block], l2Block.checkpointNumber), CommitteeAttestationsAndSigners.empty(testSignatureContext), Signature.empty(), + { txTimeoutAt: pastTimeout }, ); const result = await rotatingPublisher.sendRequests(); - expect(forwardSpy).toHaveBeenCalledTimes(1); + expect(result).toBeUndefined(); + expect(forwardSpy).not.toHaveBeenCalled(); expect(getNextPublisher).not.toHaveBeenCalled(); - // Result contains the reverted receipt (no rotation) - expect(result?.result).toMatchObject({ receipt: { status: 'reverted' } }); }); - }); - it('does not send propose tx if rollup validation fails', async () => { - l1TxUtils.simulate.mockRejectedValueOnce(new Error('Test error')); + it('stops rotating once txTimeoutAt elapses mid-rotation', async () => { + // First forward throws; getNextPublisher rotates to a new publisher; but by then the + // deadline has elapsed and the rotation loop should bail before the second forward call. + const futureTimeout = new Date(Date.now() + 100); // will elapse during the await below + forwardSpy.mockImplementationOnce(async () => { + await new Promise(resolve => setTimeout(resolve, 200)); + throw new Error('RPC error on first'); + }); + getNextPublisher.mockResolvedValueOnce(secondL1TxUtils); - await expect( - publisher.enqueueProposeCheckpoint( + await rotatingPublisher.enqueueProposeCheckpoint( new Checkpoint(l2Block.archive, header, [l2Block], l2Block.checkpointNumber), CommitteeAttestationsAndSigners.empty(testSignatureContext), Signature.empty(), - ), - ).rejects.toThrow(); - - expect(l1TxUtils.simulate).toHaveBeenCalledTimes(1); + { txTimeoutAt: futureTimeout }, + ); + const result = await rotatingPublisher.sendRequests(); - const result = await publisher.sendRequests(); - expect(result).toEqual(undefined); - expect(forwardSpy).not.toHaveBeenCalled(); - }); + expect(result).toBeUndefined(); + // forward was attempted exactly once (the first publisher); rotation was aborted before + // the second attempt because the deadline had passed. + expect(forwardSpy).toHaveBeenCalledTimes(1); + }); - it('preCheck closure uses preCheckSimulationOverridesPlan, not the enqueue-time plan', async () => { - (publisher.epochCache.isProposerPipeliningEnabled as jest.Mock).mockReturnValue(true); + it('does not rotate when forward throws MulticallForwarderRevertedError (on-chain failure)', async () => { + forwardSpy.mockRejectedValueOnce( + new MulticallForwarderRevertedError({ ...proposeTxReceipt, status: 'reverted' }), + ); - const validateSpy = jest.spyOn(publisher, 'validateCheckpointForSubmission').mockResolvedValue(undefined); + await rotatingPublisher.enqueueProposeCheckpoint( + new Checkpoint(l2Block.archive, header, [l2Block], l2Block.checkpointNumber), + CommitteeAttestationsAndSigners.empty(testSignatureContext), + Signature.empty(), + ); + const result = await rotatingPublisher.sendRequests(); - const enqueuePlan: SimulationOverridesPlan = { - chainTipsOverride: { pending: CheckpointNumber(7) }, - pendingCheckpointState: { archive: Fr.random() }, - }; - const preCheckPlan: SimulationOverridesPlan = { - chainTipsOverride: { pending: CheckpointNumber(8) }, - }; + expect(forwardSpy).toHaveBeenCalledTimes(1); + expect(getNextPublisher).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); + }); + }); + it('does not send propose tx if rollup validation fails', async () => { await publisher.enqueueProposeCheckpoint( new Checkpoint(l2Block.archive, header, [l2Block], l2Block.checkpointNumber), CommitteeAttestationsAndSigners.empty(testSignatureContext), Signature.empty(), - { simulationOverridesPlan: enqueuePlan, preCheckSimulationOverridesPlan: preCheckPlan }, ); - // Enqueue-time validation called with the enqueue plan (plus withoutBlobCheck applied). - expect(validateSpy).toHaveBeenCalledTimes(1); - expect(validateSpy.mock.calls[0][3]).toMatchObject({ - chainTipsOverride: { pending: CheckpointNumber(7) }, - disableBlobCheck: true, + // Simulate the bundle-level validate returning a failed entry for the propose call. + // When all entries fail, bundleSimulate returns undefined and sendRequests returns undefined. + const failedResult = encodeFunctionResult({ + abi: multicall3Abi, + functionName: 'aggregate3', + result: [{ success: false, returnData: '0x' }], }); + (l1TxUtils as any).simulate.mockResolvedValueOnce({ gasUsed: 0n, result: failedResult }); - // The pending preCheck request should now run the preCheck closure with the preCheck plan. - const requests: { preCheck?: () => Promise }[] = (publisher as any).requests; - expect(requests).toHaveLength(1); - const preCheck = requests[0].preCheck; - expect(preCheck).toBeDefined(); - - validateSpy.mockClear(); - await preCheck!(); - - expect(validateSpy).toHaveBeenCalledTimes(1); - expect(validateSpy.mock.calls[0][3]).toMatchObject({ - chainTipsOverride: { pending: CheckpointNumber(8) }, - disableBlobCheck: true, - }); - // And not the enqueue plan's archive override. - expect(validateSpy.mock.calls[0][3]?.pendingCheckpointState).toBeUndefined(); + const result = await publisher.sendRequests(); + expect(result).toEqual(undefined); + expect(forwardSpy).not.toHaveBeenCalled(); + expect(l1TxUtils.simulate).toHaveBeenCalledTimes(1); }); - it('preCheck does not fall back to the enqueue plan when preCheckSimulationOverridesPlan is omitted', async () => { - (publisher.epochCache.isProposerPipeliningEnabled as jest.Mock).mockReturnValue(true); + describe('bundleSimulate second-pass re-decode', () => { + const addTwoRequests = () => { + const currentL2Slot = publisher.getCurrentL2Slot(); + publisher.addRequest({ + action: 'invalidate-by-invalid-attestation', + request: { to: mockRollupAddress, data: '0xdeadbeef' }, + lastValidL2Slot: SlotNumber(Number(currentL2Slot) + 2), + checkSuccess: () => true, + }); + publisher.addRequest({ + action: 'propose', + request: { + to: mockRollupAddress, + data: encodeFunctionData({ + abi: EmpireBaseAbi, + functionName: 'signal', + args: [EthAddress.random().toString()], + }), + }, + lastValidL2Slot: SlotNumber(Number(currentL2Slot) + 2), + checkSuccess: () => true, + }); + }; - const validateSpy = jest.spyOn(publisher, 'validateCheckpointForSubmission').mockResolvedValue(undefined); + it('drops an entry that still reverts in the second-pass re-simulate', async () => { + addTwoRequests(); + + // First simulate: invalidate succeeds, propose fails. + const firstResult = encodeFunctionResult({ + abi: multicall3Abi, + functionName: 'aggregate3', + result: [ + { success: true, returnData: '0x' }, + { success: false, returnData: '0x' }, + ], + }); + // Second simulate (reduced bundle with only invalidate): that entry also fails. + const secondResult = encodeFunctionResult({ + abi: multicall3Abi, + functionName: 'aggregate3', + result: [{ success: false, returnData: '0x' }], + }); - const enqueuePlan: SimulationOverridesPlan = { - chainTipsOverride: { pending: CheckpointNumber(7) }, - pendingCheckpointState: { archive: Fr.random() }, - }; + (l1TxUtils as any).simulate + .mockResolvedValueOnce({ gasUsed: 500_000n, result: firstResult }) + .mockResolvedValueOnce({ gasUsed: 0n, result: secondResult }); - await publisher.enqueueProposeCheckpoint( - new Checkpoint(l2Block.archive, header, [l2Block], l2Block.checkpointNumber), - CommitteeAttestationsAndSigners.empty(testSignatureContext), - Signature.empty(), - { simulationOverridesPlan: enqueuePlan }, - ); + const result = await publisher.sendRequests(); - expect(validateSpy).toHaveBeenCalledTimes(1); - expect(validateSpy.mock.calls[0][3]).toMatchObject({ - chainTipsOverride: { pending: CheckpointNumber(7) }, - disableBlobCheck: true, + // Both passes dropped everything — should abort. + expect(result).toBeUndefined(); + expect(forwardSpy).not.toHaveBeenCalled(); + expect(l1TxUtils.simulate).toHaveBeenCalledTimes(2); }); - const requests: { preCheck?: () => Promise }[] = (publisher as any).requests; - expect(requests).toHaveLength(1); - const preCheck = requests[0].preCheck; - expect(preCheck).toBeDefined(); + it('sends only survivors after second-pass re-simulate filters additional failures', async () => { + addTwoRequests(); + + // First simulate: both succeed initially. + // (Simulate a case where second-pass further trims — to test the path where + // first pass survivors differ from second pass survivors.) + const firstResult = encodeFunctionResult({ + abi: multicall3Abi, + functionName: 'aggregate3', + result: [ + { success: true, returnData: '0x' }, + { success: false, returnData: '0x' }, + ], + }); + // Second simulate (reduced bundle with only invalidate): that one succeeds. + const secondResult = encodeFunctionResult({ + abi: multicall3Abi, + functionName: 'aggregate3', + result: [{ success: true, returnData: '0x' }], + }); + + (l1TxUtils as any).simulate + .mockResolvedValueOnce({ gasUsed: 500_000n, result: firstResult }) + .mockResolvedValueOnce({ gasUsed: 300_000n, result: secondResult }); - validateSpy.mockClear(); - await preCheck!(); + forwardSpy.mockResolvedValue({ receipt: proposeTxReceipt, stats: undefined, multicallData: '0x' }); - expect(validateSpy).toHaveBeenCalledTimes(1); - const preCheckArg = validateSpy.mock.calls[0][3]; - expect(preCheckArg?.disableBlobCheck).toBe(true); - expect(preCheckArg?.chainTipsOverride).toBeUndefined(); - expect(preCheckArg?.pendingCheckpointState).toBeUndefined(); - }); + const result = await publisher.sendRequests(); - it('returns errorMsg if forwarder tx reverts', async () => { - forwardSpy.mockResolvedValue({ - receipt: { ...proposeTxReceipt, status: 'reverted' }, - errorMsg: 'Test error', + expect(result).toBeDefined(); + // Only the invalidate survivor was sent. + expect(result?.sentActions).toEqual(['invalidate-by-invalid-attestation']); + expect(forwardSpy).toHaveBeenCalledTimes(1); + expect(l1TxUtils.simulate).toHaveBeenCalledTimes(2); }); - await publisher.enqueueProposeCheckpoint( - new Checkpoint(l2Block.archive, header, [l2Block], l2Block.checkpointNumber), - CommitteeAttestationsAndSigners.empty(testSignatureContext), - Signature.empty(), - ); - const result = await publisher.sendRequests(); + it('preserves first-pass survivors when second-pass simulate returns fallback', async () => { + addTwoRequests(); + + // First simulate: propose fails, invalidate survives. + const firstResult = encodeFunctionResult({ + abi: multicall3Abi, + functionName: 'aggregate3', + result: [ + { success: true, returnData: '0x' }, + { success: false, returnData: '0x' }, + ], + }); + // Second simulate: fallback (eth_simulateV1 not supported on the reduced bundle). + (l1TxUtils as any).simulate + .mockResolvedValueOnce({ gasUsed: 500_000n, result: firstResult }) + .mockResolvedValueOnce({ gasUsed: 1_000_000n, result: '0x' }); + + forwardSpy.mockResolvedValue({ receipt: proposeTxReceipt, stats: undefined, multicallData: '0x' }); + + const result = await publisher.sendRequests(); - expect(result).not.toBeInstanceOf(FormattedViemError); - if (result instanceof FormattedViemError) { - fail('Not Expected result to be a FormattedViemError'); - } else { - expect((result as any).result.errorMsg).toEqual('Test error'); - } + // Second-pass fallback must NOT re-include the propose entry that first-pass dropped. + expect(result).toBeDefined(); + expect(result?.sentActions).toEqual(['invalidate-by-invalid-attestation']); + expect(result?.failedActions).toEqual(['propose']); + expect(forwardSpy).toHaveBeenCalledTimes(1); + expect(forwardSpy.mock.calls[0][2]?.gasLimit).toEqual(MAX_L1_TX_LIMIT); + // The forwarded bundle should only contain the survivor. + expect(forwardSpy.mock.calls[0][0]).toHaveLength(1); + expect(l1TxUtils.simulate).toHaveBeenCalledTimes(2); + }); }); it('does not send requests if interrupted', async () => { forwardSpy.mockImplementationOnce( () => - sleep(10, { receipt: proposeTxReceipt, gasPrice: { maxFeePerGas: 1n, maxPriorityFeePerGas: 1n } }) as Promise<{ + sleep(10, { + receipt: proposeTxReceipt, + stats: undefined, + multicallData: '0x', + }) as Promise<{ receipt: TransactionReceipt; - gasPrice: GasPrice; - errorMsg: undefined; + stats: undefined; + multicallData: Hex; }>, ); await publisher.enqueueProposeCheckpoint( @@ -586,64 +664,6 @@ describe('SequencerPublisher', () => { expect((publisher as any).requests.length).toEqual(0); }); - it('discards only the request whose preCheck fails before sending', async () => { - const currentL2Slot = publisher.getCurrentL2Slot(); - const keptRequest = { - to: mockGovernanceProposerAddress, - data: encodeFunctionData({ - abi: EmpireBaseAbi, - functionName: 'signal', - args: [EthAddress.random().toString()], - }), - }; - const failedRequest = { - to: mockRollupAddress, - data: encodeFunctionData({ - abi: EmpireBaseAbi, - functionName: 'signal', - args: [EthAddress.random().toString()], - }), - }; - - const keptPreCheck = jest.fn(() => Promise.resolve()); - const failedPreCheck = jest.fn(() => Promise.reject(new Error('preCheck failed'))); - - publisher.addRequest({ - action: 'vote-offenses', - request: keptRequest, - lastValidL2Slot: currentL2Slot, - preCheck: keptPreCheck, - checkSuccess: () => true, - }); - publisher.addRequest({ - action: 'governance-signal', - request: failedRequest, - lastValidL2Slot: currentL2Slot, - preCheck: failedPreCheck, - checkSuccess: () => true, - }); - - forwardSpy.mockResolvedValue({ - receipt: proposeTxReceipt, - errorMsg: undefined, - }); - - const result = await publisher.sendRequestsAt(new Date((publisher as any).dateProvider.now())); - - expect(keptPreCheck).toHaveBeenCalledTimes(1); - expect(failedPreCheck).toHaveBeenCalledTimes(1); - expect(result?.sentActions).toEqual(['vote-offenses']); - expect(forwardSpy).toHaveBeenCalledTimes(1); - expect(forwardSpy).toHaveBeenCalledWith( - [keptRequest], - l1TxUtils, - { gasLimit: undefined, txTimeoutAt: undefined }, - undefined, - mockRollupAddress, - expect.anything(), - ); - }); - it('does not send requests if no valid requests are found', async () => { publisher.addRequest({ action: 'propose', @@ -704,15 +724,18 @@ describe('SequencerPublisher', () => { forwardSpy.mockResolvedValue({ receipt: proposeTxReceipt, - errorMsg: undefined, + stats: undefined, + multicallData: '0x', }); await publisher.sendRequests(); expect(forwardSpy).toHaveBeenCalledTimes(1); - // The gas config should only include the valid request's gas (100_000), not the expired one (500_000) + // The expired request (500_000) is filtered before bundle simulate. + // Bundle simulate returns '0x' (fallback), so gasLimit comes from MAX_L1_TX_LIMIT, + // not from per-request gasConfig — the expired request's gasLimit has no effect. const txConfig = forwardSpy.mock.calls[0][2]; - expect(txConfig?.gasLimit).toEqual(100_000n); + expect(txConfig?.gasLimit).toEqual(MAX_L1_TX_LIMIT); }); it('does not signal for payload when quorum is reached', async () => { @@ -737,8 +760,8 @@ describe('SequencerPublisher', () => { it('does not signal for payload with empty code', async () => { const { govPayload } = mockGovernancePayload(); - l1TxUtils.getCode.mockReturnValue(Promise.resolve(undefined)); - ``; + // isPayloadEmpty now lives on GovernanceProposerContract, not L1TxUtils. + governanceProposerContract.isPayloadEmpty.mockResolvedValue(true); expect( await publisher.enqueueGovernanceCastSignal( diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts index 47819bcb1221..893490ae5499 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts @@ -7,12 +7,10 @@ import { type GovernanceProposerContract, MULTI_CALL_3_ADDRESS, Multicall3, - RollupContract, - SimulationOverridesBuilder, + MulticallForwarderRevertedError, + type RollupContract, type SimulationOverridesPlan, type SlashingProposerContract, - type ViemCommitteeAttestations, - type ViemHeader, buildSimulationOverridesStateOverride, } from '@aztec/ethereum/contracts'; import { type L1FeeAnalysisResult, L1FeeAnalyzer } from '@aztec/ethereum/l1-fee-analysis'; @@ -26,45 +24,60 @@ import { WEI_CONST, } from '@aztec/ethereum/l1-tx-utils'; import { FormattedViemError, formatViemError, mergeAbis, tryExtractEvent } from '@aztec/ethereum/utils'; -import { sumBigint } from '@aztec/foundation/bigint'; import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { trimmedBytesLength } from '@aztec/foundation/buffer'; import { pick } from '@aztec/foundation/collection'; import type { Fr } from '@aztec/foundation/curves/bn254'; import { TimeoutError } from '@aztec/foundation/error'; import { EthAddress } from '@aztec/foundation/eth-address'; -import { Signature, type ViemSignature } from '@aztec/foundation/eth-signature'; +import { Signature } from '@aztec/foundation/eth-signature'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { InterruptibleSleep } from '@aztec/foundation/sleep'; import { bufferToHex } from '@aztec/foundation/string'; import { type DateProvider, Timer } from '@aztec/foundation/timer'; -import { EmpireBaseAbi, ErrorsAbi, RollupAbi } from '@aztec/l1-artifacts'; +import { EmpireBaseAbi, ErrorsAbi, RollupAbi, SlashingProposerAbi } from '@aztec/l1-artifacts'; import { type ProposerSlashAction, encodeSlashConsensusVotes } from '@aztec/slasher'; import { CommitteeAttestationsAndSigners, type ValidateCheckpointResult } from '@aztec/stdlib/block'; import type { Checkpoint } from '@aztec/stdlib/checkpoint'; -import { getLastL1SlotTimestampForL2Slot, getNextL1SlotTimestamp } from '@aztec/stdlib/epoch-helpers'; +import { getNextL1SlotTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import type { CheckpointHeader } from '@aztec/stdlib/rollup'; import type { L1PublishCheckpointStats } from '@aztec/stdlib/stats'; import { type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client'; import { + type Abi, type Hex, type TransactionReceipt, type TypedDataDefinition, encodeFunctionData, keccak256, - multicall3Abi, toHex, } from 'viem'; import type { SequencerPublisherConfig } from './config.js'; import { type FailedL1Tx, type L1TxFailedStore, createL1TxFailedStore } from './l1_tx_failed_store/index.js'; +import { type DroppedRequest, SequencerBundleSimulator } from './sequencer-bundle-simulator.js'; import { SequencerPublisherMetrics } from './sequencer-publisher-metrics.js'; +/** + * Returns true if the receipt indicates a successful send AND the expected event was emitted + * by the target contract. Both pieces are required: an aggregate3 entry that reverted will + * have receipt.status === 'success' but no event log. + */ +function extractEventSuccess( + receipt: TransactionReceipt | undefined, + opts: { address: string; abi: Abi; eventName: string }, +): boolean { + if (!receipt || receipt.status !== 'success') { + return false; + } + return !!tryExtractEvent(receipt.logs, opts.address.toString() as Hex, opts.abi, opts.eventName); +} + /** Result of a sendRequests call, returned by both sendRequests() and sendRequestsAt(). */ export type SendRequestsResult = { - /** The L1 transaction receipt or error from the bundled multicall. */ - result: { receipt: TransactionReceipt; errorMsg?: string } | FormattedViemError; + /** The L1 transaction receipt from the bundled multicall. */ + result: { receipt: TransactionReceipt }; /** Actions that expired (past their deadline) before the request was sent. */ expiredActions: Action[]; /** Actions that were included in the sent L1 transaction. */ @@ -119,24 +132,16 @@ export type InvalidateCheckpointRequest = { type EnqueueProposeCheckpointOpts = { txTimeoutAt?: Date; - simulationOverridesPlan?: SimulationOverridesPlan; - /** - * Overrides to apply to the preCheck simulation right before L1 submission. - * Intentionally separate from `simulationOverridesPlan`: enqueue-time validation - * may need pipelined-parent / pretend-proof-landed overrides, but preCheck must - * reflect real L1 state to catch state drift between build and submission. - */ - preCheckSimulationOverridesPlan?: SimulationOverridesPlan; }; -interface RequestWithExpiry { +export interface RequestWithExpiry { action: Action; request: L1TxRequest; lastValidL2Slot: SlotNumber; gasConfig?: Pick; blobConfig?: L1BlobInputs; - /** Optional pre-send validation. If it rejects, the request is discarded. */ - preCheck?: () => Promise; + /** Gas consumed by validateBlobs; stashed for the bundle simulate at send time. */ + blobEvaluationGas?: bigint; checkSuccess: ( request: L1TxRequest, result?: { receipt: TransactionReceipt; stats?: TransactionStats; errorMsg?: string }, @@ -146,16 +151,12 @@ interface RequestWithExpiry { export class SequencerPublisher { private interrupted = false; private metrics: SequencerPublisherMetrics; + private bundleSimulator: SequencerBundleSimulator; public epochCache: EpochCache; private failedTxStore?: Promise; - protected governanceLog = createLogger('sequencer:publisher:governance'); - protected slashingLog = createLogger('sequencer:publisher:slashing'); - protected lastActions: Partial> = {}; - private isPayloadEmptyCache: Map = new Map(); - protected log: Logger; protected ethereumSlotDuration: bigint; protected aztecSlotDuration: bigint; @@ -165,9 +166,6 @@ export class SequencerPublisher { private blobClient: BlobClientInterface; - /** Address to use for simulations in fisherman mode (actual proposer's address) */ - private proposerAddressForSimulation?: EthAddress; - /** Optional callback to obtain a replacement publisher when the current one fails to send. */ private getNextPublisher?: (excludeAddresses: EthAddress[]) => Promise; @@ -180,12 +178,6 @@ export class SequencerPublisher { /** Interruptible sleep used by sendRequestsAt to wait until a target timestamp. */ private readonly interruptibleSleep = new InterruptibleSleep(); - // A CALL to a cold address is 2700 gas - public static MULTICALL_OVERHEAD_GAS_GUESS = 5000n; - - // Gas report for VotingWithSigTest shows a max gas of 100k, but we've seen it cost 700k+ in testnet - public static VOTE_GAS_GUESS: bigint = 800_000n; - public l1TxUtils: L1TxUtils; public rollupContract: RollupContract; public govProposerContract: GovernanceProposerContract; @@ -244,7 +236,7 @@ export class SequencerPublisher { this.l1FeeAnalyzer = new L1FeeAnalyzer( this.l1TxUtils.client, deps.dateProvider, - createLogger('sequencer:publisher:fee-analyzer'), + this.log.createChild('fee-analyzer'), ); } @@ -252,11 +244,18 @@ export class SequencerPublisher { this.feeAssetPriceOracle = new FeeAssetPriceOracle( this.l1TxUtils.client, this.rollupContract, - createLogger('sequencer:publisher:price-oracle'), + this.log.createChild('price-oracle'), ); // Initialize failed L1 tx store (optional, for test networks) this.failedTxStore = createL1TxFailedStore(config.l1TxFailedStore, this.log); + + this.bundleSimulator = new SequencerBundleSimulator({ + getL1TxUtils: () => this.l1TxUtils, + rollupContract: this.rollupContract, + epochCache: this.epochCache, + log: this.log.createChild('bundle-simulator'), + }); } /** @@ -308,14 +307,6 @@ export class SequencerPublisher { return this.l1FeeAnalyzer; } - /** - * Sets the proposer address to use for simulations in fisherman mode. - * @param proposerAddress - The actual proposer's address to use for balance lookups in simulations - */ - public setProposerAddressForSimulation(proposerAddress: EthAddress | undefined) { - this.proposerAddressForSimulation = proposerAddress; - } - public addRequest(request: RequestWithExpiry) { this.requests.push(request); } @@ -393,23 +384,26 @@ export class SequencerPublisher { /** * Sends all requests that are still valid. + * @param targetSlot - The target L2 slot for this send. When provided (pipelined path via + * sendRequestsAt), it is threaded into bundleSimulate so the block.timestamp override + * matches the slot the propose is built for. When omitted, falls back to + * getCurrentL2Slot() for the non-pipelined callers in Sequencer.doWork. * @returns one of: * - A receipt and stats if the tx succeeded * - a receipt and errorMsg if it failed on L1 * - undefined if no valid requests are found OR the tx failed to send. */ @trackSpan('SequencerPublisher.sendRequests') - public async sendRequests(): Promise { + public async sendRequests(targetSlot?: SlotNumber): Promise { const requestsToProcess = [...this.requests]; this.requests = []; if (this.interrupted || requestsToProcess.length === 0) { return undefined; } - const currentL2Slot = this.getCurrentL2Slot(); + const currentL2Slot = targetSlot ?? this.getCurrentL2Slot(); this.log.debug(`Sending requests on L2 slot ${currentL2Slot}`); const validRequests = requestsToProcess.filter(request => request.lastValidL2Slot >= currentL2Slot); - const validActions = validRequests.map(x => x.action); const expiredActions = requestsToProcess .filter(request => request.lastValidL2Slot < currentL2Slot) .map(x => x.action); @@ -432,70 +426,58 @@ export class SequencerPublisher { return undefined; } - // @note - we can only have one blob config per bundle - // find requests with gas and blob configs - // See https://github.com/AztecProtocol/aztec-packages/issues/11513 + // Collect earliest txTimeoutAt across all requests. const gasConfigs = validRequests.filter(request => request.gasConfig).map(request => request.gasConfig); - const blobConfigs = validRequests.filter(request => request.blobConfig).map(request => request.blobConfig); - - if (blobConfigs.length > 1) { - throw new Error('Multiple blob configs found'); - } - - const blobConfig = blobConfigs[0]; - - // Merge gasConfigs. Yields the sum of gasLimits, and the earliest txTimeoutAt, or undefined if no gasConfig sets them. - const gasLimits = gasConfigs.map(g => g?.gasLimit).filter((g): g is bigint => g !== undefined); - let gasLimit = gasLimits.length > 0 ? sumBigint(gasLimits) : undefined; // sum - // Cap at L1 block gas limit so the node accepts the tx ("gas limit too high" otherwise). - const maxGas = MAX_L1_TX_LIMIT; - if (gasLimit !== undefined && gasLimit > maxGas) { - this.log.debug('Capping bundled tx gas limit to L1 max', { - requested: gasLimit, - capped: maxGas, - }); - gasLimit = maxGas; - } const txTimeoutAts = gasConfigs.map(g => g?.txTimeoutAt).filter((g): g is Date => g !== undefined); - const txTimeoutAt = txTimeoutAts.length > 0 ? new Date(Math.min(...txTimeoutAts.map(g => g.getTime()))) : undefined; // earliest - const txConfig: RequestWithExpiry['gasConfig'] = { gasLimit, txTimeoutAt }; + const txTimeoutAt = txTimeoutAts.length > 0 ? new Date(Math.min(...txTimeoutAts.map(g => g.getTime()))) : undefined; // Sort the requests so that proposals always go first // This ensures the committee gets precomputed correctly validRequests.sort((a, b) => compareActions(a.action, b.action)); try { - // Capture context for failed tx backup before sending - const l1BlockNumber = await this.l1TxUtils.getBlockNumber(); - const multicallData = encodeFunctionData({ - abi: multicall3Abi, - functionName: 'aggregate3', - args: [ - validRequests.map(r => ({ - target: r.request.to!, - callData: r.request.data!, - allowFailure: true, - })), - ], - }); - const blobDataHex = blobConfig?.blobs?.map(b => toHex(b)) as Hex[] | undefined; + // Bundle-level eth_simulateV1: filters out entries that revert and derives the gasLimit. + const bundleResult = await this.bundleSimulator.simulate(validRequests, currentL2Slot); + + if (bundleResult.kind === 'aborted') { + this.logDroppedInSim(bundleResult.droppedRequests); + void this.backupDroppedInSim(bundleResult.droppedRequests); + return undefined; + } + + const { requests, droppedRequests, gasLimit } = + bundleResult.kind === 'fallback' + ? { + requests: bundleResult.requests, + droppedRequests: bundleResult.droppedRequests, + gasLimit: MAX_L1_TX_LIMIT, + } + : bundleResult; - const txContext = { multicallData, blobData: blobDataHex, l1BlockNumber }; + this.logDroppedInSim(droppedRequests); + + // Compute blobConfig from survivors (not original validRequests) so that if the propose + // entry was dropped by bundleSimulate we don't attach a blob-typed config to a non-blob tx. + const [blobConfig] = requests.filter(r => r.blobConfig).map(r => r.blobConfig); + const txConfig: RequestWithExpiry['gasConfig'] = { gasLimit, txTimeoutAt }; this.log.debug('Forwarding transactions', { - validRequests: validRequests.map(request => request.action), + requests: requests.map(request => request.action), txConfig, }); - const result = await this.forwardWithPublisherRotation(validRequests, txConfig, blobConfig); + const result = await this.forwardWithPublisherRotation(requests, txConfig, blobConfig); if (result === undefined) { return undefined; } - const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions( - validRequests, + const { successfulActions = [], failedActions = [] } = this.callbackBundledTransactions(requests, result); + const allFailedActions = [...failedActions, ...droppedRequests.map(d => d.request.action)]; + return { result, - txContext, - ); - return { result, expiredActions, sentActions: validActions, successfulActions, failedActions }; + expiredActions, + sentActions: requests.map(x => x.action), + successfulActions, + failedActions: allFailedActions, + }; } catch (err) { const viemError = formatViemError(err); this.log.error(`Failed to publish bundled transactions`, viemError); @@ -512,6 +494,38 @@ export class SequencerPublisher { } } + /** Logs entries dropped by bundle simulation as warnings on the publisher's logger. */ + private logDroppedInSim(dropped: DroppedRequest[]): void { + for (const drop of dropped) { + this.log.warn('Bundle entry dropped: action reverted in sim', { + action: drop.request.action, + revertReason: drop.revertReason ?? drop.returnData, + returnData: drop.returnData, + }); + } + } + + /** Backs up entries dropped by bundle simulation, one record per dropped action. */ + private async backupDroppedInSim(dropped: DroppedRequest[]): Promise { + if (dropped.length === 0) { + return; + } + const l1BlockNumber = await this.l1TxUtils.getBlockNumber(); + for (const { request: req } of dropped) { + this.backupFailedTx({ + id: keccak256(req.request.data!), + failureType: 'simulation', + request: { to: req.request.to! as Hex, data: req.request.data! }, + l1BlockNumber: l1BlockNumber.toString(), + error: { message: 'Bundle entry dropped: action reverted in sim' }, + context: { + actions: [req.action], + sender: this.getSenderAddress().toString(), + }, + }); + } + } + /** * Forwards transactions via Multicall3, rotating to the next available publisher if a send * failure occurs (i.e. the tx never reached the chain). @@ -522,19 +536,30 @@ export class SequencerPublisher { txConfig: RequestWithExpiry['gasConfig'], blobConfig: L1BlobInputs | undefined, ) { + if (!txConfig?.gasLimit) { + throw new Error('gasLimit is required for bundled transactions'); + } + const txConfigWithGasLimit = txConfig as L1TxConfig & { gasLimit: bigint }; + const triedAddresses: EthAddress[] = []; let currentPublisher = this.l1TxUtils; while (true) { + if (txConfig.txTimeoutAt && new Date() > txConfig.txTimeoutAt) { + this.log.warn(`Tx timeout (${txConfig.txTimeoutAt.toISOString()}) elapsed; stopping publisher rotation`, { + triedAddresses: triedAddresses.map(a => a.toString()), + }); + return undefined; + } triedAddresses.push(currentPublisher.getSenderAddress()); + try { const result = await Multicall3.forward( validRequests.map(r => r.request), currentPublisher, - txConfig, + txConfigWithGasLimit, blobConfig, - this.rollupContract.address, - this.log, + { gasLimitRequired: true }, ); this.l1TxUtils = currentPublisher; return result; @@ -542,6 +567,12 @@ export class SequencerPublisher { if (err instanceof TimeoutError) { throw err; } + if (err instanceof MulticallForwarderRevertedError) { + this.log.error('Forwarder transaction reverted on-chain; not rotating publisher', err, { + transactionHash: err.receipt.transactionHash, + }); + return undefined; + } const viemError = formatViemError(err); if (!this.getNextPublisher) { this.log.error('Failed to publish bundled transactions', viemError); @@ -562,112 +593,59 @@ export class SequencerPublisher { } /* - * Schedules sending all enqueued requests at (or after) the given timestamp. + * Schedules sending all enqueued requests at (or after) the start of the given L2 slot. + * Sleeps until one L1 slot before the L2 slot boundary so the tx has a chance of being + * picked up by the first L1 block of the L2 slot. + * NB: there is a known correctness risk — being included in the L1 block right before the + * L2 slot starts would revert propose with HeaderLib__InvalidSlotNumber. * Uses InterruptibleSleep so it can be cancelled via interrupt(). - * Returns the promise for the L1 response (caller should NOT await this in the work loop). */ - public async sendRequestsAt(submitAfter: Date): Promise { - const ms = submitAfter.getTime() - this.dateProvider.now(); - if (ms > 0) { - this.log.debug(`Sleeping ${ms}ms before sending requests`, { submitAfter }); - await this.interruptibleSleep.sleep(ms); + public async sendRequestsAt(targetSlot: SlotNumber): Promise { + const l1Constants = this.epochCache.getL1Constants(); + // Start of the target L2 slot, in ms (getTimestampForSlot returns seconds). + const startOfTargetSlotMs = Number(getTimestampForSlot(targetSlot, l1Constants)) * 1000; + // Aim to be in the mempool one L1 slot before the L2 slot starts, so we have a chance of + // being picked up by the first L1 block of the L2 slot. + const submitAfterMs = startOfTargetSlotMs - Number(this.ethereumSlotDuration) * 1000; + const sleepMs = submitAfterMs - this.dateProvider.now(); + if (sleepMs > 0) { + this.log.debug(`Sleeping ${sleepMs}ms before sending requests`, { + targetSlot, + submitAfterMs, + }); + await this.interruptibleSleep.sleep(sleepMs); } if (this.interrupted) { return undefined; } - - // Re-validate enqueued requests after the sleep (state may have changed, e.g. prune or L1 reorg) - const validRequests: RequestWithExpiry[] = []; - for (const request of this.requests) { - if (!request.preCheck) { - validRequests.push(request); - continue; - } - - try { - await request.preCheck(); - validRequests.push(request); - } catch (err) { - this.log.warn(`Pre-send validation failed for ${request.action}, discarding request`, err); - } - } - - this.requests = validRequests; - if (this.requests.length === 0) { - return undefined; - } - - return this.sendRequests(); + return this.sendRequests(targetSlot); } private callbackBundledTransactions( requests: RequestWithExpiry[], - result: { receipt: TransactionReceipt; errorMsg?: string } | FormattedViemError | undefined, - txContext: { multicallData: Hex; blobData?: Hex[]; l1BlockNumber: bigint }, + result: { receipt: TransactionReceipt; multicallData: Hex }, ) { const actionsListStr = requests.map(r => r.action).join(', '); - if (result instanceof FormattedViemError) { - this.log.error(`Failed to publish bundled transactions (${actionsListStr})`, result); - this.backupFailedTx({ - id: keccak256(txContext.multicallData), - failureType: 'send-error', - request: { to: MULTI_CALL_3_ADDRESS, data: txContext.multicallData }, - blobData: txContext.blobData, - l1BlockNumber: txContext.l1BlockNumber.toString(), - error: { message: result.message, name: result.name }, - context: { - actions: requests.map(r => r.action), - requests: requests.map(r => ({ action: r.action, to: r.request.to! as Hex, data: r.request.data! })), - sender: this.getSenderAddress().toString(), - }, - }); - return { failedActions: requests.map(r => r.action) }; - } else { - this.log.verbose(`Published bundled transactions (${actionsListStr})`, { - result, - requests: requests.map(r => ({ - ...r, - // Avoid logging large blob data - blobConfig: r.blobConfig - ? { ...r.blobConfig, blobs: r.blobConfig.blobs.map(b => ({ size: trimmedBytesLength(b) })) } - : undefined, - })), - }); - const successfulActions: Action[] = []; - const failedActions: Action[] = []; - for (const request of requests) { - if (request.checkSuccess(request.request, result)) { - successfulActions.push(request.action); - } else { - failedActions.push(request.action); - } - } - // Single backup for the whole reverted tx - if (failedActions.length > 0 && result?.receipt?.status === 'reverted') { - this.backupFailedTx({ - id: result.receipt.transactionHash, - failureType: 'revert', - request: { to: MULTI_CALL_3_ADDRESS, data: txContext.multicallData }, - blobData: txContext.blobData, - l1BlockNumber: result.receipt.blockNumber.toString(), - receipt: { - transactionHash: result.receipt.transactionHash, - blockNumber: result.receipt.blockNumber.toString(), - gasUsed: (result.receipt.gasUsed ?? 0n).toString(), - status: 'reverted', - }, - error: { message: result.errorMsg ?? 'Transaction reverted' }, - context: { - actions: failedActions, - requests: requests - .filter(r => failedActions.includes(r.action)) - .map(r => ({ action: r.action, to: r.request.to! as Hex, data: r.request.data! })), - sender: this.getSenderAddress().toString(), - }, - }); + this.log.verbose(`Published bundled transactions (${actionsListStr})`, { + result, + requests: requests.map(r => ({ + ...r, + // Avoid logging large blob data + blobConfig: r.blobConfig + ? { ...r.blobConfig, blobs: r.blobConfig.blobs.map(b => ({ size: trimmedBytesLength(b) })) } + : undefined, + })), + }); + const successfulActions: Action[] = []; + const failedActions: Action[] = []; + for (const request of requests) { + if (request.checkSuccess(request.request, result)) { + successfulActions.push(request.action); + } else { + failedActions.push(request.action); } - return { successfulActions, failedActions }; } + return { successfulActions, failedActions }; } /** @@ -677,7 +655,11 @@ export class SequencerPublisher { */ public async canProposeAt(tipArchive: Fr, msgSender: EthAddress, simulationOverridesPlan?: SimulationOverridesPlan) { // TODO: #14291 - should loop through multiple keys to check if any of them can propose - const ignoredErrors = ['SlotAlreadyInChain', 'InvalidProposer', 'InvalidArchive']; + // These errors are expected when we cannot actually propose right now — usually because our + // local view of the chain is ahead of L1 (proposed parent hasn't landed yet, or someone + // else has just landed the slot, or the archive override doesn't match). We log a warn and + // skip the proposal; we do NOT treat these as bugs. + const expectedErrors = ['SlotAlreadyInChain', 'InvalidProposer', 'InvalidArchive']; const pipelined = this.epochCache.isProposerPipeliningEnabled(); const slotOffset = pipelined ? this.aztecSlotDuration : 0n; @@ -691,8 +673,8 @@ export class SequencerPublisher { await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan), ) .catch(err => { - if (err instanceof FormattedViemError && ignoredErrors.find(e => err.message.includes(e))) { - this.log.warn(`Failed canProposeAtTime check with ${ignoredErrors.find(e => err.message.includes(e))}`, { + if (err instanceof FormattedViemError && expectedErrors.find(e => err.message.includes(e))) { + this.log.warn(`Failed canProposeAtTime check with ${expectedErrors.find(e => err.message.includes(e))}`, { error: err.message, }); } else { @@ -725,7 +707,8 @@ export class SequencerPublisher { flags, ] as const; - const ts = this.getSimulationTimestamp(header.slotNumber); + const l1Constants = this.epochCache.getL1Constants(); + const ts = getTimestampForSlot(header.slotNumber, l1Constants); const stateOverrides = await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan); let balance = 0n; if (this.config.fishermanMode) { @@ -879,35 +862,6 @@ export class SequencerPublisher { } } - /** Simulates `propose` to make sure that the checkpoint is valid for submission */ - @trackSpan('SequencerPublisher.validateCheckpointForSubmission') - public async validateCheckpointForSubmission( - checkpoint: Checkpoint, - attestationsAndSigners: CommitteeAttestationsAndSigners, - attestationsAndSignersSignature: Signature, - simulationOverridesPlan?: SimulationOverridesPlan, - ): Promise { - const blobFields = checkpoint.toBlobFields(); - const blobs = await getBlobsPerL1Block(blobFields); - const blobInput = getPrefixedEthBlobCommitments(blobs); - - const args = [ - { - header: checkpoint.header.toViem(), - archive: toHex(checkpoint.archive.root.toBuffer()), - oracleInput: { - feeAssetPriceModifier: checkpoint.feeAssetPriceModifier, - }, - }, - attestationsAndSigners.getPackedAttestations(), - attestationsAndSigners.getSigners().map(signer => signer.toString()), - attestationsAndSignersSignature.toViemSignature(), - blobInput, - ] as const; - - await this.simulateProposeTx(args, simulationOverridesPlan); - } - private async enqueueCastSignalHelper( slotNumber: SlotNumber, signalType: GovernanceSignalAction, @@ -938,7 +892,7 @@ export class SequencerPublisher { return false; } - if (await this.isPayloadEmpty(payload)) { + if (await base.isPayloadEmpty(payload)) { this.log.warn(`Skipping vote cast for payload with empty code`); return false; } @@ -981,45 +935,19 @@ export class SequencerPublisher { lastValidL2Slot: slotNumber, }); - const l1BlockNumber = await this.l1TxUtils.getBlockNumber(); - const timestamp = this.getSimulationTimestamp(slotNumber); - - try { - await this.l1TxUtils.simulate(request, { time: timestamp }, [], mergeAbis([request.abi ?? [], ErrorsAbi])); - this.log.debug(`Simulation for ${action} at slot ${slotNumber} succeeded`, { request }); - } catch (err) { - const viemError = formatViemError(err); - this.log.error(`Failed simulation for ${action} at slot ${slotNumber} (enqueuing the action anyway)`, viemError, { - simulationTimestamp: timestamp, - l1BlockNumber, - }); - this.backupFailedTx({ - id: keccak256(request.data!), - failureType: 'simulation', - request: { to: request.to!, data: request.data!, value: request.value?.toString() }, - l1BlockNumber: l1BlockNumber.toString(), - error: { message: viemError.message, name: viemError.name }, - context: { - actions: [action], - slot: slotNumber, - sender: this.getSenderAddress().toString(), - }, - }); - // Yes, we enqueue the request anyway, in case there was a bug with the simulation itself - } - // TODO(palla/slash): All votes (governance and slashing) should txTimeoutAt at the end of the slot. this.addRequest({ - gasConfig: { gasLimit: SequencerPublisher.VOTE_GAS_GUESS }, action, request, lastValidL2Slot: slotNumber, checkSuccess: (_request, result) => { const success = result && - result.receipt && - result.receipt.status === 'success' && - tryExtractEvent(result.receipt.logs, base.address.toString(), EmpireBaseAbi, 'SignalCast'); + extractEventSuccess(result.receipt, { + address: base.address.toString(), + abi: EmpireBaseAbi, + eventName: 'SignalCast', + }); const logData = { ...result, slotNumber, round, payload: payload.toString() }; if (!success) { @@ -1041,17 +969,6 @@ export class SequencerPublisher { return true; } - private async isPayloadEmpty(payload: EthAddress): Promise { - const key = payload.toString(); - const cached = this.isPayloadEmptyCache.get(key); - if (cached) { - return cached; - } - const isEmpty = !(await this.l1TxUtils.getCode(payload)); - this.isPayloadEmptyCache.set(key, isEmpty); - return isEmpty; - } - /** * Enqueues a governance castSignal transaction to cast a signal for a given slot number. * @param slotNumber - The slot number to cast a signal for. @@ -1100,10 +1017,14 @@ export class SequencerPublisher { } const votes = bufferToHex(encodeSlashConsensusVotes(action.votes)); const request = await this.slashingProposerContract.buildVoteRequestFromSigner(votes, slotNumber, signer); - await this.simulateAndEnqueueRequest( + this.enqueueRequest( 'vote-offenses', request, - (receipt: TransactionReceipt) => !!this.slashingProposerContract!.tryExtractVoteCastEvent(receipt.logs), + { + address: this.slashingProposerContract.address.toString(), + abi: SlashingProposerAbi, + eventName: 'VoteCast', + }, slotNumber, ); break; @@ -1123,11 +1044,14 @@ export class SequencerPublisher { action.round, action.committees, ); - await this.simulateAndEnqueueRequest( + this.enqueueRequest( 'execute-slash', executeRequest, - (receipt: TransactionReceipt) => - !!this.slashingProposerContract!.tryExtractRoundExecutedEvent(receipt.logs), + { + address: this.slashingProposerContract.address.toString(), + abi: SlashingProposerAbi, + eventName: 'RoundExecuted', + }, slotNumber, ); break; @@ -1143,7 +1067,7 @@ export class SequencerPublisher { return true; } - /** Simulates and enqueues a proposal for a checkpoint on L1 */ + /** Enqueues a proposal for a checkpoint on L1 */ public async enqueueProposeCheckpoint( checkpoint: Checkpoint, attestationsAndSigners: CommitteeAttestationsAndSigners, @@ -1164,61 +1088,11 @@ export class SequencerPublisher { feeAssetPriceModifier: checkpoint.feeAssetPriceModifier, }; - const simulationOverridesPlan = SimulationOverridesBuilder.from(opts.simulationOverridesPlan) - .withoutBlobCheck() - .build(); - - const preCheckSimulationOverridesPlan = SimulationOverridesBuilder.from(opts.preCheckSimulationOverridesPlan) - .withoutBlobCheck() - .build(); - - try { - // @note This will make sure that we are passing the checks for our header ASSUMING that the data is also made available - // This means that we can avoid the simulation issues in later checks. - // By simulation issue, I mean the fact that the block.timestamp is equal to the last block, not the next, which - // make time consistency checks break. - // TODO(palla): Check whether we're validating twice, once here and once within addProposeTx, since we call simulateProposeTx in both places. - await this.validateCheckpointForSubmission( - checkpoint, - attestationsAndSigners, - attestationsAndSignersSignature, - simulationOverridesPlan, - ); - } catch (err: any) { - this.log.error(`Checkpoint validation failed. ${err instanceof Error ? err.message : 'No error message'}`, err, { - ...checkpoint.getStats(), - slotNumber: checkpoint.header.slotNumber, - simulationOverridesPlan, - }); - throw err; - } - - // Build a pre-check callback that re-validates the checkpoint before L1 submission. - // During pipelining this catches stale proposals due to prunes or L1 reorgs that occur during the pipeline sleep. - let preCheck = undefined; - if (this.epochCache.isProposerPipeliningEnabled()) { - preCheck = async () => { - this.log.debug(`Re-validating checkpoint ${checkpoint.number} before L1 submission`); - await this.validateCheckpointForSubmission( - checkpoint, - attestationsAndSigners, - attestationsAndSignersSignature, - preCheckSimulationOverridesPlan, - ); - }; - } - this.log.verbose(`Enqueuing checkpoint propose transaction`, { ...checkpoint.toCheckpointInfo(), txTimeoutAt: opts.txTimeoutAt, - simulationOverridesPlan, }); - await this.addProposeTx( - checkpoint, - proposeTxArgs, - { txTimeoutAt: opts.txTimeoutAt, simulationOverridesPlan }, - preCheck, - ); + await this.addProposeTx(checkpoint, proposeTxArgs, { txTimeoutAt: opts.txTimeoutAt }); } public enqueueInvalidateCheckpoint( @@ -1229,23 +1103,22 @@ export class SequencerPublisher { return; } - // We issued the simulation against the rollup contract, so we need to account for the overhead of the multicall3 - const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil((Number(request.gasUsed) * 64) / 63))); - const { gasUsed, checkpointNumber } = request; - const logData = { gasUsed, checkpointNumber, gasLimit, opts }; + const logData = { gasUsed, checkpointNumber, opts }; this.log.verbose(`Enqueuing invalidate checkpoint request`, logData); this.addRequest({ action: `invalidate-by-${request.reason}`, request: request.request, - gasConfig: { gasLimit, txTimeoutAt: opts.txTimeoutAt }, + gasConfig: opts.txTimeoutAt ? { txTimeoutAt: opts.txTimeoutAt } : undefined, lastValidL2Slot: SlotNumber(this.getCurrentL2Slot() + 2), checkSuccess: (_req, result) => { const success = result && - result.receipt && - result.receipt.status === 'success' && - tryExtractEvent(result.receipt.logs, this.rollupContract.address, RollupAbi, 'CheckpointInvalidated'); + extractEventSuccess(result.receipt, { + address: this.rollupContract.address, + abi: RollupAbi, + eventName: 'CheckpointInvalidated', + }); if (!success) { this.log.warn(`Invalidate checkpoint ${request.checkpointNumber} failed`, { ...result, ...logData }); } else { @@ -1256,73 +1129,36 @@ export class SequencerPublisher { }); } - private async simulateAndEnqueueRequest( + /** + * Dedup-checked enqueue helper for actions that are simulated at bundle-send time rather + * than at enqueue time. Validates the (action, slot) dedup key, sets `lastActions`, and + * enqueues without a gasLimit so the bundle simulate sets the only gasLimit that matters. + */ + private enqueueRequest( action: Action, request: L1TxRequest, - checkSuccess: (receipt: TransactionReceipt) => boolean | undefined, + eventOpts: { address: string; abi: Abi; eventName: string }, slotNumber: SlotNumber, - ) { - const timestamp = this.getSimulationTimestamp(slotNumber); - const logData = { slotNumber, timestamp, gasLimit: undefined as bigint | undefined }; + ): boolean { if (this.lastActions[action] && this.lastActions[action] === slotNumber) { this.log.debug(`Skipping duplicate action ${action} for slot ${slotNumber}`); return false; } - const cachedLastActionSlot = this.lastActions[action]; this.lastActions[action] = slotNumber; - this.log.debug(`Simulating ${action} for slot ${slotNumber}`, logData); - - const l1BlockNumber = await this.l1TxUtils.getBlockNumber(); - - let gasUsed: bigint; - const simulateAbi = mergeAbis([request.abi ?? [], ErrorsAbi]); - - try { - ({ gasUsed } = await this.l1TxUtils.simulate(request, { time: timestamp }, [], simulateAbi)); - this.log.verbose(`Simulation for ${action} succeeded`, { ...logData, request, gasUsed }); - } catch (err) { - const viemError = formatViemError(err, simulateAbi); - this.log.error(`Simulation for ${action} at ${slotNumber} failed`, viemError, logData); - - this.backupFailedTx({ - id: keccak256(request.data!), - failureType: 'simulation', - request: { to: request.to!, data: request.data!, value: request.value?.toString() }, - l1BlockNumber: l1BlockNumber.toString(), - error: { message: viemError.message, name: viemError.name }, - context: { - actions: [action], - slot: slotNumber, - sender: this.getSenderAddress().toString(), - }, - }); - - return false; - } - - // We issued the simulation against the rollup contract, so we need to account for the overhead of the multicall3 - const gasLimit = this.l1TxUtils.bumpGasLimit(BigInt(Math.ceil((Number(gasUsed) * 64) / 63))); - logData.gasLimit = gasLimit; - - // Store the ABI used for simulation on the request so Multicall3.forward can decode errors - // when the tx is sent and a revert is diagnosed via simulation. - const requestWithAbi = { ...request, abi: simulateAbi }; - - this.log.debug(`Enqueuing ${action}`, logData); + this.log.debug(`Enqueuing ${action}`, { slotNumber }); this.addRequest({ action, - request: requestWithAbi, - gasConfig: { gasLimit }, + request, lastValidL2Slot: slotNumber, - checkSuccess: (_req, result) => { - const success = result && result.receipt && result.receipt.status === 'success' && checkSuccess(result.receipt); + checkSuccess: (_request, result) => { + const success = result && extractEventSuccess(result.receipt, eventOpts); if (!success) { - this.log.warn(`Action ${action} at ${slotNumber} failed`, { ...result, ...logData }); + this.log.warn(`Action ${action} at ${slotNumber} failed`, { ...result, slotNumber }); this.lastActions[action] = cachedLastActionSlot; } else { - this.log.info(`Action ${action} at ${slotNumber} succeeded`, { ...result, ...logData }); + this.log.info(`Action ${action} at ${slotNumber} succeeded`, { ...result, slotNumber }); } return !!success; }, @@ -1348,7 +1184,7 @@ export class SequencerPublisher { this.l1TxUtils.restart(); } - private async prepareProposeTx(encodedData: L1ProcessArgs, simulationOverridesPlan?: SimulationOverridesPlan) { + private async prepareProposeTx(encodedData: L1ProcessArgs) { const kzg = Blob.getViemKzgInstance(); const blobInput = getPrefixedEthBlobCommitments(encodedData.blobs); this.log.debug('Validating blob input', { blobInput }); @@ -1361,7 +1197,11 @@ export class SequencerPublisher { blobEvaluationGas = BigInt(encodedData.blobs.length) * 21_000n; this.log.debug(`Using fixed blob evaluation gas estimate in fisherman mode: ${blobEvaluationGas}`); } else { - // Normal mode - use estimateGas with blob inputs + // We call validateBlobs via estimateGas with real blob+kzg sidecars as a consistency check + // that our locally-built blob commitments match the blob data. The bundle simulate at send + // time uses eth_simulateV1, which cannot carry blob inputs, so the rollup's on-chain blob + // check is forced off there — making this the only pre-flight detector of a commitment/data + // mismatch. The returned gas estimate is stashed on the request for the bundle path to read. blobEvaluationGas = await this.l1TxUtils .estimateGas( this.getSenderAddress().toString(), @@ -1419,119 +1259,21 @@ export class SequencerPublisher { blobInput, ] as const; - const { rollupData, simulationResult } = await this.simulateProposeTx(args, simulationOverridesPlan); + const rollupData = encodeFunctionData({ abi: RollupAbi, functionName: 'propose', args }); - return { args, blobEvaluationGas, rollupData, simulationResult }; - } - - /** - * Simulates the propose tx with eth_simulateV1 - * @param args - The propose tx args - * @returns The simulation result - */ - private async simulateProposeTx( - args: readonly [ - { - readonly header: ViemHeader; - readonly archive: `0x${string}`; - readonly oracleInput: { - readonly feeAssetPriceModifier: bigint; - }; - }, - ViemCommitteeAttestations, - `0x${string}`[], // Signers - ViemSignature, - `0x${string}`, - ], - simulationOverridesPlan?: SimulationOverridesPlan, - ) { - const rollupData = encodeFunctionData({ - abi: RollupAbi, - functionName: 'propose', - args, - }); - - const stateOverrides = await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan); - // In fisherman mode, simulate as the proposer but with sufficient balance - if (this.proposerAddressForSimulation) { - stateOverrides.push({ - address: this.proposerAddressForSimulation.toString(), - balance: 10n * WEI_CONST * WEI_CONST, // 10 ETH - }); - } - - const l1BlockNumber = await this.l1TxUtils.getBlockNumber(); - const simTs = this.getSimulationTimestamp(SlotNumber.fromBigInt(args[0].header.slotNumber)); - - const simulationResult = await this.l1TxUtils - .simulate( - { - to: this.rollupContract.address, - data: rollupData, - gas: MAX_L1_TX_LIMIT, - ...(this.proposerAddressForSimulation && { from: this.proposerAddressForSimulation.toString() }), - }, - { - time: simTs, - // @note reth should have a 30m gas limit per block but throws errors that this tx is beyond limit so we increase here - gasLimit: MAX_L1_TX_LIMIT * 2n, - }, - stateOverrides, - RollupAbi, - { - // @note fallback gas estimate to use if the node doesn't support simulation API - fallbackGasEstimate: MAX_L1_TX_LIMIT, - }, - ) - .catch(err => { - // In fisherman mode, we expect ValidatorSelection__MissingProposerSignature since fisherman doesn't have proposer signature - const viemError = formatViemError(err); - if (this.config.fishermanMode && viemError.message?.includes('ValidatorSelection__MissingProposerSignature')) { - this.log.debug(`Ignoring expected ValidatorSelection__MissingProposerSignature error in fisherman mode`); - // Return a minimal simulation result with the fallback gas estimate - return { - gasUsed: MAX_L1_TX_LIMIT, - logs: [], - }; - } - this.log.error(`Failed to simulate propose tx`, viemError, { simulationTimestamp: simTs }); - this.backupFailedTx({ - id: keccak256(rollupData), - failureType: 'simulation', - request: { to: this.rollupContract.address, data: rollupData }, - l1BlockNumber: l1BlockNumber.toString(), - error: { message: viemError.message, name: viemError.name }, - context: { - actions: ['propose'], - slot: Number(args[0].header.slotNumber), - sender: this.getSenderAddress().toString(), - }, - }); - throw err; - }); - - return { rollupData, simulationResult }; + return { args, blobEvaluationGas, rollupData }; } private async addProposeTx( checkpoint: Checkpoint, encodedData: L1ProcessArgs, opts: EnqueueProposeCheckpointOpts = {}, - preCheck?: () => Promise, ): Promise { const slot = checkpoint.header.slotNumber; const timer = new Timer(); const kzg = Blob.getViemKzgInstance(); - const { rollupData, simulationResult, blobEvaluationGas } = await this.prepareProposeTx( - encodedData, - opts.simulationOverridesPlan, - ); + const { rollupData, blobEvaluationGas } = await this.prepareProposeTx(encodedData); const startBlock = await this.l1TxUtils.getBlockNumber(); - const gasLimit = this.l1TxUtils.bumpGasLimit( - BigInt(Math.ceil((Number(simulationResult.gasUsed) * 64) / 63)) + - blobEvaluationGas + - SequencerPublisher.MULTICALL_OVERHEAD_GAS_GUESS, // We issue the simulation against the rollup contract, so we need to account for the overhead of the multicall3 - ); // Send the blobs to the blob client preemptively. This helps in tests where the sequencer mistakingly thinks that the propose // tx fails but it does get mined. We make sure that the blobs are sent to the blob client regardless of the tx outcome. @@ -1548,8 +1290,8 @@ export class SequencerPublisher { data: rollupData, }, lastValidL2Slot: checkpoint.header.slotNumber, - gasConfig: { txTimeoutAt: opts.txTimeoutAt, gasLimit }, - preCheck, + gasConfig: { txTimeoutAt: opts.txTimeoutAt, gasLimit: undefined }, + blobEvaluationGas, blobConfig: { blobs: encodedData.blobs.map(b => b.data), kzg, @@ -1559,10 +1301,11 @@ export class SequencerPublisher { return false; } const { receipt, stats, errorMsg } = result; - const success = - receipt && - receipt.status === 'success' && - tryExtractEvent(receipt.logs, this.rollupContract.address, RollupAbi, 'CheckpointProposed'); + const success = extractEventSuccess(receipt, { + address: this.rollupContract.address, + abi: RollupAbi, + eventName: 'CheckpointProposed', + }); if (success) { const endBlock = receipt.blockNumber; @@ -1603,13 +1346,6 @@ export class SequencerPublisher { }); } - /** Returns the timestamp of the last L1 slot within a given L2 slot. Used as the simulation timestamp - * for eth_simulateV1 calls, since it's guaranteed to be greater than any L1 block produced during the slot. */ - private getSimulationTimestamp(slot: SlotNumber): bigint { - const l1Constants = this.epochCache.getL1Constants(); - return getLastL1SlotTimestampForL2Slot(slot, l1Constants); - } - /** Returns the timestamp of the next L1 slot boundary after now. */ private getNextL1SlotTimestamp(): bigint { const l1Constants = this.epochCache.getL1Constants(); diff --git a/yarn-project/sequencer-client/src/sequencer/chain_state_overrides.ts b/yarn-project/sequencer-client/src/sequencer/chain_state_overrides.ts index 412f1562d461..60c588faa974 100644 --- a/yarn-project/sequencer-client/src/sequencer/chain_state_overrides.ts +++ b/yarn-project/sequencer-client/src/sequencer/chain_state_overrides.ts @@ -1,135 +1,162 @@ import { RollupContract, SimulationOverridesBuilder, type SimulationOverridesPlan } from '@aztec/ethereum/contracts'; import { CheckpointNumber } from '@aztec/foundation/branded-types'; -import type { Fr } from '@aztec/foundation/curves/bn254'; import type { Logger } from '@aztec/foundation/log'; -import { computeCheckpointPayloadDigest } from '@aztec/stdlib/checkpoint'; -import type { ProposedCheckpointData } from '@aztec/stdlib/checkpoint'; +import { type ProposedCheckpointData, computeCheckpointPayloadDigest } from '@aztec/stdlib/checkpoint'; import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p'; -type PipelinedParentSimulationOverridesPlanInput = { - checkpointNumber: CheckpointNumber; - proposedCheckpointData?: ProposedCheckpointData; +type CheckpointSimulationOverridesPlanInput = { + /** Target rollup contract. */ rollup: RollupContract; - signatureContext: CoordinationSignatureContext; + /** Checkpoint number to be proposed. */ + checkpointNumber: CheckpointNumber; + /** Logger instance. */ log: Logger; /** - * Whether proposer pipelining is enabled. Controls only the parent pending/fee-header - * portion of the plan — the proven override below is independent of pipelining because - * the boundary build needs it for globals and enqueue-time validation regardless. + * The proposed parent checkpoint when pipelining. Its `checkpointNumber` must equal + * `checkpointNumber - 1`; the helper enforces this. Mutually exclusive with + * `invalidateToPendingCheckpointNumber`. + */ + proposedCheckpointData?: ProposedCheckpointData; + /** + * The pending checkpoint number we'll end up at after invalidation lands. Mutually exclusive + * with `proposedCheckpointData`. */ - pipeliningEnabled: boolean; - /** If set, also overrides `tips.proven` so `canPruneAtTime` returns false at the simulation timestamp. */ - prunePending?: { provenOverride: CheckpointNumber }; -}; - -type SubmissionSimulationOverridesPlanInput = { - pipelinedParentPlan?: SimulationOverridesPlan; invalidateToPendingCheckpointNumber?: CheckpointNumber; - lastArchiveRoot: Fr; - pipeliningEnabled: boolean; + /** + * The real on-chain pending checkpoint number (typically `syncedTo.checkpointedCheckpointNumber`). + * Used as the snapshot we pin both `pending` and `proven` to avoid prunes in simulation. + */ + checkpointedCheckpointNumber: CheckpointNumber; + /** + * Chain-level consensus signature context. Used to recompute the parent's `payloadDigest` for the + * pipelined simulation override so it matches what `propose` will write into `tempCheckpointLogs[parent]` + * once the parent lands. + */ + signatureContext: CoordinationSignatureContext; }; /** - * Builds the simulated chain view used while constructing a checkpoint proposal. May carry: - * - A pending parent override + fee header (only when pipelining is enabled). - * - A proven override (whenever `prunePending` is set, even with pipelining off — the boundary - * build needs it for the globals builder's mana-min-fee lookup and the enqueue-time - * submission simulation regardless of pipelining). + * Builds the SimulationOverridesPlan describing the simulated L1 rollup state for a checkpoint's + * enqueue-time simulations: `canProposeAt` (in Sequencer.doWork) and the propose-related sims + * (validateBlockHeader, simulateProposeTx). The plan reflects "as if our pipelined parent + * checkpoint has landed and any required invalidation has executed" — the gap that needs to be + * bridged at enqueue time. + * + * Pipelining (`proposedCheckpointData`) and invalidation (`invalidateToPendingCheckpointNumber`) + * are mutually exclusive; passing both throws. */ -export async function buildPipelinedParentSimulationOverridesPlan( - input: PipelinedParentSimulationOverridesPlanInput, +export async function buildCheckpointSimulationOverridesPlan( + input: CheckpointSimulationOverridesPlanInput, ): Promise { - const builder = new SimulationOverridesBuilder(); - - if (input.pipeliningEnabled) { - const parentCheckpointNumber = CheckpointNumber(input.checkpointNumber - 1); - builder.withChainTips({ pending: parentCheckpointNumber }); - - if (input.proposedCheckpointData) { - const { header, archive, checkpointOutHash, feeAssetPriceModifier } = input.proposedCheckpointData; - builder.withPendingArchive(archive.root).withPendingTempCheckpointLogFields({ - headerHash: header.hash(), - outHash: checkpointOutHash, - slotNumber: header.slotNumber, - payloadDigest: computeCheckpointPayloadDigest({ - header, - archiveRoot: archive.root, - feeAssetPriceModifier, - signatureContext: input.signatureContext, - }), - }); - } - - const pendingFeeHeader = await computePipelinedParentFeeHeader(input); - if (pendingFeeHeader) { - builder.withPendingFeeHeader(pendingFeeHeader); - } + if (input.proposedCheckpointData && input.invalidateToPendingCheckpointNumber !== undefined) { + throw new Error( + 'Error in buildCheckpointSimulationOverridesPlan: proposedCheckpointData and invalidateToPendingCheckpointNumber are mutually exclusive', + ); } - if (input.prunePending) { - builder.withChainTips({ proven: input.prunePending.provenOverride }); + const builder = new SimulationOverridesBuilder(); + const pendingCheckpointNumber = derivePendingCheckpointNumber(input); + + // Override the latest checkpoint number when invalidating or pipelining, so our checkpoint + // follows from it. We also override the proven chain tip so we dont need to worry about + // prunes kicking in that would break out simulation if there's a prune pending. We always + // assume that a proof will land in time. If we don't have a pending checkpoint number to force, + // we still set both tips to the current checkpoint number to avoid the prune trigger. + const overridenChainTip = pendingCheckpointNumber ?? input.checkpointedCheckpointNumber; + builder.withChainTips({ pending: overridenChainTip, proven: overridenChainTip }); + + if (input.proposedCheckpointData) { + const { header, archive, checkpointOutHash, feeAssetPriceModifier } = input.proposedCheckpointData; + builder.withPendingArchive(archive.root); + // Override every locally-derivable `tempCheckpointLogs[parent]` field that L1 will eventually + // write. `slotNumber` is load-bearing for `STFLib.canPruneAtTime`: without it the cell reads + // slotNumber 0, the contract treats the pending tip as belonging to an expired epoch, and + // `getEffectivePendingCheckpointNumber` silently collapses pending back to proven — producing + // a spurious `Rollup__InvalidArchive` against the on-chain genesis archive. The other fields + // (headerHash, outHash, payloadDigest) are not strictly load-bearing for `canProposeAt` / + // `validateBlockHeader`, but mirroring the full cell keeps the simulation byte-faithful with + // what the actual `propose()` send will observe, which is a defense against future reads + // taking dependencies on them. + builder.withPendingTempCheckpointLogFields({ + headerHash: header.hash(), + outHash: checkpointOutHash, + slotNumber: header.slotNumber, + payloadDigest: computeCheckpointPayloadDigest({ + header, + archiveRoot: archive.root, + feeAssetPriceModifier, + signatureContext: input.signatureContext, + }), + }); + + const feeHeader = await computePipelinedParentFeeHeader({ + checkpointNumber: input.checkpointNumber, + proposedCheckpointData: input.proposedCheckpointData, + rollup: input.rollup, + log: input.log, + }); + if (feeHeader) { + builder.withPendingFeeHeader(feeHeader); + } } return builder.build(); } -/** Builds the simulated chain view used when validating and enqueueing checkpoint submission. */ -export function buildSubmissionSimulationOverridesPlan( - input: SubmissionSimulationOverridesPlanInput, -): SimulationOverridesPlan | undefined { - const pendingCheckpointNumber = - input.invalidateToPendingCheckpointNumber ?? input.pipelinedParentPlan?.chainTipsOverride?.pending; - - const builder = SimulationOverridesBuilder.from(input.pipelinedParentPlan); - if (pendingCheckpointNumber !== undefined) { - builder.withChainTips({ pending: pendingCheckpointNumber }); +function derivePendingCheckpointNumber(input: CheckpointSimulationOverridesPlanInput): CheckpointNumber | undefined { + if (input.invalidateToPendingCheckpointNumber !== undefined) { + return input.invalidateToPendingCheckpointNumber; } - - if (input.pipeliningEnabled && pendingCheckpointNumber !== undefined) { - builder.withPendingArchive(input.lastArchiveRoot); + if (!input.proposedCheckpointData) { + return undefined; } - - return builder.build(); + if (input.checkpointNumber < 1) { + throw new Error(`Cannot build simulation override for checkpoint ${input.checkpointNumber}: no parent exists`); + } + const expectedParent = CheckpointNumber(input.checkpointNumber - 1); + if (input.proposedCheckpointData.checkpointNumber !== expectedParent) { + throw new Error( + `Cannot build simulation override for checkpoint ${input.checkpointNumber}: proposedCheckpointData.checkpointNumber (${input.proposedCheckpointData.checkpointNumber}) does not match expected parent ${expectedParent}`, + ); + } + return expectedParent; } type PipelinedParentFeeHeaderInput = { checkpointNumber: CheckpointNumber; - proposedCheckpointData?: ProposedCheckpointData; + proposedCheckpointData: ProposedCheckpointData; rollup: RollupContract; log: Logger; }; -/** Derives the pending parent fee header used during pipelined proposal simulation. */ +/** + * Derives the pending parent fee header used during pipelined proposal simulation. Returns + * `undefined` only when no grandparent exists (i.e. the proposed parent is the genesis + * checkpoint); all other failure modes (missing grandparent state, missing fee header, RPC + * errors) throw so callers don't silently desync the fee-header override. + */ export async function computePipelinedParentFeeHeader(input: PipelinedParentFeeHeaderInput) { - if (!input.proposedCheckpointData || input.checkpointNumber < 2) { + if (input.checkpointNumber < 2) { return undefined; } const grandparentCheckpointNumber = CheckpointNumber(input.checkpointNumber - 2); - try { - const [grandparentCheckpoint, manaTarget] = await Promise.all([ - input.rollup.getCheckpoint(grandparentCheckpointNumber), - input.rollup.getManaTarget(), - ]); + const [grandparentCheckpoint, manaTarget] = await Promise.all([ + input.rollup.getCheckpoint(grandparentCheckpointNumber), + input.rollup.getManaTarget(), + ]); - if (!grandparentCheckpoint?.feeHeader) { - input.log.error( - `Grandparent checkpoint or feeHeader missing for checkpoint ${grandparentCheckpointNumber.toString()}`, - ); - return undefined; - } - - return RollupContract.computeChildFeeHeader( - grandparentCheckpoint.feeHeader, - input.proposedCheckpointData.totalManaUsed, - input.proposedCheckpointData.feeAssetPriceModifier, - manaTarget, - ); - } catch (err) { - input.log.error( - `Failed to derive pipelined parent fee header for checkpoint ${grandparentCheckpointNumber.toString()}: ${err}`, + if (!grandparentCheckpoint?.feeHeader) { + throw new Error( + `Grandparent checkpoint or feeHeader missing for checkpoint ${grandparentCheckpointNumber.toString()}`, ); - return undefined; } + + return RollupContract.computeChildFeeHeader( + grandparentCheckpoint.feeHeader, + input.proposedCheckpointData.totalManaUsed, + input.proposedCheckpointData.feeAssetPriceModifier, + manaTarget, + ); } diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts index 83345c8eef76..8661d7c0850b 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts @@ -69,10 +69,7 @@ import { mockTxIterator, setupTxsAndBlock, } from '../test/utils.js'; -import { - buildPipelinedParentSimulationOverridesPlan, - computePipelinedParentFeeHeader, -} from './chain_state_overrides.js'; +import { buildCheckpointSimulationOverridesPlan, computePipelinedParentFeeHeader } from './chain_state_overrides.js'; import { CheckpointProposalJob } from './checkpoint_proposal_job.js'; import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js'; import type { SequencerEvents } from './events.js'; @@ -188,8 +185,15 @@ describe('CheckpointProposalJob', () => { publisher.enqueueProposeCheckpoint.mockResolvedValue(undefined); publisher.enqueueGovernanceCastSignal.mockResolvedValue(true); publisher.enqueueSlashingActions.mockResolvedValue(true); + + // Default rollup contract reads used by pipelined fee-header derivation. Tests that exercise + // the failure modes override these via jest.spyOn. + jest.spyOn(publisher.rollupContract, 'getCheckpoint').mockResolvedValue({ + feeHeader: { manaUsed: 0n, excessMana: 0n, ethPerFeeAsset: 1n, congestionCost: 0n, proverCost: 0n }, + } as any); + jest.spyOn(publisher.rollupContract, 'getManaTarget').mockResolvedValue(10_000n); publisher.sendRequestsAt.mockResolvedValue({ - result: { receipt: { status: 'success' } as TransactionReceipt, errorMsg: undefined }, + result: { receipt: { status: 'success' } as TransactionReceipt }, successfulActions: ['propose'], failedActions: [], sentActions: ['propose'], @@ -369,6 +373,8 @@ describe('CheckpointProposalJob', () => { checkpointBuilder.seedBlocks([block], [txs]); validatorClient.collectAttestations.mockResolvedValue(getAttestations(block)); epochCache.isProposerPipeliningEnabled.mockReturnValue(true); + // We build checkpoint 2 on top of proposed parent at checkpoint 1. + checkpointNumber = CheckpointNumber(2); const checkpoint = await createCheckpointProposalJob({ targetSlot: SlotNumber(newSlotNumber + 1), @@ -772,6 +778,7 @@ describe('CheckpointProposalJob', () => { overrides?.targetEpoch ?? epoch, checkpointNumber, lastBlockNumber, + CheckpointNumber(checkpointNumber - 1), proposer, publisher, attestorAddress, @@ -824,10 +831,10 @@ describe('CheckpointProposalJob', () => { proverCost: 10n, }; - it('returns undefined when proposedCheckpointData is not set', async () => { + it('returns undefined when checkpoint number is below 2 (genesis grandparent)', async () => { const result = await computePipelinedParentFeeHeader({ - checkpointNumber: pipelinedCheckpointNumber, - proposedCheckpointData: undefined, + checkpointNumber: CheckpointNumber(1), + proposedCheckpointData: pendingData, rollup: publisher.rollupContract, log: createLogger('test'), }); @@ -863,152 +870,155 @@ describe('CheckpointProposalJob', () => { expect(result).toEqual(expected); }); - it('returns undefined when grandparent checkpoint is not found', async () => { + it('throws when grandparent checkpoint is not found', async () => { mockRollup({ grandparentCheckpoint: undefined }); - const result = await computePipelinedParentFeeHeader({ - checkpointNumber: pipelinedCheckpointNumber, - proposedCheckpointData: pendingData, - rollup: publisher.rollupContract, - log: createLogger('test'), - }); - expect(result).toBeUndefined(); + await expect( + computePipelinedParentFeeHeader({ + checkpointNumber: pipelinedCheckpointNumber, + proposedCheckpointData: pendingData, + rollup: publisher.rollupContract, + log: createLogger('test'), + }), + ).rejects.toThrow(/Grandparent checkpoint or feeHeader missing/); }); - it('returns undefined when grandparent checkpoint has no feeHeader', async () => { + it('throws when grandparent checkpoint has no feeHeader', async () => { mockRollup({ grandparentCheckpoint: { feeHeader: undefined } }); - const result = await computePipelinedParentFeeHeader({ - checkpointNumber: pipelinedCheckpointNumber, - proposedCheckpointData: pendingData, - rollup: publisher.rollupContract, - log: createLogger('test'), - }); - expect(result).toBeUndefined(); + await expect( + computePipelinedParentFeeHeader({ + checkpointNumber: pipelinedCheckpointNumber, + proposedCheckpointData: pendingData, + rollup: publisher.rollupContract, + log: createLogger('test'), + }), + ).rejects.toThrow(/Grandparent checkpoint or feeHeader missing/); }); - it('returns undefined when rollup calls throw', async () => { + it('propagates errors from rollup calls', async () => { jest.spyOn(publisher.rollupContract, 'getCheckpoint').mockRejectedValue(new Error('rpc error')); - const result = await computePipelinedParentFeeHeader({ - checkpointNumber: pipelinedCheckpointNumber, - proposedCheckpointData: pendingData, - rollup: publisher.rollupContract, - log: createLogger('test'), - }); - expect(result).toBeUndefined(); + await expect( + computePipelinedParentFeeHeader({ + checkpointNumber: pipelinedCheckpointNumber, + proposedCheckpointData: pendingData, + rollup: publisher.rollupContract, + log: createLogger('test'), + }), + ).rejects.toThrow(/rpc error/); }); }); - describe('buildPipelinedParentSimulationOverridesPlan', () => { + describe('buildCheckpointSimulationOverridesPlan', () => { const checkpointNumberUnderTest = CheckpointNumber(2); - it('sets pending override for the parent checkpoint when pipelining is enabled', async () => { - const plan = await buildPipelinedParentSimulationOverridesPlan({ - checkpointNumber: checkpointNumberUnderTest, - proposedCheckpointData: undefined, - rollup: publisher.rollupContract, - signatureContext, - log: createLogger('test'), - pipeliningEnabled: true, - }); - expect(plan?.chainTipsOverride?.pending).toEqual(CheckpointNumber(1)); - expect(plan?.chainTipsOverride?.proven).toBeUndefined(); - }); + const grandparentFeeHeader: FeeHeader = { + manaUsed: 3000n, + excessMana: 1000n, + ethPerFeeAsset: 500n, + congestionCost: 50n, + proverCost: 10n, + }; - it('returns undefined when pipelining off and no prunePending', async () => { - const plan = await buildPipelinedParentSimulationOverridesPlan({ - checkpointNumber: checkpointNumberUnderTest, - proposedCheckpointData: undefined, - rollup: publisher.rollupContract, - signatureContext, - log: createLogger('test'), - pipeliningEnabled: false, - }); - expect(plan).toBeUndefined(); - }); + function mockGrandparentFeeHeader() { + jest + .spyOn(publisher.rollupContract, 'getCheckpoint') + .mockResolvedValue({ feeHeader: grandparentFeeHeader } as any); + jest.spyOn(publisher.rollupContract, 'getManaTarget').mockResolvedValue(10_000n); + } - it('returns plan with proven-only override when pipelining off and prunePending is set', async () => { - const plan = await buildPipelinedParentSimulationOverridesPlan({ - checkpointNumber: checkpointNumberUnderTest, - proposedCheckpointData: undefined, - rollup: publisher.rollupContract, - signatureContext, - log: createLogger('test'), - pipeliningEnabled: false, - prunePending: { provenOverride: CheckpointNumber(0) }, - }); - expect(plan?.chainTipsOverride?.pending).toBeUndefined(); - expect(plan?.chainTipsOverride?.proven).toEqual(CheckpointNumber(0)); - }); + function makeProposedParent(checkpointNumber: CheckpointNumber): ProposedCheckpointData { + return { + checkpointNumber, + header: CheckpointHeader.empty(), + archive: new AppendOnlyTreeSnapshot(Fr.random(), 1), + checkpointOutHash: Fr.random(), + startBlock: BlockNumber(1), + blockCount: 1, + totalManaUsed: 5000n, + feeAssetPriceModifier: 100n, + }; + } - it('attaches both parent and proven overrides when pipelining on and prunePending is set', async () => { - const plan = await buildPipelinedParentSimulationOverridesPlan({ + it('pins both pending and proven to the snapshot when no proposed/invalidate input is provided', async () => { + const plan = await buildCheckpointSimulationOverridesPlan({ checkpointNumber: checkpointNumberUnderTest, - proposedCheckpointData: undefined, + checkpointedCheckpointNumber: CheckpointNumber(4), rollup: publisher.rollupContract, signatureContext, log: createLogger('test'), - pipeliningEnabled: true, - prunePending: { provenOverride: CheckpointNumber(0) }, }); - expect(plan?.chainTipsOverride?.pending).toEqual(CheckpointNumber(1)); - expect(plan?.chainTipsOverride?.proven).toEqual(CheckpointNumber(0)); + expect(plan?.chainTipsOverride?.pending).toEqual(CheckpointNumber(4)); + expect(plan?.chainTipsOverride?.proven).toEqual(CheckpointNumber(4)); + expect(plan?.pendingCheckpointState).toBeUndefined(); }); - it('populates the per-checkpoint state from proposedCheckpointData when pipelining is enabled', async () => { - const proposedHeader = CheckpointHeader.empty({ slotNumber: SlotNumber(123) }); - const proposedArchive = new AppendOnlyTreeSnapshot(Fr.random(), 1); - const proposedOutHash = Fr.random(); - const proposedFeeHeader: FeeHeader = { - manaUsed: 3000n, - excessMana: 1000n, - ethPerFeeAsset: 500n, - congestionCost: 50n, - proverCost: 10n, - }; - jest.spyOn(publisher.rollupContract, 'getCheckpoint').mockResolvedValue({ feeHeader: proposedFeeHeader } as any); - jest.spyOn(publisher.rollupContract, 'getManaTarget').mockResolvedValue(10_000n); - - const proposedData: ProposedCheckpointData = { - checkpointNumber: CheckpointNumber(1), - header: proposedHeader, - archive: proposedArchive, - checkpointOutHash: proposedOutHash, - startBlock: BlockNumber(1), - blockCount: 1, - totalManaUsed: 5000n, - feeAssetPriceModifier: 100n, - }; + it('overrides the full pending checkpoint cell from a pipelined parent', async () => { + mockGrandparentFeeHeader(); + const proposedData = makeProposedParent(CheckpointNumber(1)); - const plan = await buildPipelinedParentSimulationOverridesPlan({ - checkpointNumber: CheckpointNumber(2), + const plan = await buildCheckpointSimulationOverridesPlan({ + checkpointNumber: checkpointNumberUnderTest, proposedCheckpointData: proposedData, + checkpointedCheckpointNumber: CheckpointNumber(0), rollup: publisher.rollupContract, signatureContext, log: createLogger('test'), - pipeliningEnabled: true, }); expect(plan?.chainTipsOverride?.pending).toEqual(CheckpointNumber(1)); - expect(plan?.pendingCheckpointState?.archive).toEqual(proposedArchive.root); - expect(plan?.pendingCheckpointState?.headerHash).toEqual(proposedHeader.hash()); - expect(plan?.pendingCheckpointState?.outHash).toEqual(proposedOutHash); - expect(plan?.pendingCheckpointState?.slotNumber).toEqual(SlotNumber(123)); + expect(plan?.chainTipsOverride?.proven).toEqual(CheckpointNumber(1)); + expect(plan?.pendingCheckpointState?.archive).toEqual(proposedData.archive.root); + expect(plan?.pendingCheckpointState?.slotNumber).toEqual(proposedData.header.slotNumber); + expect(plan?.pendingCheckpointState?.headerHash).toEqual(proposedData.header.hash()); + expect(plan?.pendingCheckpointState?.outHash).toEqual(proposedData.checkpointOutHash); expect(plan?.pendingCheckpointState?.payloadDigest).toBeDefined(); expect(plan?.pendingCheckpointState?.feeHeader).toBeDefined(); }); - it('omits per-checkpoint state when proposedCheckpointData is undefined', async () => { - const plan = await buildPipelinedParentSimulationOverridesPlan({ + it('throws when the pipelined parent does not match the expected parent checkpoint', async () => { + const proposedData = makeProposedParent(CheckpointNumber(5)); + + await expect( + buildCheckpointSimulationOverridesPlan({ + checkpointNumber: checkpointNumberUnderTest, + proposedCheckpointData: proposedData, + checkpointedCheckpointNumber: CheckpointNumber(0), + rollup: publisher.rollupContract, + signatureContext, + log: createLogger('test'), + }), + ).rejects.toThrow(/does not match expected parent/); + }); + + it('throws when both proposedCheckpointData and invalidateToPendingCheckpointNumber are provided', async () => { + const proposedData = makeProposedParent(CheckpointNumber(1)); + + await expect( + buildCheckpointSimulationOverridesPlan({ + checkpointNumber: checkpointNumberUnderTest, + proposedCheckpointData: proposedData, + invalidateToPendingCheckpointNumber: CheckpointNumber(0), + checkpointedCheckpointNumber: CheckpointNumber(0), + rollup: publisher.rollupContract, + signatureContext, + log: createLogger('test'), + }), + ).rejects.toThrow(/mutually exclusive/); + }); + + it('sets pending and proven from an invalidation rollback without archive/fee overrides', async () => { + const plan = await buildCheckpointSimulationOverridesPlan({ checkpointNumber: checkpointNumberUnderTest, - proposedCheckpointData: undefined, + invalidateToPendingCheckpointNumber: CheckpointNumber(0), + checkpointedCheckpointNumber: CheckpointNumber(2), rollup: publisher.rollupContract, signatureContext, log: createLogger('test'), - pipeliningEnabled: true, }); - expect(plan?.chainTipsOverride?.pending).toEqual(CheckpointNumber(1)); + expect(plan?.chainTipsOverride?.pending).toEqual(CheckpointNumber(0)); + expect(plan?.chainTipsOverride?.proven).toEqual(CheckpointNumber(0)); expect(plan?.pendingCheckpointState).toBeUndefined(); }); }); diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts index de0d8138b7c2..23ed57fb1e9a 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts @@ -301,6 +301,7 @@ describe('CheckpointProposalJob Timing Tests', () => { epoch, checkpointNumber, BlockNumber.ZERO, + CheckpointNumber(checkpointNumber - 1), proposer, publisher, attestorAddress, @@ -405,7 +406,7 @@ describe('CheckpointProposalJob Timing Tests', () => { publisher.enqueueGovernanceCastSignal.mockResolvedValue(true); publisher.enqueueSlashingActions.mockResolvedValue(true); publisher.sendRequestsAt.mockResolvedValue({ - result: { receipt: { status: 'success' } as any, errorMsg: undefined }, + result: { receipt: { status: 'success' } as any }, successfulActions: ['propose'], failedActions: [], sentActions: ['propose'], @@ -1047,6 +1048,7 @@ describe('CheckpointProposalJob Timing Tests', () => { epoch, checkpointNumber, BlockNumber.ZERO, + CheckpointNumber(checkpointNumber - 1), proposer, publisher, attestorAddress, diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 6c8af0a66a07..e436ff196583 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -66,10 +66,7 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js'; import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js'; -import { - buildPipelinedParentSimulationOverridesPlan, - buildSubmissionSimulationOverridesPlan, -} from './chain_state_overrides.js'; +import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js'; import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js'; import { CheckpointVoter } from './checkpoint_voter.js'; import { SequencerInterruptedError } from './errors.js'; @@ -110,11 +107,12 @@ export class CheckpointProposalJob implements Traceable { private pendingL1Submission: Promise | undefined; /** - * Build-time chain state overrides used both during build (globals + invariant checks) and - * later for enqueue-time submission validation. May carry the pipelined parent override, the - * pretend-proof-landed (`proven`) override at an epoch boundary, or both. + * Chain state overrides built once per slot in proposeCheckpoint after the checkpoint is + * complete. Carries the pending parent override (archive + slot + fee header) for pipelining, + * or the invalidation pending override when rolling back. Consumed by + * publisher.validateBlockHeader before broadcast. */ - private pipelinedParentSimulationOverridesPlan?: SimulationOverridesPlan; + private checkpointSimulationOverridesPlan?: SimulationOverridesPlan; private getSignatureContext(): CoordinationSignatureContext { return this.signatureContext; @@ -126,6 +124,7 @@ export class CheckpointProposalJob implements Traceable { private readonly targetEpoch: EpochNumber, private readonly checkpointNumber: CheckpointNumber, private readonly syncedToBlockNumber: BlockNumber, + private readonly checkpointedCheckpointNumber: CheckpointNumber, // TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows. private readonly proposer: EthAddress | undefined, private readonly publisher: SequencerPublisher, @@ -153,7 +152,6 @@ export class CheckpointProposalJob implements Traceable { public readonly tracer: Tracer, bindings?: LoggerBindings, private readonly proposedCheckpointData?: ProposedCheckpointData, - private readonly prunePending?: { provenOverride: CheckpointNumber }, ) { this.log = createLogger('sequencer:checkpoint-proposal', { ...bindings, @@ -215,11 +213,7 @@ export class CheckpointProposalJob implements Traceable { // signature verification to fail silently inside Multicall3. Delay submission to the // start of `targetSlot` so the tx mines in the slot the vote was signed for. if (!this.config.fishermanMode) { - const isPipelining = this.epochCache.isProposerPipeliningEnabled(); - const submitAfter = isPipelining - ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000) - : this.dateProvider.nowAsDate(); - this.pendingL1Submission = this.publisher.sendRequestsAt(submitAfter).then(() => {}); + this.pendingL1Submission = this.publisher.sendRequestsAt(this.targetSlot).then(() => {}); } return undefined; } @@ -278,12 +272,7 @@ export class CheckpointProposalJob implements Traceable { } // Send whatever was enqueued: votes + (propose | invalidation | nothing). - // Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise. - const submitAfter = isPipelining - ? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000) - : new Date(this.dateProvider.now()); - - const l1Response = await this.publisher.sendRequestsAt(submitAfter); + const l1Response = await this.publisher.sendRequestsAt(this.targetSlot); const proposedAction = l1Response?.successfulActions.find(a => a === 'propose'); if (proposedAction) { this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, { @@ -363,25 +352,8 @@ export class CheckpointProposalJob implements Traceable { } } - const isPipelining = this.epochCache.isProposerPipeliningEnabled(); - const enqueueSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({ - pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan, - invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber, - lastArchiveRoot: checkpoint.header.lastArchiveRoot, - pipeliningEnabled: isPipelining, - }); - - const preCheckSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({ - pipelinedParentPlan: undefined, - invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber, - lastArchiveRoot: checkpoint.header.lastArchiveRoot, - pipeliningEnabled: isPipelining, - }); - await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, { txTimeoutAt, - simulationOverridesPlan: enqueueSimulationOverridesPlan, - preCheckSimulationOverridesPlan, }); } @@ -563,25 +535,26 @@ export class CheckpointProposalJob implements Traceable { this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint); } - // Create checkpoint builder for the slot. - // When pipelining, force the proposed checkpoint number and fee header to our parent so the - // fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands. + // Build the simulation plan for this slot. When pipelining, this overrides L1's view of + // pending/archive/fee-header to "as if the proposed parent had landed", so both the + // mana-min-fee simulation (in the globals builder) and the pre-broadcast + // validateBlockHeader see the chain tip the eventual L1 send will see. const isPipelining = this.epochCache.isProposerPipeliningEnabled(); - this.pipelinedParentSimulationOverridesPlan = await buildPipelinedParentSimulationOverridesPlan({ + this.checkpointSimulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({ checkpointNumber: this.checkpointNumber, - proposedCheckpointData: this.proposedCheckpointData, + proposedCheckpointData: isPipelining ? this.proposedCheckpointData : undefined, + invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber, + checkpointedCheckpointNumber: this.checkpointedCheckpointNumber, rollup: this.publisher.rollupContract, signatureContext: this.signatureContext, log: this.log, - pipeliningEnabled: isPipelining, - prunePending: this.prunePending, }); const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables( coinbase, feeRecipient, this.targetSlot, - this.pipelinedParentSimulationOverridesPlan, + this.checkpointSimulationOverridesPlan, ); // Collect L1 to L2 messages for the checkpoint and compute their hash @@ -606,7 +579,7 @@ export class CheckpointProposalJob implements Traceable { // Anchor the modifier to the predicted parent fee header: L1 will apply it against // that, not against the latest published checkpoint (which lags by one under pipelining). const predictedParentEthPerFeeAssetE12 = - this.pipelinedParentSimulationOverridesPlan?.pendingCheckpointState?.feeHeader?.ethPerFeeAsset; + this.checkpointSimulationOverridesPlan?.pendingCheckpointState?.feeHeader?.ethPerFeeAsset; const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(predictedParentEthPerFeeAssetE12); // Create a long-lived forked world state for the checkpoint builder @@ -763,6 +736,25 @@ export class CheckpointProposalJob implements Traceable { return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() }; } + // Validate the header against L1 state before broadcasting. + // If this fails the slot is aborted before any gossip work; state drift between here + // and the eventual L1 send is caught by the bundle simulate at send time. + try { + await this.publisher.validateBlockHeader(checkpoint.header, this.checkpointSimulationOverridesPlan); + } catch (err) { + this.log.error(`Pre-broadcast header validation failed for slot ${this.targetSlot}; aborting`, err, { + slot: this.targetSlot, + checkpointNumber: this.checkpointNumber, + }); + this.metrics.recordCheckpointProposalFailed('header_validation_failed'); + this.eventEmitter.emit('header-validation-failed', { + slot: this.targetSlot, + checkpointNumber: this.checkpointNumber, + reason: err instanceof Error ? err.message : String(err), + }); + return undefined; + } + // Create the checkpoint proposal and broadcast it const proposal = await this.validatorClient.createCheckpointProposal( checkpoint.header, diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_voter.ha.integration.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_voter.ha.integration.test.ts index 3a641f2bb8b3..d60279e32230 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_voter.ha.integration.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_voter.ha.integration.test.ts @@ -138,12 +138,16 @@ describe('CheckpointVoter HA Integration', () => { txUtils.client = { account: validatorAccount, getCode: () => Promise.resolve('0x1234' as `0x${string}`), + getGasPrice: () => Promise.resolve(1n), + getBlock: () => Promise.resolve({ timestamp: 0n } as any), } as any; txUtils.getSenderAddress.mockReturnValue(EthAddress.fromString(validatorAccount.address)); + txUtils.getSenderBalance.mockResolvedValue(10_000_000_000_000_000_000n); // 10 ETH txUtils.simulate.mockResolvedValue({ gasUsed: 100000n, result: '0x', }); + (txUtils as any).bumpGasLimit = (val: bigint) => val + (val * 20n) / 100n; // Mock getCode to return non-empty bytecode for governance/slashing payloads txUtils.getCode.mockResolvedValue('0x1234' as any); return txUtils; @@ -690,7 +694,8 @@ describe('CheckpointVoter HA Integration', () => { status: 'success', logs: [], } as any, - errorMsg: undefined, + stats: undefined, + multicallData: '0x', }); // Each node enqueues their respective votes diff --git a/yarn-project/sequencer-client/src/sequencer/events.ts b/yarn-project/sequencer-client/src/sequencer/events.ts index a0fa73c011e4..8c5b55f551d6 100644 --- a/yarn-project/sequencer-client/src/sequencer/events.ts +++ b/yarn-project/sequencer-client/src/sequencer/events.ts @@ -18,10 +18,14 @@ export type SequencerEvents = { * * - `hadProposedParent` indicates whether the build saw a proposed (pipelined) parent * checkpoint that hasn't landed on L1 yet. - * - `provenOverride` is the assumed proven checkpoint number when the proven-override - * for a pending prune was applied; `undefined` when no override was applied. - * - `simulatedPending` is the pending checkpoint passed to L1 simulation (when - * pipelining or invalidating; undefined otherwise). + * - `provenOverride` is the assumed proven checkpoint number pinned for the L1 + * simulation. The plan always pins both chain tips to short-circuit `canPruneAtTime`, + * so this is populated whenever a simulation plan was built — the value either + * matches the on-chain proven snapshot (defensive pin) or the assumed-proven + * checkpoint when building optimistically across a pruning boundary. + * - `simulatedPending` is the pending checkpoint passed to L1 simulation. The plan + * always pins both chain tips to short-circuit `canPruneAtTime`, so this reflects + * either the pipelined/invalidated tip or the on-chain pending snapshot. */ ['preparing-checkpoint']: (args: { targetSlot: SlotNumber; @@ -35,6 +39,17 @@ export type SequencerEvents = { ['block-build-failed']: (args: { reason: string; slot: SlotNumber }) => void; ['block-proposed']: (args: { blockNumber: BlockNumber; slot: SlotNumber; buildSlot: SlotNumber }) => void; ['checkpoint-empty']: (args: { slot: SlotNumber }) => void; + /** + * Emitted when the proposer's pre-broadcast `validateBlockHeader` simulation fails. This is a + * last-chance check before we gossip a checkpoint proposal: a failure here means the header + * would not be accepted by L1 (e.g. archive mismatch, stale chain tip, or some other state + * drift between when we built the checkpoint and when we are about to broadcast it). + */ + ['header-validation-failed']: (args: { + slot: SlotNumber; + checkpointNumber: CheckpointNumber; + reason: string; + }) => void; ['checkpoint-publish-failed']: (args: { slot: SlotNumber; successfulActions?: Action[]; diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index b2f0828f5341..d17475a5fee7 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -222,7 +222,7 @@ describe('sequencer', () => { publisher.enqueueGovernanceCastSignal.mockResolvedValue(true); publisher.enqueueSlashingActions.mockResolvedValue(true); publisher.sendRequestsAt.mockResolvedValue({ - result: { receipt: { status: 'success' } as any, errorMsg: undefined }, + result: { receipt: { status: 'success' } as any }, successfulActions: ['propose'], failedActions: [], sentActions: ['propose'], @@ -242,6 +242,11 @@ describe('sequencer', () => { rollupContract = mockDeep(); rollupContract.isEscapeHatchOpen.mockResolvedValue(false); + // Default rollup reads used by pipelined fee-header derivation. + rollupContract.getCheckpoint.mockResolvedValue({ + feeHeader: { manaUsed: 0n, excessMana: 0n, ethPerFeeAsset: 1n, congestionCost: 0n, proverCost: 0n }, + } as any); + rollupContract.getManaTarget.mockResolvedValue(10_000n); globalVariableBuilder = mock(); globalVariableBuilder.buildGlobalVariables.mockResolvedValue(globalVariables); @@ -563,7 +568,7 @@ describe('sequencer', () => { pub.enqueueGovernanceCastSignal.mockResolvedValue(true); pub.enqueueSlashingActions.mockResolvedValue(true); pub.sendRequestsAt.mockResolvedValue({ - result: { receipt: { status: 'success' } as any, errorMsg: undefined }, + result: { receipt: { status: 'success' } as any }, successfulActions: ['propose'], failedActions: [], sentActions: ['propose'], @@ -671,7 +676,10 @@ describe('sequencer', () => { expect(slasherClient.getProposerActions).toHaveBeenCalledWith(SlotNumber(1)); expect(publisher.enqueueSlashingActions).toHaveBeenCalled(); expect(publisher.enqueueGovernanceCastSignal).toHaveBeenCalled(); - expect(publisher.sendRequests).toHaveBeenCalled(); + // Submission goes through sendRequestsAt so the bundle simulate's block.timestamp + // override matches the slot the EIP-712 signatures were generated for. + expect(publisher.sendRequestsAt).toHaveBeenCalled(); + expect(publisher.sendRequests).not.toHaveBeenCalled(); // But checkpoint proposal must not start expect(publisher.enqueueProposeCheckpoint).not.toHaveBeenCalled(); @@ -694,16 +702,16 @@ describe('sequencer', () => { await sequencer.work(); expect(publisher.enqueueSlashingActions).toHaveBeenCalledTimes(1); - expect(publisher.sendRequests).toHaveBeenCalledTimes(1); + expect(publisher.sendRequestsAt).toHaveBeenCalledTimes(1); publisher.enqueueSlashingActions.mockClear(); - publisher.sendRequests.mockClear(); + publisher.sendRequestsAt.mockClear(); slasherClient.getProposerActions.mockClear(); await sequencer.work(); expect(slasherClient.getProposerActions).not.toHaveBeenCalled(); expect(publisher.enqueueSlashingActions).not.toHaveBeenCalled(); - expect(publisher.sendRequests).not.toHaveBeenCalled(); + expect(publisher.sendRequestsAt).not.toHaveBeenCalled(); }); }); @@ -757,7 +765,8 @@ describe('sequencer', () => { expect.any(EthAddress), expect.any(Function), ); - expect(publisher.sendRequests).toHaveBeenCalled(); + // Votes are submitted via sendRequestsAt (fire-and-forget, scheduled at target slot start). + expect(publisher.sendRequestsAt).toHaveBeenCalled(); }); it('should not vote when sync fails and within time limit', async () => { @@ -817,18 +826,19 @@ describe('sequencer', () => { // First attempt should succeed await sequencer.work(); expect(publisher.enqueueSlashingActions).toHaveBeenCalledTimes(1); - expect(publisher.sendRequests).toHaveBeenCalledTimes(1); + // Votes are submitted via sendRequestsAt (fire-and-forget, scheduled at target slot start). + expect(publisher.sendRequestsAt).toHaveBeenCalledTimes(1); // Reset mocks publisher.enqueueSlashingActions.mockClear(); - publisher.sendRequests.mockClear(); + publisher.sendRequestsAt.mockClear(); slasherClient.getProposerActions.mockClear(); // Second attempt in the same slot should be skipped await sequencer.work(); expect(slasherClient.getProposerActions).not.toHaveBeenCalled(); expect(publisher.enqueueSlashingActions).not.toHaveBeenCalled(); - expect(publisher.sendRequests).not.toHaveBeenCalled(); + expect(publisher.sendRequestsAt).not.toHaveBeenCalled(); }); }); @@ -1114,7 +1124,7 @@ describe('sequencer', () => { const simulationOverridesPlan = publisher.canProposeAt.mock.calls.at(-1)?.[2]; expect(simulationOverridesPlan?.chainTipsOverride?.pending).toEqual(CheckpointNumber(1)); - expect(simulationOverridesPlan?.pendingCheckpointState?.archive).toEqual(expect.anything()); + // The archive root is passed directly as the first arg to canProposeAt (not inside the plan). }); it('skips proposal when checkpoint exceeds pipeline depth', async () => { @@ -1177,15 +1187,19 @@ describe('sequencer', () => { expect(publisher.canProposeAt).not.toHaveBeenCalled(); }); - it('calls L1 check without archive override when no proposed checkpoint', async () => { + it('pins both chain tips to the on-chain pending snapshot when no proposed checkpoint applies', async () => { await setupSingleTxBlock(); await sequencer.work(); - expect(publisher.canProposeAt.mock.calls.at(-1)?.[2]).toBeUndefined(); + // The default `getL2Tips` mock has checkpointed.checkpoint.number == CheckpointNumber.ZERO. + const plan = publisher.canProposeAt.mock.calls.at(-1)?.[2]; + expect(plan?.chainTipsOverride?.pending).toEqual(CheckpointNumber.ZERO); + expect(plan?.chainTipsOverride?.proven).toEqual(CheckpointNumber.ZERO); + expect(plan?.pendingCheckpointState).toBeUndefined(); }); - it('calls L1 check without overrides when not pipelining', async () => { + it('pins both chain tips to the on-chain pending snapshot when not pipelining', async () => { await setupSingleTxBlock(); // Override back to non-pipelining @@ -1204,23 +1218,13 @@ describe('sequencer', () => { await sequencer.work(); - expect(publisher.canProposeAt.mock.calls.at(-1)?.[2]).toBeUndefined(); - }); - - it('attaches proven override equal to real pending when isPruneDueAtSlot returns true', async () => { - await setupSingleTxBlock(); - - // No proposed checkpoint, so we exercise the standalone proven override path. - // The default `getL2Tips` mock has checkpointed.checkpoint.number == CheckpointNumber.ZERO. - l2BlockSource.isPruneDueAtSlot.mockResolvedValue(true); - - await sequencer.work(); - const plan = publisher.canProposeAt.mock.calls.at(-1)?.[2]; + expect(plan?.chainTipsOverride?.pending).toEqual(CheckpointNumber.ZERO); expect(plan?.chainTipsOverride?.proven).toEqual(CheckpointNumber.ZERO); + expect(plan?.pendingCheckpointState).toBeUndefined(); }); - it('uses the simulated pending as the proven override when the caller overrides pending', async () => { + it('mirrors pending onto proven when the caller overrides pending via pipelining', async () => { await setupSingleTxBlock(); // Set up a pipelined parent (pending override = parentCheckpointNumber = 1). @@ -1278,9 +1282,6 @@ describe('sequencer', () => { feeAssetPriceModifier: 0n, } satisfies ProposedCheckpointData); - // The sequencer sets proven == simulated pending so canPruneAtTime short-circuits to false. - l2BlockSource.isPruneDueAtSlot.mockResolvedValue(true); - await sequencer.work(); const plan = publisher.canProposeAt.mock.calls.at(-1)?.[2]; @@ -1288,52 +1289,25 @@ describe('sequencer', () => { expect(plan?.chainTipsOverride?.proven).toEqual(CheckpointNumber(1)); }); - it('does not attach proven override when isPruneDueAtSlot returns false', async () => { - await setupSingleTxBlock(); - - l2BlockSource.isPruneDueAtSlot.mockResolvedValue(false); - - await sequencer.work(); - - const plan = publisher.canProposeAt.mock.calls.at(-1)?.[2]; - expect(plan?.chainTipsOverride?.proven).toBeUndefined(); - }); - - it('emits preparing-checkpoint with provenOverride when prune is due', async () => { + it('emits preparing-checkpoint with snapshot-pinned tips when no override applies', async () => { await setupSingleTxBlock(); - l2BlockSource.isPruneDueAtSlot.mockResolvedValue(true); - const events: any[] = []; sequencer.on('preparing-checkpoint', args => events.push(args)); await sequencer.work(); expect(events).toHaveLength(1); + // With no pipelined or invalidation override, both `pending` and `proven` are pinned to the + // on-chain pending snapshot (checkpointedCheckpointNumber) so `canPruneAtTime` short-circuits + // and a live re-read inside `makeChainTipsOverride` can't reintroduce a phantom prune. + // `provenOverride` mirrors the pinned proven tip whenever a plan was built. expect(events[0]).toEqual({ targetSlot: SlotNumber(2), checkpointNumber: expect.anything(), hadProposedParent: false, provenOverride: CheckpointNumber.ZERO, - simulatedPending: undefined, - }); - }); - - it('emits preparing-checkpoint without provenOverride when no prune is due', async () => { - await setupSingleTxBlock(); - - l2BlockSource.isPruneDueAtSlot.mockResolvedValue(false); - - const events: any[] = []; - sequencer.on('preparing-checkpoint', args => events.push(args)); - - await sequencer.work(); - - expect(events).toHaveLength(1); - expect(events[0]).toMatchObject({ - targetSlot: SlotNumber(2), - hadProposedParent: false, - provenOverride: undefined, + simulatedPending: CheckpointNumber.ZERO, }); }); }); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index e49c922f378c..83f9cb174b97 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -1,6 +1,6 @@ import { getKzg } from '@aztec/blob-lib'; import type { EpochCache } from '@aztec/epoch-cache'; -import { NoCommitteeError, type RollupContract, SimulationOverridesBuilder } from '@aztec/ethereum/contracts'; +import { NoCommitteeError, type RollupContract } from '@aztec/ethereum/contracts'; import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { merge, omit, pick } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; @@ -14,7 +14,7 @@ import type { SlasherClientInterface } from '@aztec/slasher'; import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block'; import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint'; import type { ChainConfig } from '@aztec/stdlib/config'; -import { getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; +import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers'; import { type ResolvedSequencerConfig, type SequencerConfig, @@ -33,7 +33,7 @@ import { DefaultSequencerConfig } from '../config.js'; import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js'; import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js'; import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js'; -import { buildPipelinedParentSimulationOverridesPlan } from './chain_state_overrides.js'; +import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js'; import { CheckpointProposalJob } from './checkpoint_proposal_job.js'; import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js'; import { CheckpointVoter } from './checkpoint_voter.js'; @@ -374,71 +374,69 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter TypedEventEmitter { - this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, { slot }); - }); - } else { - await publisher.sendRequests(); - } + void publisher.sendRequestsAt(targetSlot).catch(err => { + this.log.error(`Failed to publish votes despite sync failure for slot ${slot}`, err, { slot }); + }); } /** @@ -930,7 +921,12 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter Date: Wed, 13 May 2026 14:46:11 -0300 Subject: [PATCH 03/17] refactor(stdlib): remove deprecated RevertCode/TxExecutionResult aliases (#23249) ## Motivation The `RevertCode` and `TxExecutionResult` types each carried three deprecated aliases (`APP_LOGIC_REVERTED`, `TEARDOWN_REVERTED`, `BOTH_REVERTED`) that all collapse to the same `REVERTED` value. Keeping them around adds noise, requires `no-duplicate-enum-values` eslint suppressions, and lets new code keep reaching for the old names. ## Approach Removed the deprecated members from both enums and rewrote every call site to use `REVERTED` directly. Tests, fixtures, and a stale doc reference were updated to match. ## Changes - **stdlib**: Drop deprecated `APP_LOGIC_REVERTED`/`TEARDOWN_REVERTED`/`BOTH_REVERTED` from `RevertCode` and `TxExecutionResult`. - **simulator, pxe, aztec.js, end-to-end (tests)**: Replace remaining references with `REVERTED`. - **simulator/docs**: Update a stale `APP_LOGIC_REVERTED` reference in the public-tx-simulation doc. --- yarn-project/aztec.js/src/utils/node.test.ts | 4 ++-- .../src/e2e_cross_chain_messaging/l1_to_l2.test.ts | 2 +- .../contract_class_registration.test.ts | 6 +++--- .../src/e2e_deploy_contract/legacy.test.ts | 2 +- .../end-to-end/src/e2e_double_spend.test.ts | 2 +- .../epochs_proof_public_cross_chain.test.ts | 2 +- .../end-to-end/src/e2e_fees/failures.test.ts | 8 ++++---- .../tagging_store/sender_tagging_store.test.ts | 2 +- .../sync_sender_tagging_indexes.test.ts | 6 +++--- .../utils/get_status_change_of_pending.test.ts | 6 +++--- .../simulator/docs/avm/public-tx-simulation.md | 2 +- .../apps_tests/deployments.test.ts | 2 +- .../public_processor/public_processor.test.ts | 2 +- .../public_tx_simulator.test.ts | 12 ++++++------ yarn-project/stdlib/src/avm/revert_code.ts | 6 ------ yarn-project/stdlib/src/tx/tx_receipt.test.ts | 14 ++------------ yarn-project/stdlib/src/tx/tx_receipt.ts | 9 --------- 17 files changed, 31 insertions(+), 56 deletions(-) diff --git a/yarn-project/aztec.js/src/utils/node.test.ts b/yarn-project/aztec.js/src/utils/node.test.ts index 2bdfc8cc3699..7ae9528aae00 100644 --- a/yarn-project/aztec.js/src/utils/node.test.ts +++ b/yarn-project/aztec.js/src/utils/node.test.ts @@ -42,7 +42,7 @@ describe('waitForTx', () => { const revertedReceipt = new TxReceipt( txHash, TxStatus.CHECKPOINTED, - TxExecutionResult.APP_LOGIC_REVERTED, + TxExecutionResult.REVERTED, undefined, undefined, undefined, @@ -56,7 +56,7 @@ describe('waitForTx', () => { const revertedReceipt = new TxReceipt( txHash, TxStatus.CHECKPOINTED, - TxExecutionResult.APP_LOGIC_REVERTED, + TxExecutionResult.REVERTED, undefined, undefined, undefined, diff --git a/yarn-project/end-to-end/src/e2e_cross_chain_messaging/l1_to_l2.test.ts b/yarn-project/end-to-end/src/e2e_cross_chain_messaging/l1_to_l2.test.ts index 294069db6b86..648893b498af 100644 --- a/yarn-project/end-to-end/src/e2e_cross_chain_messaging/l1_to_l2.test.ts +++ b/yarn-project/end-to-end/src/e2e_cross_chain_messaging/l1_to_l2.test.ts @@ -270,7 +270,7 @@ describe('e2e_cross_chain_messaging l1_to_l2', () => { expect(block!.checkpointNumber).toEqual(msgCheckpointNumber); expect(block!.indexWithinCheckpoint).toEqual(IndexWithinCheckpoint.ZERO); } else { - expect(receipt.executionResult).toEqual(TxExecutionResult.APP_LOGIC_REVERTED); + expect(receipt.executionResult).toEqual(TxExecutionResult.REVERTED); } } await t.context.watcher.markAsProven(); diff --git a/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts b/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts index cb0a5b7cb54b..f9771f89e663 100644 --- a/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts +++ b/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts @@ -161,7 +161,7 @@ describe('e2e_deploy_contract contract class registration', () => { const { receipt } = await contract.methods .increment_public_value(whom, 10) .send({ from: defaultAccountAddress, wait: { dontThrowOnRevert: true } }); - expect(receipt.executionResult).toEqual(TxExecutionResult.APP_LOGIC_REVERTED); + expect(receipt.executionResult).toEqual(TxExecutionResult.REVERTED); // Meanwhile we check we didn't increment the value expect( @@ -205,7 +205,7 @@ describe('e2e_deploy_contract contract class registration', () => { const { receipt } = await contract.methods .public_constructor(whom, 43) .send({ from: defaultAccountAddress, wait: { dontThrowOnRevert: true } }); - expect(receipt.executionResult).toEqual(TxExecutionResult.APP_LOGIC_REVERTED); + expect(receipt.executionResult).toEqual(TxExecutionResult.REVERTED); expect( (await contract.methods.get_public_value(whom).simulate({ from: defaultAccountAddress })).result, ).toEqual(0n); @@ -256,7 +256,7 @@ describe('e2e_deploy_contract contract class registration', () => { const { receipt: tx } = await instance.methods .increment_public_value_no_init_check(whom, 10) .send({ from: defaultAccountAddress, wait: { dontThrowOnRevert: true } }); - expect(tx.executionResult).toEqual(TxExecutionResult.APP_LOGIC_REVERTED); + expect(tx.executionResult).toEqual(TxExecutionResult.REVERTED); }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_deploy_contract/legacy.test.ts b/yarn-project/end-to-end/src/e2e_deploy_contract/legacy.test.ts index a4d1c1e0748d..07f124da9414 100644 --- a/yarn-project/end-to-end/src/e2e_deploy_contract/legacy.test.ts +++ b/yarn-project/end-to-end/src/e2e_deploy_contract/legacy.test.ts @@ -122,7 +122,7 @@ describe('e2e_deploy_contract legacy', () => { expect(goodTxReceipt!.blockNumber).toEqual(expect.any(Number)); expect(badTxReceipt!.blockNumber).toEqual(expect.any(Number)); - expect(badTxReceipt!.executionResult).toEqual(TxExecutionResult.APP_LOGIC_REVERTED); + expect(badTxReceipt!.executionResult).toEqual(TxExecutionResult.REVERTED); const badInstance = await badDeploy.getInstance(); // But the bad tx did not deploy the class diff --git a/yarn-project/end-to-end/src/e2e_double_spend.test.ts b/yarn-project/end-to-end/src/e2e_double_spend.test.ts index 3cc69dec717d..ee59a97b6e63 100644 --- a/yarn-project/end-to-end/src/e2e_double_spend.test.ts +++ b/yarn-project/end-to-end/src/e2e_double_spend.test.ts @@ -46,7 +46,7 @@ describe('e2e_double_spend', () => { // tx will be included in a block but with app logic reverted await expect( contract.methods.emit_nullifier_public(nullifier).send({ from: defaultAccountAddress }), - ).rejects.toThrow(TxExecutionResult.APP_LOGIC_REVERTED); + ).rejects.toThrow(TxExecutionResult.REVERTED); }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts index 0d9b27000373..33743e47394e 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts @@ -98,7 +98,7 @@ describe('e2e_epochs/epochs_proof_public_cross_chain', () => { globalLeafIndex.toBigInt(), ) .send({ from: context.accounts[0], wait: { dontThrowOnRevert: true } }); - expect(failedReceipt.executionResult).toBe(TxExecutionResult.APP_LOGIC_REVERTED); + expect(failedReceipt.executionResult).toBe(TxExecutionResult.REVERTED); logger.info(`Test succeeded`); }); diff --git a/yarn-project/end-to-end/src/e2e_fees/failures.test.ts b/yarn-project/end-to-end/src/e2e_fees/failures.test.ts index 554bb03f16ea..97119290d014 100644 --- a/yarn-project/end-to-end/src/e2e_fees/failures.test.ts +++ b/yarn-project/end-to-end/src/e2e_fees/failures.test.ts @@ -98,7 +98,7 @@ describe('e2e_fees failures', () => { wait: { dontThrowOnRevert: true }, }); - expect(txReceipt.executionResult).toBe(TxExecutionResult.APP_LOGIC_REVERTED); + expect(txReceipt.executionResult).toBe(TxExecutionResult.REVERTED); const { sequencerBlockRewards } = await t.getBlockRewards(); @@ -201,7 +201,7 @@ describe('e2e_fees failures', () => { wait: { dontThrowOnRevert: true }, }); - expect(txReceipt.executionResult).toBe(TxExecutionResult.APP_LOGIC_REVERTED); + expect(txReceipt.executionResult).toBe(TxExecutionResult.REVERTED); const feeAmount = txReceipt.transactionFee!; // and thus we paid the fee @@ -298,7 +298,7 @@ describe('e2e_fees failures', () => { }, wait: { dontThrowOnRevert: true }, }); - expect(receipt.executionResult).toEqual(TxExecutionResult.TEARDOWN_REVERTED); + expect(receipt.executionResult).toEqual(TxExecutionResult.REVERTED); expect(receipt.transactionFee).toBeGreaterThan(0n); await expectMapping( @@ -346,7 +346,7 @@ describe('e2e_fees failures', () => { wait: { dontThrowOnRevert: true }, }); - expect(receipt.executionResult).toBe(TxExecutionResult.BOTH_REVERTED); + expect(receipt.executionResult).toBe(TxExecutionResult.REVERTED); expect(receipt.transactionFee).toBeGreaterThan(0n); await t.context.watcher.trigger(); diff --git a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts index b2800582f02d..6c9e4a430d33 100644 --- a/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts +++ b/yarn-project/pxe/src/storage/tagging_store/sender_tagging_store.test.ts @@ -494,7 +494,7 @@ describe('SenderTaggingStore', () => { describe('finalizePendingIndexesOfAPartiallyRevertedTx', () => { function makeTxEffect(txHash: TxHash, siloedTags: SiloedTag[]): TxEffect { return new TxEffect( - RevertCode.APP_LOGIC_REVERTED, + RevertCode.REVERTED, txHash, Fr.ZERO, [Fr.random()], // noteHashes (at least 1 nullifier required below, not here) diff --git a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts index dedfacbf5dda..e522282e3ad2 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts @@ -300,12 +300,12 @@ describe('syncSenderTaggingIndexes', () => { ); }); - // Mock getTxReceipt to return FINALIZED with APP_LOGIC_REVERTED + // Mock getTxReceipt to return FINALIZED with REVERTED aztecNode.getTxReceipt.mockResolvedValue( new TxReceipt( revertedTxHash, TxStatus.FINALIZED, - TxExecutionResult.APP_LOGIC_REVERTED, + TxExecutionResult.REVERTED, undefined, undefined, undefined, @@ -315,7 +315,7 @@ describe('syncSenderTaggingIndexes', () => { // Mock getTxEffect to return a TxEffect where only the tag at index 4 survived (non-revertible phase) const txEffect = new TxEffect( - RevertCode.APP_LOGIC_REVERTED, + RevertCode.REVERTED, revertedTxHash, Fr.ZERO, [Fr.random()], // noteHashes diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts index 676b491d8910..2842a8554eb7 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts @@ -55,7 +55,7 @@ describe('getStatusChangeOfPending', () => { new TxReceipt( hash, TxStatus.FINALIZED, - TxExecutionResult.APP_LOGIC_REVERTED, + TxExecutionResult.REVERTED, undefined, undefined, undefined, @@ -67,7 +67,7 @@ describe('getStatusChangeOfPending', () => { new TxReceipt( hash, TxStatus.FINALIZED, - TxExecutionResult.TEARDOWN_REVERTED, + TxExecutionResult.REVERTED, undefined, undefined, undefined, @@ -79,7 +79,7 @@ describe('getStatusChangeOfPending', () => { new TxReceipt( hash, TxStatus.FINALIZED, - TxExecutionResult.BOTH_REVERTED, + TxExecutionResult.REVERTED, undefined, undefined, undefined, diff --git a/yarn-project/simulator/docs/avm/public-tx-simulation.md b/yarn-project/simulator/docs/avm/public-tx-simulation.md index 54a27fbeafd8..1896a7fff258 100644 --- a/yarn-project/simulator/docs/avm/public-tx-simulation.md +++ b/yarn-project/simulator/docs/avm/public-tx-simulation.md @@ -35,7 +35,7 @@ The app logic phase contains the main application functionality. This is where m - State changes from app logic are rolled back - Side effects from private's revertible portion are also discarded - Teardown still executes -- The transaction appears on-chain with `APP_LOGIC_REVERTED` status +- The transaction appears on-chain with `REVERTED` status ### TEARDOWN Phase (Revertible, Always Runs) diff --git a/yarn-project/simulator/src/public/public_processor/apps_tests/deployments.test.ts b/yarn-project/simulator/src/public/public_processor/apps_tests/deployments.test.ts index 1183b99f0cfd..d8a6353581f3 100644 --- a/yarn-project/simulator/src/public/public_processor/apps_tests/deployments.test.ts +++ b/yarn-project/simulator/src/public/public_processor/apps_tests/deployments.test.ts @@ -249,7 +249,7 @@ describe.each([ expect(processedTxs[0].revertCode).toEqual(RevertCode.OK); // Second tx should revert in app logic (failed transfer) - expect(processedTxs[1].revertCode).toEqual(RevertCode.APP_LOGIC_REVERTED); + expect(processedTxs[1].revertCode).toEqual(RevertCode.REVERTED); // Third tx should succeed (mint), proving first contract is still accessible expect(processedTxs[2].revertCode).toEqual(RevertCode.OK); diff --git a/yarn-project/simulator/src/public/public_processor/public_processor.test.ts b/yarn-project/simulator/src/public/public_processor/public_processor.test.ts index abc3aedf918e..505622e69541 100644 --- a/yarn-project/simulator/src/public/public_processor/public_processor.test.ts +++ b/yarn-project/simulator/src/public/public_processor/public_processor.test.ts @@ -136,7 +136,7 @@ describe('public_processor', () => { it('runs a tx with reverted enqueued public calls', async function () { const tx = await mockTxWithPublicCalls(); - mockedEnqueuedCallsResult.revertCode = RevertCode.APP_LOGIC_REVERTED; + mockedEnqueuedCallsResult.revertCode = RevertCode.REVERTED; const [processed, failed] = await processor.process([tx]); diff --git a/yarn-project/simulator/src/public/public_tx_simulator/public_tx_simulator.test.ts b/yarn-project/simulator/src/public/public_tx_simulator/public_tx_simulator.test.ts index 337bd982431d..64a6cf8b585b 100644 --- a/yarn-project/simulator/src/public/public_tx_simulator/public_tx_simulator.test.ts +++ b/yarn-project/simulator/src/public/public_tx_simulator/public_tx_simulator.test.ts @@ -691,7 +691,7 @@ describe('public_tx_simulator', () => { const txResult = await simulator.simulate(tx); - expect(txResult.revertCode).toEqual(RevertCode.APP_LOGIC_REVERTED); + expect(txResult.revertCode).toEqual(RevertCode.REVERTED); // tx reports app logic failure expect(txResult.findRevertReason()).toEqual(appLogicFailure); @@ -812,7 +812,7 @@ describe('public_tx_simulator', () => { const txResult = await simulator.simulate(tx); - expect(txResult.revertCode).toEqual(RevertCode.TEARDOWN_REVERTED); + expect(txResult.revertCode).toEqual(RevertCode.REVERTED); expect(txResult.findRevertReason()).toEqual(teardownFailure); const expectedSetupGas = enqueuedCallGasUsed; @@ -921,7 +921,7 @@ describe('public_tx_simulator', () => { const txResult = await simulator.simulate(tx); - expect(txResult.revertCode).toEqual(RevertCode.BOTH_REVERTED); + expect(txResult.revertCode).toEqual(RevertCode.REVERTED); // tx reports app logic failure expect(txResult.findRevertReason()).toEqual(appLogicFailure); @@ -1246,7 +1246,7 @@ describe('public_tx_simulator', () => { }); const txResult = await simulator.simulate(tx); - expect(txResult.revertCode).toEqual(RevertCode.APP_LOGIC_REVERTED); + expect(txResult.revertCode).toEqual(RevertCode.REVERTED); const revertReason = txResult.findRevertReason(); expect(revertReason).toBeDefined(); expect(revertReason?.getOriginalMessage()).toContain(new NullifierLimitReachedError().message); @@ -1269,7 +1269,7 @@ describe('public_tx_simulator', () => { throw new NoteHashLimitReachedError(); }); const txResult = await simulator.simulate(tx); - expect(txResult.revertCode).toEqual(RevertCode.APP_LOGIC_REVERTED); + expect(txResult.revertCode).toEqual(RevertCode.REVERTED); const revertReason = txResult.findRevertReason(); expect(revertReason).toBeDefined(); expect(revertReason?.getOriginalMessage()).toContain(new NoteHashLimitReachedError().message); @@ -1296,7 +1296,7 @@ describe('public_tx_simulator', () => { }); const txResult = await simulator.simulate(tx); - expect(txResult.revertCode).toEqual(RevertCode.APP_LOGIC_REVERTED); + expect(txResult.revertCode).toEqual(RevertCode.REVERTED); const revertReason = txResult.findRevertReason(); expect(revertReason).toBeDefined(); expect(revertReason?.getOriginalMessage()).toContain(new L2ToL1MessageLimitReachedError().message); diff --git a/yarn-project/stdlib/src/avm/revert_code.ts b/yarn-project/stdlib/src/avm/revert_code.ts index 810c779d563b..23d054af08e7 100644 --- a/yarn-project/stdlib/src/avm/revert_code.ts +++ b/yarn-project/stdlib/src/avm/revert_code.ts @@ -28,12 +28,6 @@ export class RevertCode { } static readonly OK: RevertCode = new RevertCode(RevertCodeEnum.OK); static readonly REVERTED: RevertCode = new RevertCode(RevertCodeEnum.REVERTED); - /** @deprecated Use REVERTED instead. */ - static readonly APP_LOGIC_REVERTED: RevertCode = RevertCode.REVERTED; - /** @deprecated Use REVERTED instead. */ - static readonly TEARDOWN_REVERTED: RevertCode = RevertCode.REVERTED; - /** @deprecated Use REVERTED instead. */ - static readonly BOTH_REVERTED: RevertCode = RevertCode.REVERTED; public getCode(): RevertCodeEnum { return this.code; diff --git a/yarn-project/stdlib/src/tx/tx_receipt.test.ts b/yarn-project/stdlib/src/tx/tx_receipt.test.ts index 8be605399c4f..0c2044f37fad 100644 --- a/yarn-project/stdlib/src/tx/tx_receipt.test.ts +++ b/yarn-project/stdlib/src/tx/tx_receipt.test.ts @@ -42,22 +42,12 @@ describe('TxReceipt', () => { }); it('isSuccess returns false for reverted execution', () => { - const receipt = new TxReceipt( - TxHash.random(), - TxStatus.PROPOSED, - TxExecutionResult.APP_LOGIC_REVERTED, - undefined, - ); + const receipt = new TxReceipt(TxHash.random(), TxStatus.PROPOSED, TxExecutionResult.REVERTED, undefined); expect(receipt.hasExecutionSucceeded()).toBe(false); }); it('isReverted returns true for reverted execution', () => { - const receipt = new TxReceipt( - TxHash.random(), - TxStatus.PROPOSED, - TxExecutionResult.APP_LOGIC_REVERTED, - undefined, - ); + const receipt = new TxReceipt(TxHash.random(), TxStatus.PROPOSED, TxExecutionResult.REVERTED, undefined); expect(receipt.hasExecutionReverted()).toBe(true); }); diff --git a/yarn-project/stdlib/src/tx/tx_receipt.ts b/yarn-project/stdlib/src/tx/tx_receipt.ts index 348806510344..446855f44237 100644 --- a/yarn-project/stdlib/src/tx/tx_receipt.ts +++ b/yarn-project/stdlib/src/tx/tx_receipt.ts @@ -32,15 +32,6 @@ export const SortedTxStatuses: TxStatus[] = [ export enum TxExecutionResult { SUCCESS = 'success', REVERTED = 'reverted', - /** @deprecated Use REVERTED instead. */ - // eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values - APP_LOGIC_REVERTED = 'reverted', - /** @deprecated Use REVERTED instead. */ - // eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values - TEARDOWN_REVERTED = 'reverted', - /** @deprecated Use REVERTED instead. */ - // eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values - BOTH_REVERTED = 'reverted', } /** From 498b9610095ab4e1d54126e94a696241832df3b0 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 14 May 2026 05:00:12 -0300 Subject: [PATCH 04/17] test(e2e): fix race in 'proposer invalidates multiple checkpoints' (#23259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses a config-timing race in `epochs_invalidate_block.parallel.test.ts > "proposer invalidates multiple checkpoints"` that caused intermittent CI failures with `expect(validCount).toBeLessThan(quorum)` (e.g. 5/6 attestations when quorum=5). ## The race The test reads `currentSlot` via `monitor.run()` right after waiting for the first checkpoint to land — that read can land anywhere within the current L2 slot, including near its end. It then computes `badSlot1 = currentSlot + 2` and races to push malicious config (`skipCollectingAttestations: true`, …) to that slot's proposer via `await node.setConfig({...})`. `CheckpointProposalJob` is constructed with `this.config` passed by reference (`sequencer-client/src/sequencer/sequencer.ts:559`), and `Sequencer.updateConfig` reassigns `this.config = merge(...)` rather than mutating, so a job built before `setConfig` lands keeps the old config object. Under proposer pipelining (`PROPOSER_PIPELINING_SLOT_OFFSET = 1`, `epoch-cache/src/epoch_cache.ts:26`), the job for `badSlot1` is built during the last L1 slot of L2 slot `badSlot1 - 1`. With 32s L2 slots and 8s L1 slots, that's ~24s into the previous L2 slot — so if `currentSlot` was read late, badSlot1's proposer can snapshot the old config before our `setConfig` round-trip completes. ## Fix - Wait for an L2 slot boundary (`monitor.waitUntilNextL2Slot()`) before reading `currentSlot`, so we start from the beginning of a slot rather than wherever we happened to land. - Bump the gap from `+2/+3` to `+3/+4` for a second slot of margin. Cost is up to one additional L2 slot of test runtime in the worst case; the existing 8-slot wait window for both checkpoints still fits. --- .../epochs_invalidate_block.parallel.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts index 1f952fd76ef6..431632c757b2 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts @@ -378,12 +378,19 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // Wait for at least one checkpoint to be mined so that any in-progress slot has completed const initialCheckpointNumber = (await nodes[0].getChainTips()).checkpointed.checkpoint.number; await test.waitUntilCheckpointNumber(CheckpointNumber(initialCheckpointNumber + 1), test.L2_SLOT_DURATION_IN_S * 4); + + // Align to the start of an L2 slot before computing the bad slots, so we have a generous + // buffer to push the malicious config to badSlot1's proposer before it snapshots its config + // into a new CheckpointProposalJob. Under proposer pipelining, that job is built during the + // last L1 slot of the previous L2 slot (when getEpochAndSlotInNextL1Slot first returns the + // proposer's target slot), so the practical window is somewhat less than a full L2 slot. + await test.monitor.waitUntilNextL2Slot(); const { l2SlotNumber: currentSlot } = await test.monitor.run(); logger.warn(`First checkpoint mined, current slot is ${currentSlot}`); - // Pick the next two slots after the current one, with a 1-slot gap to account for pipelining - const badSlot1 = SlotNumber.add(currentSlot, 2); - const badSlot2 = SlotNumber.add(currentSlot, 3); + // Pick the next two slots with a 2-slot gap to account for pipelining plus a margin + const badSlot1 = SlotNumber.add(currentSlot, 3); + const badSlot2 = SlotNumber.add(currentSlot, 4); const badSlots = [badSlot1, badSlot2]; const badProposers = await Promise.all(badSlots.map(s => test.epochCache.getProposerAttesterAddressInSlot(s))); From 437eacbdfe3e3abf5355bfcee25f48d5c88d5426 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 14 May 2026 09:04:41 +0100 Subject: [PATCH 05/17] fix: clean up old jobs regardless of pending status (#23260) . --- .test_patterns.yml | 8 -------- .../src/proving_broker/proving_broker.test.ts | 19 ++++++++++--------- .../src/proving_broker/proving_broker.ts | 8 +++++--- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/.test_patterns.yml b/.test_patterns.yml index 10cd59cca8fd..389e57e035cd 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -197,14 +197,6 @@ tests: - *phil - *palla - # http://ci.aztec-labs.com/64a972aafaa40dd0 - # ProvingBroker › Retries › does not retry if job is stale — kv-store closes - # before the broker's final reportProvingJobError write lands. - - regex: "prover-client/src/proving_broker/proving_broker.test.ts" - error_regex: "does not retry if job is stale|Store is closed" - owners: - - *alex - # Nightly GKE tests - regex: "spartan/bootstrap.sh" owners: diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts index 34fc09fe7b06..4652413271d4 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.test.ts @@ -856,7 +856,7 @@ describe.each([ await assertJobTransition(id, 'in-progress', 'in-queue'); }); - it('cancel stale jobs that time out', async () => { + it('cleans up stale in-progress jobs before deleting their epoch database', async () => { const id = makeRandomProvingJobId(); await broker.enqueueProvingJob({ id, @@ -887,10 +887,9 @@ describe.each([ inputsUri: makeInputsUri(), }); - // advance time again so job times out. Since the job was in-progress, it won't be cleaned up as stale - // but will be rejected when it times out - await sleep(jobTimeoutMs + brokerIntervalMs); - await assertJobStatus(id, 'rejected'); + // the epoch-1 database is old enough to delete, so the broker closes any remaining epoch-1 jobs + await (broker as any).cleanupPass(); + await assertJobStatus(id, 'not-found'); }); it('rejects jobs that time out more than maxRetries times', async () => { @@ -1070,13 +1069,15 @@ describe.each([ inputsUri: makeInputsUri(), }); - await sleep(brokerIntervalMs); + await (broker as any).cleanupPass(); + await assertJobStatus(id, 'not-found'); - // job was in-progress so it won't be cleaned up as stale, but will be rejected on error + // the epoch-1 database has been deleted, so late worker reports are ignored + jest.spyOn(database, 'setProvingJobError'); await broker.reportProvingJobError(id, 'test error', true); + expect(database.setProvingJobError).not.toHaveBeenCalled(); await expect(broker.getProvingJobStatus(id)).resolves.toEqual({ - status: 'rejected', - reason: 'test error', + status: 'not-found', }); }); }); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.ts index decb4835eff3..27364938d5e1 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.ts @@ -319,6 +319,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr } private cleanUpProvingJobState(ids: ProvingJobId[]) { + const idsToClean = new Set(ids); for (const id of ids) { this.jobsCache.delete(id); const deferred = this.promises.get(id); @@ -331,6 +332,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr this.retries.delete(id); this.enqueuedAt.delete(id); } + this.completedJobNotifications = this.completedJobNotifications.filter(id => !idsToClean.has(id)); } #getProvingJobStatus(id: ProvingJobId): ProvingJobStatus { @@ -598,21 +600,21 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr } private async cleanupPass() { - this.cleanupStaleJobs(); this.reEnqueueExpiredJobs(); const oldestEpochToKeep = this.oldestEpochToKeep(); if (oldestEpochToKeep > 0) { + this.cleanupJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep)); await this.database.deleteAllProvingJobsOlderThanEpoch(EpochNumber(oldestEpochToKeep)); this.logger.trace(`Deleted all epochs older than ${oldestEpochToKeep}`); } } - private cleanupStaleJobs() { + private cleanupJobsOlderThanEpoch(epochNumber: EpochNumber) { const jobIds = Array.from(this.jobsCache.keys()); const jobsToClean: ProvingJobId[] = []; for (const id of jobIds) { const job = this.jobsCache.get(id)!; - if (this.isJobStale(job) && !this.inProgress.has(id) && !this.resultsCache.has(id)) { + if (job.epochNumber < epochNumber) { jobsToClean.push(id); } } From db4bd983cd8395e218f16379900c5319857df524 Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 14 May 2026 09:57:03 -0300 Subject: [PATCH 06/17] refactor(p2p): remove unused sendBatchRequest (#23273) `sendBatchRequest` became unused after removing the slow tx flow and the old tx reqresp method. This PR removes sendBatchRequest and cleans up code that becomes unused. It does NOT remove subprotocol validator registration/etc from reqresp. This might be done in a follow-up depending on how https://linear.app/aztec-labs/issue/A-1014/block-txs-reqresp-validator-validaterequestedblocktxs-is-never-invoked becomes solved. --- .../p2p/src/client/p2p_client.test.ts | 1 - .../p2p_client.integration_reqresp.test.ts | 70 ----- yarn-project/p2p/src/errors/reqresp.error.ts | 25 -- .../p2p/src/services/dummy_service.ts | 23 -- .../p2p/src/services/libp2p/libp2p_service.ts | 15 - .../batch_connection_sampler.test.ts | 256 ------------------ .../batch_connection_sampler.ts | 161 ----------- .../p2p/src/services/reqresp/interface.ts | 8 - .../p2p/src/services/reqresp/reqresp.test.ts | 130 +-------- .../p2p/src/services/reqresp/reqresp.ts | 237 +--------------- yarn-project/p2p/src/services/service.ts | 17 -- .../p2p/src/test-helpers/mock-pubsub.ts | 60 +--- .../telemetry-client/src/attributes.ts | 1 - .../telemetry-client/src/telemetry.ts | 1 - 14 files changed, 12 insertions(+), 993 deletions(-) delete mode 100644 yarn-project/p2p/src/services/reqresp/connection-sampler/batch_connection_sampler.test.ts delete mode 100644 yarn-project/p2p/src/services/reqresp/connection-sampler/batch_connection_sampler.ts diff --git a/yarn-project/p2p/src/client/p2p_client.test.ts b/yarn-project/p2p/src/client/p2p_client.test.ts index 86df6d146a25..115e4744585f 100644 --- a/yarn-project/p2p/src/client/p2p_client.test.ts +++ b/yarn-project/p2p/src/client/p2p_client.test.ts @@ -41,7 +41,6 @@ describe('P2P Client', () => { txPool.addPendingTxs.mockResolvedValue({ accepted: [], ignored: [], rejected: [] }); p2pService = mock(); - p2pService.sendBatchRequest.mockResolvedValue([]); l1Constants = EmptyL1RollupConstants; txCollection = mock(); diff --git a/yarn-project/p2p/src/client/test/p2p_client.integration_reqresp.test.ts b/yarn-project/p2p/src/client/test/p2p_client.integration_reqresp.test.ts index c6454f17a1d2..ac3cc50c88e5 100644 --- a/yarn-project/p2p/src/client/test/p2p_client.integration_reqresp.test.ts +++ b/yarn-project/p2p/src/client/test/p2p_client.integration_reqresp.test.ts @@ -113,44 +113,6 @@ describe('p2p client integration reqresp', () => { return (p2pService as any).node.peerId; }; - it('can request txs from peers via mock reqresp', async () => { - const numberOfNodes = 2; - const mockGossipSubNetwork = new MockGossipSubNetwork(); - - const testConfig = { - p2pBaseConfig: { ...p2pBaseConfig, rollupVersion: 1 }, - mockAttestationPool: attestationPool, - mockTxPool: txPool, - mockEpochCache: epochCache, - mockWorldState: worldState, - alwaysTrueVerifier: true, - mockGossipSubNetwork, - logger, - }; - - const clientsAndConfig = await makeAndStartTestP2PClients(numberOfNodes, testConfig); - clients = clientsAndConfig.map(c => c.client); - - await sleep(1000); - - // Create a mock tx and configure the shared pool to return it - const tx = await createMockTxWithMetadata(testConfig.p2pBaseConfig); - const txHash = tx.getTxHash(); - - txPool.getTxByHash.mockImplementation((hash: TxHash) => Promise.resolve(hash.equals(txHash) ? tx : undefined)); - - // Request the tx from node-2, which will route to node-1 via the mock network - const reqresp = getReqResp(clients[1]); - const responses = await reqresp.sendBatchRequest(ReqRespSubProtocol.TX, [new TxHashArray(txHash)], undefined); - - expect(responses).toHaveLength(1); - const txArray = responses[0] as TxArray; - expect(txArray).toHaveLength(1); - - const receivedTxHash = txArray[0].getTxHash(); - expect(receivedTxHash.toString()).toEqual(txHash.toString()); - }); - it('sendRequestToPeer routes to the correct peer handler', async () => { const numberOfNodes = 2; const mockGossipSubNetwork = new MockGossipSubNetwork(); @@ -197,36 +159,4 @@ describe('p2p client integration reqresp', () => { expect(receivedTxHash.toString()).toEqual(txHash.toString()); } }); - - it('reqresp returns empty when peer has no matching txs', async () => { - const numberOfNodes = 2; - const mockGossipSubNetwork = new MockGossipSubNetwork(); - - const testConfig = { - p2pBaseConfig: { ...p2pBaseConfig, rollupVersion: 1 }, - mockAttestationPool: attestationPool, - mockTxPool: txPool, - mockEpochCache: epochCache, - mockWorldState: worldState, - alwaysTrueVerifier: true, - mockGossipSubNetwork, - logger, - }; - - const clientsAndConfig = await makeAndStartTestP2PClients(numberOfNodes, testConfig); - clients = clientsAndConfig.map(c => c.client); - - await sleep(1000); - - // Request a random tx hash that no peer has - const randomTxHash = TxHash.random(); - const reqresp = getReqResp(clients[1]); - const responses = await reqresp.sendBatchRequest(ReqRespSubProtocol.TX, [new TxHashArray(randomTxHash)], undefined); - - // The handler returns an empty TxArray (serialized as a 4-byte vector with count 0), - // so sendBatchRequest includes it as a response with an empty TxArray. - expect(responses).toHaveLength(1); - const txArray = responses[0] as TxArray; - expect(txArray).toHaveLength(0); - }); }); diff --git a/yarn-project/p2p/src/errors/reqresp.error.ts b/yarn-project/p2p/src/errors/reqresp.error.ts index 21749b7473d2..23827d882b9e 100644 --- a/yarn-project/p2p/src/errors/reqresp.error.ts +++ b/yarn-project/p2p/src/errors/reqresp.error.ts @@ -8,28 +8,3 @@ export class IndividualReqRespTimeoutError extends Error { super(`Request to peer timed out`); } } - -/** Collective request timeout error - * - * This error will be thrown when a req resp request times out regardless of the peer. - * @category Errors - */ -export class CollectiveReqRespTimeoutError extends Error { - constructor() { - super(`Request to all peers timed out`); - } -} - -/** Invalid response error - * - * This error will be thrown when a response is received that is not valid. - * - * This error does not need to be punished as message validators will handle punishing invalid - * requests - * @category Errors - */ -export class InvalidResponseError extends Error { - constructor() { - super(`Invalid response received`); - } -} diff --git a/yarn-project/p2p/src/services/dummy_service.ts b/yarn-project/p2p/src/services/dummy_service.ts index d89cfcc59635..bd54a67fff93 100644 --- a/yarn-project/p2p/src/services/dummy_service.ts +++ b/yarn-project/p2p/src/services/dummy_service.ts @@ -119,19 +119,6 @@ export class DummyP2PService implements P2PService { return Promise.resolve(undefined); } - /** - * Sends a batch request to a peer. - * @param _protocol - The protocol to send the request on. - * @param _requests - The requests to send. - * @returns The responses from the peer, otherwise undefined. - */ - public sendBatchRequest( - _protocol: Protocol, - _requests: InstanceType[], - ): Promise[]> { - return Promise.resolve([]); - } - public sendRequestToPeer( _peerId: PeerId, _subProtocol: ReqRespSubProtocol, @@ -306,16 +293,6 @@ export class DummyReqResp implements ReqRespInterface { ): Promise | undefined> { return Promise.resolve(undefined); } - sendBatchRequest( - _subProtocol: SubProtocol, - _requests: InstanceType[], - _pinnedPeer: PeerId | undefined, - _timeoutMs?: number, - _maxPeers?: number, - _maxRetryAttempts?: number, - ): Promise[]> { - return Promise.resolve([]); - } public sendRequestToPeer( _peerId: PeerId, _subProtocol: ReqRespSubProtocol, diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts index 1cf314d4c835..bababe8fbbdd 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts @@ -101,7 +101,6 @@ import { type ReqRespSubProtocolHandlers, type ReqRespSubProtocolValidators, StatusMessage, - type SubProtocolMap, ValidationError, pingHandler, reqGoodbyeHandler, @@ -702,20 +701,6 @@ export class LibP2PService extends WithTracer implements P2PService { setImmediate(() => void safeJob()); } - /** - * Send a batch of requests to peers, and return the responses - * @param protocol - The request response protocol to use - * @param requests - The requests to send to the peers - * @returns The responses to the requests - */ - sendBatchRequest( - protocol: SubProtocol, - requests: InstanceType[], - pinnedPeerId: PeerId | undefined, - ): Promise[]> { - return this.reqresp.sendBatchRequest(protocol, requests, pinnedPeerId); - } - public sendRequestToPeer( peerId: PeerId, subProtocol: ReqRespSubProtocol, diff --git a/yarn-project/p2p/src/services/reqresp/connection-sampler/batch_connection_sampler.test.ts b/yarn-project/p2p/src/services/reqresp/connection-sampler/batch_connection_sampler.test.ts deleted file mode 100644 index 9432ac297e22..000000000000 --- a/yarn-project/p2p/src/services/reqresp/connection-sampler/batch_connection_sampler.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { describe, expect, it, jest } from '@jest/globals'; -import { createSecp256k1PeerId } from '@libp2p/peer-id-factory'; -import type { Libp2p } from 'libp2p'; - -import { BatchConnectionSampler } from './batch_connection_sampler.js'; -import { ConnectionSampler, type RandomSampler } from './connection_sampler.js'; - -describe('BatchConnectionSampler', () => { - const mockRandomSampler = { - random: jest.fn(), - } as jest.Mocked; - - let peers: Awaited>[]; - let libp2p: jest.Mocked; - let connectionSampler: ConnectionSampler; - - beforeEach(async () => { - jest.clearAllMocks(); - - // Create a set of test peers - peers = await Promise.all(new Array(5).fill(0).map(() => createSecp256k1PeerId())); - - // Mock libp2p to return our test peers - libp2p = { - getPeers: jest.fn().mockImplementation(() => [...peers]), - } as unknown as jest.Mocked; - - // Create a real connection sampler with mocked random sampling - connectionSampler = new ConnectionSampler(libp2p, mockRandomSampler, undefined, { cleanupIntervalMs: 1000 }); - }); - - afterEach(async () => { - await connectionSampler.stop(); - }); - - it('initializes with correct number of peers and request distribution', () => { - // Mock random to return sequential indices - mockRandomSampler.random.mockImplementation(_ => 0); - - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 10, /* maxPeers */ 3); - - expect(sampler.activePeerCount).toBe(3); - expect(sampler.requestsPerBucket).toBe(3); // floor(10/3) = 3 - }); - - it('assigns requests to peers deterministically with wraparound', () => { - // Mock to return first two peers - mockRandomSampler.random.mockImplementation(() => 0); - - // With 5 requests and 2 peers: - // floor(5/2) = 2 requests per peer - // Peer 0: 0,1,4 (gets extra from wraparound) - // Peer 1: 2,3 - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 5, /* maxPeers */ 2); - const assignments = new Array(5).fill(0).map((_, i) => sampler.getPeerForRequest(i)); - - // First peer gets first bucket and wraparound - expect(assignments[0]).toBe(peers[0]); // First bucket - expect(assignments[1]).toBe(peers[0]); // First bucket - expect(assignments[4]).toBe(peers[0]); // Wraparound - - // Second peer gets middle bucket - expect(assignments[2]).toBe(peers[1]); - expect(assignments[3]).toBe(peers[1]); - }); - - it('handles peer removal and replacement', () => { - mockRandomSampler.random.mockImplementation(_ => 0); - - // With 4 requests and 2 peers: - // floor(4/2) = 2 requests per peer - // Initial distribution: - // Peer 0: 0,1 - // Peer 1: 2,3 - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 4, /* maxPeers */ 2); - - const initialPeer = sampler.getPeerForRequest(0); - expect(initialPeer).toBe(peers[0]); - - // Mock random to return the third peer - mockRandomSampler.random.mockImplementation(_ => 2); - sampler.removePeerAndReplace(peers[0]); - - // After replacement: - // Replacement peer should handle the same bucket - const newPeer = sampler.getPeerForRequest(0); - expect(newPeer).toBe(peers[2]); - expect(sampler.getPeerForRequest(1)).toBe(peers[2]); - - // Other peer's bucket remains unchanged - expect(sampler.getPeerForRequest(2)).toBe(peers[1]); - expect(sampler.getPeerForRequest(3)).toBe(peers[1]); - }); - - it('handles peer removal and replacement - no replacement available', () => { - mockRandomSampler.random.mockImplementation(() => 0); - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 4, /* maxPeers */ 2); - - expect(sampler.activePeerCount).toBe(2); - expect(sampler.getPeerForRequest(0)).toBe(peers[0]); - - // Will sample no peers - libp2p.getPeers.mockReturnValue([]); - - // Remove peer 0, its requests will be distributed to peer 1 - sampler.removePeerAndReplace(peers[0]); - // Decrease the number of active peers - expect(sampler.activePeerCount).toBe(1); - - expect(sampler.getPeerForRequest(0)).toBe(peers[1]); - }); - - it('distributes requests according to documentation example', () => { - mockRandomSampler.random.mockImplementation(() => 0); - - // Example from doc comment: - // Peers: [P1] [P2] [P3] - // Requests: 0,1,2,9 | 3,4,5 | 6,7,8 - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 10, /* maxPeers */ 3); - - expect(sampler.activePeerCount).toBe(3); - expect(sampler.requestsPerBucket).toBe(3); // floor(10/3) = 3 - - // P1's bucket (0-2) plus wraparound (9) - expect(sampler.getPeerForRequest(0)).toBe(peers[0]); - expect(sampler.getPeerForRequest(1)).toBe(peers[0]); - expect(sampler.getPeerForRequest(2)).toBe(peers[0]); - expect(sampler.getPeerForRequest(9)).toBe(peers[0]); // Wraparound - - // P2's bucket (3-5) - expect(sampler.getPeerForRequest(3)).toBe(peers[1]); - expect(sampler.getPeerForRequest(4)).toBe(peers[1]); - expect(sampler.getPeerForRequest(5)).toBe(peers[1]); - - // P3's bucket (6-8) - expect(sampler.getPeerForRequest(6)).toBe(peers[2]); - expect(sampler.getPeerForRequest(7)).toBe(peers[2]); - expect(sampler.getPeerForRequest(8)).toBe(peers[2]); - }); - - it('same number of requests per peers', () => { - mockRandomSampler.random.mockImplementation(() => 0); - - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 2, /* maxPeers */ 2); - expect(sampler.requestsPerBucket).toBe(1); - expect(sampler.activePeerCount).toBe(2); - - expect(sampler.getPeerForRequest(0)).toBe(peers[0]); - expect(sampler.getPeerForRequest(1)).toBe(peers[1]); - }); - - it('handles edge cases, 0 peers, smaller batch than max peers', () => { - mockRandomSampler.random.mockImplementation(() => 0); - libp2p.getPeers.mockReturnValue([]); - - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 5, /* maxPeers */ 2); - expect(sampler.activePeerCount).toBe(0); - expect(sampler.getPeerForRequest(0)).toBeUndefined(); - - mockRandomSampler.random.mockImplementation(() => 0); - - libp2p.getPeers.mockImplementation(() => [...peers]); - const samplerWithMorePeers = new BatchConnectionSampler(connectionSampler, /* batchSize */ 2, /* maxPeers */ 3); - expect(samplerWithMorePeers.requestsPerBucket).toBe(1); // floor(2/3) = 0 - // First two requests go to first two peers - expect(samplerWithMorePeers.getPeerForRequest(0)).toBe(peers[0]); - expect(samplerWithMorePeers.getPeerForRequest(1)).toBe(peers[1]); - }); - - it('skips failed peer-index combinations and tries next peer', () => { - mockRandomSampler.random.mockImplementation(() => 0); - - // 6 requests across 3 peers (2 per peer) - // Peer 0: 0,1 Peer 1: 2,3 Peer 2: 4,5 - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 6, /* maxPeers */ 3); - - // Initially, request 0 goes to peer 0 - expect(sampler.getPeerForRequest(0)).toBe(peers[0]); - - // Mark peer 0 as failed for index 0 - sampler.markPeerFailedForIndex(peers[0], 0); - - // Now request 0 should go to the next peer (peer 1) - expect(sampler.getPeerForRequest(0)).toBe(peers[1]); - - // Mark peer 1 as also failed for index 0 - sampler.markPeerFailedForIndex(peers[1], 0); - - // Now request 0 should go to peer 2 - expect(sampler.getPeerForRequest(0)).toBe(peers[2]); - - // Request 1 should still go to peer 0 (only index 0 was failed) - expect(sampler.getPeerForRequest(1)).toBe(peers[0]); - }); - - it('samples new peer when all batch peers have failed for an index', () => { - mockRandomSampler.random.mockImplementation(() => 0); - - // 4 requests across 2 peers (peers[0] and peers[1]) - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 4, /* maxPeers */ 2); - expect(sampler.activePeerCount).toBe(2); - - // Mark both batch peers as failed for index 0 - sampler.markPeerFailedForIndex(peers[0], 0); - sampler.markPeerFailedForIndex(peers[1], 0); - - // Should sample a new peer (peers[2]) and return it - mockRandomSampler.random.mockImplementation(() => 2); - expect(sampler.getPeerForRequest(0)).toBe(peers[2]); - expect(sampler.activePeerCount).toBe(3); // New peer was added to batch - - // Other indices still work with original peers - expect(sampler.getPeerForRequest(1)).toBe(peers[0]); - expect(sampler.getPeerForRequest(2)).toBe(peers[1]); - }); - - it('returns undefined when all peers exhausted and no new peers available', () => { - mockRandomSampler.random.mockImplementation(() => 0); - - // 4 requests across 2 peers - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 4, /* maxPeers */ 2); - - // Mark both peers as failed for index 0 - sampler.markPeerFailedForIndex(peers[0], 0); - sampler.markPeerFailedForIndex(peers[1], 0); - - // No more peers available to sample - libp2p.getPeers.mockReturnValue([peers[0], peers[1]]); // Only return already-used peers - - // No peer available for index 0 - expect(sampler.getPeerForRequest(0)).toBeUndefined(); - }); - - it('failed peer-index tracking survives peer replacement', () => { - mockRandomSampler.random.mockImplementation(() => 0); - - // 4 requests across 2 peers - const sampler = new BatchConnectionSampler(connectionSampler, /* batchSize */ 4, /* maxPeers */ 2); - - // Mark peer 0 as failed for index 0 - sampler.markPeerFailedForIndex(peers[0], 0); - - // Request 0 now goes to peer 1 - expect(sampler.getPeerForRequest(0)).toBe(peers[1]); - - // Replace peer 0 with peer 2 - mockRandomSampler.random.mockImplementation(() => 2); - sampler.removePeerAndReplace(peers[0]); - - // Request 0 should still go to peer 1 (the replacement peer 2 is now in slot 0, - // but peer 0's failure record should not affect the new peer) - // Actually, the failure is tracked by peer ID, so peer 2 is a fresh peer - // Request 0's primary is now peer 2 (in slot 0), which hasn't failed - expect(sampler.getPeerForRequest(0)).toBe(peers[2]); - }); -}); diff --git a/yarn-project/p2p/src/services/reqresp/connection-sampler/batch_connection_sampler.ts b/yarn-project/p2p/src/services/reqresp/connection-sampler/batch_connection_sampler.ts deleted file mode 100644 index 42424551e696..000000000000 --- a/yarn-project/p2p/src/services/reqresp/connection-sampler/batch_connection_sampler.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { createLogger } from '@aztec/foundation/log'; - -import type { PeerId } from '@libp2p/interface'; - -import type { ConnectionSampler } from './connection_sampler.js'; - -/** - * Manages batches of peers for parallel request processing. - * Tracks active peers and provides deterministic peer assignment for requests. - * - * Example with 3 peers and 10 requests: - * - * Peers: [P1] [P2] [P3] - * ↓ ↓ ↓ ↓ ↓ ↓ ↓ ↓ ↓ ↓ - * Requests: 0,1,2,9 | 3,4,5 | 6,7,8 - * - * Each peer handles a bucket of consecutive requests. - * If a peer fails, it is replaced while maintaining the same bucket. - */ -export class BatchConnectionSampler { - private readonly batch: PeerId[] = []; - private readonly requestsPerPeer: number; - /** Tracks peer-index combinations that returned empty/invalid responses */ - private readonly failedPeerIndices: Map> = new Map(); - - constructor( - private readonly connectionSampler: ConnectionSampler, - batchSize: number, - maxPeers: number, - exclude?: PeerId[], - private readonly logger = createLogger('p2p:reqresp:batch-connection-sampler'), - ) { - if (maxPeers <= 0) { - throw new Error('Max peers cannot be 0'); - } - if (batchSize <= 0) { - throw new Error('Batch size cannot be 0'); - } - - // Calculate how many requests each peer should handle, cannot be 0 - this.requestsPerPeer = Math.max(1, Math.floor(batchSize / maxPeers)); - - // Sample initial peers - const excluding = exclude && new Map(exclude.map(peerId => [peerId.toString(), true] as const)); - this.batch = this.connectionSampler.samplePeersBatch(maxPeers, excluding); - } - - /** - * Gets the peer responsible for handling a specific request index. - * If the primary peer has previously failed for this index, tries other peers. - * If all batch peers have failed, attempts to sample a new peer. - * - * @param index - The request index - * @returns The peer assigned to handle this request, or undefined if no peer available - */ - getPeerForRequest(index: number): PeerId | undefined { - if (this.batch.length === 0) { - return undefined; - } - - // Calculate which peer bucket this index belongs to - const primaryPeerIndex = Math.floor(index / this.requestsPerPeer) % this.batch.length; - - // Try peers starting from primary, wrapping around - for (let offset = 0; offset < this.batch.length; offset++) { - const peerIndex = (primaryPeerIndex + offset) % this.batch.length; - const peer = this.batch[peerIndex]; - const peerKey = peer.toString(); - - const failedIndices = this.failedPeerIndices.get(peerKey); - if (!failedIndices || !failedIndices.has(index)) { - return peer; - } - } - - // All batch peers have failed for this index - try to sample a new peer - const newPeer = this.sampleNewPeer(); - if (newPeer) { - return newPeer; - } - - return undefined; - } - - /** - * Attempts to sample a new peer that isn't already in the batch. - * If successful, adds the peer to the batch. - * - * @returns The new peer if one was sampled, undefined otherwise - */ - private sampleNewPeer(): PeerId | undefined { - // Exclude all current batch peers - const excluding = new Map(this.batch.map(p => [p.toString(), true] as const)); - const newPeer = this.connectionSampler.getPeer(excluding); - - if (newPeer) { - this.batch.push(newPeer); - this.logger.trace('Sampled new peer for exhausted index', { newPeer: newPeer.toString() }); - return newPeer; - } - - return undefined; - } - - /** - * Marks that a peer returned an empty/invalid response for a specific request index. - * The peer will not be assigned this index again. - * - * @param peerId - The peer that failed - * @param index - The request index that failed - */ - markPeerFailedForIndex(peerId: PeerId, index: number): void { - const peerKey = peerId.toString(); - let failedIndices = this.failedPeerIndices.get(peerKey); - if (!failedIndices) { - failedIndices = new Set(); - this.failedPeerIndices.set(peerKey, failedIndices); - } - failedIndices.add(index); - this.logger.trace('Marked peer failed for index', { peerId: peerKey, index }); - } - - /** - * Removes a peer and replaces it with a new one, maintaining the same position - * in the batch array to keep request distribution consistent - * - * @param peerId - The peer to remove and replace - */ - removePeerAndReplace(peerId: PeerId): void { - const index = this.batch.findIndex(p => p === peerId); - if (index === -1) { - return; - } - - const excluding = new Map([[peerId.toString(), true]]); - const newPeer = this.connectionSampler.getPeer(excluding); // Q: Shouldn't we accumulate all excluded peers? Otherwise the sampler could return us a previously excluded peer? - - if (newPeer) { - this.batch[index] = newPeer; - this.logger.trace('Replaced peer', { peerId, newPeer }); - } else { - // If we couldn't get a replacement, remove the peer and compact the array - this.batch.splice(index, 1); - this.logger.trace('Removed peer', { peerId }); - } - } - - /** - * Gets the number of active peers - */ - get activePeerCount(): number { - return this.batch.length; - } - - /** - * Gets the number of requests each peer is assigned to handle - */ - get requestsPerBucket(): number { - return this.requestsPerPeer; - } -} diff --git a/yarn-project/p2p/src/services/reqresp/interface.ts b/yarn-project/p2p/src/services/reqresp/interface.ts index 016525a98919..6c64d1efd567 100644 --- a/yarn-project/p2p/src/services/reqresp/interface.ts +++ b/yarn-project/p2p/src/services/reqresp/interface.ts @@ -254,14 +254,6 @@ export interface ReqRespInterface { validator?: ReqRespSubProtocolValidators[ReqRespSubProtocol], ): Promise; stop(): Promise; - sendBatchRequest( - subProtocol: SubProtocol, - requests: InstanceType[], - pinnedPeer: PeerId | undefined, - timeoutMs?: number, - maxPeers?: number, - maxRetryAttempts?: number, - ): Promise[]>; sendRequestToPeer( peerId: PeerId, subProtocol: ReqRespSubProtocol, diff --git a/yarn-project/p2p/src/services/reqresp/reqresp.test.ts b/yarn-project/p2p/src/services/reqresp/reqresp.test.ts index 2ddf4d9a2cbe..e72926f21c31 100644 --- a/yarn-project/p2p/src/services/reqresp/reqresp.test.ts +++ b/yarn-project/p2p/src/services/reqresp/reqresp.test.ts @@ -1,4 +1,3 @@ -import { times } from '@aztec/foundation/collection'; import { sleep } from '@aztec/foundation/sleep'; import { PeerErrorSeverity } from '@aztec/stdlib/p2p'; import { mockTx } from '@aztec/stdlib/testing'; @@ -18,7 +17,7 @@ import { } from '../../test-helpers/reqresp-nodes.js'; import type { PeerManager } from '../peer-manager/peer_manager.js'; import type { PeerScoring } from '../peer-manager/peer_scoring.js'; -import { type ReqRespResponse, ReqRespSubProtocol, RequestableBuffer } from './interface.js'; +import { type ReqRespResponse, ReqRespSubProtocol } from './interface.js'; import { GoodByeReason, reqGoodbyeHandler } from './protocols/goodbye.js'; import { ReqRespStatus } from './status.js'; @@ -465,133 +464,6 @@ describe('ReqResp', () => { expectSuccess(txResp); }); }); - - describe('Batch requests', () => { - it('should send a batch request between many peers', async () => { - const batchSize = 9; - nodes = await createNodes(peerScoring, 3); - - await startNodes(nodes); - await sleep(500); - await connectToPeers(nodes); - await sleep(500); - - const sendRequestToPeerSpy = jest.spyOn(nodes[0].req, 'sendRequestToPeer'); - - const requests = Array.from({ length: batchSize }, _ => RequestableBuffer.fromBuffer(Buffer.from(`ping`))); - const expectResponses = Array.from({ length: batchSize }, _ => RequestableBuffer.fromBuffer(Buffer.from(`pong`))); - - const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); - expect(res).toEqual(expectResponses); - - // Expect one request to have been sent to each peer - expect(sendRequestToPeerSpy).toHaveBeenCalledTimes(batchSize); - expect(sendRequestToPeerSpy).toHaveBeenCalledWith( - expect.objectContaining({ - publicKey: nodes[1].p2p.peerId.publicKey, - }), - ReqRespSubProtocol.PING, - Buffer.from('ping'), - ); - expect(sendRequestToPeerSpy).toHaveBeenCalledWith( - expect.objectContaining({ - publicKey: nodes[2].p2p.peerId.publicKey, - }), - ReqRespSubProtocol.PING, - Buffer.from('ping'), - ); - }); - - it('should send a batch request with a pinned peer', async () => { - const batchSize = 9; - nodes = await createNodes(peerScoring, 4, { - // Bump rate limits so the pinned peer can respond - [ReqRespSubProtocol.PING]: { - peerLimit: { quotaTimeMs: 1000, quotaCount: 50 }, - globalLimit: { quotaTimeMs: 1000, quotaCount: 50 }, - }, - }); - - await startNodes(nodes); - await sleep(500); - await connectToPeers(nodes); - await sleep(500); - - const sendRequestToPeerSpy = jest.spyOn(nodes[0].req, 'sendRequestToPeer'); - - const requests = times(batchSize, i => RequestableBuffer.fromBuffer(Buffer.from(`ping${i}`))); - const expectResponses = times(batchSize, _ => RequestableBuffer.fromBuffer(Buffer.from(`pong`))); - - const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, nodes[1].p2p.peerId); - expect(res).toEqual(expectResponses); - - // Expect pinned peer to have received all requests - for (let i = 0; i < batchSize; i++) { - expect(sendRequestToPeerSpy).toHaveBeenCalledWith( - expect.objectContaining({ publicKey: nodes[1].p2p.peerId.publicKey }), - ReqRespSubProtocol.PING, - Buffer.from(`ping${i}`), - ); - } - - // Expect at least one request to have been sent to each other peer - expect(sendRequestToPeerSpy).toHaveBeenCalledWith( - expect.objectContaining({ publicKey: nodes[2].p2p.peerId.publicKey }), - ReqRespSubProtocol.PING, - expect.any(Buffer), - ); - - expect(sendRequestToPeerSpy).toHaveBeenCalledWith( - expect.objectContaining({ publicKey: nodes[3].p2p.peerId.publicKey }), - ReqRespSubProtocol.PING, - expect.any(Buffer), - ); - }); - - it('should stop after max retry attempts', async () => { - const batchSize = 12; - const failedIndices = [10, 11]; - nodes = await createNodes(peerScoring, 3); - - await startNodes(nodes); - await sleep(500); - await connectToPeers(nodes); - await sleep(500); - - const requests = Array.from({ length: batchSize }, (_, i) => - RequestableBuffer.fromBuffer(Buffer.from(`ping${i}`)), - ); - - // Mock sendRequestToPeer so that specific requests always fail with RATE_LIMIT_EXCEEDED, - // regardless of which peer they're sent to. This removes the timing dependency on the - // GCRA rate limiter leaking tokens between retries. - const originalSend = nodes[0].req.sendRequestToPeer.bind(nodes[0].req); - const sendSpy = jest - .spyOn(nodes[0].req, 'sendRequestToPeer') - .mockImplementation((peer: PeerId, protocol: ReqRespSubProtocol, buffer: Buffer) => { - const msg = buffer.toString(); - if (failedIndices.some(i => msg === `ping${i}`)) { - return Promise.resolve({ status: ReqRespStatus.RATE_LIMIT_EXCEEDED, data: Buffer.alloc(0) }); - } - return originalSend(peer, protocol, buffer); - }); - - const res = await nodes[0].req.sendBatchRequest(ReqRespSubProtocol.PING, requests, undefined); - - // 10 succeed, 2 permanently fail after all retry attempts are exhausted - const successes = res.filter(r => r !== undefined); - expect(successes).toHaveLength(batchSize - failedIndices.length); - expect(successes).toEqual( - times(batchSize - failedIndices.length, () => RequestableBuffer.fromBuffer(Buffer.from(`pong`))), - ); - - // Verify retries actually happened — those 2 requests were attempted more than once - const failedCalls = sendSpy.mock.calls.filter(([, , buf]) => - failedIndices.some(i => (buf as Buffer).toString() === `ping${i}`), - ); - expect(failedCalls.length).toBeGreaterThan(failedIndices.length); - }); - }); }); function expectSuccess(res: ReqRespResponse): asserts res is { status: ReqRespStatus.SUCCESS; data: Buffer } { diff --git a/yarn-project/p2p/src/services/reqresp/reqresp.ts b/yarn-project/p2p/src/services/reqresp/reqresp.ts index ba3fe8e518f5..c706feb14eb1 100644 --- a/yarn-project/p2p/src/services/reqresp/reqresp.ts +++ b/yarn-project/p2p/src/services/reqresp/reqresp.ts @@ -1,5 +1,4 @@ // @attribution: lodestar impl for inspiration -import { compactArray } from '@aztec/foundation/collection'; import { AbortError, TimeoutError } from '@aztec/foundation/error'; import { createLogger } from '@aztec/foundation/log'; import { executeTimeout } from '@aztec/foundation/timer'; @@ -11,11 +10,7 @@ import type { Libp2p } from 'libp2p'; import { pipeline } from 'node:stream/promises'; import type { Uint8ArrayList } from 'uint8arraylist'; -import { - CollectiveReqRespTimeoutError, - IndividualReqRespTimeoutError, - InvalidResponseError, -} from '../../errors/reqresp.error.js'; +import { IndividualReqRespTimeoutError } from '../../errors/reqresp.error.js'; import { OversizedSnappyResponseError, SnappyTransform } from '../encoding.js'; import type { PeerScoring } from '../peer-manager/peer_scoring.js'; import { @@ -23,7 +18,6 @@ import { DEFAULT_REQRESP_DIAL_TIMEOUT_MS, type P2PReqRespConfig, } from './config.js'; -import { BatchConnectionSampler } from './connection-sampler/batch_connection_sampler.js'; import { ConnectionSampler, RandomSampler } from './connection-sampler/connection_sampler.js'; import { DEFAULT_SUB_PROTOCOL_VALIDATORS, @@ -35,9 +29,7 @@ import { type ReqRespSubProtocolRateLimits, type ReqRespSubProtocolValidators, type ShouldRejectPeer, - type SubProtocolMap, UNAUTHENTICATED_ALLOWED_PROTOCOLS, - responseFromBuffer, subProtocolSizeCalculators, } from './interface.js'; import { ReqRespMetrics } from './metrics.js'; @@ -46,7 +38,7 @@ import { RequestResponseRateLimiter, prettyPrintRateLimitStatus, } from './rate-limiter/rate_limiter.js'; -import { ReqRespStatus, ReqRespStatusError, parseStatusChunk, prettyPrintReqRespStatus } from './status.js'; +import { ReqRespStatus, ReqRespStatusError, parseStatusChunk } from './status.js'; /** * The Request Response Service @@ -188,225 +180,6 @@ export class ReqResp implements ReqRespInterface { // NOTE: We assume libp2p instance is managed by the caller } - /** - * Request multiple messages over the same sub protocol, balancing the requests across peers. - * - * @devnote - * - The function prioritizes sending requests to free peers using a batch sampling strategy. - * - If a peer fails to respond or returns an invalid response, it is removed from the sampling pool and replaced. - * - The function stops retrying once all requests are processed, no active peers remain, or the maximum retry attempts are reached. - * - Responses are validated using a custom validator for the sub-protocol.* - * - * Requests are sent in parallel to each peer, but multiple requests are sent to the same peer in series - * - If a peer fails to respond or returns an invalid response, it is removed from the sampling pool and replaced. - * - The function stops retrying once all requests are processed, no active peers remain, or the maximum retry attempts are reached. - * - Responses are validated using a custom validator for the sub-protocol.* - * - * @param subProtocol - * @param requests - * @param timeoutMs - * @param maxPeers - * @returns - * - * @throws {CollectiveReqRespTimeoutError} - If the request batch exceeds the specified timeout (`timeoutMs`). - */ - @trackSpan( - 'ReqResp.sendBatchRequest', - (subProtocol: ReqRespSubProtocol, requests: InstanceType[]) => ({ - [Attributes.P2P_REQ_RESP_PROTOCOL]: subProtocol, - [Attributes.P2P_REQ_RESP_BATCH_REQUESTS_COUNT]: requests.length, - }), - ) - async sendBatchRequest( - subProtocol: SubProtocol, - requests: InstanceType[], - pinnedPeer: PeerId | undefined, - timeoutMs = 10000, - maxPeers = Math.max(10, Math.ceil(requests.length / 3)), - maxRetryAttempts = 3, - ): Promise[]> { - const responseValidator = this.subProtocolValidators[subProtocol] ?? DEFAULT_SUB_PROTOCOL_VALIDATORS[subProtocol]; - const responses: InstanceType[] = new Array(requests.length); - const requestBuffers = requests.map(req => req.toBuffer()); - const isEmptyResponse = (value: unknown): boolean => { - // Some responses serialize to a non-empty buffer even when they contain no items (e.g., empty TxArray). - if (!value || typeof value !== 'object') { - return false; - } - const length = (value as { length?: number }).length; - return typeof length === 'number' && length === 0; - }; - - const requestFunction = async (signal: AbortSignal) => { - // Track which requests still need to be processed - const pendingRequestIndices = new Set(requestBuffers.map((_, i) => i)); - - // Create batch sampler with the total number of requests and max peers - const batchSampler = new BatchConnectionSampler( - this.connectionSampler, - requests.length, - maxPeers, - compactArray([pinnedPeer]), // Exclude pinned peer from sampling, we will forcefully send all requests to it - createLogger(`${this.logger.module}:batch-connection-sampler`), - ); - - if (batchSampler.activePeerCount === 0 && !pinnedPeer) { - this.logger.warn('No active peers to send requests to'); - return []; - } - - // This is where it gets fun - // The outer loop is the retry loop, we will continue to retry until we process all indices we have - // not received a response for, or we have reached the max retry attempts - - // The inner loop is the batch loop, we will process all requests for each peer in parallel - // We will then process the results of the requests, and resample any peers that failed to respond - // We will continue to retry until we have processed all indices, or we have reached the max retry attempts - - let retryAttempts = 0; - while (pendingRequestIndices.size > 0 && batchSampler.activePeerCount > 0 && retryAttempts < maxRetryAttempts) { - if (signal.aborted) { - throw new AbortError('Batch request aborted'); - } - // Process requests in parallel for each available peer - type BatchEntry = { peerId: PeerId; indices: number[] }; - const requestBatches = new Map(); - - // Group requests by peer - for (const requestIndex of pendingRequestIndices) { - const peer = batchSampler.getPeerForRequest(requestIndex); - if (!peer) { - // No peer available for this specific index (all peers exhausted for it) - // Skip this index for now - it stays in pendingRequestIndices for retry - continue; - } - const peerAsString = peer.toString(); - if (!requestBatches.has(peerAsString)) { - requestBatches.set(peerAsString, { peerId: peer, indices: [] }); - } - requestBatches.get(peerAsString)!.indices.push(requestIndex); - } - - // If there is a pinned peer, we will always send every request to that peer - // We use the default limits for the subprotocol to avoid hitting the rate limiter - if (pinnedPeer) { - const limit = this.rateLimiter.getRateLimits(subProtocol).peerLimit.quotaCount; - requestBatches.set(pinnedPeer.toString(), { - peerId: pinnedPeer, - indices: Array.from(pendingRequestIndices.values()).slice(0, limit), - }); - } - - // If no requests could be assigned (all peers exhausted for all indices), exit early - if (requestBatches.size === 0) { - this.logger.warn('No peers available for any pending request indices, stopping batch request'); - break; - } - - // Make parallel requests for each peer's batch - // A batch entry will look something like this: - // PeerId0: [0, 1, 2, 3] - // PeerId1: [4, 5, 6, 7] - - // Peer Id 0 will send requests 0, 1, 2, 3 in serial - // while simultaneously Peer Id 1 will send requests 4, 5, 6, 7 in serial - - const batchResults = await Promise.all( - Array.from(requestBatches.entries()).map(async ([peerAsString, { peerId: peer, indices }]) => { - try { - const markIndexFailed = (index: number) => batchSampler.markPeerFailedForIndex(peer, index); - // Requests all going to the same peer are sent synchronously - const peerResults: { index: number; response: InstanceType }[] = - []; - let shouldReplacePeer = false; - const handleFailure = (status: ReqRespStatus, index: number) => { - this.logger.warn( - `Request to peer ${peerAsString} failed with status ${prettyPrintReqRespStatus(status)}`, - ); - markIndexFailed(index); - return status === ReqRespStatus.RATE_LIMIT_EXCEEDED; - }; - - for (const index of indices) { - this.logger.trace(`Sending request ${index} to peer ${peerAsString}`); - const response = await this.sendRequestToPeer(peer, subProtocol, requestBuffers[index]); - - // Check the status of the response buffer - if (response.status !== ReqRespStatus.SUCCESS) { - shouldReplacePeer = handleFailure(response.status, index); - if (shouldReplacePeer) { - break; - } - continue; - } - - if (response.data.length === 0) { - markIndexFailed(index); - continue; - } - - const object = responseFromBuffer(subProtocol, response.data); - if (isEmptyResponse(object)) { - markIndexFailed(index); - continue; - } - - const isValid = await responseValidator(requests[index], object, peer); - if (!isValid) { - markIndexFailed(index); - continue; - } - - peerResults.push({ index, response: object }); - } - - // If peer had a hard failure (rate limit), replace it for future iterations - if (shouldReplacePeer) { - this.logger.warn(`Peer ${peerAsString} hit a hard failure, removing from sampler`); - batchSampler.removePeerAndReplace(peer); - } - - return { peer, results: peerResults }; - } catch (error) { - this.logger.warn(`Failed batch request to peer ${peerAsString}:`, error); - batchSampler.removePeerAndReplace(peer); - return { peer, results: [] }; - } - }), - ); - - // Process results - for (const { results } of batchResults) { - for (const { index, response } of results) { - if (response) { - responses[index] = response; - pendingRequestIndices.delete(index); - } - } - } - - retryAttempts++; - } - - if (retryAttempts >= maxRetryAttempts) { - this.logger.warn(`Max retry attempts ${maxRetryAttempts} reached for batch request`); - } - - return responses; - }; - - try { - return await executeTimeout[]>( - requestFunction, - timeoutMs, - () => new CollectiveReqRespTimeoutError(), - ); - } catch (e: any) { - this.logger.warn(`${e.message} | subProtocol: ${subProtocol}`); - return []; - } - } - /** * Sends a request to a specific peer * @@ -785,12 +558,6 @@ export class ReqResp implements ReqRespInterface { return undefined; } - // We do not punish a collective timeout, as the node triggers this interupt, independent of the peer's behaviour - if (e instanceof CollectiveReqRespTimeoutError || e instanceof InvalidResponseError) { - this.logger.debug(`Non-punishable error in ${subProtocol}: ${e.message}`, logTags); - return undefined; - } - // Invalid status byte: the peer sent a status byte that doesn't match any known status code. // This is a protocol violation, penalize harshly. if (e instanceof ReqRespStatusError) { diff --git a/yarn-project/p2p/src/services/service.ts b/yarn-project/p2p/src/services/service.ts index e3b7590e83b1..127481ae39c4 100644 --- a/yarn-project/p2p/src/services/service.ts +++ b/yarn-project/p2p/src/services/service.ts @@ -21,7 +21,6 @@ import type { ReqRespSubProtocol, ReqRespSubProtocolHandler, ReqRespSubProtocolValidators, - SubProtocolMap, } from './reqresp/interface.js'; import type { AuthRequest, AuthResponse } from './reqresp/protocols/auth.js'; @@ -100,22 +99,6 @@ export interface P2PService { */ propagate(message: T): Promise; - /** - * Send a batch of requests to peers, and return the responses - * - * @param protocol - The request response protocol to use - * @param requests - The requests to send to the peers - * @returns The responses to the requests - */ - sendBatchRequest( - protocol: Protocol, - requests: InstanceType[], - pinnedPeerId?: PeerId, - timeoutMs?: number, - maxPeers?: number, - maxRetryAttempts?: number, - ): Promise[]>; - // Leaky abstraction: fix https://github.com/AztecProtocol/aztec-packages/issues/7963 registerBlockReceivedCallback(callback: P2PBlockReceivedCallback): void; diff --git a/yarn-project/p2p/src/test-helpers/mock-pubsub.ts b/yarn-project/p2p/src/test-helpers/mock-pubsub.ts index a537a25c5e35..cb06d1a8c0e8 100644 --- a/yarn-project/p2p/src/test-helpers/mock-pubsub.ts +++ b/yarn-project/p2p/src/test-helpers/mock-pubsub.ts @@ -23,15 +23,13 @@ import type { MemPools } from '../mem_pools/interface.js'; import { DummyPeerDiscoveryService, DummyPeerManager, LibP2PService } from '../services/index.js'; import type { P2PReqRespConfig } from '../services/reqresp/config.js'; import type { ConnectionSampler } from '../services/reqresp/connection-sampler/connection_sampler.js'; -import { - type ReqRespInterface, - type ReqRespResponse, - type ReqRespSubProtocol, - type ReqRespSubProtocolHandler, - type ReqRespSubProtocolHandlers, - type ReqRespSubProtocolValidators, - type SubProtocolMap, - responseFromBuffer, +import type { + ReqRespInterface, + ReqRespResponse, + ReqRespSubProtocol, + ReqRespSubProtocolHandler, + ReqRespSubProtocolHandlers, + ReqRespSubProtocolValidators, } from '../services/reqresp/interface.js'; import { ReqRespStatus } from '../services/reqresp/status.js'; import { GossipSubEvent } from '../types/index.js'; @@ -89,8 +87,8 @@ export function getMockPubSubP2PServiceFactory( /** * Mock implementation of ReqRespInterface that routes requests to other peers' handlers through the mock network. - * When a peer calls sendBatchRequest, the mock iterates over network peers and invokes their registered handler - * for the sub-protocol, simulating the request-response protocol without actual libp2p streams. + * When a peer calls sendRequestToPeer, the mock looks up the target peer's registered handler for the + * sub-protocol and invokes it, simulating the request-response protocol without actual libp2p streams. */ class MockReqResp implements ReqRespInterface { private handlers: Partial = {}; @@ -132,46 +130,6 @@ class MockReqResp implements ReqRespInterface { return this.handlers[subProtocol]; } - async sendBatchRequest( - subProtocol: SubProtocol, - requests: InstanceType[], - pinnedPeer: PeerId | undefined, - _timeoutMs?: number, - _maxPeers?: number, - _maxRetryAttempts?: number, - ): Promise[]> { - const responses: InstanceType[] = []; - const peers = this.network.getReqRespPeers().filter(p => !p.peerId.equals(this.peerId)); - const targetPeers = pinnedPeer ? peers.filter(p => p.peerId.equals(pinnedPeer)) : peers; - const delayMs = this.network.getPropagationDelayMs(); - - if (delayMs > 0) { - await sleep(delayMs); - } - - for (const request of requests) { - const requestBuffer = request.toBuffer(); - for (const peer of targetPeers) { - const handler = peer.getHandler(subProtocol); - if (!handler) { - continue; - } - try { - const responseBuffer = await handler(this.peerId, requestBuffer); - if (responseBuffer.length > 0) { - const response = responseFromBuffer(subProtocol, responseBuffer); - responses.push(response as InstanceType); - break; - } - } catch (err) { - this.logger.debug(`Mock reqresp handler error from peer ${peer.peerId}`, { err }); - } - } - } - - return responses; - } - async sendRequestToPeer( peerId: PeerId, subProtocol: ReqRespSubProtocol, diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts index df2686e844f3..adf11f85dced 100644 --- a/yarn-project/telemetry-client/src/attributes.ts +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -96,7 +96,6 @@ export const VALIDATOR_STATUS = 'aztec.validator_status'; export const P2P_ID = 'aztec.p2p.id'; export const P2P_REQ_RESP_PROTOCOL = 'aztec.p2p.req_resp.protocol'; -export const P2P_REQ_RESP_BATCH_REQUESTS_COUNT = 'aztec.p2p.req_resp.batch_requests_count'; /** The state of a peer (Healthy, Disconnect, Banned) */ export const P2P_PEER_SCORE_STATE = 'aztec.p2p.peer_score_state'; export const POOL_NAME = 'aztec.pool.name'; diff --git a/yarn-project/telemetry-client/src/telemetry.ts b/yarn-project/telemetry-client/src/telemetry.ts index 5e304b61619c..25e1fd07149c 100644 --- a/yarn-project/telemetry-client/src/telemetry.ts +++ b/yarn-project/telemetry-client/src/telemetry.ts @@ -48,7 +48,6 @@ type BannedMetricAttributeNames = (typeof Attributes)[ | 'TX_HASH' | 'PROVING_JOB_ID' | 'P2P_ID' - | 'P2P_REQ_RESP_BATCH_REQUESTS_COUNT' | 'TARGET_ADDRESS' | 'MANA_USED' | 'TOTAL_INSTRUCTIONS']; From 6eedaa5e6c78e38e2723ea98b04ad74752957c00 Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 14 May 2026 12:21:59 -0300 Subject: [PATCH 07/17] chore(p2p): remove proposal_tx_collector leftovers (#23276) ProposalTxCollector doesn't exist anymore. Clean up unused files. Rename bench that is now only testing BatchTxRequester. --- yarn-project/bootstrap.sh | 2 +- ...client.batch_tx_requester.bench.README.md} | 76 ++-- ...p_client.batch_tx_requester.bench.test.ts} | 8 +- .../proposal_tx_collector_worker.ts | 345 ------------------ .../proposal_tx_collector_worker_protocol.ts | 40 -- .../p2p/src/services/reqresp/README.md | 14 - .../p2p/src/services/reqresp/reqresp.ts | 12 +- 7 files changed, 35 insertions(+), 462 deletions(-) rename yarn-project/p2p/src/client/test/{tx_proposal_collector/README.md => p2p_client.batch_tx_requester.bench.README.md} (71%) rename yarn-project/p2p/src/client/test/{tx_proposal_collector/p2p_client.proposal_tx_collector.bench.test.ts => p2p_client.batch_tx_requester.bench.test.ts} (96%) delete mode 100644 yarn-project/p2p/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts delete mode 100644 yarn-project/p2p/src/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.ts diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index d263e662de9c..5bdb3de38595 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -245,7 +245,7 @@ function bench_cmds { echo "$hash BENCH_OUTPUT=bench-out/kv_store.bench.json yarn-project/scripts/run_test.sh kv-store/src/bench/map_bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/tx_pool_v2.bench.json yarn-project/scripts/run_test.sh p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/tx_validator.bench.json yarn-project/scripts/run_test.sh p2p/src/msg_validators/tx_validator/tx_validator_bench.test.ts" - echo "$hash:ISOLATE=1:CPUS=16:MEM=32g:TIMEOUT=1800 BENCH_OUTPUT=bench-out/p2p_client_proposal_tx_collector.bench.json yarn-project/scripts/run_test.sh p2p/src/client/test/tx_proposal_collector/p2p_client.proposal_tx_collector.bench.test.ts" + echo "$hash:ISOLATE=1:CPUS=16:MEM=32g:TIMEOUT=1800 BENCH_OUTPUT=bench-out/p2p_client_batch_tx_requester.bench.json yarn-project/scripts/run_test.sh p2p/src/client/test/p2p_client.batch_tx_requester.bench.test.ts" echo "$hash BENCH_OUTPUT=bench-out/tx.bench.json yarn-project/scripts/run_test.sh stdlib/src/tx/tx_bench.test.ts" echo "$hash:ISOLATE=1:CPUS=10:MEM=16g:LOG_LEVEL=silent BENCH_OUTPUT=bench-out/proving_broker.bench.json yarn-project/scripts/run_test.sh prover-client/src/test/proving_broker_testbench.test.ts" echo "$hash:ISOLATE=1:CPUS=16:MEM=16g BENCH_OUTPUT=bench-out/avm_bulk_test.bench.json yarn-project/scripts/run_test.sh bb-prover/src/avm_proving_tests/avm_bulk.test.ts" diff --git a/yarn-project/p2p/src/client/test/tx_proposal_collector/README.md b/yarn-project/p2p/src/client/test/p2p_client.batch_tx_requester.bench.README.md similarity index 71% rename from yarn-project/p2p/src/client/test/tx_proposal_collector/README.md rename to yarn-project/p2p/src/client/test/p2p_client.batch_tx_requester.bench.README.md index 3a489503faab..50867738fbb6 100644 --- a/yarn-project/p2p/src/client/test/tx_proposal_collector/README.md +++ b/yarn-project/p2p/src/client/test/p2p_client.batch_tx_requester.bench.README.md @@ -1,6 +1,6 @@ -# ProposalTxCollector Benchmarks +# BatchTxRequester Benchmarks -This benchmark suite measures **how quickly a proposer node can fetch missing transactions from P2P peers** when building a block proposal. It compares two alternative transaction-collection implementations under several controlled "who-has-which-txs" distributions. +This benchmark suite measures **how quickly a proposer node can fetch missing transactions from P2P peers** when building a block proposal under several controlled "who-has-which-txs" distributions. ## Purpose @@ -10,12 +10,6 @@ This benchmark answers: - How long does it take to fetch **N missing txs** (N ∈ **{10, 50, 100, 500}**)? - How do different **peer availability patterns** affect performance? -- Which collector strategy performs better under each pattern? - -The suite compares two collectors: - -- **`BatchTxRequesterCollector`** (collector type: `batch-requester`) -- **`SendBatchRequestCollector`** (collector type: `send-batch-request`) ## Architecture @@ -24,7 +18,7 @@ The benchmark runs a small simulated network on localhost: ``` ┌─────────────────────────────────────────────────────────────────────┐ │ Test Process (Driver) │ -│ p2p_client.proposal_tx_collector.bench.test.ts │ +│ p2p_client.batch_tx_requester.bench.test.ts │ │ ┌─────────────────────────────────────────────────────────────┐ │ │ │ WorkerClientManager │ │ │ │ (src/testbench/worker_client_manager.ts) │ │ @@ -34,7 +28,7 @@ The benchmark runs a small simulated network on localhost: │ ▼ ▼ ▼ │ │ ┌───────────┐ ┌───────────┐ ┌───────────┐ │ │ │ Worker 0 │◄──────►│ Worker 1 │◄──────►│ Worker N-1│ │ -│ │ (Collector│ P2P │(Responder)│ P2P │(Responder)│ │ +│ │(Aggregator│ P2P │(Responder)│ P2P │(Responder)│ │ │ │ Node) │ │ │ │ │ │ │ │ TxPool:[] │ │ TxPool: │ │ TxPool: │ │ │ │ │ │ [txs...] │ │ [txs...] │ │ @@ -54,12 +48,12 @@ Using separate OS processes makes the setup closer to real networking behavior ( The network is intentionally asymmetric: -- **Worker 0 is the collector/proposer node** +- **Worker 0 is the aggregator/proposer node** - Starts with an **empty tx pool** (`[]`) - - Is the only worker instructed to run the collector for each `BENCH_REQRESP` command + - Is the only worker instructed to run `BatchTxRequester` for each `BENCH_REQRESP` command - **Workers 1..N-1 are responder peers** - Locally generate and filter txs according to the distribution pattern - - Respond to req/resp queries made by Worker 0's collector + - Respond to req/resp queries made by Worker 0's `BatchTxRequester` This models a proposer that has only `txHashes` in a proposal and must fetch the full tx bodies from the network. @@ -72,7 +66,7 @@ Each benchmark case generates `missingTxCount` mock txs and assigns them to peer **Every responder peer has every transaction.** - Simulates the best-case: high replication / high gossip success -- Expectation: collector should quickly succeed; differences mostly reflect collector overhead and batching strategy +- Expectation: the requester should quickly succeed; differences mostly reflect requester overhead and batching strategy ### `sparse` @@ -81,7 +75,7 @@ Each benchmark case generates `missingTxCount` mock txs and assigns them to peer Each responder is bucketed and holds txs whose index falls into its bucket or the "next" bucket (striped by tx index). - Simulates partial propagation, churn, or uneven mempool convergence -- Expectation: collector must query multiple peers and cope with "misses" +- Expectation: the requester must query multiple peers and cope with "misses" ### `pinned-only` @@ -92,33 +86,13 @@ Each responder is bucketed and holds txs whose index falls into its bucket or th > **Guardrail:** the pinned peer index must be within `(0, numberOfPeers)` (Worker 0 cannot be pinned). -## Collectors Under Test - -### `BatchTxRequesterCollector` (`batch-requester`) - -```typescript -new BatchTxRequesterCollector(p2pService, logger, new DateProvider()) -``` - -Uses the P2P service plus internal logic to fetch missing txs, coordinating requests in a batched or staged way. - -### `SendBatchRequestCollector` (`send-batch-request`) - -```typescript -const maxPeers = 10; -const maxRetryAttempts = Math.max(peerIds.length, 3); -new SendBatchRequestCollector(p2pService, maxPeers, maxRetryAttempts) -``` - -Explicitly caps the number of peers it will involve (`maxPeers`) and uses a retry budget derived from peer count. - ## Test Parameters | Parameter | Value | Description | |-----------|-------|-------------| | `PEERS_PER_RUN` | 30 | Number of worker processes spawned | | `MISSING_TX_COUNTS` | 10, 50, 100, 500 | Number of missing transactions to fetch | -| `TIMEOUT_MS` | 30,000 ms | Collector timeout per case | +| `TIMEOUT_MS` | 30,000 ms | Per-case timeout for the requester | | `TEST_TIMEOUT_MS` | 600,000 ms | Overall Jest timeout (10 minutes) | ## Running @@ -127,13 +101,13 @@ From the p2p package: ```bash cd yarn-project/p2p -yarn test src/client/test/tx_proposal_collector/p2p_client.proposal_tx_collector.bench.test.ts +yarn test src/client/test/p2p_client.batch_tx_requester.bench.test.ts ``` Or from repo root: ```bash -yarn test p2p_client.proposal_tx_collector.bench.test.ts +yarn test p2p_client.batch_tx_requester.bench.test.ts ``` The benchmark is intentionally long due to spawning many processes and running multiple cases. @@ -145,14 +119,12 @@ The benchmark is intentionally long due to spawning many processes and running m If no env vars are set, the suite prints a table: ``` -| Collector | Distribution | Missing | Duration (ms) | Fetched | Success | -|---------------------|--------------|---------|---------------|---------|---------| -| batch-requester | pinned-only | 10 | 123 | 10 | Yes | -| send-batch-request | pinned-only | 10 | 145 | 10 | Yes | +| Distribution | Missing | Duration (ms) | Fetched | Success | +|--------------|---------|---------------|---------|---------| +| pinned-only | 10 | 123 | 10 | Yes | +| pinned-only | 50 | 145 | 50 | Yes | ``` -Plus a comparison summary stating which collector was faster per `(distribution, missing)` pair. - ### JSON metrics (for CI/dashboards) ```bash @@ -160,8 +132,8 @@ BENCH_OUTPUT=/path/results.json yarn test ... ``` Writes JSON metrics like: -- `ProposalTxCollector///missing_/duration` (ms) -- `ProposalTxCollector///missing_/fetched` (txs) +- `BatchTxRequester//missing_/duration` (ms) +- `BatchTxRequester//missing_/fetched` (txs) ### Markdown file output @@ -175,14 +147,14 @@ Writes the pretty table + summary to disk. For each case the benchmark records: -- `durationMs`: wall-clock time spent inside the collector call -- `fetchedCount`: how many txs were returned by the collector +- `durationMs`: wall-clock time spent inside the requester call +- `fetchedCount`: how many txs were returned by the requester - `success`: `fetchedCount === missingTxCount` **Guidelines:** - **Always check `Success` first.** A faster run that fetched fewer txs is not a win. -- Compare collectors **within the same distribution + missing count** only. +- Compare runs **within the same distribution + missing count** only. - Expect `pinned-only` to highlight pinned-peer behavior (fast if pinned peer is used effectively; slow if the algorithm wastes time sampling other peers). - Expect `sparse` to be the most "network-like" stress case, since many peers won't have each requested tx. @@ -193,7 +165,7 @@ Inside each worker, the benchmark intentionally reduces variability: - **Unlimited rate limits** are installed so the req/resp rate limiter doesn't dominate results - **Deterministic tx generation** ensures all workers see the same tx set without large IPC payloads -This makes the benchmark better for *comparing collectors* (A vs B), but it is **not** a perfect model of production networking conditions. +This makes the benchmark better for tracking regressions, but it is **not** a perfect model of production networking conditions. ## Limitations @@ -207,9 +179,7 @@ This benchmark does **not** measure: | File | Purpose | |------|---------| -| `p2p_client.proposal_tx_collector.bench.test.ts` | Test suite (cases, distributions, output formatting) | -| `proposal_tx_collector_worker.ts` | Collector-specific worker implementation | -| `proposal_tx_collector_worker_protocol.ts` | IPC message types and serialization | +| `p2p_client.batch_tx_requester.bench.test.ts` | Test suite (cases, distributions, output formatting) | | `src/testbench/worker_client_manager.ts` | Worker process manager (forking, IPC, orchestration) | | `src/testbench/p2p_client_testbench_worker.ts` | General testbench worker implementation | | `src/test-helpers/testbench-utils.ts` | Shared mocks and utilities (InMemoryTxPool, InMemoryAttestationPool, etc.) | diff --git a/yarn-project/p2p/src/client/test/tx_proposal_collector/p2p_client.proposal_tx_collector.bench.test.ts b/yarn-project/p2p/src/client/test/p2p_client.batch_tx_requester.bench.test.ts similarity index 96% rename from yarn-project/p2p/src/client/test/tx_proposal_collector/p2p_client.proposal_tx_collector.bench.test.ts rename to yarn-project/p2p/src/client/test/p2p_client.batch_tx_requester.bench.test.ts index 148783fbd1ed..d14db02583a7 100644 --- a/yarn-project/p2p/src/client/test/tx_proposal_collector/p2p_client.proposal_tx_collector.bench.test.ts +++ b/yarn-project/p2p/src/client/test/p2p_client.batch_tx_requester.bench.test.ts @@ -9,7 +9,7 @@ import { type DistributionPattern, WorkerClientManager, testChainConfig, -} from '../../../testbench/worker_client_manager.js'; +} from '../../testbench/worker_client_manager.js'; const TEST_TIMEOUT_MS = 600_000; // 10 minutes jest.setTimeout(TEST_TIMEOUT_MS); @@ -75,7 +75,7 @@ const CASES: readonly BenchmarkCase[] = BASE_SCENARIOS.flatMap(base => })), ); -describe('ProposalTxCollector Benchmarks', () => { +describe('BatchTxRequester Benchmarks', () => { const results: BenchmarkResult[] = []; let logger: Logger; @@ -181,7 +181,7 @@ function toPrettyString(benchResults: BenchmarkResult[]): string { lines.push(''); lines.push('='.repeat(80)); - lines.push('ProposalTxCollector Benchmark Results'); + lines.push('BatchTxRequester Benchmark Results'); lines.push('='.repeat(80)); lines.push(''); lines.push('| Distribution | Missing | Duration (ms) | Fetched | Success |'); @@ -212,7 +212,7 @@ function toBenchmarkJSON(benchResults: BenchmarkResult[], indent = 2): string { const metrics: JsonBenchmarkResult[] = []; for (const result of benchResults) { - const baseName = `ProposalTxCollector/${result.distribution}/missing_${result.missingTxCount}`; + const baseName = `BatchTxRequester/${result.distribution}/missing_${result.missingTxCount}`; metrics.push( { name: `${baseName}/duration`, diff --git a/yarn-project/p2p/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts b/yarn-project/p2p/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts deleted file mode 100644 index ae8121da7d8d..000000000000 --- a/yarn-project/p2p/src/client/test/tx_proposal_collector/proposal_tx_collector_worker.ts +++ /dev/null @@ -1,345 +0,0 @@ -import { MockL2BlockSource } from '@aztec/archiver/test'; -import { SecretValue } from '@aztec/foundation/config'; -import { createLogger } from '@aztec/foundation/log'; -import { sleep } from '@aztec/foundation/sleep'; -import { DateProvider, Timer, executeTimeout } from '@aztec/foundation/timer'; -import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; -import type { L2BlockSource } from '@aztec/stdlib/block'; -import type { ContractDataSource } from '@aztec/stdlib/contract'; -import { GasFees } from '@aztec/stdlib/gas'; -import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/server'; -import type { DataStoreConfig } from '@aztec/stdlib/kv-store'; -import { PeerErrorSeverity } from '@aztec/stdlib/p2p'; -import type { Tx, TxValidationResult } from '@aztec/stdlib/tx'; -import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client'; - -import type { PeerId } from '@libp2p/interface'; -import { peerIdFromString } from '@libp2p/peer-id'; - -import type { P2PConfig } from '../../../config.js'; -import { BatchTxRequester } from '../../../services/reqresp/batch-tx-requester/batch_tx_requester.js'; -import type { IBatchRequestTxValidator } from '../../../services/reqresp/batch-tx-requester/tx_validator.js'; -import { RateLimitStatus } from '../../../services/reqresp/rate-limiter/rate_limiter.js'; -import { RequestTracker } from '../../../services/tx_collection/request_tracker.js'; -import { - AlwaysTrueCircuitVerifier, - BENCHMARK_CONSTANTS, - InMemoryAttestationPool, - InMemoryTxPool, - UNLIMITED_RATE_LIMIT_QUOTA, - calculateInternalTimeout, - createMockEpochCache, - createMockWorldStateSynchronizer, -} from '../../../test-helpers/index.js'; -import { createP2PClient } from '../../index.js'; -import type { P2PClient } from '../../p2p_client.js'; -import { - type WorkerCommand, - type WorkerResponse, - deserializeBlockProposal, - deserializeTx, - deserializeTxHash, -} from './proposal_tx_collector_worker_protocol.js'; - -let client: P2PClient | undefined; -let txPool: InMemoryTxPool | undefined; -let attestationPool: InMemoryAttestationPool | undefined; -let logger = createLogger('p2p:proposal-bench'); -let kvStore: Awaited> | undefined; -let ipcDisconnected = false; - -function ensureClient(): P2PClient { - if (!client || !txPool) { - throw new Error('Worker client not started'); - } - return client; -} - -function isIpcDisconnectError(err: unknown): boolean { - const code = (err as NodeJS.ErrnoException | undefined)?.code; - return code === 'EPIPE' || code === 'ERR_IPC_CHANNEL_CLOSED'; -} - -function sendMessage(message: WorkerResponse): Promise { - const send = process.send; - if (!send || !process.connected || ipcDisconnected) { - return Promise.resolve(); - } - - return new Promise(resolve => { - const fallbackTimeout = setTimeout(() => resolve(), 2000); - try { - send.call(process, message, undefined, undefined, err => { - clearTimeout(fallbackTimeout); - if (!err) { - resolve(); - return; - } - if (isIpcDisconnectError(err)) { - ipcDisconnected = true; - resolve(); - return; - } - logger.warn('Failed to send IPC message', { error: err?.message ?? String(err) }); - resolve(); - }); - } catch (err: any) { - clearTimeout(fallbackTimeout); - if (isIpcDisconnectError(err)) { - ipcDisconnected = true; - resolve(); - return; - } - logger.warn('Failed to send IPC message', { error: err?.message ?? String(err) }); - resolve(); - } - }); -} - -async function startClient(config: P2PConfig, clientIndex: number) { - txPool = new InMemoryTxPool(); - attestationPool = new InMemoryAttestationPool(); - const epochCache = createMockEpochCache(); - const worldState = createMockWorldStateSynchronizer(); - const l2BlockSource = new MockL2BlockSource(); - const proofVerifier = new AlwaysTrueCircuitVerifier(); - kvStore = await openTmpStore(`proposal-bench-${clientIndex}`, true, BENCHMARK_CONSTANTS.KV_STORE_MAP_SIZE_KB); - logger = createLogger(`p2p:proposal-bench:${clientIndex}`); - - const telemetry = getTelemetryClient(); - const deps = { - txPool, - attestationPool, - store: kvStore, - logger, - }; - - client = await createP2PClient( - config as P2PConfig & DataStoreConfig, - l2BlockSource as L2BlockSource & ContractDataSource, - proofVerifier as ClientProtocolCircuitVerifier, - worldState, - epochCache, - { getCurrentMinFees: () => Promise.resolve(GasFees.empty()) }, - 'proposal-tx-collector-bench-worker', - new DateProvider(), - telemetry as TelemetryClient, - deps, - await l2BlockSource.getInitialHeader().hash(), - ); - - await client.start(); - installUnlimitedRateLimits(); - - for (let i = 0; i < 120; i++) { - if (client.isReady()) { - return; - } - await sleep(500); - } - - throw new Error('Timed out waiting for P2P client readiness'); -} - -function installSamplerOverrides(peerList: ReturnType[]) { - const reqResp = (ensureClient() as any).p2pService.reqresp as any; - const sampler = reqResp.connectionSampler as any; - - sampler.getPeerListSortedByConnectionCountAsc = (excluding?: Set) => { - if (!excluding || excluding.size === 0) { - return peerList; - } - return peerList.filter(peerId => !excluding.has(peerId.toString())); - }; - sampler.samplePeersBatch = (numberToSample: number, excluding?: Map) => { - const filtered = peerList.filter(peerId => !excluding?.has(peerId.toString())); - return filtered.slice(0, Math.min(numberToSample, filtered.length)); - }; - sampler.getPeer = (excluding?: Map) => { - const filtered = peerList.filter(peerId => !excluding?.has(peerId.toString())); - return filtered[0]; - }; -} - -function installUnlimitedRateLimits() { - const reqResp = (ensureClient() as any).p2pService.reqresp as any; - const rateLimiter = reqResp.rateLimiter as any; - - rateLimiter.getRateLimits = () => UNLIMITED_RATE_LIMIT_QUOTA; - rateLimiter.allow = () => RateLimitStatus.Allowed; -} - -async function runCollector(cmd: Extract) { - const { txHashes, blockProposal, pinnedPeerId, peerIds, timeoutMs } = cmd; - const reqResp = (ensureClient() as any).p2pService.reqresp as any; - const peerList = peerIds.map(peerId => peerIdFromString(peerId)); - - installSamplerOverrides(peerList); - installUnlimitedRateLimits(); - - const p2pService = { - reqResp, - connectionSampler: { - getPeerListSortedByConnectionCountAsc: () => peerList, - }, - txValidatorConfig: { - l1ChainId: 1, - rollupVersion: 1, - proofVerifier: { - verifyProof: () => Promise.resolve({ valid: true, durationMs: 0, totalDurationMs: 0 }), - stop: () => Promise.resolve(), - }, - }, - peerScoring: { - penalizePeer: (_peerId: PeerId, _penalty: PeerErrorSeverity) => {}, - }, - }; - - const parsedTxHashes = txHashes.map(deserializeTxHash); - const parsedProposal = deserializeBlockProposal(blockProposal); - const pinnedPeer = pinnedPeerId ? peerIdFromString(pinnedPeerId) : undefined; - - const timer = new Timer(); - let fetchedCount = 0; - - const internalTimeoutMs = calculateInternalTimeout(timeoutMs); - - const noopTxValidator: IBatchRequestTxValidator = { - validateRequestedTx: (_tx: Tx): Promise => Promise.resolve({ result: 'valid' }), - validateRequestedTxs: (txs: Tx[]): Promise => - Promise.resolve(txs.map(() => ({ result: 'valid' }))), - }; - - try { - const fetched = await executeTimeout( - (_signal: AbortSignal) => { - const tracker = RequestTracker.create(parsedTxHashes, new Date(Date.now() + internalTimeoutMs)); - const batchRequester = new BatchTxRequester( - tracker, - parsedProposal, - pinnedPeer, - p2pService, - logger, - new DateProvider(), - { txValidator: noopTxValidator }, - ); - return BatchTxRequester.collectAllTxs(batchRequester.run()); - }, - timeoutMs, - () => new Error(`Collector timed out after ${timeoutMs}ms`), - ); - fetchedCount = fetched.length; - } catch (err: any) { - logger.warn(`Collector error: ${err?.message ?? String(err)}`); - } - - return { durationMs: timer.ms(), fetchedCount }; -} - -async function stopClient() { - if (!client) { - return; - } - await client.stop(); - if (kvStore?.close) { - await kvStore.close(); - } - client = undefined; - txPool = undefined; - attestationPool = undefined; -} - -function gracefulExit(code: number = 0) { - try { - if (process.connected) { - process.disconnect(); - } - } catch { - // IPC channel already closed - } - setTimeout(() => process.exit(code), 5000).unref(); -} - -process.on('disconnect', () => { - ipcDisconnected = true; - void stopClient(); -}); - -process.on('error', err => { - if (isIpcDisconnectError(err)) { - ipcDisconnected = true; - return; - } - logger.warn('Worker process error', { error: err?.message ?? String(err) }); -}); - -process.on('message', (msg: WorkerCommand) => { - void (async () => { - if (!msg || typeof msg !== 'object') { - return; - } - - const requestId = msg.requestId; - - try { - switch (msg.type) { - case 'START': { - const rawConfig = msg.config; - const config: P2PConfig = { - ...rawConfig, - peerIdPrivateKey: rawConfig.peerIdPrivateKey ? new SecretValue(rawConfig.peerIdPrivateKey) : undefined, - } as P2PConfig; - - await startClient(config, msg.clientIndex); - const peerId = (ensureClient() as any).p2pService.node.peerId.toString(); - await sendMessage({ type: 'READY', requestId, peerId }); - break; - } - case 'SET_TXS': { - if (!txPool) { - throw new Error('Tx pool not initialized'); - } - const txs = msg.txs.map(deserializeTx); - const count = msg.mode === 'append' ? txPool.appendTxs(txs) : txPool.setTxs(txs); - await sendMessage({ type: 'TXS_SET', requestId, count }); - break; - } - case 'SET_BLOCK_PROPOSAL': { - if (!attestationPool) { - throw new Error('Attestation pool not initialized'); - } - const proposal = deserializeBlockProposal(msg.blockProposal); - await attestationPool.tryAddBlockProposal(proposal); - await sendMessage({ type: 'BLOCK_PROPOSAL_SET', requestId, archiveRoot: proposal.archive.toString() }); - break; - } - case 'RUN_COLLECTOR': { - const { durationMs, fetchedCount } = await runCollector(msg); - await sendMessage({ type: 'COLLECTOR_RESULT', requestId, durationMs, fetchedCount }); - break; - } - case 'GET_PEER_COUNT': { - const peers = await ensureClient().getPeers(); - await sendMessage({ type: 'PEER_COUNT', requestId, count: peers.length }); - break; - } - case 'STOP': { - await stopClient(); - await sendMessage({ type: 'STOPPED', requestId }); - gracefulExit(0); - break; - } - default: { - const _exhaustive: never = msg; - throw new Error(`Unknown command: ${(msg as { type?: string }).type}`); - } - } - } catch (err: any) { - await sendMessage({ type: 'ERROR', requestId, error: err?.message ?? String(err) }); - if (msg.type === 'START') { - await stopClient(); - gracefulExit(1); - } - } - })(); -}); diff --git a/yarn-project/p2p/src/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.ts b/yarn-project/p2p/src/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.ts deleted file mode 100644 index 9db03cdcfb7d..000000000000 --- a/yarn-project/p2p/src/client/test/tx_proposal_collector/proposal_tx_collector_worker_protocol.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { BlockProposal } from '@aztec/stdlib/p2p'; -import { Tx, TxHash } from '@aztec/stdlib/tx'; - -import type { P2PConfig } from '../../../config.js'; - -export type SerializedP2PConfig = Omit & { peerIdPrivateKey?: string }; - -export type WorkerCommand = - | { type: 'START'; requestId: string; clientIndex: number; config: SerializedP2PConfig } - | { type: 'SET_TXS'; requestId: string; txs: string[]; mode?: 'replace' | 'append' } - | { type: 'SET_BLOCK_PROPOSAL'; requestId: string; blockProposal: string } - | { - type: 'RUN_COLLECTOR'; - requestId: string; - txHashes: string[]; - blockProposal: string; - pinnedPeerId?: string; - peerIds: string[]; - timeoutMs: number; - } - | { type: 'GET_PEER_COUNT'; requestId: string } - | { type: 'STOP'; requestId: string }; - -export type WorkerResponse = - | { type: 'READY'; requestId: string; peerId: string } - | { type: 'TXS_SET'; requestId: string; count: number } - | { type: 'BLOCK_PROPOSAL_SET'; requestId: string; archiveRoot: string } - | { type: 'COLLECTOR_RESULT'; requestId: string; durationMs: number; fetchedCount: number } - | { type: 'PEER_COUNT'; requestId: string; count: number } - | { type: 'STOPPED'; requestId: string } - | { type: 'ERROR'; requestId: string; error: string }; - -export const serializeTx = (tx: Tx) => tx.toBuffer().toString('hex'); -export const deserializeTx = (hex: string) => Tx.fromBuffer(Buffer.from(hex, 'hex')); - -export const serializeTxHash = (txHash: TxHash) => txHash.toString(); -export const deserializeTxHash = (hex: string) => TxHash.fromString(hex); - -export const serializeBlockProposal = (proposal: BlockProposal) => proposal.toBuffer().toString('hex'); -export const deserializeBlockProposal = (hex: string) => BlockProposal.fromBuffer(Buffer.from(hex, 'hex')); diff --git a/yarn-project/p2p/src/services/reqresp/README.md b/yarn-project/p2p/src/services/reqresp/README.md index 982e00a28e74..fcd67f06899b 100644 --- a/yarn-project/p2p/src/services/reqresp/README.md +++ b/yarn-project/p2p/src/services/reqresp/README.md @@ -46,7 +46,6 @@ Per-protocol size limits checked via preamble before decompression. | Error Type | Severity | |------------|----------| | GOODBYE subprotocol errors | None | -| `CollectiveReqRespTimeoutError` / `InvalidResponseError` | None | | `AbortError` / connection close / muxer closed | None | | `ECONNRESET` / `EPIPE` / `ECONNREFUSED` / `ERR_UNEXPECTED_EOF` | HighToleranceError | | `ERR_UNSUPPORTED_PROTOCOL` | HighToleranceError | @@ -183,19 +182,6 @@ Protected peers (private/trusted/preferred) are always considered "authenticated Conditional registration: BLOCK_TXS handler only registered when `config.disableTransactions` is false. Otherwise peers get `ERR_UNSUPPORTED_PROTOCOL`. -**Requester side via `sendBatchRequest`** (Snappy limit: `max(N, 1) * 512 + 1` KB): - -| Rule | Consequence | File | -|------|-------------|------| -| Archive root must match request | MidToleranceError | `libp2p_service.ts` (`validateRequestedBlockTxs`) | -| BitVector length must match request | MidToleranceError | same | -| No duplicate tx hashes | MidToleranceError | same | -| Tx count within bounds | MidToleranceError | same | -| Local block proposal must exist for archive root | Rejected (no penalty) | same | -| All tx hashes must be in proposal's tx list at allowed indices | LowToleranceError | same | -| Txs in strictly increasing index order | LowToleranceError | same | -| Each tx passes well-formedness (Metadata [4 fields], Size, Data, Proof) | LowToleranceError | same | - **Requester side via `BatchTxRequester`** (separate validation path): | Rule | Consequence | File | diff --git a/yarn-project/p2p/src/services/reqresp/reqresp.ts b/yarn-project/p2p/src/services/reqresp/reqresp.ts index c706feb14eb1..2218f33033f3 100644 --- a/yarn-project/p2p/src/services/reqresp/reqresp.ts +++ b/yarn-project/p2p/src/services/reqresp/reqresp.ts @@ -44,7 +44,7 @@ import { ReqRespStatus, ReqRespStatusError, parseStatusChunk } from './status.js * The Request Response Service * * It allows nodes to request specific information from their peers, its use case covers recovering - * information that was missed during a syncronisation or a gossip event. + * information that was missed during a synchronisation or a gossip event. * * This service implements the request response sub protocol, it is heavily inspired from * ethereum implementations of the same name. @@ -126,7 +126,8 @@ export class ReqResp implements ReqRespInterface { Object.assign(this.subProtocolHandlers, subProtocolHandlers); Object.assign(this.subProtocolValidators, subProtocolValidators); - // Register all protocol handlers + // Register streamHandler with libp2p. + // The streamHandler is responsible for reading the incoming stream, determining the protocol, then triggering the appropriate handler. for (const subProtocol of Object.keys(subProtocolHandlers)) { this.logger.debug(`Registering handler for sub protocol ${subProtocol}`); await this.libp2p.handle( @@ -530,13 +531,13 @@ export class ReqResp implements ReqRespInterface { ): PeerErrorSeverity | undefined { const logTags = { peerId: peerId.toString(), subProtocol }; - //Punishable error - peer should never send badly formed request + // Punishable error - peer should never send badly formed request if (e instanceof ReqRespStatusError && e.status === ReqRespStatus.BADLY_FORMED_REQUEST) { this.logger.debug(`Punishable error in ${subProtocol}: ${e.cause}`, logTags); return PeerErrorSeverity.LowToleranceError; } - //TODO: (mralj): think if we should penalize peer here based on connection errors + // TODO: (mralj): think if we should penalize peer here based on connection errors return undefined; } @@ -577,7 +578,8 @@ export class ReqResp implements ReqRespInterface { /* * Errors specific to connection handling - * These can happen both when sending request and response*/ + * These can happen both when sending request and response. + */ private categorizeConnectionErrors( e: any, peerId: PeerId, From 816aef39bc17454f4e5f2d973cc73a8285c9319e Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 14 May 2026 16:26:14 +0100 Subject: [PATCH 08/17] feat: slash truncated checkpoint proposals (#23250) Fix A-520 --- .../aztec-node/templates/_pod-template.yaml | 4 + spartan/aztec-node/values.yaml | 1 + spartan/environments/network-defaults.yml | 5 + spartan/scripts/deploy_network.sh | 1 + spartan/terraform/deploy-aztec-infra/main.tf | 1 + .../terraform/deploy-aztec-infra/variables.tf | 6 + .../aztec-node/src/aztec-node/server.ts | 15 + ..._invalid_checkpoint_proposal_slash.test.ts | 388 ++++++++++++++++++ yarn-project/foundation/src/config/env_var.ts | 1 + yarn-project/p2p/src/client/p2p_client.ts | 21 +- .../attestation_pool/attestation_pool.ts | 185 ++++++--- .../attestation_pool_test_suite.ts | 72 ++++ .../p2p/src/test-helpers/testbench-utils.ts | 28 +- yarn-project/slasher/README.md | 7 + yarn-project/slasher/src/config.ts | 6 + yarn-project/slasher/src/index.ts | 1 + ...nvalid_checkpoint_proposal_watcher.test.ts | 237 +++++++++++ ...ted_invalid_checkpoint_proposal_watcher.ts | 191 +++++++++ .../src/interfaces/aztec-node-admin.test.ts | 1 + yarn-project/stdlib/src/interfaces/p2p.ts | 8 + yarn-project/stdlib/src/interfaces/slasher.ts | 2 + .../stdlib/src/slashing/helpers.test.ts | 19 + yarn-project/stdlib/src/slashing/helpers.ts | 4 + yarn-project/stdlib/src/slashing/types.ts | 7 + .../txe/src/state_machine/dummy_p2p_client.ts | 15 +- 25 files changed, 1155 insertions(+), 71 deletions(-) create mode 100644 yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts create mode 100644 yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts create mode 100644 yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts diff --git a/spartan/aztec-node/templates/_pod-template.yaml b/spartan/aztec-node/templates/_pod-template.yaml index 67bfaec31a23..e395a80da696 100644 --- a/spartan/aztec-node/templates/_pod-template.yaml +++ b/spartan/aztec-node/templates/_pod-template.yaml @@ -237,6 +237,10 @@ spec: - name: SLASH_INVALID_BLOCK_PENALTY value: {{ .Values.node.slash.invalidBlockPenalty | quote }} {{- end }} + {{- if .Values.node.slash.invalidCheckpointProposalPenalty }} + - name: SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY + value: {{ .Values.node.slash.invalidCheckpointProposalPenalty | quote }} + {{- end }} {{- if .Values.node.slash.proposeInvalidAttestationsPenalty }} - name: SLASH_PROPOSE_INVALID_ATTESTATIONS_PENALTY value: {{ .Values.node.slash.proposeInvalidAttestationsPenalty | quote }} diff --git a/spartan/aztec-node/values.yaml b/spartan/aztec-node/values.yaml index c16fb422b64c..30c32c5d33d2 100644 --- a/spartan/aztec-node/values.yaml +++ b/spartan/aztec-node/values.yaml @@ -152,6 +152,7 @@ node: inactivityPenalty: "" inactivityTargetPercentage: "" invalidBlockPenalty: "" + invalidCheckpointProposalPenalty: "" proposeInvalidAttestationsPenalty: "" attestDescendantOfInvalidPenalty: "" attestInvalidCheckpointProposalPenalty: "" diff --git a/spartan/environments/network-defaults.yml b/spartan/environments/network-defaults.yml index 3bfe0cd37aaa..93e6a668c647 100644 --- a/spartan/environments/network-defaults.yml +++ b/spartan/environments/network-defaults.yml @@ -143,6 +143,8 @@ slasher: &slasher SLASH_UNKNOWN_PENALTY: 10e18 # Penalty for broadcasting an invalid block. SLASH_INVALID_BLOCK_PENALTY: 10e18 + # Penalty for broadcasting an invalid checkpoint proposal. + SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0 # L2 slots grace period before considering an offense expired. SLASH_GRACE_PERIOD_L2_SLOTS: 0 @@ -247,6 +249,7 @@ networks: SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 10e18 SLASH_UNKNOWN_PENALTY: 10e18 SLASH_INVALID_BLOCK_PENALTY: 10e18 + SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0 SLASH_GRACE_PERIOD_L2_SLOTS: 0 ENABLE_VERSION_CHECK: true @@ -293,6 +296,7 @@ networks: SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 10e18 SLASH_UNKNOWN_PENALTY: 10e18 SLASH_INVALID_BLOCK_PENALTY: 10e18 + SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0 SLASH_GRACE_PERIOD_L2_SLOTS: 64 ENABLE_VERSION_CHECK: true @@ -353,4 +357,5 @@ networks: SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 2000e18 SLASH_UNKNOWN_PENALTY: 2000e18 SLASH_INVALID_BLOCK_PENALTY: 2000e18 + SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY: 0 SLASH_GRACE_PERIOD_L2_SLOTS: 1200 diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index edd3526eb4b8..8b809b90261a 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -595,6 +595,7 @@ SLASH_ATTEST_DESCENDANT_OF_INVALID_PENALTY = ${SLASH_ATTEST_DESCENDANT_OF_INVALI SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY = ${SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY:-null} SLASH_UNKNOWN_PENALTY = ${SLASH_UNKNOWN_PENALTY:-null} SLASH_INVALID_BLOCK_PENALTY = ${SLASH_INVALID_BLOCK_PENALTY:-null} +SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY = ${SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY:-null} SLASH_OFFENSE_EXPIRATION_ROUNDS = ${SLASH_OFFENSE_EXPIRATION_ROUNDS:-null} SLASH_MAX_PAYLOAD_SIZE = ${SLASH_MAX_PAYLOAD_SIZE:-null} OTEL_COLLECTOR_ENDPOINT = "${OTEL_COLLECTOR_ENDPOINT}" diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index 13ea3870380f..52b80e924a06 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -209,6 +209,7 @@ locals { "validator.slash.attestInvalidCheckpointProposalPenalty" = var.SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY "validator.slash.unknownPenalty" = var.SLASH_UNKNOWN_PENALTY "validator.slash.invalidBlockPenalty" = var.SLASH_INVALID_BLOCK_PENALTY + "validator.slash.invalidCheckpointProposalPenalty" = var.SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY "validator.slash.offenseExpirationRounds" = var.SLASH_OFFENSE_EXPIRATION_ROUNDS "validator.slash.maxPayloadSize" = var.SLASH_MAX_PAYLOAD_SIZE "validator.node.env.TRANSACTIONS_DISABLED" = var.TRANSACTIONS_DISABLED diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index 538f37fd0b23..18cd6e7406e8 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -520,6 +520,12 @@ variable "SLASH_INVALID_BLOCK_PENALTY" { nullable = true } +variable "SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY" { + description = "The slash invalid checkpoint proposal penalty" + type = string + nullable = true +} + variable "SLASH_OFFENSE_EXPIRATION_ROUNDS" { description = "The slash offense expiration rounds" type = string diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 817ce3f0bf1d..3c4f21f7f689 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -42,6 +42,7 @@ import { import { PublicContractsDB, PublicProcessorFactory } from '@aztec/simulator/server'; import { AttestationsBlockWatcher, + BroadcastedInvalidCheckpointProposalWatcher, EpochPruneWatcher, type SlasherClientInterface, type Watcher, @@ -720,6 +721,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb let validatorsSentinel: Awaited> | undefined; let epochPruneWatcher: EpochPruneWatcher | undefined; let attestationsBlockWatcher: AttestationsBlockWatcher | undefined; + let broadcastedInvalidCheckpointProposalWatcher: BroadcastedInvalidCheckpointProposalWatcher | undefined; if (!proverOnly) { validatorsSentinel = await createSentinel(epochCache, archiver, p2pClient, config); @@ -739,6 +741,15 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb watchers.push(epochPruneWatcher); } + if (config.slashBroadcastedInvalidCheckpointProposalPenalty > 0n) { + broadcastedInvalidCheckpointProposalWatcher = new BroadcastedInvalidCheckpointProposalWatcher( + p2pClient, + epochCache, + config, + ); + watchers.push(broadcastedInvalidCheckpointProposalWatcher); + } + // We assume we want to slash for invalid attestations unless all max penalties are set to 0 if (config.slashProposeInvalidAttestationsPenalty > 0n || config.slashAttestDescendantOfInvalidPenalty > 0n) { attestationsBlockWatcher = new AttestationsBlockWatcher(archiver, epochCache, config); @@ -762,6 +773,10 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb await attestationsBlockWatcher.start(); started.push(attestationsBlockWatcher); } + if (broadcastedInvalidCheckpointProposalWatcher) { + await broadcastedInvalidCheckpointProposalWatcher.start(); + started.push(broadcastedInvalidCheckpointProposalWatcher); + } log.info(`All p2p services started`); }) .catch(err => log.error('Failed to start p2p services after archiver sync', err)); diff --git a/yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts new file mode 100644 index 000000000000..51395cbc47cb --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts @@ -0,0 +1,388 @@ +import type { AztecNodeService } from '@aztec/aztec-node'; +import { Fr } from '@aztec/aztec.js/fields'; +import { BlockNumber, EpochNumber, IndexWithinCheckpoint, SlotNumber } from '@aztec/foundation/branded-types'; +import { Buffer32 } from '@aztec/foundation/buffer'; +import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer'; +import { retryUntil } from '@aztec/foundation/retry'; +import { sleep } from '@aztec/foundation/sleep'; +import { OffenseType } from '@aztec/slasher'; +import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p'; +import { + makeBlockHeader, + makeBlockProposal, + makeCheckpointHeader, + makeCheckpointProposal, +} from '@aztec/stdlib/testing'; +import { TxHash } from '@aztec/stdlib/tx'; + +import { jest } from '@jest/globals'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { P2PNetworkTest } from '../e2e_p2p/p2p_network.js'; +import { awaitCommitteeExists } from '../e2e_p2p/shared.js'; +import { shouldCollectMetrics } from '../fixtures/fixtures.js'; +import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNode } from '../fixtures/setup_p2p_test.js'; +import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; + +const TEST_TIMEOUT = 1_000_000; + +jest.setTimeout(TEST_TIMEOUT); + +const NUM_VALIDATORS = 1; +const BOOT_NODE_UDP_PORT = 4900; +const COMMITTEE_SIZE = NUM_VALIDATORS; +const ETHEREUM_SLOT_DURATION = 4; +const AZTEC_EPOCH_DURATION = 2; +const AZTEC_SLOT_DURATION = ETHEREUM_SLOT_DURATION * AZTEC_EPOCH_DURATION; +const SLASHING_QUORUM = 5; +const SLASHING_ROUND_SIZE = 8; +const TERMINAL_BLOCK_INDEX = IndexWithinCheckpoint(1); +const HIGHER_BLOCK_INDEX = IndexWithinCheckpoint(2); + +const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'broadcasted-invalid-checkpoint-proposal-slash-')); + +type SlashOffense = Awaited>[number]; + +function getAttesterSigner(validatorIndex: number) { + const privateKey = getPrivateKeyFromIndex(ATTESTER_PRIVATE_KEYS_START_INDEX + validatorIndex)!; + return new Secp256k1Signer(Buffer32.fromBuffer(privateKey)); +} + +function findBroadcastedInvalidCheckpointOffense( + offenses: SlashOffense[], + validator: string, + slot: SlotNumber, +): SlashOffense | undefined { + return offenses.find( + offense => + offense.validator.toString() === validator && + offense.offenseType === OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL && + offense.epochOrSlot === BigInt(slot), + ); +} + +async function awaitBroadcastedInvalidCheckpointOffense({ + node, + validator, + slot, +}: { + node: AztecNodeService; + validator: string; + slot: SlotNumber; +}) { + return await retryUntil( + async () => { + const offenses = await node.getSlashOffenses('all'); + return findBroadcastedInvalidCheckpointOffense(offenses, validator, slot); + }, + `A-520 offense for slot ${slot}`, + AZTEC_SLOT_DURATION * 3, + 1, + ); +} + +async function expectNoBroadcastedInvalidCheckpointOffense({ + node, + validator, + slot, +}: { + node: AztecNodeService; + validator: string; + slot: SlotNumber; +}) { + // The watcher polls every second with this test's slot timing; wait long enough + // for the closed slot to be scanned before asserting no offense was recorded. + await sleep(2_000); + const offenses = await node.getSlashOffenses('all'); + expect(findBroadcastedInvalidCheckpointOffense(offenses, validator, slot)).toBeUndefined(); +} + +async function awaitRetainedProposalsForSlot({ + node, + slot, + blockCount, + checkpointCount, +}: { + node: AztecNodeService; + slot: SlotNumber; + blockCount: number; + checkpointCount: number; +}) { + return await retryUntil( + async () => { + const proposals = await node.getP2P().getProposalsForSlot(slot); + return proposals.blockProposals.length === blockCount && proposals.checkpointProposals.length === checkpointCount + ? proposals + : undefined; + }, + `retained proposals for slot ${slot}`, + 5, + 0.2, + ); +} + +async function makeBlock({ + signer, + signatureContext, + targetSlot, + indexWithinCheckpoint, + seed, +}: { + signer: Secp256k1Signer; + signatureContext: CoordinationSignatureContext; + targetSlot: SlotNumber; + indexWithinCheckpoint: IndexWithinCheckpoint; + seed: number; +}) { + return await makeBlockProposal({ + blockHeader: makeBlockHeader(seed, { + blockNumber: BlockNumber(seed), + slotNumber: targetSlot, + }), + indexWithinCheckpoint, + txHashes: [TxHash.random()], + archiveRoot: Fr.random(), + signer, + signatureContext, + }); +} + +async function makeInvalidCheckpointProposals({ + signer, + signatureContext, + targetSlot, + seed, + includeTerminalBlockAsLastBlock = false, +}: { + signer: Secp256k1Signer; + signatureContext: CoordinationSignatureContext; + targetSlot: SlotNumber; + seed: number; + includeTerminalBlockAsLastBlock?: boolean; +}) { + const earlierBlock = await makeBlock({ + signer, + signatureContext, + targetSlot, + indexWithinCheckpoint: IndexWithinCheckpoint(0), + seed, + }); + const terminalBlock = await makeBlock({ + signer, + signatureContext, + targetSlot, + indexWithinCheckpoint: TERMINAL_BLOCK_INDEX, + seed: seed + 1, + }); + const higherBlock = await makeBlock({ + signer, + signatureContext, + targetSlot, + indexWithinCheckpoint: HIGHER_BLOCK_INDEX, + seed: seed + 2, + }); + const checkpoint = await makeCheckpointProposal({ + signer, + checkpointHeader: makeCheckpointHeader(seed, { slotNumber: targetSlot }), + archiveRoot: terminalBlock.archive, + lastBlock: includeTerminalBlockAsLastBlock + ? { + blockHeader: terminalBlock.blockHeader, + indexWithinCheckpoint: terminalBlock.indexWithinCheckpoint, + txHashes: terminalBlock.txHashes, + } + : undefined, + signatureContext, + }); + + return { earlierBlock, terminalBlock, higherBlock, checkpoint }; +} + +describe('e2e_slashing_broadcasted_invalid_checkpoint_proposal_slash', () => { + let t: P2PNetworkTest; + let nodes: AztecNodeService[] = []; + + const slashingUnit = BigInt(1e14); + + beforeEach(async () => { + t = await P2PNetworkTest.create({ + testName: 'e2e_slashing_broadcasted_invalid_checkpoint_proposal_slash', + numberOfNodes: 0, + numberOfValidators: NUM_VALIDATORS, + basePort: BOOT_NODE_UDP_PORT, + metricsPort: shouldCollectMetrics(), + initialConfig: { + anvilSlotsInAnEpoch: 4, + listenAddress: '127.0.0.1', + aztecEpochDuration: AZTEC_EPOCH_DURATION, + ethereumSlotDuration: ETHEREUM_SLOT_DURATION, + aztecSlotDuration: AZTEC_SLOT_DURATION, + aztecTargetCommitteeSize: COMMITTEE_SIZE, + aztecProofSubmissionEpochs: 1024, + enableProposerPipelining: false, + mockGossipSubNetwork: true, + slashingQuorum: SLASHING_QUORUM, + slashingRoundSizeInEpochs: SLASHING_ROUND_SIZE / AZTEC_EPOCH_DURATION, + slashAmountSmall: slashingUnit, + slashAmountMedium: slashingUnit * 2n, + slashAmountLarge: slashingUnit * 3n, + slashPrunePenalty: 0n, + slashDataWithholdingPenalty: 0n, + slashInactivityPenalty: 0n, + slashBroadcastedInvalidBlockPenalty: 0n, + slashBroadcastedInvalidCheckpointProposalPenalty: slashingUnit, + slashDuplicateProposalPenalty: 0n, + slashDuplicateAttestationPenalty: 0n, + slashProposeInvalidAttestationsPenalty: 0n, + slashAttestDescendantOfInvalidPenalty: 0n, + slashAttestInvalidCheckpointProposalPenalty: 0n, + slashUnknownPenalty: 0n, + slashSelfAllowed: true, + }, + }); + + await t.setup(); + await t.applyBaseSetup(); + }); + + afterEach(async () => { + await t.stopNodes(nodes); + if (t.monitor) { + await t.teardown(); + } + fs.rmSync(`${DATA_DIR}-0`, { recursive: true, force: true, maxRetries: 3 }); + }); + + const setupNodeAndValidator = async () => { + const { rollup } = await t.getContracts(); + + await t.ctx.cheatCodes.rollup.advanceToEpoch(EpochNumber(4)); + await t.ctx.cheatCodes.rollup.debugRollup(); + + const node = await createNode( + { + ...t.ctx.aztecNodeConfig, + dontStartSequencer: true, + enableProposerPipelining: false, + slashBroadcastedInvalidCheckpointProposalPenalty: slashingUnit, + slashSelfAllowed: true, + }, + t.ctx.dateProvider, + BOOT_NODE_UDP_PORT + 1, + t.bootstrapNodeEnr, + 0, + t.genesis, + `${DATA_DIR}-0`, + shouldCollectMetrics(), + ); + nodes = [node]; + + await retryUntil(() => node.isReady(), 'node ready', 30, 0.5); + await awaitCommitteeExists({ rollup, logger: t.logger }); + + const currentSlot = await rollup.getSlotNumber(); + expect(currentSlot).toBeGreaterThan(2); + + const signer = getAttesterSigner(0); + const validator = t.validators[0].attester.toString(); + const signatureContext: CoordinationSignatureContext = { + chainId: t.ctx.aztecNodeConfig.l1ChainId, + rollupAddress: t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress, + }; + + return { node, currentSlot, signer, validator, signatureContext }; + }; + + it('slashes a validator that broadcasts a checkpoint truncated below its own retained block proposal', async () => { + const { node, currentSlot, signer, validator, signatureContext } = await setupNodeAndValidator(); + const targetSlot = SlotNumber(Number(currentSlot) - 2); + + const alreadyRetainedProposals = await makeInvalidCheckpointProposals({ + signer, + signatureContext, + targetSlot, + seed: 0xa520, + }); + + await node.getP2P().broadcastProposal(alreadyRetainedProposals.earlierBlock); + await node.getP2P().broadcastProposal(alreadyRetainedProposals.terminalBlock); + await node.getP2P().broadcastProposal(alreadyRetainedProposals.higherBlock); + await node.getP2P().broadcastCheckpointProposal(alreadyRetainedProposals.checkpoint); + + const firstProposals = await awaitRetainedProposalsForSlot({ + node, + slot: targetSlot, + blockCount: 3, + checkpointCount: 1, + }); + expect(firstProposals.blockProposals.map(proposal => proposal.getSender()?.toString())).toEqual([ + validator, + validator, + validator, + ]); + expect(firstProposals.checkpointProposals[0].getSender()?.toString()).toEqual(validator); + + const firstOffense = await awaitBroadcastedInvalidCheckpointOffense({ + node, + validator, + slot: targetSlot, + }); + expect(firstOffense.amount).toEqual(slashingUnit); + }); + + it('does not slash a valid checkpoint whose lastBlock supplies the terminal proposal until a delayed higher-index block is retained', async () => { + const { node, currentSlot, signer, validator, signatureContext } = await setupNodeAndValidator(); + const targetSlot = SlotNumber(Number(currentSlot) - 2); + const lateHigherBlockProposals = await makeInvalidCheckpointProposals({ + signer, + signatureContext, + targetSlot, + seed: 0xa530, + includeTerminalBlockAsLastBlock: true, + }); + + await node.getP2P().broadcastProposal(lateHigherBlockProposals.earlierBlock); + await node.getP2P().broadcastCheckpointProposal(lateHigherBlockProposals.checkpoint); + + const validProposals = await awaitRetainedProposalsForSlot({ + node, + slot: targetSlot, + blockCount: 2, + checkpointCount: 1, + }); + expect(validProposals.blockProposals.map(proposal => proposal.getSender()?.toString())).toEqual([ + validator, + validator, + ]); + const terminalProposal = validProposals.blockProposals.find( + proposal => proposal.indexWithinCheckpoint === TERMINAL_BLOCK_INDEX, + ); + expect(terminalProposal?.archive.toString()).toEqual(lateHigherBlockProposals.terminalBlock.archive.toString()); + expect(terminalProposal?.getSender()?.toString()).toEqual(validator); + expect(validProposals.checkpointProposals[0].getSender()?.toString()).toEqual(validator); + await expectNoBroadcastedInvalidCheckpointOffense({ node, validator, slot: targetSlot }); + + await node.getP2P().broadcastProposal(lateHigherBlockProposals.higherBlock); + + const invalidProposals = await awaitRetainedProposalsForSlot({ + node, + slot: targetSlot, + blockCount: 3, + checkpointCount: 1, + }); + expect(invalidProposals.blockProposals.map(proposal => proposal.getSender()?.toString())).toEqual([ + validator, + validator, + validator, + ]); + + const offense = await awaitBroadcastedInvalidCheckpointOffense({ + node, + validator, + slot: targetSlot, + }); + expect(offense.amount).toEqual(slashingUnit); + }); +}); diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 8cf507f49279..65d020ac5f49 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -245,6 +245,7 @@ export type EnvVar = | 'SLASH_INACTIVITY_TARGET_PERCENTAGE' | 'SLASH_INACTIVITY_CONSECUTIVE_EPOCH_THRESHOLD' | 'SLASH_INVALID_BLOCK_PENALTY' + | 'SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY' | 'SLASH_DUPLICATE_PROPOSAL_PENALTY' | 'SLASH_DUPLICATE_ATTESTATION_PENALTY' | 'SLASH_OVERRIDE_PAYLOAD' diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 9e7c486033fc..2816d013159e 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -34,7 +34,7 @@ import type { ENR } from '@nethermindeth/enr'; import { type P2PConfig, getP2PDefaultConfig } from '../config.js'; import { TxPoolError } from '../errors/tx-pool.error.js'; -import type { AttestationPoolApi } from '../mem_pools/attestation_pool/attestation_pool.js'; +import type { AttestationPoolApi, ProposalsForSlot } from '../mem_pools/attestation_pool/attestation_pool.js'; import type { MemPools } from '../mem_pools/interface.js'; import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js'; import type { AuthRequest, StatusMessage } from '../services/index.js'; @@ -371,8 +371,21 @@ export class P2PClient extends WithTracer implements P2P { // Store our own last-block proposal so we can respond to req/resp requests for it. await this.attestationPool.tryAddBlockProposal(blockProposal); } + const checkpointCore = proposal.toCore(); + const { count } = await this.attestationPool.tryAddCheckpointProposal(checkpointCore); + if (count > 1) { + if (this.config.broadcastEquivocatedProposals) { + this.log.warn(`Broadcasting equivocated checkpoint proposal for slot ${proposal.slotNumber}`, { + slot: proposal.slotNumber, + archive: proposal.archive.toString(), + count, + }); + } else { + throw new Error(`Attempted to broadcast a duplicate checkpoint proposal for slot ${proposal.slotNumber}`); + } + } // Gossipsub doesn't deliver own messages, so fire the all-nodes handler locally - await this.p2pService.notifyOwnCheckpointProposal(proposal.toCore()); + await this.p2pService.notifyOwnCheckpointProposal(checkpointCore); return this.p2pService.propagate(proposal); } @@ -394,6 +407,10 @@ export class P2PClient extends WithTracer implements P2P { return this.attestationPool.addOwnCheckpointAttestations(attestations); } + public getProposalsForSlot(slot: SlotNumber): Promise { + return this.attestationPool.getProposalsForSlot(slot); + } + public hasBlockProposalsForSlot(slot: SlotNumber): Promise { return this.attestationPool.hasBlockProposalsForSlot(slot); } diff --git a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts index 109e472aa35f..d98e80e2fbeb 100644 --- a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts +++ b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts @@ -25,6 +25,11 @@ export type TryAddResult = { count: number; }; +export type ProposalsForSlot = { + blockProposals: BlockProposal[]; + checkpointProposals: CheckpointProposalCore[]; +}; + export const MAX_CHECKPOINT_PROPOSALS_PER_SLOT = 2; export const MAX_BLOCK_PROPOSALS_PER_POSITION = 2; /** Maximum attestations a single signer can make per slot before being rejected. */ @@ -35,6 +40,7 @@ export type AttestationPoolApi = Pick< AttestationPool, | 'tryAddBlockProposal' | 'getBlockProposalByArchive' + | 'getProposalsForSlot' | 'tryAddCheckpointProposal' | 'getCheckpointProposal' | 'addOwnCheckpointAttestations' @@ -57,6 +63,8 @@ export type AttestationPoolApi = Pick< * one attestation per (slot, signer)). Distinct *signed payload hashes* arriving at * the same position are tracked in the matching index multimap so the equivocation * count reaches 2 even when archive collides on `feeAssetPriceModifier` variants. + * Proposal bytes are retained per accepted payload hash, up to the same + * equivocation caps, for slashing watchers that need signed P2P proposals. */ export class AttestationPool { private metrics: PoolInstrumentation; @@ -75,22 +83,25 @@ export class AttestationPool { // Stores the first proposal seen per slot. private checkpointProposalPerSlot: AztecAsyncMap; + // Checkpoint proposals from `${paddedSlot}-${payloadHash}` to serialized CheckpointProposalCore. + // Stores every accepted distinct payload up to MAX_CHECKPOINT_PROPOSALS_PER_SLOT. + private checkpointProposalsPerSlotAndHash: AztecAsyncMap; + // Distinct payload hashes seen per slot. Hash collision = duplicate. // Hash count reaching 2 = equivocation. // Key: slot number, Value: CheckpointProposalHash (`0x`-prefixed hex) private checkpointProposalHashesPerSlot: AztecAsyncMultiMap; - // Block proposals from positionKey to serialized BlockProposal. - // Stores the first proposal seen per (slot, indexWithinCheckpoint). - private blockProposalPerSlotAndIndex: AztecAsyncMap; + // Block proposals from `${paddedSlot}-${paddedIndex}-${payloadHash}` to serialized BlockProposal. + // Stores every accepted distinct payload up to MAX_BLOCK_PROPOSALS_PER_POSITION. + private blockProposalsPerSlotIndexAndHash: AztecAsyncMap; // Distinct payload hashes seen per (slot, indexWithinCheckpoint). // Key: slot * (1 << INDEX_BITS) + indexWithinCheckpoint, Value: BlockProposalHash (`0x`-prefixed hex) private blockProposalHashesPerSlotAndIndex: AztecAsyncMultiMap; - // Secondary index from archive root to positionKey, so that the block-txs req/resp - // handler can still resolve a stored proposal by archive root. - private blockProposalSlotAndIndexPerArchive: AztecAsyncMap; + // Secondary index from archive root to all retained block proposal keys. + private blockProposalKeysPerArchive: AztecAsyncMultiMap; constructor( private store: AztecAsyncKVStore, @@ -98,9 +109,9 @@ export class AttestationPool { private log = createLogger('aztec:attestation_pool'), ) { // Initialize block proposal storage - this.blockProposalPerSlotAndIndex = store.openMap('proposals'); + this.blockProposalsPerSlotIndexAndHash = store.openMap('block_proposals_by_slot_index_and_hash'); this.blockProposalHashesPerSlotAndIndex = store.openMultiMap('block_proposals_for_slot_and_index'); - this.blockProposalSlotAndIndexPerArchive = store.openMap('block_proposals_by_archive'); + this.blockProposalKeysPerArchive = store.openMultiMap('block_proposals_by_archive'); // Initialize checkpoint attestations storage this.attestationPerSlotAndSigner = store.openMap('checkpoint_attestations'); @@ -108,6 +119,7 @@ export class AttestationPool { // Initialize checkpoint proposal storage this.checkpointProposalPerSlot = store.openMap('checkpoint_proposals'); + this.checkpointProposalsPerSlotAndHash = store.openMap('checkpoint_proposals_by_slot_and_hash'); this.checkpointProposalHashesPerSlot = store.openMultiMap('checkpoint_proposals_for_slot'); this.metrics = new PoolInstrumentation(telemetry, PoolName.ATTESTATION_POOL, this.poolStats); @@ -124,7 +136,7 @@ export class AttestationPool { for await (const _ of this.attestationPerSlotAndSigner.entriesAsync()) { return false; } - for await (const _ of this.blockProposalPerSlotAndIndex.entriesAsync()) { + for await (const _ of this.blockProposalsPerSlotIndexAndHash.entriesAsync()) { return false; } return true; @@ -143,6 +155,35 @@ export class AttestationPool { return slot.toString().padStart(AttestationPool.SLOT_PAD_DIGITS, '0'); } + /** Fixed-width decimal index string for use in composite string keys. */ + private indexPaddedKey(indexWithinCheckpoint: number): string { + return indexWithinCheckpoint.toString().padStart(4, '0'); + } + + /** Key for retained block proposals. */ + private getBlockProposalKey( + slot: SlotNumber | number, + indexWithinCheckpoint: number, + payloadHash: BlockProposalHash, + ): string { + return `${this.slotPaddedKey(slot)}-${this.indexPaddedKey(indexWithinCheckpoint)}-${payloadHash}`; + } + + /** Range bounds for all retained block proposals in a slot. */ + private getBlockProposalKeyRangeForSlot(slot: SlotNumber): { start: string; end: string } { + return { start: `${this.slotPaddedKey(slot)}-`, end: `${this.slotPaddedKey(slot + 1)}-` }; + } + + /** Key for retained checkpoint proposals. */ + private getCheckpointProposalKey(slot: SlotNumber | number, payloadHash: CheckpointProposalHash): string { + return `${this.slotPaddedKey(slot)}-${payloadHash}`; + } + + /** Range bounds for all retained checkpoint proposals in a slot. */ + private getCheckpointProposalKeyRangeForSlot(slot: SlotNumber): { start: string; end: string } { + return { start: `${this.slotPaddedKey(slot)}-`, end: `${this.slotPaddedKey(slot + 1)}-` }; + } + /** Key for the per-(slot, signer) attestation main store and equivocation index. */ private getSlotSignerKey(slot: SlotNumber, signerAddress: string): string { return `${this.slotPaddedKey(slot)}-${signerAddress}`; @@ -185,8 +226,7 @@ export class AttestationPool { * - Detects duplicates by signed-payload hash (not archive); a re-broadcast of the * exact same signed payload returns `alreadyExists: true`. * - Distinct payload hashes at the same `(slot, indexWithinCheckpoint)` are tracked - * in the equivocation index. The first hash also stores the proposal bytes; later - * distinct hashes only bump `count` so libp2p can fire its duplicate callback. + * in the equivocation index and retained up to the cap. * * @param blockProposal - The block proposal to add * @returns Result indicating whether the proposal was added and duplicate detection info @@ -210,14 +250,13 @@ export class AttestationPool { // Track the new payload hash for equivocation detection. await this.blockProposalHashesPerSlotAndIndex.set(positionKey, payloadHash); - - // Only the first distinct payload at this position is stored; later equivocations - // are detected via the multimap but their payload bytes are not retained. - const alreadyHasStored = await this.blockProposalPerSlotAndIndex.hasAsync(positionKey); - if (!alreadyHasStored) { - await this.blockProposalPerSlotAndIndex.set(positionKey, blockProposal.withoutSignedTxs().toBuffer()); - await this.blockProposalSlotAndIndexPerArchive.set(blockProposal.archive.toString(), positionKey); - } + const proposalKey = this.getBlockProposalKey( + blockProposal.slotNumber, + blockProposal.indexWithinCheckpoint, + payloadHash, + ); + await this.blockProposalsPerSlotIndexAndHash.set(proposalKey, blockProposal.withoutSignedTxs().toBuffer()); + await this.blockProposalKeysPerArchive.set(blockProposal.archive.toString(), proposalKey); this.log.debug( `Added block proposal for slot ${blockProposal.slotNumber} and index ${blockProposal.indexWithinCheckpoint}`, @@ -226,7 +265,6 @@ export class AttestationPool { payloadHash, slotNumber: blockProposal.slotNumber, indexWithinCheckpoint: blockProposal.indexWithinCheckpoint, - stored: !alreadyHasStored, }, ); @@ -237,40 +275,57 @@ export class AttestationPool { /** * Get block proposal by archive root. * - * Resolves the archive root to its `(slot, indexWithinCheckpoint)` via a secondary - * index, then fetches the stored proposal (if any). Returns the *first* proposal - * seen at that position, even if a later equivocating payload was tracked. - * Validates that the stored proposal's archive matches the requested one before - * returning, guarding against secondary-index corruption or position-key reuse. + * Resolves the archive root through the archive index and returns the first + * retained proposal for that archive. This lookup is used by block-txs req/resp, + * where any retained proposal for the requested archive gives the tx hash list. * * @param archiveRoot - The archive root to look up * @return The block proposal if it exists and its archive matches, otherwise undefined. */ public async getBlockProposalByArchive(archiveRoot: string): Promise { - const positionKey = await this.blockProposalSlotAndIndexPerArchive.getAsync(archiveRoot); - if (positionKey === undefined) { - return undefined; - } - const buffer = await this.blockProposalPerSlotAndIndex.getAsync(positionKey); - if (!buffer || buffer.length === 0) { - return undefined; + for await (const proposalKey of this.blockProposalKeysPerArchive.getValuesAsync(archiveRoot)) { + const buffer = await this.blockProposalsPerSlotIndexAndHash.getAsync(proposalKey); + if (!buffer || buffer.length === 0) { + continue; + } + try { + const proposal = BlockProposal.fromBuffer(buffer); + if (proposal.archive.toString() === archiveRoot) { + return proposal; + } + } catch { + continue; + } } - let proposal: BlockProposal; - try { - proposal = BlockProposal.fromBuffer(buffer); - } catch { - return undefined; + return undefined; + } + + /** Returns retained signed proposals for a slot. */ + public async getProposalsForSlot(slot: SlotNumber): Promise { + const blockProposals: BlockProposal[] = []; + const checkpointProposals: CheckpointProposalCore[] = []; + + for await (const [_, buffer] of this.blockProposalsPerSlotIndexAndHash.entriesAsync( + this.getBlockProposalKeyRangeForSlot(slot), + )) { + try { + blockProposals.push(BlockProposal.fromBuffer(buffer)); + } catch { + continue; + } } - const storedArchive = proposal.archive.toString(); - if (storedArchive !== archiveRoot) { - this.log.warn(`Stored block proposal archive does not match requested archive root`, { - requestedArchive: archiveRoot, - storedArchive, - positionKey, - }); - return undefined; + + for await (const [_, buffer] of this.checkpointProposalsPerSlotAndHash.entriesAsync( + this.getCheckpointProposalKeyRangeForSlot(slot), + )) { + try { + checkpointProposals.push(CheckpointProposal.fromBuffer(buffer)); + } catch { + continue; + } } - return proposal; + + return { blockProposals, checkpointProposals }; } /** Checks if any block proposals exist for a given slot (at index 0). */ @@ -286,8 +341,8 @@ export class AttestationPool { * - Detects duplicates by signed-payload hash (not archive); a re-broadcast of the * exact same signed payload returns `alreadyExists: true`. * - Distinct payload hashes at the same slot are tracked in the equivocation index. - * Only the first distinct payload's bytes are stored; later distinct hashes bump - * `count` so libp2p can fire its duplicate callback. + * Distinct payload bytes are retained up to the same cap so slashing watchers + * can recover signed proposals. * * Note: This method only handles the CheckpointProposalCore. If the original * CheckpointProposal contains a lastBlock, the caller should extract it via @@ -313,9 +368,13 @@ export class AttestationPool { // Track the new payload hash for equivocation detection. await this.checkpointProposalHashesPerSlot.set(slot, payloadHash); + await this.checkpointProposalsPerSlotAndHash.set( + this.getCheckpointProposalKey(slot, payloadHash), + proposal.toBuffer(), + ); // Only the first distinct payload at this slot is stored; later equivocations - // are detected via the multimap but their payload bytes are not retained. + // are retained in `checkpointProposalsPerSlotAndHash`. const alreadyHasStored = await this.checkpointProposalPerSlot.hasAsync(slot); if (!alreadyHasStored) { await this.checkpointProposalPerSlot.set(slot, proposal.toBuffer()); @@ -471,22 +530,30 @@ export class AttestationPool { } } + for await (const key of this.checkpointProposalsPerSlotAndHash.keysAsync({ + end: `${oldestSlotPadded}-`, + })) { + await this.checkpointProposalsPerSlotAndHash.delete(key); + } + // Delete block proposals for slots < oldestSlot, using blockProposalHashesPerSlotAndIndex as index. // Key format: slot * (1 << INDEX_BITS) + indexWithinCheckpoint const blockPositionEndKey = oldestSlot * (1 << AttestationPool.INDEX_BITS); for await (const positionKey of this.blockProposalHashesPerSlotAndIndex.keysAsync({ end: blockPositionEndKey })) { await this.blockProposalHashesPerSlotAndIndex.delete(positionKey); - const stored = await this.blockProposalPerSlotAndIndex.getAsync(positionKey); - if (stored) { - try { - const proposal = BlockProposal.fromBuffer(stored); - await this.blockProposalSlotAndIndexPerArchive.delete(proposal.archive.toString()); - } catch { - // ignore decode errors when cleaning up - } - await this.blockProposalPerSlotAndIndex.delete(positionKey); - numberOfBlockProposals++; + } + + for await (const [key, buffer] of this.blockProposalsPerSlotIndexAndHash.entriesAsync({ + end: `${oldestSlotPadded}-`, + })) { + try { + const proposal = BlockProposal.fromBuffer(buffer); + await this.blockProposalKeysPerArchive.deleteValue(proposal.archive.toString(), key); + } catch { + // ignore decode errors when cleaning up } + await this.blockProposalsPerSlotIndexAndHash.delete(key); + numberOfBlockProposals++; } }); diff --git a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts index 7265d2e52a42..33e092856237 100644 --- a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +++ b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts @@ -246,6 +246,45 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo expect(retrievedProposal!.toBuffer()).toEqual(proposal.toBuffer()); expect(retrievedProposal!.getSender()?.toString()).toBe(signers[0].address.toString()); }); + + it('should retain an exact duplicate block proposal only once', async () => { + const slotNumber = 420; + const proposal = await mockBlockProposalForPool(signers[0], slotNumber); + + await ap.tryAddBlockProposal(proposal); + await ap.tryAddBlockProposal(proposal); + + const proposals = await ap.getProposalsForSlot(SlotNumber(slotNumber)); + expect(proposals.blockProposals.map(proposal => proposal.toBuffer())).toEqual([ + proposal.withoutSignedTxs().toBuffer(), + ]); + }); + + it('should retain all accepted block proposals at a position', async () => { + const slotNumber = 420; + const blockHeader = makeBlockHeader(1, { slotNumber: SlotNumber(slotNumber) }); + const proposal1 = await makeBlockProposal({ + signer: signers[0], + blockHeader, + archiveRoot: Fr.random(), + indexWithinCheckpoint: IndexWithinCheckpoint(1), + }); + const proposal2 = await makeBlockProposal({ + signer: signers[0], + blockHeader, + archiveRoot: Fr.random(), + indexWithinCheckpoint: IndexWithinCheckpoint(1), + }); + + await ap.tryAddBlockProposal(proposal1); + await ap.tryAddBlockProposal(proposal2); + + const proposals = await ap.getProposalsForSlot(SlotNumber(slotNumber)); + expect(proposals.blockProposals.map(proposal => proposal.toBuffer())).toEqual( + expect.arrayContaining([proposal1.withoutSignedTxs().toBuffer(), proposal2.withoutSignedTxs().toBuffer()]), + ); + expect(await ap.getBlockProposalByArchive(proposal2.archive.toString())).toBeDefined(); + }); }); describe('CheckpointProposal in attestation pool', () => { @@ -353,6 +392,11 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo const retrievedProposal = await ap.getCheckpointProposal(SlotNumber(slotNumber)); expect(retrievedProposal!.toBuffer()).toEqual(proposal1.toBuffer()); + + const proposals = await ap.getProposalsForSlot(SlotNumber(slotNumber)); + expect(proposals.checkpointProposals.map(proposal => proposal.toBuffer())).toEqual( + expect.arrayContaining([proposal1.toBuffer(), proposal2.toBuffer()]), + ); }); it('should detect equivocation when only feeAssetPriceModifier differs', async () => { @@ -385,6 +429,34 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo expect(result2.count).toBe(2); }); + it('should delete retained proposals older than a given slot', async () => { + const oldSlot = 100; + const newSlot = 200; + const oldBlock = await mockBlockProposalForPool(signers[0], oldSlot); + const newBlock = await mockBlockProposalForPool(signers[1], newSlot); + const oldCheckpoint = await mockCheckpointProposalForPool(signers[0], oldSlot); + const newCheckpoint = await mockCheckpointProposalForPool(signers[1], newSlot); + + await ap.tryAddBlockProposal(oldBlock); + await ap.tryAddBlockProposal(newBlock); + await ap.tryAddCheckpointProposal(oldCheckpoint); + await ap.tryAddCheckpointProposal(newCheckpoint); + + await ap.deleteOlderThan(SlotNumber(newSlot)); + + expect(await ap.getProposalsForSlot(SlotNumber(oldSlot))).toEqual({ + blockProposals: [], + checkpointProposals: [], + }); + const newProposals = await ap.getProposalsForSlot(SlotNumber(newSlot)); + expect(newProposals.blockProposals.map(proposal => proposal.toBuffer())).toContainEqual( + newBlock.withoutSignedTxs().toBuffer(), + ); + expect(newProposals.checkpointProposals.map(proposal => proposal.toBuffer())).toContainEqual( + newCheckpoint.toBuffer(), + ); + }); + it('should return added=false when exceeding capacity', async () => { const slotNumber = 420; diff --git a/yarn-project/p2p/src/test-helpers/testbench-utils.ts b/yarn-project/p2p/src/test-helpers/testbench-utils.ts index 17bd755a724c..3bba29f5b65e 100644 --- a/yarn-project/p2p/src/test-helpers/testbench-utils.ts +++ b/yarn-project/p2p/src/test-helpers/testbench-utils.ts @@ -4,12 +4,7 @@ import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import type { Logger } from '@aztec/foundation/log'; import type { L2Block, L2BlockId } from '@aztec/stdlib/block'; import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; -import type { - BlockProposal, - CheckpointAttestation, - CheckpointProposal, - CheckpointProposalCore, -} from '@aztec/stdlib/p2p'; +import type { BlockProposal, CheckpointAttestation, CheckpointProposalCore } from '@aztec/stdlib/p2p'; import { type BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx'; import EventEmitter from 'events'; @@ -215,6 +210,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 { */ export class InMemoryAttestationPool { private proposals = new Map(); + private checkpoints = new Map(); tryAddBlockProposal(blockProposal: BlockProposal): Promise { const id = blockProposal.archive.toString(); @@ -230,12 +226,25 @@ export class InMemoryAttestationPool { return Promise.resolve(this.proposals.get(id)); } - tryAddCheckpointProposal(_proposal: CheckpointProposal): Promise { + tryAddCheckpointProposal(proposal: CheckpointProposalCore): Promise { + const proposals = this.checkpoints.get(proposal.slotNumber) ?? []; + proposals.push(proposal); + this.checkpoints.set(proposal.slotNumber, proposals); return Promise.resolve({ added: true, alreadyExists: false, count: 1 }); } - getCheckpointProposal(_slot: SlotNumber): Promise { - return Promise.resolve(undefined); + getCheckpointProposal(slot: SlotNumber): Promise { + return Promise.resolve(this.checkpoints.get(slot)?.[0]); + } + + getProposalsForSlot(slot: SlotNumber): Promise<{ + blockProposals: BlockProposal[]; + checkpointProposals: CheckpointProposalCore[]; + }> { + return Promise.resolve({ + blockProposals: [...this.proposals.values()].filter(proposal => proposal.slotNumber === slot), + checkpointProposals: this.checkpoints.get(slot) ?? [], + }); } async addOwnCheckpointAttestations(_attestations: CheckpointAttestation[]): Promise {} @@ -267,6 +276,7 @@ export class InMemoryAttestationPool { resetState(): void { this.proposals.clear(); + this.checkpoints.clear(); } } diff --git a/yarn-project/slasher/README.md b/yarn-project/slasher/README.md index fd8aa439b041..e62427fab4f9 100644 --- a/yarn-project/slasher/README.md +++ b/yarn-project/slasher/README.md @@ -134,6 +134,12 @@ List of all slashable offenses in the system: **Target**: Committee members who attested in the invalid proposal slot. **Time Unit**: Slot-based offense. +### BROADCASTED_INVALID_CHECKPOINT_PROPOSAL +**Description**: A proposer broadcast a checkpoint proposal that terminates before a higher-index block proposal signed by the same proposer in the same slot. +**Detection**: BroadcastedInvalidCheckpointProposalWatcher scans retained P2P proposals and compares checkpoint archive roots to signed block proposals from the same slot and signer. +**Target**: Proposer who broadcast the truncated checkpoint proposal. +**Time Unit**: Slot-based offense. + ## Configuration ### L1 System Settings (L1ContractsConfig) @@ -167,6 +173,7 @@ These settings are configured locally on each validator node: - `slashDataWithholdingPenalty`: Penalty for DATA_WITHHOLDING - `slashInactivityPenalty`: Penalty for INACTIVITY - `slashBroadcastedInvalidBlockPenalty`: Penalty for BROADCASTED_INVALID_BLOCK_PROPOSAL +- `slashBroadcastedInvalidCheckpointProposalPenalty`: Penalty for BROADCASTED_INVALID_CHECKPOINT_PROPOSAL - `slashDuplicateProposalPenalty`: Penalty for DUPLICATE_PROPOSAL - `slashProposeInvalidAttestationsPenalty`: Penalty for PROPOSED_INSUFFICIENT_ATTESTATIONS and PROPOSED_INCORRECT_ATTESTATIONS - `slashAttestDescendantOfInvalidPenalty`: Penalty for ATTESTED_DESCENDANT_OF_INVALID diff --git a/yarn-project/slasher/src/config.ts b/yarn-project/slasher/src/config.ts index 26102d3bb805..441ee79e6551 100644 --- a/yarn-project/slasher/src/config.ts +++ b/yarn-project/slasher/src/config.ts @@ -21,6 +21,7 @@ export const DefaultSlasherConfig: SlasherConfig = { slashInactivityTargetPercentage: slasherDefaultEnv.SLASH_INACTIVITY_TARGET_PERCENTAGE, slashInactivityConsecutiveEpochThreshold: slasherDefaultEnv.SLASH_INACTIVITY_CONSECUTIVE_EPOCH_THRESHOLD, slashBroadcastedInvalidBlockPenalty: BigInt(slasherDefaultEnv.SLASH_INVALID_BLOCK_PENALTY), + slashBroadcastedInvalidCheckpointProposalPenalty: BigInt(slasherDefaultEnv.SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY), slashDuplicateProposalPenalty: BigInt(slasherDefaultEnv.SLASH_DUPLICATE_PROPOSAL_PENALTY), slashDuplicateAttestationPenalty: BigInt(slasherDefaultEnv.SLASH_DUPLICATE_ATTESTATION_PENALTY), slashInactivityPenalty: BigInt(slasherDefaultEnv.SLASH_INACTIVITY_PENALTY), @@ -81,6 +82,11 @@ export const slasherConfigMappings: ConfigMappingsType = { description: 'Penalty amount for slashing a validator for an invalid block proposed via p2p.', ...bigintConfigHelper(DefaultSlasherConfig.slashBroadcastedInvalidBlockPenalty), }, + slashBroadcastedInvalidCheckpointProposalPenalty: { + env: 'SLASH_INVALID_CHECKPOINT_PROPOSAL_PENALTY', + description: 'Penalty amount for slashing a validator for an invalid checkpoint proposal proposed via p2p.', + ...bigintConfigHelper(DefaultSlasherConfig.slashBroadcastedInvalidCheckpointProposalPenalty), + }, slashDuplicateProposalPenalty: { env: 'SLASH_DUPLICATE_PROPOSAL_PENALTY', description: 'Penalty amount for slashing a validator for sending duplicate proposals.', diff --git a/yarn-project/slasher/src/index.ts b/yarn-project/slasher/src/index.ts index 797815fceec6..b59b375fb98f 100644 --- a/yarn-project/slasher/src/index.ts +++ b/yarn-project/slasher/src/index.ts @@ -1,6 +1,7 @@ export * from './config.js'; export * from './watchers/epoch_prune_watcher.js'; export * from './watchers/attestations_block_watcher.js'; +export * from './watchers/broadcasted_invalid_checkpoint_proposal_watcher.js'; export * from './slasher_client.js'; export * from './slash_offenses_collector.js'; export * from './slasher_client_interface.js'; diff --git a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts new file mode 100644 index 000000000000..1bc4c1654823 --- /dev/null +++ b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts @@ -0,0 +1,237 @@ +import type { EpochCacheInterface } from '@aztec/epoch-cache'; +import { IndexWithinCheckpoint, SlotNumber } from '@aztec/foundation/branded-types'; +import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { EmptyL1RollupConstants } from '@aztec/stdlib/epoch-helpers'; +import type { P2PClient } from '@aztec/stdlib/interfaces/server'; +import type { BlockProposal, CheckpointProposalCore } from '@aztec/stdlib/p2p'; +import { OffenseType } from '@aztec/stdlib/slashing'; +import { + makeBlockHeader, + makeBlockProposal, + makeCheckpointHeader, + makeCheckpointProposal, +} from '@aztec/stdlib/testing'; + +import { jest } from '@jest/globals'; +import { type MockProxy, mock } from 'jest-mock-extended'; + +import { DefaultSlasherConfig, type SlasherConfig } from '../config.js'; +import { WANT_TO_SLASH_EVENT, type WantToSlashArgs } from '../watcher.js'; +import { BroadcastedInvalidCheckpointProposalWatcher } from './broadcasted_invalid_checkpoint_proposal_watcher.js'; + +describe('BroadcastedInvalidCheckpointProposalWatcher', () => { + let p2pClient: MockProxy>; + let epochCache: MockProxy>; + let config: SlasherConfig; + let watcher: BroadcastedInvalidCheckpointProposalWatcher; + let handler: jest.MockedFunction<(args: WantToSlashArgs[]) => void>; + + beforeEach(() => { + p2pClient = mock>(); + epochCache = mock>(); + epochCache.getCurrentAndNextSlot.mockReturnValue({ currentSlot: SlotNumber(12), nextSlot: SlotNumber(13) }); + epochCache.getL1Constants.mockReturnValue({ + ...EmptyL1RollupConstants, + epochDuration: 8, + ethereumSlotDuration: 12, + }); + config = { + ...DefaultSlasherConfig, + slashBroadcastedInvalidCheckpointProposalPenalty: 11n, + }; + watcher = new BroadcastedInvalidCheckpointProposalWatcher(p2pClient, epochCache, config, 4); + handler = jest.fn(); + watcher.on(WANT_TO_SLASH_EVENT, handler); + }); + + const makeBlocks = async (signer: Secp256k1Signer, slot: SlotNumber, count: number): Promise => + await Promise.all( + Array.from({ length: count }, (_, index) => + makeBlockProposal({ + signer, + blockHeader: makeBlockHeader(index + 1, { slotNumber: slot }), + archiveRoot: Fr.random(), + indexWithinCheckpoint: IndexWithinCheckpoint(index), + }), + ), + ); + + const makeCheckpointCore = async ( + signer: Secp256k1Signer, + slot: SlotNumber, + terminalBlock: BlockProposal, + includeLastBlock = false, + ): Promise => { + const checkpoint = await makeCheckpointProposal({ + signer, + checkpointHeader: makeCheckpointHeader(1, { slotNumber: slot }), + archiveRoot: terminalBlock.archive, + lastBlock: includeLastBlock + ? { + blockHeader: terminalBlock.blockHeader, + indexWithinCheckpoint: terminalBlock.indexWithinCheckpoint, + txHashes: terminalBlock.txHashes, + } + : undefined, + }); + return checkpoint.toCore(); + }; + + const mockProposals = ( + slot: SlotNumber, + blockProposals: BlockProposal[], + checkpointProposals: CheckpointProposalCore[], + ) => + p2pClient.getProposalsForSlot.mockImplementation(querySlot => + Promise.resolve( + querySlot === slot ? { blockProposals, checkpointProposals } : { blockProposals: [], checkpointProposals: [] }, + ), + ); + + it('slashes when higher-index block proposals arrive before a truncated checkpoint proposal', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const checkpoint = await makeCheckpointCore(signer, slot, blocks[1]); + mockProposals(slot, blocks, [checkpoint]); + + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledWith([ + { + validator: signer.address, + amount: 11n, + offenseType: OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: 10n, + }, + ]); + }); + + it('slashes when a higher-index proposal arrives after an earlier non-slashing scan', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const checkpoint = await makeCheckpointCore(signer, slot, blocks[1]); + mockProposals(slot, blocks.slice(0, 2), [checkpoint]); + + await watcher.scanSlot(slot); + expect(handler).not.toHaveBeenCalled(); + + mockProposals(slot, blocks, [checkpoint]); + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledTimes(1); + expect(handler.mock.calls[0][0][0].validator).toEqual(signer.address); + }); + + it('infers the terminal proposal from a retained block reconstructed out of embedded lastBlock', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const checkpointWithLastBlock = await makeCheckpointProposal({ + signer, + checkpointHeader: makeCheckpointHeader(1, { slotNumber: slot }), + archiveRoot: blocks[1].archive, + lastBlock: { + blockHeader: blocks[1].blockHeader, + indexWithinCheckpoint: blocks[1].indexWithinCheckpoint, + txHashes: blocks[1].txHashes, + }, + }); + mockProposals(slot, [checkpointWithLastBlock.getBlockProposal()!, blocks[2]], [checkpointWithLastBlock.toCore()]); + + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledTimes(1); + expect(handler.mock.calls[0][0][0].validator).toEqual(signer.address); + }); + + it('does not slash when the checkpoint terminates at the highest known block', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const checkpoint = await makeCheckpointCore(signer, slot, blocks[3]); + mockProposals(slot, blocks, [checkpoint]); + + await watcher.scanSlot(slot); + + expect(handler).not.toHaveBeenCalled(); + }); + + it('does not slash without a matching signed terminal block proposal', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const missingTerminal = await makeBlockProposal({ + signer, + blockHeader: makeBlockHeader(99, { slotNumber: slot }), + archiveRoot: Fr.random(), + indexWithinCheckpoint: IndexWithinCheckpoint(1), + }); + const checkpoint = await makeCheckpointCore(signer, slot, missingTerminal); + mockProposals(slot, blocks, [checkpoint]); + + await watcher.scanSlot(slot); + + expect(handler).not.toHaveBeenCalled(); + }); + + it('does not slash when the higher-index block is signed by a different validator', async () => { + const signer = Secp256k1Signer.random(); + const otherSigner = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 2); + const higherBlock = (await makeBlocks(otherSigner, slot, 3))[2]; + const checkpoint = await makeCheckpointCore(signer, slot, blocks[1]); + mockProposals(slot, [...blocks, higherBlock], [checkpoint]); + + await watcher.scanSlot(slot); + + expect(handler).not.toHaveBeenCalled(); + }); + + it('does not emit duplicate offenses on repeated scans', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const checkpoint = await makeCheckpointCore(signer, slot, blocks[1]); + mockProposals(slot, blocks, [checkpoint]); + + await watcher.scanSlot(slot); + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledTimes(1); + }); + + it('scans a lookback of closed slots', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const checkpoint = await makeCheckpointCore(signer, slot, blocks[1]); + mockProposals(slot, blocks, [checkpoint]); + + await watcher.scan(); + + expect(p2pClient.getProposalsForSlot).toHaveBeenCalledWith(SlotNumber(7)); + expect(p2pClient.getProposalsForSlot).toHaveBeenCalledWith(SlotNumber(10)); + expect(handler).toHaveBeenCalledTimes(1); + }); + + it('only expands beyond the lookback for newly closed slots', async () => { + p2pClient.getProposalsForSlot.mockResolvedValue({ blockProposals: [], checkpointProposals: [] }); + + await watcher.scan(); + p2pClient.getProposalsForSlot.mockClear(); + epochCache.getCurrentAndNextSlot.mockReturnValue({ currentSlot: SlotNumber(13), nextSlot: SlotNumber(14) }); + + await watcher.scan(); + + expect(p2pClient.getProposalsForSlot.mock.calls.map(([slot]) => slot)).toEqual([ + SlotNumber(8), + SlotNumber(9), + SlotNumber(10), + SlotNumber(11), + ]); + }); +}); diff --git a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts new file mode 100644 index 000000000000..66d651c45bea --- /dev/null +++ b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts @@ -0,0 +1,191 @@ +import type { EpochCacheInterface } from '@aztec/epoch-cache'; +import { SlotNumber } from '@aztec/foundation/branded-types'; +import { merge, pick } from '@aztec/foundation/collection'; +import type { EthAddress } from '@aztec/foundation/eth-address'; +import { type Logger, createLogger } from '@aztec/foundation/log'; +import { RunningPromise } from '@aztec/foundation/running-promise'; +import type { P2PClient, SlasherConfig } from '@aztec/stdlib/interfaces/server'; +import type { BlockProposal, CheckpointProposalCore } from '@aztec/stdlib/p2p'; +import { OffenseType } from '@aztec/stdlib/slashing'; + +import EventEmitter from 'node:events'; + +import { WANT_TO_SLASH_EVENT, type WantToSlashArgs, type Watcher, type WatcherEmitter } from '../watcher.js'; + +const BroadcastedInvalidCheckpointProposalWatcherConfigKeys = [ + 'slashBroadcastedInvalidCheckpointProposalPenalty', +] as const; + +const SCAN_SLOT_LAG = 1; +const DEFAULT_SCAN_SLOT_LOOKBACK = 4; + +type BroadcastedInvalidCheckpointProposalWatcherConfig = Pick< + SlasherConfig, + (typeof BroadcastedInvalidCheckpointProposalWatcherConfigKeys)[number] +>; + +type ProposalsForSlot = Awaited>; +type P2PProposalsForSlotSource = Pick; + +type SignedBlockProposal = { + proposal: BlockProposal; + signer: EthAddress; +}; + +/** Detects truncated-checkpoint proposal offenses from retained signed P2P proposals. */ +export class BroadcastedInvalidCheckpointProposalWatcher + extends (EventEmitter as new () => WatcherEmitter) + implements Watcher +{ + private readonly log: Logger = createLogger('broadcasted-invalid-checkpoint-proposal-watcher'); + private readonly runningPromise: RunningPromise; + private readonly emittedOffenses = new Set(); + private readonly scanSlotLookback: number; + private config: BroadcastedInvalidCheckpointProposalWatcherConfig; + private lastScannedSlot: SlotNumber | undefined; + + constructor( + private readonly p2pClient: P2PProposalsForSlotSource, + private readonly epochCache: Pick, + config: BroadcastedInvalidCheckpointProposalWatcherConfig, + scanSlotLookback = DEFAULT_SCAN_SLOT_LOOKBACK, + ) { + super(); + const constants = epochCache.getL1Constants(); + this.config = pick(config, ...BroadcastedInvalidCheckpointProposalWatcherConfigKeys); + this.scanSlotLookback = Math.max(1, scanSlotLookback); + const intervalMs = Math.max(1000, (constants.ethereumSlotDuration * 1000) / 4); + this.runningPromise = new RunningPromise(() => this.scan(), this.log, intervalMs); + this.log.info('BroadcastedInvalidCheckpointProposalWatcher initialized', { + scanSlotLookback: this.scanSlotLookback, + }); + } + + public updateConfig(config: Partial): void { + this.config = merge(this.config, pick(config, ...BroadcastedInvalidCheckpointProposalWatcherConfigKeys)); + this.log.verbose('BroadcastedInvalidCheckpointProposalWatcher config updated', this.config); + } + + public start(): Promise { + this.runningPromise.start(); + return Promise.resolve(); + } + + public stop(): Promise { + return this.runningPromise.stop(); + } + + /** Scans newly closed slots, plus a small lookback for late-arriving proposals. */ + public async scan(): Promise { + if (this.config.slashBroadcastedInvalidCheckpointProposalPenalty <= 0n) { + return; + } + + const { currentSlot } = this.epochCache.getCurrentAndNextSlot(); + if (currentSlot <= SlotNumber(SCAN_SLOT_LAG)) { + return; + } + + const newestSlotToConsider = SlotNumber(currentSlot - 1 - SCAN_SLOT_LAG); + const oldestLookbackSlot = SlotNumber(Math.max(0, newestSlotToConsider - this.scanSlotLookback + 1)); + const oldestUnscannedSlot = + this.lastScannedSlot === undefined ? oldestLookbackSlot : SlotNumber(this.lastScannedSlot + 1); + const oldestSlot = SlotNumber(Math.min(oldestLookbackSlot, oldestUnscannedSlot)); + for (let slot = oldestSlot; slot <= newestSlotToConsider; slot++) { + await this.scanSlot(SlotNumber(slot)); + } + this.lastScannedSlot = newestSlotToConsider; + } + + /** Scans a single slot. Public for tests. */ + public async scanSlot(slot: SlotNumber): Promise { + if (this.config.slashBroadcastedInvalidCheckpointProposalPenalty <= 0n) { + return; + } + + const proposals = await this.p2pClient.getProposalsForSlot(slot); + const slashArgs = this.getSlashArgsForProposals(slot, proposals).filter(args => this.markAsNewOffense(args)); + if (slashArgs.length === 0) { + return; + } + + this.log.info(`Detected broadcasted invalid checkpoint proposal offense`, { + slot, + offenses: slashArgs.map(args => ({ + validator: args.validator.toString(), + offenseType: args.offenseType, + epochOrSlot: args.epochOrSlot, + })), + }); + this.emit(WANT_TO_SLASH_EVENT, slashArgs); + } + + private getSlashArgsForProposals(slot: SlotNumber, proposals: ProposalsForSlot): WantToSlashArgs[] { + const offenders = this.findOffenders(proposals.blockProposals, proposals.checkpointProposals); + // we expect one proposer per slot today. + return [...offenders.values()].map(validator => ({ + validator, + amount: this.config.slashBroadcastedInvalidCheckpointProposalPenalty, + offenseType: OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: BigInt(slot), + })); + } + + private findOffenders(blockProposals: BlockProposal[], checkpointProposals: CheckpointProposalCore[]) { + const blocksBySigner = this.getSignedBlocksBySigner(blockProposals); + const offenders = new Map(); + + for (const checkpoint of checkpointProposals) { + const checkpointSigner = checkpoint.getSender(); + if (!checkpointSigner) { + continue; + } + + const signerKey = checkpointSigner.toString(); + const signerBlocks = blocksBySigner.get(signerKey) ?? []; + const terminalBlocks = signerBlocks.filter( + ({ proposal }) => proposal.slotNumber === checkpoint.slotNumber && proposal.archive.equals(checkpoint.archive), + ); + if (terminalBlocks.length === 0) { + continue; + } + + const hasTruncatedHigherBlock = terminalBlocks.some(terminalBlock => + signerBlocks.some( + ({ proposal }) => + proposal.slotNumber === checkpoint.slotNumber && + proposal.indexWithinCheckpoint > terminalBlock.proposal.indexWithinCheckpoint, + ), + ); + if (hasTruncatedHigherBlock) { + offenders.set(signerKey, checkpointSigner); + } + } + + return offenders; + } + + private getSignedBlocksBySigner(blockProposals: BlockProposal[]): Map { + const blocksBySigner = new Map(); + for (const proposal of blockProposals) { + const signer = proposal.getSender(); + if (!signer) { + continue; + } + const signerKey = signer.toString(); + const signerBlocks = blocksBySigner.get(signerKey) ?? []; + signerBlocks.push({ proposal, signer }); + blocksBySigner.set(signerKey, signerBlocks); + } + return blocksBySigner; + } + + private markAsNewOffense(args: WantToSlashArgs): boolean { + const key = `${args.validator.toString()}-${args.offenseType}-${args.epochOrSlot}`; + if (this.emittedOffenses.has(key)) { + return false; + } + this.emittedOffenses.add(key); + return true; + } +} diff --git a/yarn-project/stdlib/src/interfaces/aztec-node-admin.test.ts b/yarn-project/stdlib/src/interfaces/aztec-node-admin.test.ts index 127d58eb453e..43de98324029 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node-admin.test.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node-admin.test.ts @@ -113,6 +113,7 @@ class MockAztecNodeAdmin implements AztecNodeAdmin { slashInactivityConsecutiveEpochThreshold: 1, slashInactivityPenalty: 1000n, slashBroadcastedInvalidBlockPenalty: 1n, + slashBroadcastedInvalidCheckpointProposalPenalty: 1n, slashDuplicateProposalPenalty: 1n, slashDuplicateAttestationPenalty: 1n, slashAttestInvalidCheckpointProposalPenalty: 1000n, diff --git a/yarn-project/stdlib/src/interfaces/p2p.ts b/yarn-project/stdlib/src/interfaces/p2p.ts index a729ecf22938..1b02398d51d2 100644 --- a/yarn-project/stdlib/src/interfaces/p2p.ts +++ b/yarn-project/stdlib/src/interfaces/p2p.ts @@ -2,7 +2,9 @@ import type { CheckpointProposalHash, SlotNumber } from '@aztec/foundation/brand import { z } from 'zod'; +import type { BlockProposal } from '../p2p/block_proposal.js'; import { CheckpointAttestation } from '../p2p/checkpoint_attestation.js'; +import type { CheckpointProposalCore } from '../p2p/checkpoint_proposal.js'; import { type ApiSchemaFor, optional, schemas } from '../schemas/index.js'; import { Tx } from '../tx/tx.js'; import { TxHash } from '../tx/tx_hash.js'; @@ -67,6 +69,12 @@ export interface P2PApi { export interface P2PClient extends P2PApi { /** Manually adds checkpoint attestations to the p2p client attestation pool. */ addOwnCheckpointAttestations(attestations: CheckpointAttestation[]): Promise; + + /** Returns retained signed proposals for a slot. */ + getProposalsForSlot(slot: SlotNumber): Promise<{ + blockProposals: BlockProposal[]; + checkpointProposals: CheckpointProposalCore[]; + }>; } export const P2PApiSchema: ApiSchemaFor = { diff --git a/yarn-project/stdlib/src/interfaces/slasher.ts b/yarn-project/stdlib/src/interfaces/slasher.ts index 9e71e16e0f16..0dc264a0c183 100644 --- a/yarn-project/stdlib/src/interfaces/slasher.ts +++ b/yarn-project/stdlib/src/interfaces/slasher.ts @@ -14,6 +14,7 @@ export interface SlasherConfig { slashDataWithholdingPenalty: bigint; slashInactivityPenalty: bigint; slashBroadcastedInvalidBlockPenalty: bigint; + slashBroadcastedInvalidCheckpointProposalPenalty: bigint; slashDuplicateProposalPenalty: bigint; slashDuplicateAttestationPenalty: bigint; slashProposeInvalidAttestationsPenalty: bigint; @@ -37,6 +38,7 @@ export const SlasherConfigSchema = zodFor()( slashInactivityConsecutiveEpochThreshold: z.number(), slashInactivityPenalty: schemas.BigInt, slashProposeInvalidAttestationsPenalty: schemas.BigInt, + slashBroadcastedInvalidCheckpointProposalPenalty: schemas.BigInt, slashDuplicateProposalPenalty: schemas.BigInt, slashDuplicateAttestationPenalty: schemas.BigInt, slashAttestDescendantOfInvalidPenalty: schemas.BigInt, diff --git a/yarn-project/stdlib/src/slashing/helpers.test.ts b/yarn-project/stdlib/src/slashing/helpers.test.ts index cc2d7c00e56d..98588f87d9f1 100644 --- a/yarn-project/stdlib/src/slashing/helpers.test.ts +++ b/yarn-project/stdlib/src/slashing/helpers.test.ts @@ -199,6 +199,7 @@ describe('SlashingHelpers', () => { const penalty = getPenaltyForOffense(OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, { slashAttestDescendantOfInvalidPenalty: 1n, slashBroadcastedInvalidBlockPenalty: 2n, + slashBroadcastedInvalidCheckpointProposalPenalty: 11n, slashDuplicateProposalPenalty: 3n, slashDuplicateAttestationPenalty: 4n, slashAttestInvalidCheckpointProposalPenalty: 5n, @@ -211,5 +212,23 @@ describe('SlashingHelpers', () => { expect(penalty).toBe(5n); }); + + it('returns the configured penalty for broadcasting invalid checkpoint proposal', () => { + const penalty = getPenaltyForOffense(OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, { + slashAttestDescendantOfInvalidPenalty: 1n, + slashBroadcastedInvalidBlockPenalty: 2n, + slashBroadcastedInvalidCheckpointProposalPenalty: 11n, + slashDuplicateProposalPenalty: 3n, + slashDuplicateAttestationPenalty: 4n, + slashAttestInvalidCheckpointProposalPenalty: 5n, + slashPrunePenalty: 6n, + slashDataWithholdingPenalty: 7n, + slashUnknownPenalty: 8n, + slashInactivityPenalty: 9n, + slashProposeInvalidAttestationsPenalty: 10n, + }); + + expect(penalty).toBe(11n); + }); }); }); diff --git a/yarn-project/stdlib/src/slashing/helpers.ts b/yarn-project/stdlib/src/slashing/helpers.ts index 21ca279597f1..135c0d247c29 100644 --- a/yarn-project/stdlib/src/slashing/helpers.ts +++ b/yarn-project/stdlib/src/slashing/helpers.ts @@ -50,6 +50,7 @@ export function getPenaltyForOffense( SlasherConfig, | 'slashAttestDescendantOfInvalidPenalty' | 'slashBroadcastedInvalidBlockPenalty' + | 'slashBroadcastedInvalidCheckpointProposalPenalty' | 'slashDuplicateProposalPenalty' | 'slashDuplicateAttestationPenalty' | 'slashAttestInvalidCheckpointProposalPenalty' @@ -74,6 +75,8 @@ export function getPenaltyForOffense( return config.slashAttestDescendantOfInvalidPenalty; case OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL: return config.slashBroadcastedInvalidBlockPenalty; + case OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL: + return config.slashBroadcastedInvalidCheckpointProposalPenalty; case OffenseType.DUPLICATE_PROPOSAL: return config.slashDuplicateProposalPenalty; case OffenseType.DUPLICATE_ATTESTATION: @@ -94,6 +97,7 @@ export function getTimeUnitForOffense(offense: OffenseType): 'epoch' | 'slot' { switch (offense) { case OffenseType.ATTESTED_DESCENDANT_OF_INVALID: case OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL: + case OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL: case OffenseType.DUPLICATE_PROPOSAL: case OffenseType.DUPLICATE_ATTESTATION: case OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL: diff --git a/yarn-project/stdlib/src/slashing/types.ts b/yarn-project/stdlib/src/slashing/types.ts index 6a72b45c061b..531489009866 100644 --- a/yarn-project/stdlib/src/slashing/types.ts +++ b/yarn-project/stdlib/src/slashing/types.ts @@ -26,6 +26,8 @@ export enum OffenseType { DUPLICATE_ATTESTATION = 9, /** A committee member attested to a checkpoint proposal in a slot with an invalid block proposal */ ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL = 10, + /** A proposer broadcast a checkpoint proposal truncated before a higher-index block proposal in the same slot */ + BROADCASTED_INVALID_CHECKPOINT_PROPOSAL = 11, } export function getOffenseTypeName(offense: OffenseType) { @@ -52,6 +54,8 @@ export function getOffenseTypeName(offense: OffenseType) { return 'duplicate_attestation'; case OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL: return 'attested_to_invalid_checkpoint_proposal'; + case OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL: + return 'broadcasted_invalid_checkpoint_proposal'; default: throw new Error(`Unknown offense type: ${offense}`); } @@ -71,6 +75,7 @@ export const OffenseToBigInt: Record = { [OffenseType.DUPLICATE_PROPOSAL]: 8n, [OffenseType.DUPLICATE_ATTESTATION]: 9n, [OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL]: 10n, + [OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL]: 11n, }; export function bigIntToOffense(offense: bigint): OffenseType { @@ -97,6 +102,8 @@ export function bigIntToOffense(offense: bigint): OffenseType { return OffenseType.DUPLICATE_ATTESTATION; case 10n: return OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL; + case 11n: + return OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL; default: throw new Error(`Unknown offense: ${offense}`); } diff --git a/yarn-project/txe/src/state_machine/dummy_p2p_client.ts b/yarn-project/txe/src/state_machine/dummy_p2p_client.ts index 25bc4085dc75..85a84a659282 100644 --- a/yarn-project/txe/src/state_machine/dummy_p2p_client.ts +++ b/yarn-project/txe/src/state_machine/dummy_p2p_client.ts @@ -18,7 +18,13 @@ import type { } from '@aztec/p2p'; import type { EthAddress, L2BlockStreamEvent, L2Tips } from '@aztec/stdlib/block'; import type { ITxProvider, PeerInfo } from '@aztec/stdlib/interfaces/server'; -import type { BlockProposal, CheckpointAttestation, CheckpointProposal, TopicType } from '@aztec/stdlib/p2p'; +import type { + BlockProposal, + CheckpointAttestation, + CheckpointProposal, + CheckpointProposalCore, + TopicType, +} from '@aztec/stdlib/p2p'; import type { BlockHeader, Tx, TxHash } from '@aztec/stdlib/tx'; export class DummyP2P implements P2P { @@ -159,6 +165,13 @@ export class DummyP2P implements P2P { throw new Error('DummyP2P does not implement "addOwnCheckpointAttestations"'); } + public getProposalsForSlot(_slot: SlotNumber): Promise<{ + blockProposals: BlockProposal[]; + checkpointProposals: CheckpointProposalCore[]; + }> { + return Promise.resolve({ blockProposals: [], checkpointProposals: [] }); + } + public getL2BlockHash(_number: number): Promise { throw new Error('DummyP2P does not implement "getL2BlockHash"'); } From 55df9620c58b1ad69f0d8de289e9fb7564d88541 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 14 May 2026 17:42:31 +0100 Subject: [PATCH 09/17] refactor: remove unused map in attestation pool (#23284) Remove the single-checkpoint-proposal map in favour of the "by hash" variant. --- .../attestation_pool/attestation_pool.ts | 65 ++++++++----------- .../attestation_pool_test_suite.ts | 7 +- .../p2p/src/test-helpers/testbench-utils.ts | 2 +- 3 files changed, 32 insertions(+), 42 deletions(-) diff --git a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts index d98e80e2fbeb..b2491ebee79d 100644 --- a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts +++ b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts @@ -14,7 +14,7 @@ import { PoolInstrumentation, PoolName, type PoolStatsCallback } from '../instru /** Result of trying to add an item (proposal or attestation) to the pool */ export type TryAddResult = { - /** Whether the item was added to a main store. False when the slot/position/(slot,signer) already had a stored entry, even if a new equivocation hash was tracked. */ + /** Whether the item was accepted into pool state. False when it already existed, was invalid, or hit a cap. */ added: boolean; /** Whether the exact signed payload (matched by payload hash) already existed in the pool. */ alreadyExists: boolean; @@ -58,13 +58,11 @@ export type AttestationPoolApi = Pick< * Attestations and proposals observed via the p2p network are stored for requests * from the validator to produce a block, or to serve to other peers. * - * Equivocation detection: each main store holds at most one entry per equivocation - * position (one checkpoint proposal per slot, one block proposal per (slot, position), - * one attestation per (slot, signer)). Distinct *signed payload hashes* arriving at - * the same position are tracked in the matching index multimap so the equivocation - * count reaches 2 even when archive collides on `feeAssetPriceModifier` variants. - * Proposal bytes are retained per accepted payload hash, up to the same - * equivocation caps, for slashing watchers that need signed P2P proposals. + * Equivocation detection: distinct *signed payload hashes* arriving at the same + * position are tracked in the matching index multimap so the equivocation count + * reaches 2 even when archive collides on `feeAssetPriceModifier` variants. + * Proposal bytes are retained per accepted payload hash, up to the same equivocation + * caps, for slashing watchers that need signed P2P proposals. */ export class AttestationPool { private metrics: PoolInstrumentation; @@ -79,10 +77,6 @@ export class AttestationPool { // Key: `${paddedSlot}-${signerAddress}`, Value: CheckpointProposalHash (`0x`-prefixed hex) private attestationHashesPerSlotAndSigner: AztecAsyncMultiMap; - // Checkpoint proposals from slot number to serialized CheckpointProposal. - // Stores the first proposal seen per slot. - private checkpointProposalPerSlot: AztecAsyncMap; - // Checkpoint proposals from `${paddedSlot}-${payloadHash}` to serialized CheckpointProposalCore. // Stores every accepted distinct payload up to MAX_CHECKPOINT_PROPOSALS_PER_SLOT. private checkpointProposalsPerSlotAndHash: AztecAsyncMap; @@ -118,7 +112,6 @@ export class AttestationPool { this.attestationHashesPerSlotAndSigner = store.openMultiMap('checkpoint_attestations_per_slot_and_signer'); // Initialize checkpoint proposal storage - this.checkpointProposalPerSlot = store.openMap('checkpoint_proposals'); this.checkpointProposalsPerSlotAndHash = store.openMap('checkpoint_proposals_by_slot_and_hash'); this.checkpointProposalHashesPerSlot = store.openMultiMap('checkpoint_proposals_for_slot'); @@ -133,13 +126,13 @@ export class AttestationPool { /** Returns whether the pool is empty. */ public async isEmpty(): Promise { - for await (const _ of this.attestationPerSlotAndSigner.entriesAsync()) { - return false; - } - for await (const _ of this.blockProposalsPerSlotIndexAndHash.entriesAsync()) { - return false; - } - return true; + const [attestationCount, blockProposalCount, checkpointProposalCount] = await Promise.all([ + this.attestationPerSlotAndSigner.sizeAsync(), + this.blockProposalsPerSlotIndexAndHash.sizeAsync(), + this.checkpointProposalsPerSlotAndHash.sizeAsync(), + ]); + + return attestationCount === 0 && blockProposalCount === 0 && checkpointProposalCount === 0; } /** Number of bits reserved for indexWithinCheckpoint in position keys. */ @@ -373,18 +366,10 @@ export class AttestationPool { proposal.toBuffer(), ); - // Only the first distinct payload at this slot is stored; later equivocations - // are retained in `checkpointProposalsPerSlotAndHash`. - const alreadyHasStored = await this.checkpointProposalPerSlot.hasAsync(slot); - if (!alreadyHasStored) { - await this.checkpointProposalPerSlot.set(slot, proposal.toBuffer()); - } - this.log.debug(`Added checkpoint proposal for slot ${slot}`, { archive: proposal.archive.toString(), payloadHash, slotNumber: slot, - stored: !alreadyHasStored, }); return { added: true, alreadyExists: false, count: count + 1 }; @@ -392,7 +377,9 @@ export class AttestationPool { } /** - * Get the (first) checkpoint proposal stored for the given slot. + * Get a retained checkpoint proposal stored for the given slot. + * If multiple proposals were retained for an equivocation, returns the lowest + * payload hash deterministically. * * Returns a CheckpointProposalCore (without lastBlock info) since the lastBlock * is extracted and stored separately as a BlockProposal when added. @@ -401,13 +388,16 @@ export class AttestationPool { * @return The checkpoint proposal core if one is stored, otherwise undefined. */ public async getCheckpointProposal(slot: SlotNumber): Promise { - const buffer = await this.checkpointProposalPerSlot.getAsync(slot); - try { - if (buffer && buffer.length > 0) { - return CheckpointProposal.fromBuffer(buffer); + for await (const [_, buffer] of this.checkpointProposalsPerSlotAndHash.entriesAsync( + this.getCheckpointProposalKeyRangeForSlot(slot), + )) { + try { + if (buffer && buffer.length > 0) { + return CheckpointProposal.fromBuffer(buffer); + } + } catch { + continue; } - } catch { - return undefined; } return undefined; @@ -524,16 +514,13 @@ export class AttestationPool { // Delete checkpoint proposals for slots < oldestSlot. for await (const slot of this.checkpointProposalHashesPerSlot.keysAsync({ end: oldestSlot })) { await this.checkpointProposalHashesPerSlot.delete(slot); - if (await this.checkpointProposalPerSlot.hasAsync(slot)) { - await this.checkpointProposalPerSlot.delete(slot); - numberOfCheckpointProposals++; - } } for await (const key of this.checkpointProposalsPerSlotAndHash.keysAsync({ end: `${oldestSlotPadded}-`, })) { await this.checkpointProposalsPerSlotAndHash.delete(key); + numberOfCheckpointProposals++; } // Delete block proposals for slots < oldestSlot, using blockProposalHashesPerSlotAndIndex as index. diff --git a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts index 33e092856237..19180d9d156d 100644 --- a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts +++ b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool_test_suite.ts @@ -385,13 +385,16 @@ export function describeAttestationPool(getAttestationPool: () => AttestationPoo const result2 = await ap.tryAddCheckpointProposal(proposal2); // The second distinct payload is tracked as an equivocation, count goes to 2, - // but its bytes are not retained — the first proposal stays in the main store. + // and both accepted payloads are retained by payload hash. expect(result2.added).toBe(true); expect(result2.alreadyExists).toBe(false); expect(result2.count).toBe(2); const retrievedProposal = await ap.getCheckpointProposal(SlotNumber(slotNumber)); - expect(retrievedProposal!.toBuffer()).toEqual(proposal1.toBuffer()); + const expectedProposal = [proposal1, proposal2].sort((a, b) => + a.getPayloadHash().localeCompare(b.getPayloadHash()), + )[0]; + expect(retrievedProposal!.toBuffer()).toEqual(expectedProposal.toBuffer()); const proposals = await ap.getProposalsForSlot(SlotNumber(slotNumber)); expect(proposals.checkpointProposals.map(proposal => proposal.toBuffer())).toEqual( diff --git a/yarn-project/p2p/src/test-helpers/testbench-utils.ts b/yarn-project/p2p/src/test-helpers/testbench-utils.ts index 3bba29f5b65e..2c1d982f92fb 100644 --- a/yarn-project/p2p/src/test-helpers/testbench-utils.ts +++ b/yarn-project/p2p/src/test-helpers/testbench-utils.ts @@ -271,7 +271,7 @@ export class InMemoryAttestationPool { } isEmpty(): Promise { - return Promise.resolve(this.proposals.size === 0); + return Promise.resolve(this.proposals.size === 0 && this.checkpoints.size === 0); } resetState(): void { From 78c6e8178adc6d439c59416f95443ca85830b4d4 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 14 May 2026 18:51:22 -0300 Subject: [PATCH 10/17] chore(p2p): assert last block in checkpoint proposal is correct (#23274) Checks that inHash, archive, and sig ctx match. Should catch errors during construction. --- .../stdlib/src/p2p/checkpoint_proposal.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/yarn-project/stdlib/src/p2p/checkpoint_proposal.ts b/yarn-project/stdlib/src/p2p/checkpoint_proposal.ts index 2595068e7f6d..286be1c2b64f 100644 --- a/yarn-project/stdlib/src/p2p/checkpoint_proposal.ts +++ b/yarn-project/stdlib/src/p2p/checkpoint_proposal.ts @@ -27,6 +27,7 @@ import { type CoordinationSignatureType, EMPTY_COORDINATION_SIGNATURE_CONTEXT, type Signable, + coordinationSignatureContextEquals, getCoordinationSignatureTypedData, readCoordinationSignatureContext, recoverCoordinationSigner, @@ -99,9 +100,28 @@ export class CheckpointProposal extends Gossipable implements Signable { public readonly signatureContext: CoordinationSignatureContext, /** Optional last block info, including its own signature for BlockProposal extraction */ - public readonly lastBlock?: CheckpointLastBlock, + public readonly lastBlock?: CheckpointLastBlock | BlockProposal, ) { super(); + + // Check that last block properties match those of the checkpoint. + if (lastBlock && 'inHash' in lastBlock && !lastBlock.inHash.equals(checkpointHeader.inHash)) { + throw new Error( + `CheckpointProposal lastBlock inHash ${lastBlock.inHash} does not match checkpoint inHash ${checkpointHeader.inHash}`, + ); + } + if (lastBlock && 'archiveRoot' in lastBlock && !lastBlock.archiveRoot.equals(archive)) { + throw new Error( + `CheckpointProposal lastBlock archive ${lastBlock.archiveRoot} does not match checkpoint archive ${archive}`, + ); + } + if ( + lastBlock && + 'signatureContext' in lastBlock && + !coordinationSignatureContextEquals(lastBlock.signatureContext, signatureContext) + ) { + throw new Error(`CheckpointProposal lastBlock signatureContext does not match checkpoint signatureContext`); + } } override generateP2PMessageIdentifier(): Promise { From 0da3480364ac15cdc8c35e423a7e3073f3f08b3d Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 14 May 2026 18:57:21 -0300 Subject: [PATCH 11/17] refactor(l1-tx-utils): use DateProvider for fail-fast timeout check (#23257) Addresses [Phil's review comment](https://github.com/AztecProtocol/aztec-packages/pull/23165#discussion_r3235804064) on #23165: uses the injected `DateProvider` instead of `new Date()` for the pre-gas-estimation timeout check in `L1TxUtils.sendTransaction`, so tests can drive the clock. --- .../ethereum/src/l1_tx_utils/l1_tx_utils.ts | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts index 5acf9209114f..47c2af0cf8f1 100644 --- a/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts +++ b/yarn-project/ethereum/src/l1_tx_utils/l1_tx_utils.ts @@ -213,6 +213,19 @@ export class L1TxUtils extends ReadOnlyL1TxUtils { return await this.signTransaction(txRequest as TransactionSerializable); } + private async checkInterruptedOrTimedOut(gasConfig: Pick): Promise { + if (this.interrupted) { + throw new InterruptError(`Transaction sending is interrupted`); + } + const now = new Date(await this.getL1Timestamp()); + if (gasConfig.txTimeoutAt && now > gasConfig.txTimeoutAt) { + throw new TimeoutError( + `Transaction timed out before sending (now ${now.toISOString()} > timeoutAt ${gasConfig.txTimeoutAt.toISOString()})`, + ); + } + return now; + } + /** * Sends a transaction with gas estimation and pricing * @param request - The transaction request (to, data, value) @@ -225,23 +238,15 @@ export class L1TxUtils extends ReadOnlyL1TxUtils { blobInputs?: L1BlobInputs, stateChange: TxUtilsState = TxUtilsState.SENT, ): Promise<{ txHash: Hex; state: L1TxState }> { - if (this.interrupted) { - throw new InterruptError(`Transaction sending is interrupted`); - } - - // Fail fast before doing any work (gas estimation, balance check) if the caller's deadline - // has already passed. The same check is repeated after gas estimation in case it took long - // enough to push us past the deadline. - if (gasConfigOverrides?.txTimeoutAt && new Date() > gasConfigOverrides.txTimeoutAt) { - throw new TimeoutError( - `Transaction timed out before sending (now ${new Date().toISOString()} > timeoutAt ${gasConfigOverrides.txTimeoutAt.toISOString()})`, - ); - } - try { const gasConfig = merge(this.config, gasConfigOverrides); const account = this.getSenderAddress().toString(); + // Fail fast before doing any work (gas estimation, balance check) if we've been interrupted + // or if the caller's deadline has already passed. The same check is repeated after gas + // estimation in case it took long enough to push us past the deadline. + await this.checkInterruptedOrTimedOut(gasConfig); + let gasLimit: bigint; if (this.debugMaxGasLimit) { gasLimit = MAX_L1_TX_LIMIT; @@ -254,16 +259,7 @@ export class L1TxUtils extends ReadOnlyL1TxUtils { const gasPrice = await this.getGasPrice(gasConfig, !!blobInputs); - if (this.interrupted) { - throw new InterruptError(`Transaction sending is interrupted`); - } - - const now = new Date(await this.getL1Timestamp()); - if (gasConfig.txTimeoutAt && now > gasConfig.txTimeoutAt) { - throw new TimeoutError( - `Transaction timed out before sending (now ${now.toISOString()} > timeoutAt ${gasConfig.txTimeoutAt.toISOString()})`, - ); - } + const now = await this.checkInterruptedOrTimedOut(gasConfig); let txHash: Hex; let nonce: number; From fff23f9012fed1be9ca0615bc5a1c24b9c83ab13 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 14 May 2026 22:13:05 -0300 Subject: [PATCH 12/17] feat(sandbox): support proposer pipelining in local network (#23277) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The local network sandbox (`aztec start --local-network`) historically ran without proposer pipelining, so the compose-routed e2e suite (`src/composed/*`, `src/guides/*`, cli-wallet flows, docs examples, playground) never exercised the pipelined sequencer path. Turning pipelining on revealed that each L2 slot took a full real-time slot (~72 s) before the L1 multicall fired, blowing up sandbox boot from ~30 s to ~5 min, because the existing `AnvilTestWatcher` triggers don't fire in the pipelined-publish window. ## Approach First commit flips `SEQ_ENABLE_PROPOSER_PIPELINING=true` on the three sandbox-test compose envs so every compose-routed test runs through the pipelined path. Second commit teaches `AnvilTestWatcher` about the proposer's target slot by hooking the sequencer's `block-proposed` event in `createLocalNetwork`; when the proposer has built a block destined for a slot beyond L1, the watcher warps L1 (and, via `cheatcodes.warp`, the injected date provider) forward, waking the pipelined publisher's `sendRequestsAt` sleep and the upstream `waitForValidParentCheckpointOnL1` wait. `block-proposed` is used rather than the cleaner `state-changed → PUBLISHING_CHECKPOINT` because the latter only fires *after* `waitForValidParentCheckpointOnL1` unblocks — which is what we are trying to break — so it would be circular. ## Changes - **yarn-project/end-to-end, docs/examples/ts, playground (compose)**: add `SEQ_ENABLE_PROPOSER_PIPELINING=true` to the `local-network` / `aztec` service env so every compose-routed sandbox test runs pipelined. - **yarn-project/aztec (`AnvilTestWatcher`)**: new `setProposedTargetSlot` setter and a `warpTimeIfNeeded` branch (gated on `isLocalNetwork`) that warps L1 to the target slot's timestamp when it's ahead of L1. - **yarn-project/aztec (`createLocalNetwork`)**: subscribe to the sequencer's `block-proposed` event and forward `slot` to the watcher. Verified locally: sandbox boot drops from ~5 min back to ~27 s under pipelining, and `e2e_local_network_example.test.ts` (both tests) passes in ~33 s. --- aztec-up/test/amm_flow.sh | 2 ++ aztec-up/test/basic_install.sh | 1 + aztec-up/test/bridge_and_claim.sh | 2 ++ docs/examples/ts/docker-compose.yml | 1 + playground/docker-compose.yml | 1 + .../aztec/src/local-network/local-network.ts | 15 +++++++++ .../aztec/src/testing/anvil_test_watcher.ts | 32 +++++++++++++++++++ .../end-to-end/scripts/docker-compose.yml | 1 + 8 files changed, 55 insertions(+) diff --git a/aztec-up/test/amm_flow.sh b/aztec-up/test/amm_flow.sh index d58b05276f70..75fda916c58f 100755 --- a/aztec-up/test/amm_flow.sh +++ b/aztec-up/test/amm_flow.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail +export SEQ_ENABLE_PROPOSER_PIPELINING=true + # Start local network and wait for port to open. aztec start --local-network & local_network_pid=$! diff --git a/aztec-up/test/basic_install.sh b/aztec-up/test/basic_install.sh index 497aeda2b873..ce4f7397f506 100755 --- a/aztec-up/test/basic_install.sh +++ b/aztec-up/test/basic_install.sh @@ -10,6 +10,7 @@ echo export LOG_LEVEL=silent export PXE_PROVER=none +export SEQ_ENABLE_PROPOSER_PIPELINING=true # Start local network and wait for port to open. aztec start --local-network & diff --git a/aztec-up/test/bridge_and_claim.sh b/aztec-up/test/bridge_and_claim.sh index b073daadb3c8..f26f5d664338 100755 --- a/aztec-up/test/bridge_and_claim.sh +++ b/aztec-up/test/bridge_and_claim.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail +export SEQ_ENABLE_PROPOSER_PIPELINING=true + # Start local network and wait for port to open. aztec start --local-network & local_network_pid=$! diff --git a/docs/examples/ts/docker-compose.yml b/docs/examples/ts/docker-compose.yml index d881961f4f74..247b321c6912 100644 --- a/docs/examples/ts/docker-compose.yml +++ b/docs/examples/ts/docker-compose.yml @@ -28,6 +28,7 @@ services: WS_BLOCK_CHECK_INTERVAL_MS: 500 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 P2P_MIN_TX_POOL_AGE_MS: 0 + SEQ_ENABLE_PROPOSER_PIPELINING: 'true' HARDWARE_CONCURRENCY: ${HARDWARE_CONCURRENCY:-} docs-examples: diff --git a/playground/docker-compose.yml b/playground/docker-compose.yml index d48663150b0d..7d86f4bd03ba 100644 --- a/playground/docker-compose.yml +++ b/playground/docker-compose.yml @@ -27,6 +27,7 @@ services: WS_BLOCK_CHECK_INTERVAL_MS: 50 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 P2P_MIN_TX_POOL_AGE_MS: 0 + SEQ_ENABLE_PROPOSER_PIPELINING: 'true' healthcheck: test: ['CMD', 'curl', '-fSs', 'http://127.0.0.1:8080/status'] interval: 3s diff --git a/yarn-project/aztec/src/local-network/local-network.ts b/yarn-project/aztec/src/local-network/local-network.ts index 6293df2653c5..a771e611b09c 100644 --- a/yarn-project/aztec/src/local-network/local-network.ts +++ b/yarn-project/aztec/src/local-network/local-network.ts @@ -206,6 +206,21 @@ export async function createLocalNetwork(config: Partial = { SequencerState.SYNCHRONIZING, ]); watcher?.setIsSequencerBuilding(() => !idleStates.has(sequencer.getState())); + // Under proposer pipelining the L1 publish for slot N happens during wall-clock slot N, + // but the proposer for slot N has already built the checkpoint during slot N-1 and is + // waiting for L1 to advance. We need to fast-forward L1 to wake that wait — and the wait + // we have to break first is `waitForValidParentCheckpointOnL1`, which blocks the + // checkpoint_proposal_job's background submission task until the archiver has synced past + // the build slot. That wait happens *before* `PUBLISHING_CHECKPOINT` is set, so a hook on + // that state transition would be circular (L1 has to advance before the state we'd use to + // advance L1 fires). The earliest pre-wait signal is `block-proposed`, which the sequencer + // emits once each block is built. In sandbox single-block-per-slot mode this is + // effectively "checkpoint built", and the watcher warp is harmless if a subsequent + // assembly/validation/parent-wait step aborts: L1 just sits one slot ahead, which the + // cascade absorbs. + if (watcher) { + sequencer.on('block-proposed', ({ slot }) => watcher!.setProposedTargetSlot(Number(slot))); + } } let epochTestSettler: EpochTestSettler | undefined; diff --git a/yarn-project/aztec/src/testing/anvil_test_watcher.ts b/yarn-project/aztec/src/testing/anvil_test_watcher.ts index e2f9c8ed2cbb..81505d5296fa 100644 --- a/yarn-project/aztec/src/testing/anvil_test_watcher.ts +++ b/yarn-project/aztec/src/testing/anvil_test_watcher.ts @@ -44,6 +44,12 @@ export class AnvilTestWatcher { // Tracks when we first observed the current unfilled slot with pending txs (real wall time). private unfilledSlotFirstSeen?: { slot: number; realTime: number }; + // Latest target slot for which the proposer has built a block destined for L1 but which has + // not yet been committed. Set by the proposer-pipelining hook from `block-proposed` events so + // the watcher can advance L1 (and the injected date provider) to the target slot ahead of the + // publisher's `sendRequestsAt` sleep, instead of waiting a full wall-clock slot. + private proposedTargetSlot?: number; + constructor( private cheatcodes: EthCheatCodes, rollupAddress: EthAddress, @@ -86,6 +92,18 @@ export class AnvilTestWatcher { this.isSequencerBuilding = fn; } + /** + * Records the target slot for which the proposer has built a block destined for L1. Used by + * the local-network watcher to fast-forward L1 (and the injected date provider) ahead of the + * pipelined publisher's `sendRequestsAt` sleep so it ends promptly instead of waiting a full + * wall-clock slot. Only ratchets up — late warps for stale slots are no-ops. + */ + setProposedTargetSlot(slot: number) { + if (this.proposedTargetSlot === undefined || slot > this.proposedTargetSlot) { + this.proposedTargetSlot = slot; + } + } + async start() { if (this.filledRunningPromise) { throw new Error('Watcher already watching for filled slot'); @@ -177,6 +195,20 @@ export class AnvilTestWatcher { return; } + // Pipelined-publish shortcut: if the proposer has built a block destined for a slot + // beyond the current L1 slot, fast-forward L1 to that slot's timestamp so the publisher's + // `sendRequestsAt(targetSlot)` sleep ends and the multicall mines inside the target slot. + // Without this, the publisher waits up to a full real-time slot for wall clock to catch up. + if (this.proposedTargetSlot !== undefined && this.proposedTargetSlot > currentSlot) { + const targetSlotTimestamp = Number( + await this.rollup.read.getTimestampForSlot([BigInt(this.proposedTargetSlot)]), + ); + if (await this.warpToTimestamp(targetSlotTimestamp)) { + this.logger.info(`Warped L1 to target slot ${this.proposedTargetSlot} for pipelined publish`); + } + return; + } + // If there are pending txs and the sequencer missed them, warp quickly (after a 2s real-time debounce) so the // sequencer can retry in the next slot. Without this, we'd have to wait a full real-time slot duration (~36s) for // the dateProvider to catch up to the next slot timestamp. We skip the warp if the sequencer is actively building diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 528efb33a286..e394e2610220 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -28,6 +28,7 @@ services: WS_BLOCK_CHECK_INTERVAL_MS: 500 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 P2P_MIN_TX_POOL_AGE_MS: 0 + SEQ_ENABLE_PROPOSER_PIPELINING: 'true' HARDWARE_CONCURRENCY: ${HARDWARE_CONCURRENCY:-} end-to-end: From 3ba6dbe6820f9e7fa92b6d8d65ccbb51541b9403 Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Thu, 14 May 2026 23:26:43 -0400 Subject: [PATCH 13/17] test(e2e): fix race in broadcasted_invalid_block_proposal_slash under pipelining (#23302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes the `e2e_p2p_broadcasted_invalid_block_proposal_slash` failure that has been blocking the `merge-train/spartan` train (run https://github.com/AztecProtocol/aztec-packages/actions/runs/25896899879, test log http://ci.aztec-labs.com/2bf4e2cd2d9e7944). The test creates the malicious proposer first (auto-starting its sequencer) and only later creates the honest nodes and waits for P2P mesh. Under `enableProposerPipelining: true` (turned on for this test by #23070), the malicious proposer is selected for the very next slot, builds + broadcasts the invalid proposal one slot ahead, and lands the broadcast before the honest validators have joined the mesh. They then reject it at the gossipsub `checkpoint_proposal_validator` with `Penalizing peer for invalid slot number` (since their target slot has already moved past), so the `state_mismatch` slashing path never runs. The malicious sequencer then gets stuck on the failed publish (`Awaiting pending L1 payload submission`) and never proposes again before the test times out on `awaitOffenseDetected`. This is the same race that #23070 fixed in `duplicate_proposal_slash.test.ts`; the same pattern is applied here: - Create both the invalid proposer and the honest nodes with `dontStartSequencer: true`. - After P2P mesh connectivity + committee formation, use `advanceToEpochBeforeProposer` to land one epoch before an epoch where the invalid proposer is scheduled. - Start all sequencers, then `advanceToEpoch(targetEpoch, { offset: -AZTEC_SLOT_DURATION })` so the malicious slot fires while every node is online and at the same wall-clock slot. - After `awaitOffenseDetected` on one node, poll `getSlashOffenses` across **all** nodes for `BROADCASTED_INVALID_BLOCK_PROPOSAL` — under pipelining a given receiver may have already advanced past the build slot when the proposal arrives, so we need to catch whichever node was still in the build slot. The on-chain slash assertion (`rollup.listenToSlash`) is preserved unchanged. Full failure analysis: https://gist.github.com/AztecBot/39b69c1117f419145938ccd2c198f8e9 ## Test plan - CI: `e2e_p2p_broadcasted_invalid_block_proposal_slash` passes on `merge-train/spartan`. - Local `./bootstrap.sh ci` / `fast` / `build` are not runnable in this container (no Docker socket and `$HOME` not writable for the container UID — `yarn install` fails on `corepack` mkdir, parallel-bootstrap can't create `~/.parallel`). Fix is a direct port of a pattern already shipping green on `next` via the sibling `duplicate_proposal_slash.test.ts`. ClaudeBox log: https://claudebox.work/s/06a4929a1971beaf?run=1 --- ...asted_invalid_block_proposal_slash.test.ts | 93 +++++++++++-------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts index 02dd223b3b86..2915bca2ce86 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts @@ -1,4 +1,5 @@ import type { AztecNodeService } from '@aztec/aztec-node'; +import type { TestAztecNodeService } from '@aztec/aztec-node/test'; import { EthAddress } from '@aztec/aztec.js/addresses'; import { EpochNumber } from '@aztec/foundation/branded-types'; import { promiseWithResolvers } from '@aztec/foundation/promise'; @@ -13,7 +14,7 @@ import path from 'path'; import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { createNodes } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest } from './p2p_network.js'; -import { awaitCommitteeExists, awaitOffenseDetected } from './shared.js'; +import { advanceToEpochBeforeProposer, awaitCommitteeExists, awaitOffenseDetected } from './shared.js'; const TEST_TIMEOUT = 1_000_000; @@ -114,10 +115,14 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { t.logger.warn('Creating nodes'); - // Create first node that broadcasts invalid proposals + // Create first node that broadcasts invalid proposals. Keep its sequencer stopped until + // every node has joined the P2P mesh; otherwise (under proposer pipelining) the invalid + // proposer can publish its sole bad block to slot N before the honest nodes are connected, + // and they will reject the proposal as "invalid slot number" instead of slashing it. const invalidProposerConfig = { ...t.ctx.aztecNodeConfig, broadcastInvalidBlockProposal: true, + dontStartSequencer: true, }; const invalidProposerNodes = await createNodes( invalidProposerConfig, @@ -134,9 +139,9 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { const invalidProposerAddress = invalidProposerNodes[0].getSequencer()!.validatorAddresses![0]; t.logger.warn(`Invalid proposer address: ${invalidProposerAddress.toString()}`); - // Create remaining honest nodes + // Create remaining honest nodes, also with sequencers stopped, for the same reason. const honestNodes = await createNodes( - t.ctx.aztecNodeConfig, + { ...t.ctx.aztecNodeConfig, dontStartSequencer: true }, t.ctx.dateProvider, t.bootstrapNodeEnr, NUM_VALIDATORS - 1, @@ -149,42 +154,39 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { nodes = [...invalidProposerNodes, ...honestNodes]; - // Wait for P2P mesh to be fully formed before proceeding + // Wait for P2P mesh to be fully formed before starting sequencers await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS); await awaitCommitteeExists({ rollup, logger: t.logger }); - const startSlot = await rollup.getSlotNumber(); - const proposerEarliestSlot = startSlot + 1; - - // Wait until the bad proposer has had a slot - await retryUntil( - async () => { - const currentSlot = await rollup.getSlotNumber(); - return currentSlot >= proposerEarliestSlot; - }, - 'Wait for next slot...', - TEST_TIMEOUT / 1000, - ETHEREUM_SLOT_DURATION, - ); - - await retryUntil( - async () => { - const currentProposer = await rollup.getCurrentProposer(); - if (!currentProposer.equals(invalidProposerAddress)) { - t.logger.info( - `Current proposer: ${currentProposer}, waiting for malicious proposer ${invalidProposerAddress} to get a slot...`, - ); - return false; - } - return true; - }, - 'Wait for malicious proposer slot...', - TEST_TIMEOUT / 1000, - ETHEREUM_SLOT_DURATION, - ); + // Find an epoch where the invalid proposer is selected, stopping one epoch before so + // we have time to start sequencers before the target epoch arrives. + const epochCache = (honestNodes[0] as TestAztecNodeService).epochCache; + const { targetEpoch } = await advanceToEpochBeforeProposer({ + epochCache, + cheatCodes: t.ctx.cheatCodes.rollup, + targetProposer: invalidProposerAddress, + logger: t.logger, + }); - const offenses = await awaitOffenseDetected({ + // Start all sequencers while still one epoch before the target + t.logger.warn('Starting all sequencers'); + await Promise.all(nodes.map(n => n.getSequencer()!.start())); + + // Now warp to one slot before the target epoch — sequencers are already running. + // Under proposer pipelining, the invalid proposer begins building for the first slot + // of the target epoch one slot earlier; warping to the start of the epoch would force + // the bad proposal to serialize past the slot boundary, after which honest receivers + // reject it as late. + t.logger.warn(`Advancing to one slot before target epoch ${targetEpoch}`); + await t.ctx.cheatCodes.rollup.advanceToEpoch(targetEpoch, { offset: -AZTEC_SLOT_DURATION }); + + // Wait for offense to be detected. Under proposer pipelining, the invalid block proposal is + // broadcast at the slot boundary while a receiver's wall clock may have already advanced + // past the build slot — when that happens, the honest node rejects the gossip with "invalid + // slot number" before slashing logic runs. Collect offenses from every node so we catch + // whichever node managed to process the proposal while still in the build slot. + await awaitOffenseDetected({ epochDuration: t.ctx.aztecNodeConfig.aztecEpochDuration, logger: t.logger, nodeAdmin: nodes[1], // Use honest node to check for offenses @@ -193,10 +195,23 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { timeoutSeconds: AZTEC_SLOT_DURATION * 16, }); - // Check offense is correct - expect(offenses).toHaveLength(1); - expect(offenses[0].offenseType).toEqual(OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL); - expect(offenses[0].validator.toString()).toEqual(t.validators[0].attester.toString()); + const invalidBlockOffenses = await retryUntil( + async () => { + const allOffenses = (await Promise.all(nodes.map(n => n.getSlashOffenses('all')))).flat(); + const filtered = allOffenses.filter(o => o.offenseType === OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL); + if (filtered.length > 0) { + return filtered; + } + }, + 'broadcasted invalid block proposal offense', + AZTEC_SLOT_DURATION * 4, + ); + + t.logger.warn(`Collected broadcasted invalid block proposal offenses`, { invalidBlockOffenses }); + expect(invalidBlockOffenses.length).toBeGreaterThan(0); + for (const offense of invalidBlockOffenses) { + expect(offense.validator.toString()).toEqual(invalidProposerAddress.toString()); + } // Check slash is recorded on chain const slashPromise = promiseWithResolvers<{ amount: bigint; attester: EthAddress }>(); From e0f2339e2ffb60a6a0e68b45922e663e9df6475e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 15 May 2026 05:29:51 -0300 Subject: [PATCH 14/17] fix(archiver): atomic getter for L2 tips (#23295) Prevents the archiver from reporting invalid L2 tips by querying all chain tips within a db transaction. Moves the responsibility of assembling the tip data to the block store itself to minimize the number of queries to the db. Clamps proven and finalized tips such that an incorrect L1 sync still results in finalized <= proven <= checkpoints. And adds explicit assertions that tips are ordered. Also adds a guard in the tips store that prevents from deleting block hashes that are still alive by a given chain tip, instead of assuming that the finalized chain tip is always the oldest one. This should catch errors where the block stream breaks due to a finalized chain tip running ahead of a proven chain tip. Note that this PR does NOT enforce ordering at the L2Tips struct itself, since consumers (ie the ones that report the "local" chain tips) may break this contract (see A-1061). This PR is a simpler alternative to #22964. Fixes A-1018. --- .../archiver/src/store/block_store.ts | 203 ++++++++++++++++-- .../archiver/src/store/l2_tips_cache.ts | 125 +---------- .../l2_block_stream/l2_tips_store_base.ts | 25 ++- 3 files changed, 218 insertions(+), 135 deletions(-) diff --git a/yarn-project/archiver/src/store/block_store.ts b/yarn-project/archiver/src/store/block_store.ts index 46490f35bd77..6e68cd2e41c8 100644 --- a/yarn-project/archiver/src/store/block_store.ts +++ b/yarn-project/archiver/src/store/block_store.ts @@ -13,7 +13,10 @@ import { BlockHash, Body, CommitteeAttestation, + GENESIS_CHECKPOINT_HEADER_HASH, L2Block, + type L2TipId, + type L2Tips, type ValidateCheckpointResult, deserializeValidateCheckpointResult, serializeValidateCheckpointResult, @@ -1129,6 +1132,174 @@ export class BlockStore { return typeof lastBlockNumber === 'number' ? BlockNumber(lastBlockNumber) : BlockNumber(INITIAL_L2_BLOCK_NUM - 1); } + /** + * Resolves all five L2 chain tips (proposed, proposedCheckpoint, checkpointed, proven, finalized) + * in a single read-only transaction so the snapshot is internally consistent. Each underlying + * record is read at most once: latest block, latest confirmed checkpoint, and latest pending + * checkpoint are each loaded directly (no separate "find the number, then look up data" hop), + * the proven/finalized checkpoint singletons are read once and their storage entries are + * reused if they coincide with the latest checkpoint, and per-tip block hashes are deduped + * when two tips land on the same block (e.g. finalized == proven, or proposedCheckpoint falls + * back to checkpointed when no pending checkpoint exists). + * + * The result is guaranteed to satisfy `finalized <= proven <= checkpointed <= proposed` (by + * block number). Genesis is represented by `(INITIAL_L2_BLOCK_NUM - 1)` and the supplied + * `genesisBlockHash`, paired with the synthetic genesis checkpoint id. + * + * @param genesisBlockHash - Block hash to report for the synthetic pre-initial block (used when + * a tip is still at genesis). + */ + async getL2TipsData(genesisBlockHash: BlockHash): Promise { + return await this.db.transactionAsync(async () => { + // Define genesis tips + const genesisBlockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1); + const genesisCheckpointNumber = CheckpointNumber(INITIAL_CHECKPOINT_NUMBER - 1); + const genesisBlockId = { number: genesisBlockNumber, hash: genesisBlockHash.toString() }; + const genesisCheckpointId = { + number: genesisCheckpointNumber, + hash: GENESIS_CHECKPOINT_HEADER_HASH.toString(), + }; + const genesisTip: L2TipId = { block: genesisBlockId, checkpoint: genesisCheckpointId }; + + // Load latest block and checkpoint entries + const [latestBlockEntry] = await toArray(this.#blocks.entriesAsync({ reverse: true, limit: 1 })); + const [proposedCheckpointEntry] = await toArray( + this.#proposedCheckpoints.entriesAsync({ reverse: true, limit: 1 }), + ); + const [latestCheckpointEntry] = await toArray(this.#checkpoints.entriesAsync({ reverse: true, limit: 1 })); + const latestCheckpointNumber = latestCheckpointEntry + ? CheckpointNumber(latestCheckpointEntry[0]) + : genesisCheckpointNumber; + + // Load proven and finalized checkpoint number pointers + const [provenRaw, finalizedRaw] = await Promise.all([ + this.#lastProvenCheckpoint.getAsync(), + this.#lastFinalizedCheckpoint.getAsync(), + ]); + + // Clamp to enforce finalized <= proven <= checkpointed. + const provenCheckpointNumber = CheckpointNumber(Math.min(provenRaw ?? 0, latestCheckpointNumber)); + const finalizedCheckpointNumber = CheckpointNumber(Math.min(finalizedRaw ?? 0, provenCheckpointNumber)); + + // Avoid loading the same checkpoint more than once + const checkpointStorageCache = new Map(); + if (latestCheckpointEntry) { + checkpointStorageCache.set(CheckpointNumber(latestCheckpointEntry[0]), latestCheckpointEntry[1]); + } + const loadCheckpointStorage = async (n: CheckpointNumber): Promise => { + if (n === 0) { + return undefined; + } + if (!checkpointStorageCache.has(n)) { + const checkpointStorage = await this.#checkpoints.getAsync(n); + if (!checkpointStorage) { + throw new CheckpointNotFoundError(n); + } + checkpointStorageCache.set(n, checkpointStorage); + } + return checkpointStorageCache.get(n)!; + }; + + // Load proven and finalized checkpoint storage entries + const provenCheckpoint = await loadCheckpointStorage(provenCheckpointNumber); + const finalizedCheckpoint = await loadCheckpointStorage(finalizedCheckpointNumber); + + // Avoid loading the same block hash multiple times when tips land on the same block + const blockHashCache = new Map(); + blockHashCache.set(genesisBlockNumber, genesisBlockHash.toString()); + if (latestBlockEntry) { + blockHashCache.set(latestBlockEntry[0], BlockHash.fromBuffer(latestBlockEntry[1].blockHash).toString()); + } + const loadBlockHash = async (n: BlockNumber): Promise => { + if (!blockHashCache.has(n)) { + const blockStorage = await this.#blocks.getAsync(n); + if (!blockStorage) { + throw new BlockNotFoundError(n); + } + const blockHash = BlockHash.fromBuffer(blockStorage.blockHash).toString(); + blockHashCache.set(n, blockHash); + } + return blockHashCache.get(n)!; + }; + + // Build proposed chain tip (this one has block only, no checkpoint) + const proposedBlockId = + latestBlockEntry === undefined + ? genesisBlockId + : { + number: BlockNumber(latestBlockEntry[0]), + hash: BlockHash.fromBuffer(latestBlockEntry[1].blockHash).toString(), + }; + + // Build other tips from checkpoint data, reading corresponding block data from the cache + const buildTipFromCheckpoint = async ( + stored: ProposedCheckpointStorage | CheckpointStorage | undefined, + ): Promise => { + if (!stored) { + return genesisTip; + } + const blockNumber = BlockNumber(stored.startBlock + stored.blockCount - 1); + const blockHash = await loadBlockHash(blockNumber); + const header = CheckpointHeader.fromBuffer(stored.header); + return { + block: { number: blockNumber, hash: blockHash }, + checkpoint: { number: CheckpointNumber(stored.checkpointNumber), hash: header.hash().toString() }, + }; + }; + + const checkpointedTip = await buildTipFromCheckpoint(latestCheckpointEntry?.[1]); + const provenTip = await buildTipFromCheckpoint(provenCheckpoint); + const finalizedTip = await buildTipFromCheckpoint(finalizedCheckpoint); + + // Proposed checkpoint falls back to the checkpoint tip if it's not set. And if local storage is + // inconsistent and the proposed checkpoint is behind the checkpointed tip, we patch that and + // report the checkpointed tip as the proposed checkpoint to maintain the invariant. + const proposedCheckpointTip = + proposedCheckpointEntry === undefined || proposedCheckpointEntry[0] <= latestCheckpointNumber + ? checkpointedTip + : await buildTipFromCheckpoint(proposedCheckpointEntry[1]); + + // A checkpointed block past the latest stored block would mean a checkpoint + // references blocks that aren't in blocks. + if (proposedBlockId.number < checkpointedTip.block.number) { + throw new Error( + `Inconsistent block store: latest block ${proposedBlockId.number} is behind checkpointed block ${checkpointedTip.block.number}`, + ); + } + + // Assert that checkpoint numbers are increasing + if ( + finalizedTip.checkpoint.number > provenTip.checkpoint.number || + provenTip.checkpoint.number > checkpointedTip.checkpoint.number || + checkpointedTip.checkpoint.number > proposedCheckpointTip.checkpoint.number + ) { + throw new Error( + `Inconsistent checkpoint numbers in chain tips: finalized=${finalizedTip.checkpoint.number} proven=${provenTip.checkpoint.number} checkpointed=${checkpointedTip.checkpoint.number} proposed=${proposedCheckpointTip.checkpoint.number}`, + ); + } + + // Assert block numbers are increasing + if ( + finalizedTip.block.number > provenTip.block.number || + provenTip.block.number > checkpointedTip.block.number || + checkpointedTip.block.number > proposedCheckpointTip.block.number || + proposedCheckpointTip.block.number > proposedBlockId.number + ) { + throw new Error( + `Inconsistent block numbers in chain tips: finalized=${finalizedTip.block.number} proven=${provenTip.block.number} checkpointed=${checkpointedTip.block.number} proposedCheckpoint=${proposedCheckpointTip.block.number} proposed=${proposedBlockId.number}`, + ); + } + + return { + proposed: proposedBlockId, + proposedCheckpoint: proposedCheckpointTip, + checkpointed: checkpointedTip, + proven: provenTip, + finalized: finalizedTip, + }; + }); + } + /** * Gets the most recent L1 block processed. * @returns The L1 block that published the latest L2 block @@ -1188,13 +1359,15 @@ export class BlockStore { } async getProvenCheckpointNumber(): Promise { - const [latestCheckpointNumber, provenCheckpointNumber] = await Promise.all([ - this.getLatestCheckpointNumber(), - this.#lastProvenCheckpoint.getAsync(), - ]); - return (provenCheckpointNumber ?? 0) > latestCheckpointNumber - ? latestCheckpointNumber - : CheckpointNumber(provenCheckpointNumber ?? 0); + return await this.db.transactionAsync(async () => { + const [latestCheckpointNumber, provenCheckpointNumber] = await Promise.all([ + this.getLatestCheckpointNumber(), + this.#lastProvenCheckpoint.getAsync(), + ]); + return (provenCheckpointNumber ?? 0) > latestCheckpointNumber + ? latestCheckpointNumber + : CheckpointNumber(provenCheckpointNumber ?? 0); + }); } async setProvenCheckpointNumber(checkpointNumber: CheckpointNumber) { @@ -1203,13 +1376,15 @@ export class BlockStore { } async getFinalizedCheckpointNumber(): Promise { - const [latestCheckpointNumber, finalizedCheckpointNumber] = await Promise.all([ - this.getLatestCheckpointNumber(), - this.#lastFinalizedCheckpoint.getAsync(), - ]); - return (finalizedCheckpointNumber ?? 0) > latestCheckpointNumber - ? latestCheckpointNumber - : CheckpointNumber(finalizedCheckpointNumber ?? 0); + return await this.db.transactionAsync(async () => { + const [provenCheckpointNumber, finalizedCheckpointNumber] = await Promise.all([ + this.getProvenCheckpointNumber(), + this.#lastFinalizedCheckpoint.getAsync(), + ]); + return (finalizedCheckpointNumber ?? 0) > provenCheckpointNumber + ? provenCheckpointNumber + : CheckpointNumber(finalizedCheckpointNumber ?? 0); + }); } setFinalizedCheckpointNumber(checkpointNumber: CheckpointNumber) { diff --git a/yarn-project/archiver/src/store/l2_tips_cache.ts b/yarn-project/archiver/src/store/l2_tips_cache.ts index bc69983fc722..68fa309a005b 100644 --- a/yarn-project/archiver/src/store/l2_tips_cache.ts +++ b/yarn-project/archiver/src/store/l2_tips_cache.ts @@ -1,12 +1,4 @@ -import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; -import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; -import { - type BlockData, - type BlockHash, - type CheckpointId, - GENESIS_CHECKPOINT_HEADER_HASH, - type L2Tips, -} from '@aztec/stdlib/block'; +import type { BlockHash, L2Tips } from '@aztec/stdlib/block'; import type { BlockStore } from './block_store.js'; @@ -20,10 +12,10 @@ export class L2TipsCache { #tipsPromise: Promise | undefined; /** - * Asymmetric by design: the genesis block hash is dynamic — derived from the injected initial header, - * which depends on `genesisTimestamp` and any prefilled state. The genesis checkpoint hash is static — - * checkpoint 0 is fully synthetic (no real checkpoint header exists at 0), so it stays at the protocol - * constant `GENESIS_CHECKPOINT_HEADER_HASH`. + * The genesis block hash is dynamic — derived from the injected initial header, which depends on + * `genesisTimestamp` and any prefilled state — so it is supplied here rather than read from store. + * The genesis checkpoint hash, by contrast, is the static protocol constant and is resolved + * inside the block store. */ constructor( private blockStore: BlockStore, @@ -32,115 +24,12 @@ export class L2TipsCache { /** Returns the cached L2 tips. Loads from the block store on first call. */ public getL2Tips(): Promise { - return (this.#tipsPromise ??= this.loadFromStore()); + return (this.#tipsPromise ??= this.blockStore.getL2TipsData(this.initialBlockHash)); } /** Reloads the L2 tips from the block store. Should be called after the writer transaction has committed. */ public async refresh(): Promise { - this.#tipsPromise = this.loadFromStore(); + this.#tipsPromise = this.blockStore.getL2TipsData(this.initialBlockHash); await this.#tipsPromise; } - - private async loadFromStore(): Promise { - const [ - latestBlockNumber, - provenBlockNumber, - proposedCheckpointBlockNumber, - checkpointedBlockNumber, - finalizedBlockNumber, - ] = await Promise.all([ - this.blockStore.getLatestL2BlockNumber(), - this.blockStore.getProvenBlockNumber(), - this.blockStore.getProposedCheckpointL2BlockNumber(), - this.blockStore.getCheckpointedL2BlockNumber(), - this.blockStore.getFinalizedL2BlockNumber(), - ]); - - const genesisBlockHeader = { - blockHash: this.initialBlockHash, - checkpointNumber: CheckpointNumber.ZERO, - } as const; - const beforeInitialBlockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1); - - const getBlockData = (blockNumber: BlockNumber) => - blockNumber > beforeInitialBlockNumber - ? this.blockStore.getBlockData({ number: blockNumber }) - : genesisBlockHeader; - - const [latestBlockData, provenBlockData, proposedCheckpointBlockData, checkpointedBlockData, finalizedBlockData] = - await Promise.all( - [ - latestBlockNumber, - provenBlockNumber, - proposedCheckpointBlockNumber, - checkpointedBlockNumber, - finalizedBlockNumber, - ].map(getBlockData), - ); - - if ( - !latestBlockData || - !provenBlockData || - !finalizedBlockData || - !checkpointedBlockData || - !proposedCheckpointBlockData - ) { - throw new Error('Failed to load block data for L2 tips'); - } - - const [provenCheckpointId, finalizedCheckpointId, proposedCheckpointId, checkpointedCheckpointId] = - await Promise.all([ - this.getCheckpointIdForBlock(provenBlockData), - this.getCheckpointIdForBlock(finalizedBlockData), - this.getCheckpointIdForProposedCheckpoint(checkpointedBlockData), - this.getCheckpointIdForBlock(checkpointedBlockData), - ]); - - return { - proposed: { number: latestBlockNumber, hash: latestBlockData.blockHash.toString() }, - proven: { - block: { number: provenBlockNumber, hash: provenBlockData.blockHash.toString() }, - checkpoint: provenCheckpointId, - }, - proposedCheckpoint: { - block: { number: proposedCheckpointBlockNumber, hash: proposedCheckpointBlockData.blockHash.toString() }, - checkpoint: proposedCheckpointId, - }, - finalized: { - block: { number: finalizedBlockNumber, hash: finalizedBlockData.blockHash.toString() }, - checkpoint: finalizedCheckpointId, - }, - checkpointed: { - block: { number: checkpointedBlockNumber, hash: checkpointedBlockData.blockHash.toString() }, - checkpoint: checkpointedCheckpointId, - }, - }; - } - - private async getCheckpointIdForProposedCheckpoint( - checkpointedBlockData: Pick, - ): Promise { - const checkpointData = await this.blockStore.getLastProposedCheckpoint(); - if (!checkpointData) { - return this.getCheckpointIdForBlock(checkpointedBlockData); - } - return { - number: checkpointData.checkpointNumber, - hash: checkpointData.header.hash().toString(), - }; - } - - private async getCheckpointIdForBlock(blockData: Pick): Promise { - const checkpointData = await this.blockStore.getCheckpointData(blockData.checkpointNumber); - if (!checkpointData) { - return { - number: CheckpointNumber.ZERO, - hash: GENESIS_CHECKPOINT_HEADER_HASH.toString(), - }; - } - return { - number: checkpointData.checkpointNumber, - hash: checkpointData.header.hash().toString(), - }; - } } diff --git a/yarn-project/stdlib/src/block/l2_block_stream/l2_tips_store_base.ts b/yarn-project/stdlib/src/block/l2_block_stream/l2_tips_store_base.ts index 676e732b665b..9637ff5fd17d 100644 --- a/yarn-project/stdlib/src/block/l2_block_stream/l2_tips_store_base.ts +++ b/yarn-project/stdlib/src/block/l2_block_stream/l2_tips_store_base.ts @@ -213,11 +213,30 @@ export abstract class L2TipsStoreBase implements L2BlockStreamEventHandler, L2Bl await this.saveTag('finalized', event.block); const finalizedCheckpointNumber = await this.getCheckpointNumberForBlock(event.block.number); - await this.deleteBlockHashesBefore(event.block.number); - await this.deleteBlockToCheckpointBefore(event.block.number); + // Cap the deletion bound at the lowest live tip. This should always be the finalized tip, but + // we have hit bugs where this is not the case. Deleting the block hash, block-to-checkpoint mapping, + // or enclosing checkpoint object for a live tip would dangle subsequent `getBlockId`/`getCheckpointId` + // lookups and lock the block stream into an error loop. + const tips = await Promise.all([ + this.getTip('proposed'), + this.getTip('proposedCheckpoint'), + this.getTip('checkpointed'), + this.getTip('proven'), + ]); + const liveTipBlocks = tips.filter((t): t is BlockNumber => t !== undefined && t > 0); + const safeBlockBound = BlockNumber(Math.min(event.block.number, ...liveTipBlocks)); + await this.deleteBlockHashesBefore(safeBlockBound); + await this.deleteBlockToCheckpointBefore(safeBlockBound); if (finalizedCheckpointNumber !== undefined) { - await this.deleteCheckpointsBefore(finalizedCheckpointNumber); + const tipCheckpoints = await Promise.all(liveTipBlocks.map(b => this.getCheckpointNumberForBlock(b))); + const safeCheckpointBound = CheckpointNumber( + Math.min( + finalizedCheckpointNumber, + ...tipCheckpoints.filter((c): c is CheckpointNumber => c !== undefined && c > 0), + ), + ); + await this.deleteCheckpointsBefore(safeCheckpointBound); } }); } From 0291cc27e48c5e1e8edae72ea4ad16d55546d344 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 15 May 2026 05:32:52 -0300 Subject: [PATCH 15/17] fix(sequencer): use targetSlot in tryVoteWhenEscapeHatchOpen under pipelining (#23296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `Sequencer.tryVoteWhenEscapeHatchOpen` constructed `CheckpointVoter` with the wall-clock `slot` and called `publisher.sendRequestsAt(slot)`. Under proposer pipelining we are the elected proposer for `slot + 1` (`targetSlot`), and the multicall is expected to mine in `targetSlot`. `EmpireBase.sol::_internalSignal`: - Verifies the EIP-712 digest against the **mining-slot** signature - Checks `msg.sender == getCurrentProposer()` for the **mining slot** Both fail under pipelining because we're the proposer for `targetSlot`, not `slot`. The multicall reverts silently inside Multicall3 and every governance/slashing entry is dropped. ## Fix Thread `targetSlot` through `tryVoteWhenEscapeHatchOpen` and use it for both: - `CheckpointVoter` (binds the EIP-712 signature to `targetSlot`) - `publisher.sendRequestsAt(targetSlot)` (delays submission so the tx mines in `targetSlot`) This mirrors `tryVoteWhenSyncFails` and `CheckpointProposalJob.execute`, which already use `targetSlot` correctly. When pipelining is disabled `targetSlot == slot` (from `epochCache.getTargetEpochAndSlotInNextL1Slot()`), so `sendRequestsAt` resolves with no extra sleep and the legacy behaviour is preserved. ## Showcase Re-enables `e2e_sequencer/escape_hatch_vote_only.test.ts` with `enableProposerPipelining: true` and `inboxLag: 2`. The test asserts `finalStats.votes >= slotsPassed` over the escape-hatch window — this assertion fails without the fix because no votes ever land. Test-side adjustments for the pipelined timing model: - Move event listener attachment to **after** the warp into the escape-hatch epoch. Checkpoint proposals in flight at warp time fail their L1 propose tx and are setup-warp artifacts, not vote-only window failures. - Snapshot `slotAtMeasurement` for the vote-count lower bound, then wait for the L1 slot to advance two more so the trailing vote (signed in build slot N for target slot N+1) has time to mine before counting. --- .../escape_hatch_vote_only.test.ts | 78 ++++++++++++------- .../src/sequencer/sequencer.ts | 34 +++++--- 2 files changed, 74 insertions(+), 38 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_sequencer/escape_hatch_vote_only.test.ts b/yarn-project/end-to-end/src/e2e_sequencer/escape_hatch_vote_only.test.ts index 401c4159176b..d235815b15ff 100644 --- a/yarn-project/end-to-end/src/e2e_sequencer/escape_hatch_vote_only.test.ts +++ b/yarn-project/end-to-end/src/e2e_sequencer/escape_hatch_vote_only.test.ts @@ -75,6 +75,10 @@ describe('e2e_escape_hatch_vote_only', () => { minTxsPerBlock: 0, enforceTimeTable: true, automineL1Setup: true, + // Pipelining opts — exercise the §6 B5 fix (tryVoteWhenEscapeHatchOpen signing/submitting for targetSlot). + // inboxLag: 2 so the sequencer sources L1->L2 messages from a sealed checkpoint when building for slot+1. + enableProposerPipelining: true, + inboxLag: 2, }); ({ @@ -142,19 +146,38 @@ describe('e2e_escape_hatch_vote_only', () => { afterEach(() => teardown()); it('casts governance signals and advances checkpoints while escape hatch is closed', async () => { + const sequencer = sequencerClient!.getSequencer(); + // Enable voting from the sequencer. await aztecNodeAdmin!.setConfig({ governanceProposerPayload: newGovernanceProposerPayloadAddress, minTxsPerBlock: 0, }); - // Set up event listeners to track sequencer behavior + // We need to set it for hatch 1, and then make a time jump. We do this such that we don't pollute the epoch cache. + // The warp must happen before we attach failure-event listeners, because any checkpoint proposal in flight at warp + // time will fail (its propose tx becomes invalid after the L1 timestamp jump) — that is a test-setup artifact, not + // a behavior we are asserting on. + if (OPEN_THE_HATCH) { + await ethCheatCodes.store( + await rollup.getEscapeHatchAddress(), + ethCheatCodes.keccak256(BigInt(EscapeHatchStorage.find(s => s.label === '$designatedProposer')!.slot), 1n), + escapeHatchProposerAddress.toField().toBigInt(), + ); + expect(await rollup.isEscapeHatchOpen(EpochNumber(Number(ESCAPE_HATCH_FREQUENCY)))).toBeTruthy(); + + logger.info(`Advancing to epoch ${ESCAPE_HATCH_FREQUENCY}`); + + await cheatCodes.rollup.advanceToEpoch(EpochNumber(Number(ESCAPE_HATCH_FREQUENCY)), { + offset: -ETHEREUM_SLOT_DURATION, + }); + } + + // Set up event listeners to track sequencer behavior during the vote-only window const failEvents: Array<{ type: keyof SequencerEvents; args: any }> = []; const blockProposedEvents: Array<{ blockNumber: any; slot: any }> = []; const checkpointPublishedEvents: Array<{ checkpoint: any; slot: any }> = []; - const sequencer = sequencerClient!.getSequencer(); - // Track failure events that indicate problems const failEventTypes: (keyof SequencerEvents)[] = [ 'block-build-failed', @@ -192,22 +215,6 @@ describe('e2e_escape_hatch_vote_only', () => { logger.warn(`Sequencer published checkpoint when escape hatch should be open`, args); }); - // We need to set it for hatch 1, and then make a time jump. We do this such that we don't pollute the epoch cache - if (OPEN_THE_HATCH) { - await ethCheatCodes.store( - await rollup.getEscapeHatchAddress(), - ethCheatCodes.keccak256(BigInt(EscapeHatchStorage.find(s => s.label === '$designatedProposer')!.slot), 1n), - escapeHatchProposerAddress.toField().toBigInt(), - ); - expect(await rollup.isEscapeHatchOpen(EpochNumber(Number(ESCAPE_HATCH_FREQUENCY)))).toBeTruthy(); - - logger.info(`Advancing to epoch ${ESCAPE_HATCH_FREQUENCY}`); - - await cheatCodes.rollup.advanceToEpoch(EpochNumber(Number(ESCAPE_HATCH_FREQUENCY)), { - offset: -ETHEREUM_SLOT_DURATION, - }); - } - const getStats = async () => ({ slot: await rollup.getSlotNumber(), epoch: await rollup.getEpochNumberForSlotNumber(await rollup.getSlotNumber()), @@ -229,20 +236,37 @@ describe('e2e_escape_hatch_vote_only', () => { 1, ); - const finalStats = await getStats(); - - // Due to the the stats not being pulled at the same time, a vote could land after the slot is fetched, but before the votes are. - // Therefore, we use the slots passed as the lower bound. - const slotsPassed = finalStats.slot - initialStats.slot; + // Snapshot the slot we will assert against now; under proposer pipelining the sequencer signs a vote in build + // slot N for target slot N+1 and submits it at the start of N+1, so the votes corresponding to slots up through + // `slotAtMeasurement` lag the current slot by one. Wait for the L1 slot to advance one more so the last + // in-flight vote (signed for `slotAtMeasurement`) has time to mine before we count votes. + const slotAtMeasurement = await rollup.getSlotNumber(); + const slotsPassed = slotAtMeasurement - initialStats.slot; expect(slotsPassed).toBeGreaterThan(0); + const drainTarget = slotAtMeasurement + 2; + await retryUntil( + () => rollup.getSlotNumber().then(s => s >= drainTarget), + 'pipelined vote drain', + AZTEC_SLOT_DURATION * 4, + 1, + ); + + const finalStats = await getStats(); expect(finalStats.votes - initialStats.votes).toBeGreaterThanOrEqual(slotsPassed); if (OPEN_THE_HATCH) { expect(finalStats.pending - initialStats.pending).toBe(0); // When escape hatch is open, sequencer should only vote, not build blocks nor checkpoints, but there should also be no failures. - expect(blockProposedEvents).toEqual([]); - expect(failEvents).toEqual([]); - expect(checkpointPublishedEvents).toEqual([]); + // Filter out events corresponding to pre-warp slots — they are checkpoint proposals that were in flight when + // the test warped past their target slot and whose L1 propose tx then fails. That's a setup artifact of the + // warp, not behavior we are asserting on in the vote-only window. + const inVoteOnlyWindow = (e: T) => { + const slotValue = (e as any).slot ?? (e as any).args?.slot; + return slotValue === undefined || Number(slotValue) >= Number(initialStats.slot); + }; + expect(blockProposedEvents.filter(inVoteOnlyWindow)).toEqual([]); + expect(failEvents.filter(inVoteOnlyWindow)).toEqual([]); + expect(checkpointPublishedEvents.filter(inVoteOnlyWindow)).toEqual([]); } else { expect(finalStats.pending - initialStats.pending).toBeGreaterThanOrEqual(slotsPassed); } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 83f9cb174b97..6dda210dad1a 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -312,7 +312,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter ({ [Attributes.SLOT_NUMBER]: slot })) protected async tryVoteWhenEscapeHatchOpen(args: { slot: SlotNumber; + targetSlot: SlotNumber; proposer: EthAddress | undefined; }): Promise { - const { slot, proposer } = args; + const { slot, targetSlot, proposer } = args; // Prevent duplicate attempts in the same slot if (this.lastSlotForFallbackVote === slot) { @@ -898,10 +899,19 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter { + this.log.error(`Failed to publish escape-hatch votes for slot ${slot}`, err, { slot, targetSlot }); + }); } /** From 1c09902e94b16adca76d28c47a0e36a29c882c7d Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 15 May 2026 05:37:01 -0300 Subject: [PATCH 16/17] fix(world-state): make fork close idempotent for pruned forks (#23298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Under proposer pipelining, the checkpoint job opens a world-state fork with `closeDelayMs: 12_000`. If a pending-chain unwind or historical prune destroys that fork on the C++ side before the delay fires, `DELETE_FORK` rejects with `"Fork not found"`, producing a stray warn log and leaking the per-fork queue entry in the JS instance — one dead entry per affected pipelined slot. ## Approach Make `close()` idempotent via an in-flight `closePromise`, and treat `"Fork not found"` as benign on close (same precedent as the existing `"Native instance is closed"` suppression — fork IDs are monotonic and never reused). Also wrap the per-fork queue cleanup in `try/finally` in both the native and IPC instances so the JS-side queue map cannot outlive the native fork on error. ## Changes - **world-state**: `MerkleTreesForkFacade.close()` is now idempotent and swallows `"Fork not found"`; per-fork queue cleanup in `NativeWorldStateInstance` and `IpcWorldStateInstance` moved to `finally`. - **world-state (tests)**: Regression test that disposes a `closeDelayMs` fork, triggers an unwind that destroys it on the C++ side, and asserts no warn is logged and the queue entry is cleaned up. Fixes A-1055 --- .../src/native/ipc_world_state_instance.ts | 47 ++++++++++-------- .../src/native/merkle_trees_facade.ts | 21 ++++++-- .../src/native/native_world_state.test.ts | 28 +++++++++++ .../src/native/native_world_state_instance.ts | 49 ++++++++++--------- 4 files changed, 97 insertions(+), 48 deletions(-) diff --git a/yarn-project/world-state/src/native/ipc_world_state_instance.ts b/yarn-project/world-state/src/native/ipc_world_state_instance.ts index 5489c80c37ce..cd4d20f1a003 100644 --- a/yarn-project/world-state/src/native/ipc_world_state_instance.ts +++ b/yarn-project/world-state/src/native/ipc_world_state_instance.ts @@ -279,28 +279,33 @@ export class IpcWorldState implements NativeWorldStateInstance { this.queues.set(forkId, requestQueue); } - const response = await requestQueue.execute( - async () => { - assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the IPC instance'); - assert.equal(this.open, true, 'IPC instance is closed'); - let response: WorldStateResponse[T]; - try { - response = await this._sendMessage(messageType, body); - } catch (error: any) { - errorHandler(error.message); - throw error; - } - return responseHandler(response); - }, - messageType, - committedOnly, - ); - - if (messageType === WorldStateMessageType.DELETE_FORK) { - await requestQueue.stop(); - this.queues.delete(forkId); + // The per-fork queue is cleaned up in `finally` even on error, so the JS-side queues map cannot outlive + // the native fork (e.g. when the native fork was already destroyed by an unwind/historical-prune and + // DELETE_FORK rejects with "Fork not found"). + try { + const response = await requestQueue.execute( + async () => { + assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the IPC instance'); + assert.equal(this.open, true, 'IPC instance is closed'); + let response: WorldStateResponse[T]; + try { + response = await this._sendMessage(messageType, body); + } catch (error: any) { + errorHandler(error.message); + throw error; + } + return responseHandler(response); + }, + messageType, + committedOnly, + ); + return response; + } finally { + if (messageType === WorldStateMessageType.DELETE_FORK) { + await requestQueue.stop(); + this.queues.delete(forkId); + } } - return response; } async close(): Promise { diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 2cc687575e8f..2d32a8def90e 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -208,6 +208,7 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations { export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations { private log = createLogger('world-state:merkle-trees-fork-facade'); + private closePromise: Promise | undefined; constructor( instance: NativeWorldStateInstance, @@ -291,8 +292,17 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr }; } - public async close(): Promise { + public close(): Promise { assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set'); + // Share the in-flight close promise across duplicate dispose calls so DELETE_FORK is sent at most once. + if (this.closePromise) { + return this.closePromise; + } + this.closePromise = this.doClose(); + return this.closePromise; + } + + private async doClose(): Promise { try { await this.instance.call(WorldStateMessageType.DELETE_FORK, { forkId: this.revision.forkId }); } catch (err: any) { @@ -301,6 +311,12 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr if (err?.message === 'Native instance is closed') { return; } + // Ignore "Fork not found": the native fork was already destroyed by a pending-chain unwind or a + // historical prune (both call C++ remove_forks_for_block). Fork IDs are monotonic and never reused, + // so swallowing this on close cannot mask a deletion of a different fork. + if (err?.message === 'Fork not found') { + return; + } throw err; } } @@ -310,9 +326,6 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr void sleep(this.opts.closeDelayMs) .then(() => this.close()) .catch(err => { - if (err && 'message' in err && err.message === 'Native instance is closed') { - return; // Ignore errors due to native instance being closed - } this.log.warn('Error closing MerkleTreesForkFacade after delay', { err }); }); } else { diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index 47ff292af6c8..5acafa7d67b6 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -14,6 +14,7 @@ import { timesAsync } from '@aztec/foundation/collection'; import { randomBytes } from '@aztec/foundation/crypto/random'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { sleep } from '@aztec/foundation/sleep'; import type { SiblingPath } from '@aztec/foundation/trees'; import { PublicDataWrite } from '@aztec/stdlib/avm'; import { L2Block } from '@aztec/stdlib/block'; @@ -937,6 +938,33 @@ describe('NativeWorldState', () => { } } }); + + // Regression test for A-1055: a delayed-close fork that the C++ side has already destroyed (via + // remove_forks_for_block on an unwind or historical prune) must dispose silently rather than logging a + // warning, and its JS-side per-fork queue entry must be cleaned up. + it('does not fail when a delayed-close fork is destroyed by a reorg before its close fires', async () => { + const baseFork = await ws.fork(); + for (let i = 0; i < 3; i++) { + const { block, messages } = await mockBlock(BlockNumber(i + 1), 1, baseFork); + await ws.handleL2BlockAndMessages(block, messages); + } + await baseFork.close(); + + const closeDelayMs = 1000; + const delayedFork = await ws.fork(undefined, { closeDelayMs }); + const forkId = (delayedFork as any).revision.forkId; + const warnSpy = jest.spyOn((delayedFork as any).log, 'warn'); + + await (delayedFork as any)[Symbol.asyncDispose](); + + await ws.unwindBlocks(BlockNumber.fromBigInt(2n)); + await expect(delayedFork.getSiblingPath(MerkleTreeId.NULLIFIER_TREE, 0n)).rejects.toThrow('Fork not found'); + + await sleep(closeDelayMs * 3); + + expect(warnSpy).not.toHaveBeenCalled(); + expect((ws as any).instance.queues.has(forkId)).toBe(false); + }); }); describe('Invalid Blocks', () => { diff --git a/yarn-project/world-state/src/native/native_world_state_instance.ts b/yarn-project/world-state/src/native/native_world_state_instance.ts index 6f4d60d0fd33..c4016ba1e477 100644 --- a/yarn-project/world-state/src/native/native_world_state_instance.ts +++ b/yarn-project/world-state/src/native/native_world_state_instance.ts @@ -184,30 +184,33 @@ export class NativeWorldState implements NativeWorldStateInstance { this.queues.set(forkId, requestQueue); } - // Enqueue the request and wait for the response - const response = await requestQueue.execute( - async () => { - assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance'); - assert.equal(this.open, true, 'Native instance is closed'); - let response: WorldStateResponse[T]; - try { - response = await this._sendMessage(messageType, body); - } catch (error: any) { - errorHandler(error.message); - throw error; - } - return responseHandler(response); - }, - messageType, - committedOnly, - ); - - // If the request was to delete the fork then we clean it up here - if (messageType === WorldStateMessageType.DELETE_FORK) { - await requestQueue.stop(); - this.queues.delete(forkId); + // Enqueue the request and wait for the response. The per-fork queue is cleaned up in `finally` even on + // error, so the JS-side queues map cannot outlive the native fork (e.g. when the native fork was already + // destroyed by an unwind/historical-prune and DELETE_FORK rejects with "Fork not found"). + try { + const response = await requestQueue.execute( + async () => { + assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance'); + assert.equal(this.open, true, 'Native instance is closed'); + let response: WorldStateResponse[T]; + try { + response = await this._sendMessage(messageType, body); + } catch (error: any) { + errorHandler(error.message); + throw error; + } + return responseHandler(response); + }, + messageType, + committedOnly, + ); + return response; + } finally { + if (messageType === WorldStateMessageType.DELETE_FORK) { + await requestQueue.stop(); + this.queues.delete(forkId); + } } - return response; } /** From b354f8d2b17984fabb7716387bc9b1e6810efe15 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Fri, 15 May 2026 08:41:37 +0000 Subject: [PATCH 17/17] fix(yarn-project): retry sqlite3mc vendor download on transient errors --- yarn-project/sqlite3mc-wasm/scripts/vendor.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yarn-project/sqlite3mc-wasm/scripts/vendor.sh b/yarn-project/sqlite3mc-wasm/scripts/vendor.sh index c4abb0a2ae28..244e7b1e6e98 100755 --- a/yarn-project/sqlite3mc-wasm/scripts/vendor.sh +++ b/yarn-project/sqlite3mc-wasm/scripts/vendor.sh @@ -75,7 +75,9 @@ WORK_DIR=$(mktemp -d) trap 'rm -rf "$WORK_DIR"' EXIT echo "==> Downloading ${ASSET}" -curl -fsSL -o "$WORK_DIR/$ASSET" "$URL" +# --retry-all-errors covers transient DNS / connection-reset failures that +# would otherwise break CI on the GitHub release-assets host. +curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors -o "$WORK_DIR/$ASSET" "$URL" echo "==> Verifying zip SHA256" ACTUAL_SHA=$(sha256sum "$WORK_DIR/$ASSET" | awk '{print $1}')