diff --git a/yarn-project/archiver/src/archiver/archiver.test.ts b/yarn-project/archiver/src/archiver/archiver.test.ts index dc69987142f1..155c874ebadd 100644 --- a/yarn-project/archiver/src/archiver/archiver.test.ts +++ b/yarn-project/archiver/src/archiver/archiver.test.ts @@ -813,6 +813,13 @@ describe('Archiver', () => { await archiver.syncImmediate(); expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(2)); + // Verify L2Tips after syncing checkpoint 2 + const lastBlockInCheckpoint2 = allCheckpoints[1].blocks[allCheckpoints[1].blocks.length - 1].number; + const tipsAtCheckpoint2 = await archiver.getL2Tips(); + expect(tipsAtCheckpoint2.proposed.number).toEqual(lastBlockInCheckpoint2); + expect(tipsAtCheckpoint2.checkpointed.block.number).toEqual(lastBlockInCheckpoint2); + expect(tipsAtCheckpoint2.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2)); + logger.warn(`Expecting prune back to checkpoint 1`); publicClient.getBlockNumber.mockResolvedValue(95n); checkpoints = checkpoints.slice(0, 1); // Keep only checkpoint 1 as the valid checkpoint @@ -820,6 +827,13 @@ describe('Archiver', () => { expect(loggerSpy).toHaveBeenCalledWith(expect.stringContaining(`L2 prune has been detected`), expect.anything()); expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(1)); + // Verify L2Tips after pruning back to checkpoint 1 + const lastBlockInCheckpoint1 = allCheckpoints[0].blocks[allCheckpoints[0].blocks.length - 1].number; + const tipsAfterPrune = await archiver.getL2Tips(); + expect(tipsAfterPrune.proposed.number).toEqual(lastBlockInCheckpoint1); + expect(tipsAfterPrune.checkpointed.block.number).toEqual(lastBlockInCheckpoint1); + expect(tipsAfterPrune.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1)); + const txHash = allCheckpoints[1].blocks[0].body.txEffects[0].txHash; expect(await archiver.getTxEffect(txHash)).resolves.toBeUndefined; expect(await archiver.getPublishedCheckpoints(CheckpointNumber(2), 1)).toEqual([]); @@ -1678,6 +1692,83 @@ describe('Archiver', () => { expect(retrievedBlock!.number).toEqual(BlockNumber(1)); }); + it('retrieves multiple blocks with getL2BlocksNew', async () => { + setupMinimalL1Mocks(); + const block1 = await makeBlock(BlockNumber(1), 0, genesisArchive); + const block2 = await makeBlock(BlockNumber(2), 1, block1.archive); + const block3 = await makeBlock(BlockNumber(3), 2, block2.archive); + + await archiver.addBlock(block1); + await archiver.addBlock(block2); + await archiver.addBlock(block3); + + const blocks = await archiver.getL2BlocksNew(BlockNumber(1), 3); + expect(blocks.length).toEqual(3); + expect(await blocks[0].hash()).toEqual(await block1.hash()); + expect(await blocks[1].hash()).toEqual(await block2.hash()); + expect(await blocks[2].hash()).toEqual(await block3.hash()); + }); + + it('retrieves blocks with limit in getL2BlocksNew', async () => { + setupMinimalL1Mocks(); + const block1 = await makeBlock(BlockNumber(1), 0, genesisArchive); + const block2 = await makeBlock(BlockNumber(2), 1, block1.archive); + const block3 = await makeBlock(BlockNumber(3), 2, block2.archive); + + await archiver.addBlock(block1); + await archiver.addBlock(block2); + await archiver.addBlock(block3); + + // Request only 2 blocks starting from block 1 + const blocks = await archiver.getL2BlocksNew(BlockNumber(1), 2); + expect(blocks.length).toEqual(2); + expect(await blocks[0].hash()).toEqual(await block1.hash()); + expect(await blocks[1].hash()).toEqual(await block2.hash()); + }); + + it('retrieves blocks starting from middle with getL2BlocksNew', async () => { + setupMinimalL1Mocks(); + const block1 = await makeBlock(BlockNumber(1), 0, genesisArchive); + const block2 = await makeBlock(BlockNumber(2), 1, block1.archive); + const block3 = await makeBlock(BlockNumber(3), 2, block2.archive); + + await archiver.addBlock(block1); + await archiver.addBlock(block2); + await archiver.addBlock(block3); + + // Start from block 2 + const blocks = await archiver.getL2BlocksNew(BlockNumber(2), 2); + expect(blocks.length).toEqual(2); + expect(await blocks[0].hash()).toEqual(await block2.hash()); + expect(await blocks[1].hash()).toEqual(await block3.hash()); + }); + + it('returns empty array when requesting blocks beyond available range', async () => { + setupMinimalL1Mocks(); + const block1 = await makeBlock(BlockNumber(1), 0, genesisArchive); + + await archiver.addBlock(block1); + + // Request blocks starting from block 5 (which doesn't exist) + const blocks = await archiver.getL2BlocksNew(BlockNumber(5), 3); + expect(blocks).toEqual([]); + }); + + it('returns partial results when limit exceeds available blocks', async () => { + setupMinimalL1Mocks(); + const block1 = await makeBlock(BlockNumber(1), 0, genesisArchive); + const block2 = await makeBlock(BlockNumber(2), 1, block1.archive); + + await archiver.addBlock(block1); + await archiver.addBlock(block2); + + // Request 10 blocks but only 2 are available + const blocks = await archiver.getL2BlocksNew(BlockNumber(1), 10); + expect(blocks.length).toEqual(2); + expect(await blocks[0].hash()).toEqual(await block1.hash()); + expect(await blocks[1].hash()).toEqual(await block2.hash()); + }); + it('blocks added via addBlock become checkpointed when checkpoint syncs from L1', async () => { // First, sync checkpoint 1 from L1 to establish a baseline const checkpoint1 = checkpoints[0]; @@ -1710,6 +1801,12 @@ describe('Archiver', () => { expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1)); const lastBlockInCheckpoint1 = checkpoint1.blocks[checkpoint1.blocks.length - 1].number; + // Verify L2Tips after syncing checkpoint 1: proposed and checkpointed should both be at checkpoint 1 + const tipsAfterCheckpoint1 = await archiver.getL2Tips(); + expect(tipsAfterCheckpoint1.proposed.number).toEqual(lastBlockInCheckpoint1); + expect(tipsAfterCheckpoint1.checkpointed.block.number).toEqual(lastBlockInCheckpoint1); + expect(tipsAfterCheckpoint1.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1)); + // Now add blocks for checkpoint 2 via addBlock (simulating local block production) const checkpoint2 = checkpoints[1]; for (const block of checkpoint2.blocks) { @@ -1721,6 +1818,12 @@ describe('Archiver', () => { expect(await archiver.getBlockNumber()).toEqual(lastBlockInCheckpoint2); expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1)); + // Verify L2Tips after adding blocks: proposed advances but checkpointed stays at checkpoint 1 + const tipsAfterAddBlock = await archiver.getL2Tips(); + expect(tipsAfterAddBlock.proposed.number).toEqual(lastBlockInCheckpoint2); + expect(tipsAfterAddBlock.checkpointed.block.number).toEqual(lastBlockInCheckpoint1); + expect(tipsAfterAddBlock.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1)); + // getCheckpointedBlock should return undefined for the new blocks since checkpoint 2 hasn't synced const firstNewBlockNumber = lastBlockInCheckpoint1 + 1; const uncheckpointedBlock = await archiver.getCheckpointedBlock(BlockNumber(firstNewBlockNumber)); @@ -1749,6 +1852,12 @@ describe('Archiver', () => { // Now the blocks should be checkpointed expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(2)); + // Verify L2Tips after syncing checkpoint 2: proposed and checkpointed should both be at checkpoint 2 + const tipsAfterCheckpoint2 = await archiver.getL2Tips(); + expect(tipsAfterCheckpoint2.proposed.number).toEqual(lastBlockInCheckpoint2); + expect(tipsAfterCheckpoint2.checkpointed.block.number).toEqual(lastBlockInCheckpoint2); + expect(tipsAfterCheckpoint2.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2)); + // getCheckpointedBlock should now work for the new blocks const checkpointedBlock = await archiver.getCheckpointedBlock(BlockNumber(firstNewBlockNumber)); expect(checkpointedBlock).toBeDefined(); @@ -1825,6 +1934,12 @@ describe('Archiver', () => { expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1)); const lastBlockInCheckpoint1 = checkpoint1.blocks[checkpoint1.blocks.length - 1].number; + // Verify L2Tips after syncing checkpoint 1: proposed and checkpointed at checkpoint 1 + const tipsAfterCheckpoint1 = await archiver.getL2Tips(); + expect(tipsAfterCheckpoint1.proposed.number).toEqual(lastBlockInCheckpoint1); + expect(tipsAfterCheckpoint1.checkpointed.block.number).toEqual(lastBlockInCheckpoint1); + expect(tipsAfterCheckpoint1.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1)); + // Now add more blocks via addBlock (simulating local block production ahead of L1) const checkpoint2 = checkpoints[1]; for (const block of checkpoint2.blocks) { @@ -1838,6 +1953,12 @@ describe('Archiver', () => { // But checkpoint number should still be 1 expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(1)); + // Verify L2Tips after adding blocks: proposed advances, checkpointed stays at checkpoint 1 + const tipsAfterAddBlock = await archiver.getL2Tips(); + expect(tipsAfterAddBlock.proposed.number).toEqual(lastBlockInCheckpoint2); + expect(tipsAfterAddBlock.checkpointed.block.number).toEqual(lastBlockInCheckpoint1); + expect(tipsAfterAddBlock.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1)); + // New blocks should not be checkpointed yet const firstNewBlockNumber = lastBlockInCheckpoint1 + 1; const uncheckpointedBlock = await archiver.getCheckpointedBlock(BlockNumber(firstNewBlockNumber)); @@ -1862,6 +1983,12 @@ describe('Archiver', () => { // Now all blocks should be checkpointed expect(await archiver.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(2)); + // Verify L2Tips after syncing checkpoint 2: both proposed and checkpointed at checkpoint 2 + const tipsAfterCheckpoint2 = await archiver.getL2Tips(); + expect(tipsAfterCheckpoint2.proposed.number).toEqual(lastBlockInCheckpoint2); + expect(tipsAfterCheckpoint2.checkpointed.block.number).toEqual(lastBlockInCheckpoint2); + expect(tipsAfterCheckpoint2.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2)); + const checkpointedBlock = await archiver.getCheckpointedBlock(BlockNumber(firstNewBlockNumber)); expect(checkpointedBlock).toBeDefined(); expect(checkpointedBlock!.checkpointNumber).toEqual(2); @@ -1871,6 +1998,371 @@ describe('Archiver', () => { // TODO(palla/reorg): Add a unit test for the archiver handleEpochPrune xit('handles an upcoming L2 prune', () => {}); + describe('getCheckpointedBlocks', () => { + it('returns checkpointed blocks with checkpoint info', async () => { + const rollupTxs = checkpoints.map(c => makeRollupTx(c)); + const blobHashes = checkpoints.map(makeVersionedBlobHashes); + + mockL1BlockNumbers(100n); + + mockRollup.read.status.mockResolvedValue([ + 0n, + GENESIS_ROOT, + 3n, + checkpoints[2].archive.root.toString(), + GENESIS_ROOT, + ]); + + checkpoints.forEach((c, i) => + makeCheckpointProposedEvent(70n + BigInt(i) * 10n, c.number, c.archive.root.toString(), blobHashes[i]), + ); + messagesPerCheckpoint.forEach((messages, i) => + makeMessageSentEvents(60n + BigInt(i) * 5n, checkpoints[i].number, messages), + ); + mockInbox.read.getState.mockResolvedValue( + makeInboxStateFromMsgCount(messagesPerCheckpoint.reduce((acc, curr) => acc + curr.length, 0)), + ); + + rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx)); + const blobsFromCheckpoints = checkpoints.map(c => makeBlobsFromCheckpoint(c)); + blobsFromCheckpoints.forEach(blobs => blobClient.getBlobSidecar.mockResolvedValueOnce(blobs)); + + await archiver.start(false); + await waitUntilArchiverCheckpoint(CheckpointNumber(3)); + + // Get checkpointed blocks starting from block 1 + const checkpointedBlocks = await archiver.getCheckpointedBlocks(BlockNumber(1), 100); + + // Should return all blocks from all checkpoints + const expectedBlocks = checkpoints.flatMap(c => c.blocks); + expect(checkpointedBlocks.length).toBe(expectedBlocks.length); + + // Verify blocks are returned in correct order and have correct checkpoint info + let blockIndex = 0; + for (let cpIdx = 0; cpIdx < checkpoints.length; cpIdx++) { + const checkpoint = checkpoints[cpIdx]; + for (let i = 0; i < checkpoint.blocks.length; i++) { + const cb = checkpointedBlocks[blockIndex]; + const expectedBlock = checkpoint.blocks[i]; + + // Verify block number matches + expect(cb.block.number).toBe(expectedBlock.number); + + // Verify checkpoint number is correct + expect(cb.checkpointNumber).toBe(checkpoint.number); + + // Verify archive root matches (more reliable than hash which depends on L1-to-L2 messages) + expect(cb.block.archive.root.toString()).toBe(expectedBlock.archive.root.toString()); + + // Verify L1 published data is present + expect(cb.l1).toBeDefined(); + expect(cb.l1.blockNumber).toBeGreaterThan(0n); + + blockIndex++; + } + } + }, 10_000); + + it('respects the limit parameter', async () => { + const rollupTxs = checkpoints.map(c => makeRollupTx(c)); + const blobHashes = checkpoints.map(makeVersionedBlobHashes); + + mockL1BlockNumbers(100n); + + mockRollup.read.status.mockResolvedValue([ + 0n, + GENESIS_ROOT, + 3n, + checkpoints[2].archive.root.toString(), + GENESIS_ROOT, + ]); + + checkpoints.forEach((c, i) => + makeCheckpointProposedEvent(70n + BigInt(i) * 10n, c.number, c.archive.root.toString(), blobHashes[i]), + ); + messagesPerCheckpoint.forEach((messages, i) => + makeMessageSentEvents(60n + BigInt(i) * 5n, checkpoints[i].number, messages), + ); + mockInbox.read.getState.mockResolvedValue( + makeInboxStateFromMsgCount(messagesPerCheckpoint.reduce((acc, curr) => acc + curr.length, 0)), + ); + + rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx)); + const blobsFromCheckpoints = checkpoints.map(c => makeBlobsFromCheckpoint(c)); + blobsFromCheckpoints.forEach(blobs => blobClient.getBlobSidecar.mockResolvedValueOnce(blobs)); + + await archiver.start(false); + await waitUntilArchiverCheckpoint(CheckpointNumber(3)); + + // Get only 2 checkpointed blocks starting from block 1 (out of 3 total) + const checkpointedBlocks = await archiver.getCheckpointedBlocks(BlockNumber(1), 2); + expect(checkpointedBlocks.length).toBe(2); + + // Verify exact block numbers (blocks 1 and 2) + expect(checkpointedBlocks[0].block.number).toBe(BlockNumber(1)); + expect(checkpointedBlocks[1].block.number).toBe(BlockNumber(2)); + + // Verify archive roots match original checkpoint blocks + expect(checkpointedBlocks[0].block.archive.root.toString()).toBe( + checkpoints[0].blocks[0].archive.root.toString(), + ); + expect(checkpointedBlocks[1].block.archive.root.toString()).toBe( + checkpoints[1].blocks[0].archive.root.toString(), + ); + + // Verify checkpoint numbers (block 1 is from checkpoint 1, block 2 is from checkpoint 2) + expect(checkpointedBlocks[0].checkpointNumber).toBe(1); + expect(checkpointedBlocks[1].checkpointNumber).toBe(2); + }, 10_000); + + it('returns blocks starting from specified block number', async () => { + const rollupTxs = checkpoints.map(c => makeRollupTx(c)); + const blobHashes = checkpoints.map(makeVersionedBlobHashes); + + mockL1BlockNumbers(100n); + + mockRollup.read.status.mockResolvedValue([ + 0n, + GENESIS_ROOT, + 3n, + checkpoints[2].archive.root.toString(), + GENESIS_ROOT, + ]); + + checkpoints.forEach((c, i) => + makeCheckpointProposedEvent(70n + BigInt(i) * 10n, c.number, c.archive.root.toString(), blobHashes[i]), + ); + messagesPerCheckpoint.forEach((messages, i) => + makeMessageSentEvents(60n + BigInt(i) * 5n, checkpoints[i].number, messages), + ); + mockInbox.read.getState.mockResolvedValue( + makeInboxStateFromMsgCount(messagesPerCheckpoint.reduce((acc, curr) => acc + curr.length, 0)), + ); + + rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx)); + const blobsFromCheckpoints = checkpoints.map(c => makeBlobsFromCheckpoint(c)); + blobsFromCheckpoints.forEach(blobs => blobClient.getBlobSidecar.mockResolvedValueOnce(blobs)); + + await archiver.start(false); + await waitUntilArchiverCheckpoint(CheckpointNumber(3)); + + // Get blocks starting from block 2 (skip block 1, get blocks 2 and 3) + const checkpointedBlocks = await archiver.getCheckpointedBlocks(BlockNumber(2), 10); + + // Should return 2 blocks (blocks 2 and 3 - since there are only 3 blocks total, 1 per checkpoint) + expect(checkpointedBlocks.length).toBe(2); + + // Verify block numbers are sequential starting from 2 + expect(checkpointedBlocks[0].block.number).toBe(BlockNumber(2)); + expect(checkpointedBlocks[1].block.number).toBe(BlockNumber(3)); + + // Verify checkpoint numbers (block 2 is from checkpoint 2, block 3 is from checkpoint 3) + expect(checkpointedBlocks[0].checkpointNumber).toBe(2); + expect(checkpointedBlocks[1].checkpointNumber).toBe(3); + + // Verify archive roots match expected blocks from checkpoints + expect(checkpointedBlocks[0].block.archive.root.toString()).toBe( + checkpoints[1].blocks[0].archive.root.toString(), + ); + expect(checkpointedBlocks[1].block.archive.root.toString()).toBe( + checkpoints[2].blocks[0].archive.root.toString(), + ); + }, 10_000); + + it('returns empty array when no checkpointed blocks exist', async () => { + mockL1BlockNumbers(100n); + mockRollup.read.status.mockResolvedValue([0n, GENESIS_ROOT, 0n, GENESIS_ROOT, GENESIS_ROOT]); + mockInbox.read.getState.mockResolvedValue(makeInboxStateFromMsgCount(0)); + + await archiver.start(false); + + const checkpointedBlocks = await archiver.getCheckpointedBlocks(BlockNumber(1), 10); + expect(checkpointedBlocks).toEqual([]); + }, 10_000); + + it('filters by proven status when proven=true', async () => { + const rollupTxs = checkpoints.map(c => makeRollupTx(c)); + const blobHashes = checkpoints.map(makeVersionedBlobHashes); + + mockL1BlockNumbers(100n); + + // Set checkpoint 1 as proven (provenCheckpointNumber = 1) + mockRollup.read.status.mockResolvedValue([ + 1n, // provenCheckpointNumber + checkpoints[0].archive.root.toString(), // provenArchive + 3n, // pendingCheckpointNumber + checkpoints[2].archive.root.toString(), // pendingArchive + checkpoints[0].archive.root.toString(), // archiveForLocalPendingCheckpointNumber + ]); + + checkpoints.forEach((c, i) => + makeCheckpointProposedEvent(70n + BigInt(i) * 10n, c.number, c.archive.root.toString(), blobHashes[i]), + ); + messagesPerCheckpoint.forEach((messages, i) => + makeMessageSentEvents(60n + BigInt(i) * 5n, checkpoints[i].number, messages), + ); + mockInbox.read.getState.mockResolvedValue( + makeInboxStateFromMsgCount(messagesPerCheckpoint.reduce((acc, curr) => acc + curr.length, 0)), + ); + + rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx)); + const blobsFromCheckpoints = checkpoints.map(c => makeBlobsFromCheckpoint(c)); + blobsFromCheckpoints.forEach(blobs => blobClient.getBlobSidecar.mockResolvedValueOnce(blobs)); + + await archiver.start(false); + await waitUntilArchiverCheckpoint(CheckpointNumber(3)); + + // Get all checkpointed blocks without proven filter + const allBlocks = await archiver.getCheckpointedBlocks(BlockNumber(1), 100); + const totalBlocks = checkpoints.reduce((acc, c) => acc + c.blocks.length, 0); + expect(allBlocks.length).toBe(totalBlocks); + + // Get only proven checkpointed blocks (should only include blocks from checkpoint 1) + const provenBlocks = await archiver.getCheckpointedBlocks(BlockNumber(1), 100, true); + const checkpoint1Blocks = checkpoints[0].blocks; + expect(provenBlocks.length).toBe(checkpoint1Blocks.length); + + // Verify all proven blocks are from checkpoint 1 and match expected blocks + for (let i = 0; i < provenBlocks.length; i++) { + const cb = provenBlocks[i]; + expect(cb.checkpointNumber).toBe(1); + expect(cb.block.number).toBe(checkpoint1Blocks[i].number); + + // Verify archive root matches (more reliable than hash which depends on L1-to-L2 messages) + expect(cb.block.archive.root.toString()).toBe(checkpoint1Blocks[i].archive.root.toString()); + } + + // Verify the last proven block number matches the last block of checkpoint 1 + const lastProvenBlock = provenBlocks[provenBlocks.length - 1]; + const lastCheckpoint1Block = checkpoint1Blocks[checkpoint1Blocks.length - 1]; + expect(lastProvenBlock.block.number).toBe(lastCheckpoint1Block.number); + }, 10_000); + }); + + describe('getL2BlocksNew with proven filter', () => { + it('filters by proven status when proven=true', async () => { + const rollupTxs = checkpoints.map(c => makeRollupTx(c)); + const blobHashes = checkpoints.map(makeVersionedBlobHashes); + + mockL1BlockNumbers(100n); + + // Set checkpoint 1 as proven (provenCheckpointNumber = 1) + mockRollup.read.status.mockResolvedValue([ + 1n, // provenCheckpointNumber + checkpoints[0].archive.root.toString(), // provenArchive + 3n, // pendingCheckpointNumber + checkpoints[2].archive.root.toString(), // pendingArchive + checkpoints[0].archive.root.toString(), // archiveForLocalPendingCheckpointNumber + ]); + + checkpoints.forEach((c, i) => + makeCheckpointProposedEvent(70n + BigInt(i) * 10n, c.number, c.archive.root.toString(), blobHashes[i]), + ); + messagesPerCheckpoint.forEach((messages, i) => + makeMessageSentEvents(60n + BigInt(i) * 5n, checkpoints[i].number, messages), + ); + mockInbox.read.getState.mockResolvedValue( + makeInboxStateFromMsgCount(messagesPerCheckpoint.reduce((acc, curr) => acc + curr.length, 0)), + ); + + rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx)); + const blobsFromCheckpoints = checkpoints.map(c => makeBlobsFromCheckpoint(c)); + blobsFromCheckpoints.forEach(blobs => blobClient.getBlobSidecar.mockResolvedValueOnce(blobs)); + + await archiver.start(false); + await waitUntilArchiverCheckpoint(CheckpointNumber(3)); + + // Get all blocks without proven filter + const allBlocks = await archiver.getL2BlocksNew(BlockNumber(1), 100); + const totalBlocks = checkpoints.reduce((acc, c) => acc + c.blocks.length, 0); + expect(allBlocks.length).toBe(totalBlocks); + + // Get only proven blocks (should only include blocks from checkpoint 1) + const provenBlocks = await archiver.getL2BlocksNew(BlockNumber(1), 100, true); + const checkpoint1Blocks = checkpoints[0].blocks; + expect(provenBlocks.length).toBe(checkpoint1Blocks.length); + + // Verify block numbers match checkpoint 1 blocks + for (let i = 0; i < provenBlocks.length; i++) { + expect(provenBlocks[i].number).toBe(checkpoint1Blocks[i].number); + + // Verify archive root matches (more reliable than hash which depends on L1-to-L2 messages) + expect(provenBlocks[i].archive.root.toString()).toBe(checkpoint1Blocks[i].archive.root.toString()); + } + + // Verify the last proven block is the last block of checkpoint 1 + const lastProvenBlockNumber = checkpoint1Blocks[checkpoint1Blocks.length - 1].number; + expect(provenBlocks[provenBlocks.length - 1].number).toBe(lastProvenBlockNumber); + + // Verify no unproven blocks are included + const unprovenBlockNumbers = checkpoints.slice(1).flatMap(c => c.blocks.map(b => b.number)); + provenBlocks.forEach(b => { + expect(unprovenBlockNumbers).not.toContain(b.number); + }); + }, 10_000); + + it('returns all blocks when proven=false or undefined', async () => { + const rollupTxs = checkpoints.map(c => makeRollupTx(c)); + const blobHashes = checkpoints.map(makeVersionedBlobHashes); + + mockL1BlockNumbers(100n); + + // Set checkpoint 1 as proven + mockRollup.read.status.mockResolvedValue([ + 1n, + checkpoints[0].archive.root.toString(), + 3n, + checkpoints[2].archive.root.toString(), + checkpoints[0].archive.root.toString(), + ]); + + checkpoints.forEach((c, i) => + makeCheckpointProposedEvent(70n + BigInt(i) * 10n, c.number, c.archive.root.toString(), blobHashes[i]), + ); + messagesPerCheckpoint.forEach((messages, i) => + makeMessageSentEvents(60n + BigInt(i) * 5n, checkpoints[i].number, messages), + ); + mockInbox.read.getState.mockResolvedValue( + makeInboxStateFromMsgCount(messagesPerCheckpoint.reduce((acc, curr) => acc + curr.length, 0)), + ); + + rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx)); + const blobsFromCheckpoints = checkpoints.map(c => makeBlobsFromCheckpoint(c)); + blobsFromCheckpoints.forEach(blobs => blobClient.getBlobSidecar.mockResolvedValueOnce(blobs)); + + await archiver.start(false); + await waitUntilArchiverCheckpoint(CheckpointNumber(3)); + + const expectedBlocks = checkpoints.flatMap(c => c.blocks); + const totalBlocks = expectedBlocks.length; + + // Get blocks with proven=false - should include all blocks + const blocksProvenFalse = await archiver.getL2BlocksNew(BlockNumber(1), 100, false); + expect(blocksProvenFalse.length).toBe(totalBlocks); + + // Verify all block numbers are present + for (let i = 0; i < blocksProvenFalse.length; i++) { + expect(blocksProvenFalse[i].number).toBe(expectedBlocks[i].number); + } + + // Get blocks with proven=undefined - should include all blocks + const blocksProvenUndefined = await archiver.getL2BlocksNew(BlockNumber(1), 100); + expect(blocksProvenUndefined.length).toBe(totalBlocks); + + // Verify all block numbers match + for (let i = 0; i < blocksProvenUndefined.length; i++) { + expect(blocksProvenUndefined[i].number).toBe(expectedBlocks[i].number); + } + + // Verify blocks include unproven blocks (from checkpoints 2 and 3) + const unprovenBlockNumbers = checkpoints.slice(1).flatMap(c => c.blocks.map(b => b.number)); + const returnedBlockNumbers = blocksProvenFalse.map(b => b.number); + unprovenBlockNumbers.forEach(unprovenNum => { + expect(returnedBlockNumbers).toContain(unprovenNum); + }); + }, 10_000); + }); + const waitUntilArchiverCheckpoint = async (checkpointNumber: CheckpointNumber) => { logger.info(`Waiting for archiver to sync to checkpoint ${checkpointNumber}`); await retryUntil(() => archiver.getSynchedCheckpointNumber().then(n => n === checkpointNumber), 'sync', 10, 0.1); diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index f10228753595..e048fa9959b8 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -1,5 +1,5 @@ import type { BlobClientInterface } from '@aztec/blob-client/client'; -import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants'; +import { GENESIS_BLOCK_HEADER_HASH, INITIAL_L2_BLOCK_NUM, INITIAL_L2_CHECKPOINT_NUM } from '@aztec/constants'; import { EpochCache } from '@aztec/epoch-cache'; import { createEthereumChain } from '@aztec/ethereum/chain'; import { BlockTagTooOldError, InboxContract, RollupContract } from '@aztec/ethereum/contracts'; @@ -593,14 +593,11 @@ export class Archiver ); const newBlocks = blockPromises.filter(isDefined).flat(); - // TODO(pw/mbps): Don't convert to legacy blocks here - const blocks: L2Block[] = (await Promise.all(newBlocks.map(x => this.getBlock(x.number)))).filter(isDefined); - // Emit an event for listening services to react to the chain prune this.emit(L2BlockSourceEvents.L2PruneDetected, { type: L2BlockSourceEvents.L2PruneDetected, epochNumber: pruneFromEpochNumber, - blocks, + blocks: newBlocks, }); this.log.debug( @@ -1392,6 +1389,16 @@ export class Archiver return publishedBlock; } + public async getL2BlocksNew(from: BlockNumber, limit: number, proven?: boolean): Promise { + const blocks = await this.store.store.getBlocks(from, limit); + + if (proven === true) { + const provenBlockNumber = await this.store.getProvenBlockNumber(); + return blocks.filter(b => b.number <= provenBlockNumber); + } + return blocks; + } + public async getBlockHeader(number: BlockNumber | 'latest'): Promise { if (number === 'latest') { number = await this.store.getSynchedL2BlockNumber(); @@ -1407,6 +1414,20 @@ export class Archiver return this.store.getCheckpointedBlock(number); } + public async getCheckpointedBlocks( + from: BlockNumber, + limit: number, + proven?: boolean, + ): Promise { + const blocks = await this.store.store.getCheckpointedBlocks(from, limit); + + if (proven === true) { + const provenBlockNumber = await this.store.getProvenBlockNumber(); + return blocks.filter(b => b.block.number <= provenBlockNumber); + } + return blocks; + } + getCheckpointedBlockByHash(blockHash: Fr): Promise { return this.store.getCheckpointedBlockByHash(blockHash); } @@ -1414,6 +1435,9 @@ export class Archiver getProvenBlockNumber(): Promise { return this.store.getProvenBlockNumber(); } + getCheckpointedBlockNumber(): Promise { + return this.store.getCheckpointedL2BlockNumber(); + } getCheckpointedBlockByArchive(archive: Fr): Promise { return this.store.getCheckpointedBlockByArchive(archive); } @@ -1524,9 +1548,10 @@ export class Archiver } async getL2Tips(): Promise { - const [latestBlockNumber, provenBlockNumber] = await Promise.all([ + const [latestBlockNumber, provenBlockNumber, checkpointedBlockNumber] = await Promise.all([ this.getBlockNumber(), this.getProvenBlockNumber(), + this.getCheckpointedBlockNumber(), ] as const); // TODO(#13569): Compute proper finalized block number based on L1 finalized block. @@ -1534,44 +1559,114 @@ export class Archiver // NOTE: update end-to-end/src/e2e_epochs/epochs_empty_blocks.test.ts as that uses finalized blocks in computations const finalizedBlockNumber = BlockNumber(Math.max(provenBlockNumber - this.l1constants.epochDuration * 2, 0)); - const [latestBlockHeader, provenBlockHeader, finalizedBlockHeader] = await Promise.all([ - latestBlockNumber > 0 ? this.getBlockHeader(latestBlockNumber) : undefined, - provenBlockNumber > 0 ? this.getBlockHeader(provenBlockNumber) : undefined, - finalizedBlockNumber > 0 ? this.getBlockHeader(finalizedBlockNumber) : undefined, - ] as const); + const beforeInitialblockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1); - if (latestBlockNumber > 0 && !latestBlockHeader) { + // Get the latest block header and checkpointed blocks for proven, finalised and checkpointed blocks + const [latestBlockHeader, provenCheckpointedBlock, finalizedCheckpointedBlock, checkpointedBlock] = + await Promise.all([ + latestBlockNumber > beforeInitialblockNumber ? this.getBlockHeader(latestBlockNumber) : undefined, + provenBlockNumber > beforeInitialblockNumber ? this.getCheckpointedBlock(provenBlockNumber) : undefined, + finalizedBlockNumber > beforeInitialblockNumber ? this.getCheckpointedBlock(finalizedBlockNumber) : undefined, + checkpointedBlockNumber > beforeInitialblockNumber + ? this.getCheckpointedBlock(checkpointedBlockNumber) + : undefined, + ] as const); + + const beforeInitialCheckpointNumber = CheckpointNumber(INITIAL_L2_CHECKPOINT_NUM - 1); + + if (latestBlockNumber > beforeInitialblockNumber && !latestBlockHeader) { throw new Error(`Failed to retrieve latest block header for block ${latestBlockNumber}`); } - if (provenBlockNumber > 0 && !provenBlockHeader) { + // Checkpointed blocks must exist for proven, finalized and checkpointed tips if they are beyond the initial block number. + if (checkpointedBlockNumber > beforeInitialblockNumber && !checkpointedBlock?.block.header) { throw new Error( - `Failed to retrieve proven block header for block ${provenBlockNumber} (latest block is ${latestBlockNumber})`, + `Failed to retrieve checkpointed block header for block ${checkpointedBlockNumber} (latest block is ${latestBlockNumber})`, ); } - if (finalizedBlockNumber > 0 && !finalizedBlockHeader) { + if (provenBlockNumber > beforeInitialblockNumber && !provenCheckpointedBlock?.block.header) { + throw new Error( + `Failed to retrieve proven checkpointed for block ${provenBlockNumber} (latest block is ${latestBlockNumber})`, + ); + } + + if (finalizedBlockNumber > beforeInitialblockNumber && !finalizedCheckpointedBlock?.block.header) { throw new Error( `Failed to retrieve finalized block header for block ${finalizedBlockNumber} (latest block is ${latestBlockNumber})`, ); } const latestBlockHeaderHash = (await latestBlockHeader?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; - const provenBlockHeaderHash = (await provenBlockHeader?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; - const finalizedBlockHeaderHash = (await finalizedBlockHeader?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; + const provenBlockHeaderHash = (await provenCheckpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; + const finalizedBlockHeaderHash = + (await finalizedCheckpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; + const checkpointedBlockHeaderHash = (await checkpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH; + + // Now attempt to retrieve checkpoints for proven, finalised and checkpointed blocks + const [[provenBlockCheckpoint], [finalizedBlockCheckpoint], [checkpointedBlockCheckpoint]] = await Promise.all([ + provenCheckpointedBlock !== undefined + ? await this.getPublishedCheckpoints(provenCheckpointedBlock?.checkpointNumber, 1) + : [undefined], + finalizedCheckpointedBlock !== undefined + ? await this.getPublishedCheckpoints(finalizedCheckpointedBlock?.checkpointNumber, 1) + : [undefined], + checkpointedBlock !== undefined + ? await this.getPublishedCheckpoints(checkpointedBlock?.checkpointNumber, 1) + : [undefined], + ]); - return { - latest: { number: latestBlockNumber, hash: latestBlockHeaderHash.toString() }, - proven: { number: provenBlockNumber, hash: provenBlockHeaderHash.toString() }, - finalized: { number: finalizedBlockNumber, hash: finalizedBlockHeaderHash.toString() }, + const initialcheckpointId = { + number: beforeInitialCheckpointNumber, + hash: '', }; + + const makeCheckpointId = (checkpoint: PublishedCheckpoint | undefined) => { + if (checkpoint === undefined) { + return initialcheckpointId; + } + return { + number: checkpoint.checkpoint.number, + hash: checkpoint.checkpoint.hash().toString(), + }; + }; + + const l2Tips: L2Tips = { + proposed: { + number: latestBlockNumber, + hash: latestBlockHeaderHash.toString(), + }, + proven: { + block: { + number: provenBlockNumber, + hash: provenBlockHeaderHash.toString(), + }, + checkpoint: makeCheckpointId(provenBlockCheckpoint), + }, + finalized: { + block: { + number: finalizedBlockNumber, + hash: finalizedBlockHeaderHash.toString(), + }, + checkpoint: makeCheckpointId(finalizedBlockCheckpoint), + }, + checkpointed: { + block: { + number: checkpointedBlockNumber, + hash: checkpointedBlockHeaderHash.toString(), + }, + checkpoint: makeCheckpointId(checkpointedBlockCheckpoint), + }, + }; + + return l2Tips; } public async rollbackTo(targetL2BlockNumber: BlockNumber): Promise { // TODO(pw/mbps): This still assumes 1 block per checkpoint const currentBlocks = await this.getL2Tips(); - const currentL2Block = currentBlocks.latest.number; - const currentProvenBlock = currentBlocks.proven.number; + const currentL2Block = currentBlocks.proposed.number; + const currentProvenBlock = currentBlocks.proven.block.number; if (targetL2BlockNumber >= currentL2Block) { throw new Error(`Target L2 block ${targetL2BlockNumber} must be less than current L2 block ${currentL2Block}`); @@ -1777,6 +1872,7 @@ export class ArchiverStoreHelper | 'addBlocks' | 'getBlock' | 'getBlocks' + | 'getCheckpointedBlocks' > { #log = createLogger('archiver:block-helper'); diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 60abca8653c7..2f333438e7f8 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -85,6 +85,14 @@ export interface ArchiverDataStore { */ getCheckpointedBlock(number: number): Promise; + /** + * Gets up to `limit` amount of checkpointed L2 blocks starting from `from`. + * @param from - Number of the first block to return (inclusive). + * @param limit - The number of blocks to return. + * @returns The requested checkpointed L2 blocks. + */ + getCheckpointedBlocks(from: number, limit: number): Promise; + /** * Returns the block for the given hash, or undefined if not exists. * @param blockHash - The block hash to return. diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts index fe49b7dd088f..af9895f6e0ac 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts @@ -503,6 +503,34 @@ export class BlockStore { ); } + /** + * Gets up to `limit` amount of Checkpointed L2 blocks starting from `from`. + * @param start - Number of the first block to return (inclusive). + * @param limit - The number of blocks to return. + * @returns The requested L2 blocks + */ + async *getCheckpointedBlocks(start: BlockNumber, limit: number): AsyncIterableIterator { + const checkpointCache = new Map(); + for await (const [blockNumber, blockStorage] of this.getBlockStorages(start, limit)) { + const block = await this.getBlockFromBlockStorage(blockNumber, blockStorage); + if (block) { + const checkpoint = + checkpointCache.get(CheckpointNumber(blockStorage.checkpointNumber)) ?? + (await this.#checkpoints.getAsync(blockStorage.checkpointNumber)); + if (checkpoint) { + checkpointCache.set(CheckpointNumber(blockStorage.checkpointNumber), checkpoint); + const checkpointedBlock = new CheckpointedL2Block( + CheckpointNumber(checkpoint.checkpointNumber), + block, + L1PublishedData.fromBuffer(checkpoint.l1), + checkpoint.attestations.map(buf => CommitteeAttestation.fromBuffer(buf)), + ); + yield checkpointedBlock; + } + } + } + } + async getCheckpointedBlockByHash(blockHash: Fr): Promise { const blockNumber = await this.#blockHashIndex.getAsync(blockHash.toString()); if (blockNumber === undefined) { diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 9c403cb349c9..d932cd16e5b4 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -234,6 +234,10 @@ export class KVArchiverDataStore implements ArchiverDataStore, ContractDataSourc return toArray(this.#blockStore.getBlocks(from, limit)); } + getCheckpointedBlocks(from: BlockNumber, limit: number): Promise { + return toArray(this.#blockStore.getCheckpointedBlocks(from, limit)); + } + /** * Gets up to `limit` amount of L2 blocks headers starting from `from`. * diff --git a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts index 83499dfa6763..f3b72a1c940f 100644 --- a/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts +++ b/yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts @@ -1,6 +1,6 @@ -import { BlockNumber, type CheckpointNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; -import type { L2Tips } from '@aztec/stdlib/block'; +import type { CheckpointId, L2BlockId, L2TipId, L2Tips } from '@aztec/stdlib/block'; import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; /** @@ -33,9 +33,15 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource { getL2Tips(): Promise { const number = this.blockNumber; - const tip = { number: BlockNumber(number), hash: new Fr(number).toString() }; + const blockId: L2BlockId = { number: BlockNumber(number), hash: new Fr(number).toString() }; + const checkpointId: CheckpointId = { + number: CheckpointNumber(number), + hash: new Fr(number + 1).toString(), + }; + const tip: L2TipId = { block: blockId, checkpoint: checkpointId }; return Promise.resolve({ - latest: tip, + proposed: blockId, + checkpointed: tip, proven: tip, finalized: tip, }); diff --git a/yarn-project/archiver/src/test/mock_l2_block_source.ts b/yarn-project/archiver/src/test/mock_l2_block_source.ts index 8120eb32c4ec..5339ba9b7e63 100644 --- a/yarn-project/archiver/src/test/mock_l2_block_source.ts +++ b/yarn-project/archiver/src/test/mock_l2_block_source.ts @@ -8,6 +8,7 @@ import { createLogger } from '@aztec/foundation/log'; import type { FunctionSelector } from '@aztec/stdlib/abi'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; import { + CheckpointedL2Block, L2Block, L2BlockHash, L2BlockNew, @@ -30,6 +31,7 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { private provenBlockNumber: number = 0; private finalizedBlockNumber: number = 0; + private checkpointedBlockNumber: number = 0; private log = createLogger('archiver:mock_l2_block_source'); @@ -64,6 +66,10 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { this.finalizedBlockNumber = finalizedBlockNumber; } + public setCheckpointedBlockNumber(checkpointedBlockNumber: number) { + this.checkpointedBlockNumber = checkpointedBlockNumber; + } + /** * Method to fetch the rollup contract address at the base-layer. * @returns The rollup address. @@ -92,9 +98,40 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { return Promise.resolve(BlockNumber(this.provenBlockNumber)); } - public getCheckpointedBlock(_number: BlockNumber) { - // In this mock, we don't track checkpointed blocks separately - return Promise.resolve(undefined); + public getCheckpointedBlock(number: BlockNumber): Promise { + if (number > this.checkpointedBlockNumber) { + return Promise.resolve(undefined); + } + const block = this.l2Blocks[number - 1]; + if (!block) { + return Promise.resolve(undefined); + } + const checkpointedBlock = new CheckpointedL2Block( + CheckpointNumber(number), + block.toL2Block(), + new L1PublishedData(BigInt(number), BigInt(number), `0x${number.toString(16).padStart(64, '0')}`), + [], + ); + return Promise.resolve(checkpointedBlock); + } + + public async getCheckpointedBlocks( + from: BlockNumber, + limit: number, + _proven?: boolean, + ): Promise { + const result: CheckpointedL2Block[] = []; + for (let i = 0; i < limit; i++) { + const blockNum = from + i; + if (blockNum > this.checkpointedBlockNumber) { + break; + } + const block = await this.getCheckpointedBlock(BlockNumber(blockNum)); + if (block) { + result.push(block); + } + } + return result; } /** @@ -151,6 +188,11 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { ); } + async getL2BlocksNew(from: BlockNumber, limit: number, proven?: boolean): Promise { + const blocks = await this.getBlocks(from, limit, proven); + return blocks.map(x => x.toL2Block()); + } + public async getPublishedBlockByHash(blockHash: Fr): Promise { for (const block of this.l2Blocks) { const hash = await block.hash(); @@ -263,29 +305,45 @@ export class MockL2BlockSource implements L2BlockSource, ContractDataSource { } async getL2Tips(): Promise { - const [latest, proven, finalized] = [ + const [latest, proven, finalized, checkpointed] = [ await this.getBlockNumber(), await this.getProvenBlockNumber(), this.finalizedBlockNumber, + this.checkpointedBlockNumber, ] as const; const latestBlock = this.l2Blocks[latest - 1]; const provenBlock = this.l2Blocks[proven - 1]; const finalizedBlock = this.l2Blocks[finalized - 1]; + const checkpointedBlock = this.l2Blocks[checkpointed - 1]; + + const latestBlockId = { + number: BlockNumber(latest), + hash: (await latestBlock?.hash())?.toString(), + }; + const provenBlockId = { + number: BlockNumber(proven), + hash: (await provenBlock?.hash())?.toString(), + }; + const finalizedBlockId = { + number: BlockNumber(finalized), + hash: (await finalizedBlock?.hash())?.toString(), + }; + const checkpointedBlockId = { + number: BlockNumber(checkpointed), + hash: (await checkpointedBlock?.hash())?.toString(), + }; + + const makeTipId = (blockId: typeof latestBlockId) => ({ + block: blockId, + checkpoint: { number: CheckpointNumber(blockId.number), hash: blockId.hash }, + }); return { - latest: { - number: BlockNumber(latest), - hash: (await latestBlock?.hash())?.toString(), - }, - proven: { - number: BlockNumber(proven), - hash: (await provenBlock?.hash())?.toString(), - }, - finalized: { - number: BlockNumber(finalized), - hash: (await finalizedBlock?.hash())?.toString(), - }, + proposed: latestBlockId, + checkpointed: makeTipId(checkpointedBlockId), + proven: makeTipId(provenBlockId), + finalized: makeTipId(finalizedBlockId), }; } diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index b0e4988bf64d..627eab7a9dfd 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -14,7 +14,7 @@ import { createEthereumChain } from '@aztec/ethereum/chain'; import { getPublicClient } from '@aztec/ethereum/client'; import { RegistryContract, RollupContract } from '@aztec/ethereum/contracts'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; -import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { compactArray, pick } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -54,9 +54,11 @@ import { type DataInBlock, type L2Block, L2BlockHash, + L2BlockNew, type L2BlockSource, type PublishedL2Block, } from '@aztec/stdlib/block'; +import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ContractClassPublic, ContractDataSource, @@ -614,6 +616,18 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { return (await this.blockSource.getPublishedBlocks(from, limit)) ?? []; } + public async getPublishedCheckpoints(from: CheckpointNumber, limit: number): Promise { + return (await this.blockSource.getPublishedCheckpoints(from, limit)) ?? []; + } + + public async getL2BlocksNew(from: BlockNumber, limit: number): Promise { + return (await this.blockSource.getL2BlocksNew(from, limit)) ?? []; + } + + public async getCheckpointedBlocks(from: BlockNumber, limit: number, proven?: boolean) { + return (await this.blockSource.getCheckpointedBlocks(from, limit, proven)) ?? []; + } + /** * Method to fetch the current min L2 fees. * @returns The current min L2 fees. @@ -1316,7 +1330,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { } // And it has an L2 block hash - const l2BlockHash = await archiver.getL2Tips().then(tips => tips.latest.hash); + const l2BlockHash = await archiver.getL2Tips().then(tips => tips.proposed.hash); if (!l2BlockHash) { this.metrics.recordSnapshotError(); throw new Error(`Archiver has no latest L2 block hash downloaded. Cannot start snapshot.`); @@ -1350,7 +1364,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { throw new Error('Archiver implementation does not support rollbacks.'); } - const finalizedBlock = await archiver.getL2Tips().then(tips => tips.finalized.number); + const finalizedBlock = await archiver.getL2Tips().then(tips => tips.finalized.block.number); if (targetBlock < finalizedBlock) { if (force) { this.log.warn(`Clearing world state database to allow rolling back behind finalized block ${finalizedBlock}`); diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.test.ts b/yarn-project/aztec-node/src/sentinel/sentinel.test.ts index cccca2afbb57..4a5c9801cc1a 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.test.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.test.ts @@ -1,5 +1,5 @@ import type { EpochCache } from '@aztec/epoch-cache'; -import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { compactArray, times } from '@aztec/foundation/collection'; import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -9,15 +9,16 @@ import { OffenseType, WANT_TO_SLASH_EVENT, type WantToSlashArgs } from '@aztec/s import type { SlasherConfig } from '@aztec/slasher/config'; import { CommitteeAttestation, + L2BlockNew, type L2BlockSource, type L2BlockStream, type L2BlockStreamEvent, - PublishedL2Block, - getAttestationInfoFromPublishedL2Block, + getAttestationInfoFromPublishedCheckpoint, } from '@aztec/stdlib/block'; -import { type L1RollupConstants, getEpochAtSlot } from '@aztec/stdlib/epoch-helpers'; +import { Checkpoint, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { type L1RollupConstants, getEpochAtSlot, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; import type { BlockAttestation } from '@aztec/stdlib/p2p'; -import { makeBlockAttestation, randomPublishedL2Block } from '@aztec/stdlib/testing'; +import { makeAttestationFromCheckpoint, makeBlockAttestation } from '@aztec/stdlib/testing'; import type { ValidatorStats, ValidatorStatusHistory, @@ -89,16 +90,26 @@ describe('sentinel', () => { describe('getSlotActivity', () => { let signers: Secp256k1Signer[]; let validators: EthAddress[]; - let block: PublishedL2Block; + let block: L2BlockNew; + let publishedCheckpoint: PublishedCheckpoint; let attestations: BlockAttestation[]; let proposer: EthAddress; let committee: EthAddress[]; + /** Helper to create and emit a chain-checkpointed event */ + const emitCheckpointEvent = async (checkpoint: Checkpoint, checkpointAttestations: CommitteeAttestation[] = []) => { + const published = new PublishedCheckpoint(checkpoint, L1PublishedData.random(), checkpointAttestations); + const lastBlock = checkpoint.blocks.at(-1)!; + const block = { number: lastBlock.number, hash: (await lastBlock.hash()).toString() }; + await sentinel.handleBlockStreamEvent({ type: 'chain-checkpointed', checkpoint: published, block }); + return published; + }; + beforeEach(async () => { signers = times(4, Secp256k1Signer.random); validators = signers.map(signer => signer.address); - block = await randomPublishedL2Block(Number(slot)); - attestations = signers.map(signer => makeBlockAttestation({ signer, archive: block.block.archive.root })); + block = await L2BlockNew.random(BlockNumber(1), { slotNumber: slot }); + attestations = signers.map(signer => makeBlockAttestation({ signer, archive: block.archive.root })); proposer = validators[0]; committee = [...validators]; @@ -106,7 +117,9 @@ describe('sentinel', () => { }); it('flags block as mined', async () => { - await sentinel.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); + // Create a checkpoint with a block at the target slot and emit chain-checkpointed event + const checkpoint = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: slot }); + await emitCheckpointEvent(checkpoint); const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); expect(activity[proposer.toString()]).toEqual('block-mined'); @@ -125,49 +138,70 @@ describe('sentinel', () => { }); it('identifies attestors from p2p and archiver', async () => { - block = await randomPublishedL2Block(Number(slot), { signers: signers.slice(0, 2) }); - const attestorsFromBlock = compactArray( - getAttestationInfoFromPublishedL2Block(block).map(info => + // Create a checkpoint with a block at the target slot + const checkpoint = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: slot }); + // Create attestations from signers 0 and 1 + const checkpointAttestations = signers.slice(0, 2).map(signer => { + const blockAttestation = makeAttestationFromCheckpoint(checkpoint, signer, signer); + return new CommitteeAttestation(signer.address, blockAttestation.signature); + }); + + // Emit the chain-checkpointed event with attestations from signers 0 and 1 + publishedCheckpoint = await emitCheckpointEvent(checkpoint, checkpointAttestations); + + const attestorsFromCheckpoint = compactArray( + getAttestationInfoFromPublishedCheckpoint(publishedCheckpoint).map(info => info.status === 'recovered-from-signature' || info.status === 'provided-as-address' ? info.address : undefined, ), ); - expect(attestorsFromBlock.map(a => a.toString())).toEqual(signers.slice(0, 2).map(a => a.address.toString())); + expect(attestorsFromCheckpoint.map(a => a.toString())).toEqual( + signers.slice(0, 2).map(a => a.address.toString()), + ); - await sentinel.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); + // P2P provides attestation from signer 2 (validator 2) p2p.getAttestationsForSlot.mockResolvedValue(attestations.slice(2, 3)); const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); + // Validator 1 attested via archiver checkpoint data expect(activity[committee[1].toString()]).toEqual('attestation-sent'); + // Validator 2 attested via p2p expect(activity[committee[2].toString()]).toEqual('attestation-sent'); + // Validator 3 has no attestation expect(activity[committee[3].toString()]).toEqual('attestation-missed'); }); it('only counts recovered-from-signature attestations, not placeholder attestations', async () => { - // Create a block with only 2 signers (validators 0 and 1), plus placeholders for 2 and 3 - block = await randomPublishedL2Block(Number(slot), { signers: signers.slice(0, 2) }); + // Create a checkpoint with only 2 signers (validators 0 and 1), plus placeholders for 2 and 3 + const checkpoint = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: slot }); + + // Create attestations from first 2 signers + const signedAttestations = signers.slice(0, 2).map(signer => { + const blockAttestation = makeAttestationFromCheckpoint(checkpoint, signer, signer); + return new CommitteeAttestation(signer.address, blockAttestation.signature); + }); // Add placeholder attestations for the missing validators (no signature) const placeholderAttestations = validators.slice(2).map(addr => CommitteeAttestation.fromAddress(addr)); - // Append placeholders to the existing attestations - const allAttestations = [...block.attestations, ...placeholderAttestations]; - block = new PublishedL2Block(block.block, block.l1, allAttestations); + // Combine signed and placeholder attestations + const allAttestations = [...signedAttestations, ...placeholderAttestations]; + + // Emit chain-checkpointed event with both signed and placeholder attestations + // The Sentinel should only count the recovered-from-signature ones + publishedCheckpoint = await emitCheckpointEvent(checkpoint, allAttestations); - // Verify that getAttestationInfoFromPublishedL2Block returns 4 entries total: + // Verify that getAttestationInfoFromPublishedCheckpoint returns 4 entries total: // - 2 with status 'recovered-from-signature' (actual attestations with valid signatures) // - 2 with status 'provided-as-address' (placeholders for missing validators) - const attestationInfo = getAttestationInfoFromPublishedL2Block(block); + const attestationInfo = getAttestationInfoFromPublishedCheckpoint(publishedCheckpoint); expect(attestationInfo).toHaveLength(4); const recoveredSignatures = attestationInfo.filter(info => info.status === 'recovered-from-signature'); const placeholders = attestationInfo.filter(info => info.status === 'provided-as-address'); expect(recoveredSignatures).toHaveLength(2); expect(placeholders).toHaveLength(2); - // After processing the block, only the validators with actual signatures should be recorded as attestors - await sentinel.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); - // No additional attestations from p2p p2p.getAttestationsForSlot.mockResolvedValue([]); @@ -183,7 +217,11 @@ describe('sentinel', () => { }); it('identifies missed attestors if block is mined', async () => { - await sentinel.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); + // Create checkpoint with a block at the target slot + const checkpoint = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: slot }); + await emitCheckpointEvent(checkpoint); + + // P2P provides attestations from validators 0, 1, 2 (not validator 3) p2p.getAttestationsForSlot.mockResolvedValue(attestations.slice(0, -1)); const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); @@ -516,13 +554,13 @@ describe('sentinel', () => { it('calls inactivity watcher with performance data', async () => { const blockNumber = BlockNumber(15); const blockHash = '0xblockhash'; - const mockBlock = await randomPublishedL2Block(blockNumber); - const slot = mockBlock.block.header.getSlot(); + const mockBlock = await L2BlockNew.random(blockNumber); + const slot = mockBlock.header.getSlot(); const epochNumber = getEpochAtSlot(slot, l1Constants); const validator1 = EthAddress.random(); const validator2 = EthAddress.random(); const validator3 = EthAddress.random(); - const headerSlots = times(l1Constants.epochDuration, i => SlotNumber(slot - i)).reverse(); + const [fromSlot, toSlot] = getSlotRangeForEpoch(epochNumber, l1Constants); epochCache.getEpochAndSlotNow.mockReturnValue({ epoch: epochNumber, @@ -530,7 +568,7 @@ describe('sentinel', () => { ts, now: ts, }); - archiver.getBlock.calledWith(blockNumber).mockResolvedValue(mockBlock.block); + archiver.getL2BlockNew.calledWith(blockNumber).mockResolvedValue(mockBlock); archiver.getL1Constants.mockResolvedValue(l1Constants); epochCache.getL1Constants.mockReturnValue(l1Constants); @@ -545,7 +583,7 @@ describe('sentinel', () => { // Validator 1 missed 1 attestation only, we won't slash them [validator1.toString()]: { address: validator1, - totalSlots: headerSlots.length, + totalSlots: l1Constants.epochDuration, missedProposals: { count: 0, currentStreak: 0, rate: 0, total: 0 }, missedAttestations: { count: 1, currentStreak: 0, rate: 1 / 8, total: 8 }, history: [], @@ -553,7 +591,7 @@ describe('sentinel', () => { // Validator 2 missed 7 out of 8, we will slash them [validator2.toString()]: { address: validator2, - totalSlots: headerSlots.length, + totalSlots: l1Constants.epochDuration, missedProposals: { count: 0, currentStreak: 0, rate: 0, total: 0 }, missedAttestations: { count: 7, currentStreak: 3, rate: 7 / 8, total: 8 }, history: [], @@ -562,7 +600,7 @@ describe('sentinel', () => { // This difference happens because we don't count attestations for a slot where there was no proposal [validator3.toString()]: { address: validator3, - totalSlots: headerSlots.length, + totalSlots: l1Constants.epochDuration, missedProposals: { count: 0, currentStreak: 0, rate: 0, total: 0 }, missedAttestations: { count: 4, currentStreak: 4, rate: 4 / 4, total: 4 }, history: [], @@ -578,8 +616,8 @@ describe('sentinel', () => { await sentinel.handleChainProven({ type: 'chain-proven', block: { number: blockNumber, hash: blockHash } }); expect(computeStatsSpy).toHaveBeenCalledWith({ - fromSlot: headerSlots[0], - toSlot: headerSlots[headerSlots.length - 1], + fromSlot, + toSlot, validators: [validator1, validator2, validator3], }); const makeInactivitySlash = (validator: EthAddress): WantToSlashArgs => ({ diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.ts b/yarn-project/aztec-node/src/sentinel/sentinel.ts index 9a3cb4e8fde4..fd6ab19f9f61 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.ts @@ -1,5 +1,5 @@ import type { EpochCache } from '@aztec/epoch-cache'; -import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { countWhile, filterAsync, fromEntries, getEntries, mapValues } from '@aztec/foundation/collection'; import { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; @@ -19,7 +19,7 @@ import { L2BlockStream, type L2BlockStreamEvent, type L2BlockStreamEventHandler, - getAttestationInfoFromPublishedL2Block, + getAttestationInfoFromPublishedCheckpoint, } from '@aztec/stdlib/block'; import { getEpochAtSlot, getSlotRangeForEpoch, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import type { @@ -44,8 +44,10 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme protected initialSlot: SlotNumber | undefined; protected lastProcessedSlot: SlotNumber | undefined; // eslint-disable-next-line aztec-custom/no-non-primitive-in-collections - protected slotNumberToBlock: Map = - new Map(); + protected slotNumberToCheckpoint: Map< + SlotNumber, + { checkpointNumber: CheckpointNumber; archive: string; attestors: EthAddress[] } + > = new Map(); constructor( protected epochCache: EpochCache, @@ -87,39 +89,46 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { await this.l2TipsStore.handleBlockStreamEvent(event); - if (event.type === 'blocks-added') { - // Store mapping from slot to archive, block number, and attestors - for (const block of event.blocks) { - this.slotNumberToBlock.set(block.block.header.getSlot(), { - blockNumber: BlockNumber(block.block.number), - archive: block.block.archive.root.toString(), - attestors: getAttestationInfoFromPublishedL2Block(block) - .filter(a => a.status === 'recovered-from-signature') - .map(a => a.address!), - }); - } - - // Prune the archive map to only keep at most N entries - const historyLength = this.store.getHistoryLength(); - if (this.slotNumberToBlock.size > historyLength) { - const toDelete = Array.from(this.slotNumberToBlock.keys()) - .sort((a, b) => Number(a - b)) - .slice(0, this.slotNumberToBlock.size - historyLength); - for (const key of toDelete) { - this.slotNumberToBlock.delete(key); - } - } + if (event.type === 'chain-checkpointed') { + this.handleCheckpoint(event); } else if (event.type === 'chain-proven') { await this.handleChainProven(event); } } + protected handleCheckpoint(event: L2BlockStreamEvent) { + if (event.type !== 'chain-checkpointed') { + return; + } + const checkpoint = event.checkpoint; + + // Store mapping from slot to archive, checkpoint number, and attestors + this.slotNumberToCheckpoint.set(checkpoint.checkpoint.header.slotNumber, { + checkpointNumber: checkpoint.checkpoint.number, + archive: checkpoint.checkpoint.archive.root.toString(), + attestors: getAttestationInfoFromPublishedCheckpoint(checkpoint) + .filter(a => a.status === 'recovered-from-signature') + .map(a => a.address!), + }); + + // Prune the archive map to only keep at most N entries + const historyLength = this.store.getHistoryLength(); + if (this.slotNumberToCheckpoint.size > historyLength) { + const toDelete = Array.from(this.slotNumberToCheckpoint.keys()) + .sort((a, b) => Number(a - b)) + .slice(0, this.slotNumberToCheckpoint.size - historyLength); + for (const key of toDelete) { + this.slotNumberToCheckpoint.delete(key); + } + } + } + protected async handleChainProven(event: L2BlockStreamEvent) { if (event.type !== 'chain-proven') { return; } - const blockNumber = BlockNumber(event.block.number); - const block = await this.archiver.getBlock(blockNumber); + const blockNumber = event.block.number; + const block = await this.archiver.getL2BlockNew(blockNumber); if (!block) { this.logger.error(`Failed to get block ${blockNumber}`, { block }); return; @@ -291,8 +300,8 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme return false; } - const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.latest.hash); - const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.latest.hash); + const archiverLastBlockHash = await this.l2TipsStore.getL2Tips().then(tip => tip.proposed.hash); + const p2pLastBlockHash = await this.p2p.getL2Tips().then(tips => tips.proposed.hash); const isP2pSynced = archiverLastBlockHash === p2pLastBlockHash; if (!isP2pSynced) { this.logger.debug(`Waiting for P2P client to sync with archiver`, { archiverLastBlockHash, p2pLastBlockHash }); @@ -331,7 +340,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme // or all attestations for all proposals in the slot if no block was mined. // We gather from both p2p (contains the ones seen on the p2p layer) and archiver // (contains the ones synced from mined blocks, which we may have missed from p2p). - const block = this.slotNumberToBlock.get(slot); + const block = this.slotNumberToCheckpoint.get(slot); const p2pAttested = await this.p2p.getAttestationsForSlot(slot, block?.archive); // Filter out attestations with invalid signatures const p2pAttestors = p2pAttested.map(a => a.getSender()).filter((s): s is EthAddress => s !== undefined); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts index dc2133dbfbb1..11003ad31d60 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts @@ -348,7 +348,7 @@ export class EpochsTestContext { ]); this.logger.info(`Wait for node synch ${blockNumber} ${type}`, { blockNumber, type, syncState, tips }); if (type === 'proven') { - synched = tips.proven.number >= blockNumber && syncState.latestBlockNumber >= blockNumber; + synched = tips.proven.block.number >= blockNumber && syncState.latestBlockNumber >= blockNumber; } else if (type === 'finalized') { synched = syncState.finalizedBlockNumber >= blockNumber; } else { diff --git a/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts b/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts index 2f2b63722893..686c34520a94 100644 --- a/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/e2e_l1_publisher/e2e_l1_publisher.test.ts @@ -202,11 +202,15 @@ describe('L1Publisher integration', () => { }, getL2Tips(): Promise { const latestBlock = blocks.at(-1); - const res = latestBlock + const blockId = latestBlock ? { number: latestBlock.number, hash: latestBlock.hash.toString() } : { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }; + const tipId = { + block: blockId, + checkpoint: { number: CheckpointNumber(blockId.number), hash: blockId.hash }, + }; - return Promise.resolve({ latest: res, proven: res, finalized: res }); + return Promise.resolve({ proposed: blockId, checkpointed: tipId, proven: tipId, finalized: tipId }); }, getBlockNumber(): Promise { return Promise.resolve(BlockNumber(blocks.at(-1)?.number ?? BlockNumber.ZERO)); diff --git a/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts b/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts index aeb9498dfbc3..cfc17ffbdefe 100644 --- a/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts +++ b/yarn-project/end-to-end/src/e2e_snapshot_sync.test.ts @@ -80,7 +80,7 @@ describe('e2e_snapshot_sync', () => { const expectNodeSyncedToL2Block = async (node: AztecNode | ProverNode, blockNumber: number) => { const tips = await node.getL2Tips(); - expect(tips.latest.number).toBeGreaterThanOrEqual(blockNumber); + expect(tips.proposed.number).toBeGreaterThanOrEqual(blockNumber); const worldState = await node.getWorldStateSyncStatus(); expect(worldState.latestBlockNumber).toBeGreaterThanOrEqual(blockNumber); }; diff --git a/yarn-project/foundation/src/branded-types/checkpoint_number.ts b/yarn-project/foundation/src/branded-types/checkpoint_number.ts index a188384fc6d0..ef349d541b30 100644 --- a/yarn-project/foundation/src/branded-types/checkpoint_number.ts +++ b/yarn-project/foundation/src/branded-types/checkpoint_number.ts @@ -94,7 +94,21 @@ CheckpointNumber.ZERO = CheckpointNumber(0); * Zod schema for parsing and validating CheckpointNumber values. * Accepts numbers, bigints, or strings and coerces them to CheckpointNumber. */ -export const CheckpointNumberSchema = z - .union([z.number(), z.bigint(), z.string()]) - .pipe(z.coerce.number().int().min(0)) - .transform(value => CheckpointNumber(value)); +function makeCheckpointNumberSchema(minValue: number) { + return z + .union([z.number(), z.bigint(), z.string()]) + .pipe(z.coerce.number().int().min(minValue)) + .transform(value => CheckpointNumber(value)); +} + +/** + * Zod schema for parsing and validating Checkpoint values. + * Accepts numbers, bigints, or strings and coerces them to CheckpointNumber. + */ +export const CheckpointNumberSchema = makeCheckpointNumberSchema(0); + +/** + * Zod schema for parsing and validating CheckpointNumber values that are strictly positive. + * Accepts numbers, bigints, or strings and coerces them to CheckpointNumber. + */ +export const CheckpointNumberPositiveSchema = makeCheckpointNumberSchema(1); diff --git a/yarn-project/foundation/src/branded-types/index.ts b/yarn-project/foundation/src/branded-types/index.ts index 7168a053b702..7bfb38583b4a 100644 --- a/yarn-project/foundation/src/branded-types/index.ts +++ b/yarn-project/foundation/src/branded-types/index.ts @@ -1,5 +1,5 @@ export { BlockNumber, BlockNumberSchema, BlockNumberPositiveSchema } from './block_number.js'; -export { CheckpointNumber, CheckpointNumberSchema } from './checkpoint_number.js'; +export { CheckpointNumber, CheckpointNumberSchema, CheckpointNumberPositiveSchema } from './checkpoint_number.js'; export { EpochNumber, EpochNumberSchema } from './epoch.js'; export { SlotNumber, SlotNumberSchema } from './slot.js'; diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.ts b/yarn-project/kv-store/src/stores/l2_tips_store.ts index 7d03a56f3430..d27cade683e5 100644 --- a/yarn-project/kv-store/src/stores/l2_tips_store.ts +++ b/yarn-project/kv-store/src/stores/l2_tips_store.ts @@ -1,6 +1,7 @@ import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants'; -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import type { + CheckpointId, L2BlockId, L2BlockStreamEvent, L2BlockStreamEventHandler, @@ -8,18 +9,29 @@ import type { L2BlockTag, L2Tips, } from '@aztec/stdlib/block'; +import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { AztecAsyncMap } from '../interfaces/map.js'; import type { AztecAsyncKVStore } from '../interfaces/store.js'; -/** Stores currently synced L2 tips and unfinalized block hashes. */ +/** Maintains and returns the current set of L2 Tips. Maintains stores of block hashes and checkpoints in order to do so. + */ export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider { private readonly l2TipsStore: AztecAsyncMap; private readonly l2BlockHashesStore: AztecAsyncMap; + private readonly l2BlockNumberToCheckpointNumberStore: AztecAsyncMap; + private readonly l2CheckpointStore: AztecAsyncMap; - constructor(store: AztecAsyncKVStore, namespace: string) { + constructor( + private store: AztecAsyncKVStore, + namespace: string, + ) { this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_')); this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_')); + this.l2BlockNumberToCheckpointNumberStore = store.openMap( + [namespace, 'l2_block_number_to_checkpoint_number'].join('_'), + ); + this.l2CheckpointStore = store.openMap([namespace, 'l2_checkpoint_store'].join('_')); } public getL2BlockHash(number: BlockNumber): Promise { @@ -27,14 +39,71 @@ export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLo } public async getL2Tips(): Promise { - return { - latest: await this.getL2Tip('latest'), - finalized: await this.getL2Tip('finalized'), - proven: await this.getL2Tip('proven'), - }; + return await this.store.transactionAsync(async () => { + const [proposedBlockId, finalizedBlockId, provenBlockId, checkpointedBlockId] = await Promise.all([ + this.getBlockId('proposed'), + this.getBlockId('finalized'), + this.getBlockId('proven'), + this.getBlockId('checkpointed'), + ]); + + const [finalizedCheckpointId, provenCheckpointId, checkpointedCheckpointId] = await Promise.all([ + this.getCheckpointId('finalized'), + this.getCheckpointId('proven'), + this.getCheckpointId('checkpointed'), + ]); + + const l2Tips: L2Tips = { + proposed: proposedBlockId, + finalized: { block: finalizedBlockId, checkpoint: finalizedCheckpointId }, + proven: { block: provenBlockId, checkpoint: provenCheckpointId }, + checkpointed: { block: checkpointedBlockId, checkpoint: checkpointedCheckpointId }, + }; + return Promise.resolve(l2Tips); + }); } - private async getL2Tip(tag: L2BlockTag): Promise { + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + switch (event.type) { + case 'blocks-added': + await this.handleBlocksAdded(event); + break; + case 'chain-checkpointed': + await this.handleChainCheckpointed(event); + break; + case 'chain-pruned': + await this.handleChainPruned(event); + break; + case 'chain-proven': + await this.handleChainProven(event); + break; + case 'chain-finalized': + await this.handleChainFinalized(event); + break; + } + } + + private async getCheckpointId(tag: L2BlockTag): Promise { + const blockNumber = await this.l2TipsStore.getAsync(tag); + if (blockNumber === undefined || blockNumber === 0) { + return { number: CheckpointNumber.ZERO, hash: '' }; + } + const checkpointNumber = await this.l2BlockNumberToCheckpointNumberStore.getAsync(blockNumber); + if (checkpointNumber === undefined) { + // No checkpoint associated with this block yet + return { number: CheckpointNumber.ZERO, hash: '' }; + } + const checkpointBuffer = await this.l2CheckpointStore.getAsync(checkpointNumber); + if (!checkpointBuffer) { + throw new Error(`Checkpoint not found for checkpoint number ${checkpointNumber}`); + } + + const checkpoint = PublishedCheckpoint.fromBuffer(checkpointBuffer); + + return { number: checkpointNumber, hash: checkpoint.checkpoint.hash().toString() }; + } + + private async getBlockId(tag: L2BlockTag): Promise { const blockNumber = await this.l2TipsStore.getAsync(tag); if (blockNumber === undefined || blockNumber === 0) { return { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }; @@ -47,29 +116,76 @@ export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLo return { number: blockNumber, hash: blockHash }; } - public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { - switch (event.type) { - case 'blocks-added': { - const blocks = event.blocks.map(b => b.block); - for (const block of blocks) { - await this.l2BlockHashesStore.set(block.number, (await block.hash()).toString()); - } - await this.l2TipsStore.set('latest', blocks.at(-1)!.number); - break; + private async handleBlocksAdded(event: L2BlockStreamEvent) { + if (event.type !== 'blocks-added') { + return; + } + // Simply add the new block hashes by the block number and update the proposed tip + await this.store.transactionAsync(async () => { + const blocks = event.blocks; + for (const block of blocks) { + await this.l2BlockHashesStore.set(block.number, (await block.hash()).toString()); } - case 'chain-pruned': - await this.saveTag('latest', event.block); - break; - case 'chain-proven': - await this.saveTag('proven', event.block); - break; - case 'chain-finalized': - await this.saveTag('finalized', event.block); - for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.block.number })) { - await this.l2BlockHashesStore.delete(key); - } - break; + await this.l2TipsStore.set('proposed', blocks.at(-1)!.number); + }); + } + + private async handleChainCheckpointed(event: L2BlockStreamEvent) { + if (event.type !== 'chain-checkpointed') { + return; + } + // Update the checkpointed chain tip and save the checkpoint + await this.store.transactionAsync(async () => { + await this.saveTag('checkpointed', event.block); + await this.saveCheckpoint(event.checkpoint); + }); + } + + private async handleChainPruned(event: L2BlockStreamEvent) { + if (event.type !== 'chain-pruned') { + return; + } + // Update the proposed and checkpointed tips + await this.store.transactionAsync(async () => { + await this.saveTag('proposed', event.block); + await this.saveTag('checkpointed', event.block); + }); + } + + private async handleChainProven(event: L2BlockStreamEvent) { + if (event.type !== 'chain-proven') { + return; } + // Updtae the proven chain tip + await this.store.transactionAsync(async () => { + await this.saveTag('proven', event.block); + }); + } + + private async handleChainFinalized(event: L2BlockStreamEvent) { + if (event.type !== 'chain-finalized') { + return; + } + await this.store.transactionAsync(async () => { + // Update the finalized tip + await this.saveTag('finalized', event.block); + // Get the checkpoint number for the finalized block + const finalizedCheckpointNumber = await this.l2BlockNumberToCheckpointNumberStore.getAsync(event.block.number); + // Clean up block hashes for blocks earlier than the finalized tip + for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.block.number })) { + await this.l2BlockHashesStore.delete(key); + } + // Clean up block-to-checkpoint mappings for blocks earlier than the finalized tip + for await (const key of this.l2BlockNumberToCheckpointNumberStore.keysAsync({ end: event.block.number })) { + await this.l2BlockNumberToCheckpointNumberStore.delete(key); + } + // Clean up checkpoints older than the finalized checkpoint + if (finalizedCheckpointNumber !== undefined) { + for await (const key of this.l2CheckpointStore.keysAsync({ end: finalizedCheckpointNumber })) { + await this.l2CheckpointStore.delete(key); + } + } + }); } private async saveTag(name: L2BlockTag, block: L2BlockId) { @@ -78,4 +194,14 @@ export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLo await this.l2BlockHashesStore.set(block.number, block.hash); } } + + private async saveCheckpoint(publishedCheckpoint: PublishedCheckpoint) { + const checkpoint = publishedCheckpoint.checkpoint; + const lastBlock = checkpoint.blocks.at(-1)!; + // Only store the mapping for the last block since tips only point to checkpoint boundaries + await Promise.all([ + this.l2BlockNumberToCheckpointNumberStore.set(lastBlock.number, checkpoint.number), + this.l2CheckpointStore.set(checkpoint.number, publishedCheckpoint.toBuffer()), + ]); + } } diff --git a/yarn-project/node-lib/src/actions/build-snapshot-metadata.ts b/yarn-project/node-lib/src/actions/build-snapshot-metadata.ts index d7c8e0d2a236..3077e490e1de 100644 --- a/yarn-project/node-lib/src/actions/build-snapshot-metadata.ts +++ b/yarn-project/node-lib/src/actions/build-snapshot-metadata.ts @@ -7,13 +7,13 @@ export async function buildSnapshotMetadata( archiver: Archiver, config: UploadSnapshotConfig, ): Promise { - const [rollupAddress, l1BlockNumber, { latest }] = await Promise.all([ + const [rollupAddress, l1BlockNumber, tips] = await Promise.all([ archiver.getRollupAddress(), archiver.getL1BlockNumber(), archiver.getL2Tips(), ] as const); - const { number: l2BlockNumber, hash: l2BlockHash } = latest; + const { number: l2BlockNumber, hash: l2BlockHash } = tips.proposed; if (!l2BlockHash) { throw new Error(`Failed to get L2 block hash from archiver.`); } diff --git a/yarn-project/p2p/src/client/p2p_client.test.ts b/yarn-project/p2p/src/client/p2p_client.test.ts index 5c0389999e0f..168da286736a 100644 --- a/yarn-project/p2p/src/client/p2p_client.test.ts +++ b/yarn-project/p2p/src/client/p2p_client.test.ts @@ -321,15 +321,19 @@ describe('P2P Client', () => { it('moves the tips on a chain reorg', async () => { blockSource.setProvenBlockNumber(0); + // Set checkpointed before starting so blocks are synced as checkpointed + blockSource.setCheckpointedBlockNumber(100); await client.start(); await advanceToProvenBlock(BlockNumber(90)); await advanceToFinalizedBlock(BlockNumber(50)); + const anyCheckpoint = { number: expect.any(Number), hash: expect.any(String) }; await expect(client.getL2Tips()).resolves.toEqual({ - latest: { number: BlockNumber(100), hash: expect.any(String) }, - proven: { number: BlockNumber(90), hash: expect.any(String) }, - finalized: { number: BlockNumber(50), hash: expect.any(String) }, + proposed: { number: BlockNumber(100), hash: expect.any(String) }, + checkpointed: { block: { number: BlockNumber(100), hash: expect.any(String) }, checkpoint: anyCheckpoint }, + proven: { block: { number: BlockNumber(90), hash: expect.any(String) }, checkpoint: anyCheckpoint }, + finalized: { block: { number: BlockNumber(50), hash: expect.any(String) }, checkpoint: anyCheckpoint }, }); blockSource.removeBlocks(10); @@ -337,19 +341,22 @@ describe('P2P Client', () => { await client.sync(); await expect(client.getL2Tips()).resolves.toEqual({ - latest: { number: BlockNumber(90), hash: expect.any(String) }, - proven: { number: BlockNumber(90), hash: expect.any(String) }, - finalized: { number: BlockNumber(50), hash: expect.any(String) }, + proposed: { number: BlockNumber(90), hash: expect.any(String) }, + checkpointed: { block: { number: BlockNumber(90), hash: expect.any(String) }, checkpoint: anyCheckpoint }, + proven: { block: { number: BlockNumber(90), hash: expect.any(String) }, checkpoint: anyCheckpoint }, + finalized: { block: { number: BlockNumber(50), hash: expect.any(String) }, checkpoint: anyCheckpoint }, }); blockSource.addBlocks([await L2Block.random(BlockNumber(91)), await L2Block.random(BlockNumber(92))]); + blockSource.setCheckpointedBlockNumber(92); await client.sync(); await expect(client.getL2Tips()).resolves.toEqual({ - latest: { number: BlockNumber(92), hash: expect.any(String) }, - proven: { number: BlockNumber(90), hash: expect.any(String) }, - finalized: { number: BlockNumber(50), hash: expect.any(String) }, + proposed: { number: BlockNumber(92), hash: expect.any(String) }, + checkpointed: { block: { number: BlockNumber(92), hash: expect.any(String) }, checkpoint: anyCheckpoint }, + proven: { block: { number: BlockNumber(90), hash: expect.any(String) }, checkpoint: anyCheckpoint }, + finalized: { block: { number: BlockNumber(50), hash: expect.any(String) }, checkpoint: anyCheckpoint }, }); }); diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 01a0366c16a3..2a2b7084d6e7 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -1,16 +1,17 @@ -import { GENESIS_BLOCK_HEADER_HASH, INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; +import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants'; import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { createLogger } from '@aztec/foundation/log'; import { DateProvider } from '@aztec/foundation/timer'; -import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncSingleton } from '@aztec/kv-store'; +import type { AztecAsyncKVStore, AztecAsyncSingleton } from '@aztec/kv-store'; +import { L2TipsKVStore } from '@aztec/kv-store/stores'; import { type EthAddress, - type L2BlockId, type L2BlockNew, type L2BlockSource, L2BlockStream, type L2BlockStreamEvent, type L2Tips, + type L2TipsStore, } from '@aztec/stdlib/block'; import type { ContractDataSource } from '@aztec/stdlib/contract'; import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; @@ -55,10 +56,7 @@ export class P2PClient private provenBlockNumberAtStart = -1; private finalizedBlockNumberAtStart = -1; - private synchedBlockHashes: AztecAsyncMap; - private synchedLatestBlockNumber: AztecAsyncSingleton; - private synchedProvenBlockNumber: AztecAsyncSingleton; - private synchedFinalizedBlockNumber: AztecAsyncSingleton; + private l2Tips: L2TipsStore; private synchedLatestSlot: AztecAsyncSingleton; private txPool: TxPool; @@ -126,11 +124,7 @@ export class P2PClient return undefined; }); - // REFACTOR: Try replacing these with an L2TipsStore - this.synchedBlockHashes = store.openMap('p2p_pool_block_hashes'); - this.synchedLatestBlockNumber = store.openSingleton('p2p_pool_last_l2_block'); - this.synchedProvenBlockNumber = store.openSingleton('p2p_pool_last_proven_l2_block'); - this.synchedFinalizedBlockNumber = store.openSingleton('p2p_pool_last_finalized_l2_block'); + this.l2Tips = new L2TipsKVStore(store, 'p2p_client'); this.synchedLatestSlot = store.openSingleton('p2p_pool_last_l2_slot'); } @@ -156,7 +150,7 @@ export class P2PClient } public getL2BlockHash(number: BlockNumber): Promise { - return this.synchedBlockHashes.getAsync(number); + return this.l2Tips.getL2BlockHash(number); } public updateP2PConfig(config: Partial): Promise { @@ -165,56 +159,20 @@ export class P2PClient return Promise.resolve(); } - public async getL2Tips(): Promise { - const latestBlockNumber = await this.getSyncedLatestBlockNum(); - let latestBlockHash: string | undefined; - - const provenBlockNumber = await this.getSyncedProvenBlockNum(); - let provenBlockHash: string | undefined; - - const finalizedBlockNumber = await this.getSyncedFinalizedBlockNum(); - let finalizedBlockHash: string | undefined; - - if (latestBlockNumber > 0) { - latestBlockHash = await this.synchedBlockHashes.getAsync(latestBlockNumber); - if (typeof latestBlockHash === 'undefined') { - throw new Error(`Block hash for latest block ${latestBlockNumber} not found in p2p client`); - } - } - - if (provenBlockNumber > 0) { - provenBlockHash = await this.synchedBlockHashes.getAsync(provenBlockNumber); - if (typeof provenBlockHash === 'undefined') { - throw new Error(`Block hash for proven block ${provenBlockNumber} not found in p2p client`); - } - } - - if (finalizedBlockNumber > 0) { - finalizedBlockHash = await this.synchedBlockHashes.getAsync(finalizedBlockNumber); - if (typeof finalizedBlockHash === 'undefined') { - throw new Error(`Block hash for finalized block ${finalizedBlockNumber} not found in p2p client`); - } - } - - const genesisHash = GENESIS_BLOCK_HEADER_HASH.toString(); - - return { - latest: { hash: latestBlockHash ?? genesisHash, number: latestBlockNumber }, - proven: { hash: provenBlockHash ?? genesisHash, number: provenBlockNumber }, - finalized: { hash: finalizedBlockHash ?? genesisHash, number: finalizedBlockNumber }, - }; + public getL2Tips(): Promise { + return this.l2Tips.getL2Tips(); } public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { this.log.debug(`Handling block stream event ${event.type}`); + switch (event.type) { case 'blocks-added': - await this.handleLatestL2Blocks(event.blocks.map(b => b.block.toL2Block())); + await this.handleLatestL2Blocks(event.blocks); break; case 'chain-finalized': { - // TODO (alexg): I think we can prune the block hashes map here - await this.setBlockHash(event.block); - const from = BlockNumber((await this.getSyncedFinalizedBlockNum()) + 1); + const oldFinalizedBlockNum = await this.getSyncedFinalizedBlockNum(); + const from = BlockNumber(oldFinalizedBlockNum + 1); const limit = event.block.number - from + 1; if (limit > 0) { const oldBlocks = await this.l2BlockSource.getBlocks(from, limit); @@ -222,28 +180,24 @@ export class P2PClient } break; } - case 'chain-proven': { - await this.setBlockHash(event.block); + case 'chain-proven': this.txCollection.stopCollectingForBlocksUpTo(event.block.number); - await this.synchedProvenBlockNumber.set(event.block.number); break; - } case 'chain-pruned': - await this.setBlockHash(event.block); this.txCollection.stopCollectingForBlocksAfter(event.block.number); await this.handlePruneL2Blocks(event.block.number); break; + case 'chain-checkpointed': + break; default: { const _: never = event; break; } } - } - private async setBlockHash(block: L2BlockId): Promise { - if (block.hash !== undefined) { - await this.synchedBlockHashes.set(block.number, block.hash.toString()); - } + // Pass the event through to our l2 tips store + await this.l2Tips.handleBlockStreamEvent(event); + await this.startServiceIfSynched(); } #assertIsReady() { @@ -267,9 +221,9 @@ export class P2PClient // get the current latest block numbers const latestBlockNumbers = await this.l2BlockSource.getL2Tips(); - this.latestBlockNumberAtStart = latestBlockNumbers.latest.number; - this.provenBlockNumberAtStart = latestBlockNumbers.proven.number; - this.finalizedBlockNumberAtStart = latestBlockNumbers.finalized.number; + this.latestBlockNumberAtStart = latestBlockNumbers.proposed.number; + this.provenBlockNumberAtStart = latestBlockNumbers.proven.block.number; + this.finalizedBlockNumberAtStart = latestBlockNumbers.finalized.block.number; const syncedLatestBlock = (await this.getSyncedLatestBlockNum()) + 1; const syncedProvenBlock = (await this.getSyncedProvenBlockNum()) + 1; @@ -638,7 +592,8 @@ export class P2PClient * @returns Block number of latest L2 Block we've synced with. */ public async getSyncedLatestBlockNum(): Promise { - return (await this.synchedLatestBlockNumber.getAsync()) ?? BlockNumber(INITIAL_L2_BLOCK_NUM - 1); + const tips = await this.l2Tips.getL2Tips(); + return tips.proposed.number; } /** @@ -646,11 +601,13 @@ export class P2PClient * @returns Block number of latest proven L2 Block we've synced with. */ public async getSyncedProvenBlockNum(): Promise { - return (await this.synchedProvenBlockNumber.getAsync()) ?? BlockNumber(INITIAL_L2_BLOCK_NUM - 1); + const tips = await this.l2Tips.getL2Tips(); + return tips.proven.block.number; } public async getSyncedFinalizedBlockNum(): Promise { - return (await this.synchedFinalizedBlockNumber.getAsync()) ?? BlockNumber(INITIAL_L2_BLOCK_NUM - 1); + const tips = await this.l2Tips.getL2Tips(); + return tips.finalized.block.number; } /** Returns latest L2 slot for which we have seen an L2 block. */ @@ -705,20 +662,8 @@ export class P2PClient await this.startCollectingMissingTxs(blocks); const lastBlock = blocks.at(-1)!; - - await Promise.all( - blocks.map(async block => - this.setBlockHash({ - number: block.number, - hash: await block.hash().then(h => h.toString()), - }), - ), - ); - - await this.synchedLatestBlockNumber.set(lastBlock.number); await this.synchedLatestSlot.set(BigInt(lastBlock.header.getSlot())); this.log.verbose(`Synched to latest block ${lastBlock.number}`); - await this.startServiceIfSynched(); } /** Request txs for unproven blocks so the prover node has more chances to get them. */ @@ -771,10 +716,7 @@ export class P2PClient await this.attestationPool.deleteAttestationsOlderThan(lastBlockSlot); - await this.synchedFinalizedBlockNumber.set(lastBlockNum); this.log.debug(`Synched to finalized block ${lastBlockNum} at slot ${lastBlockSlot}`); - - await this.startServiceIfSynched(); } /** @@ -822,18 +764,16 @@ export class P2PClient } else { await this.txPool.markMinedAsPending(minedTxsFromReorg, latestBlock); } - - await this.synchedLatestBlockNumber.set(latestBlock); - // no need to update block hashes, as they will be updated as new blocks are added } private async startServiceIfSynched() { if (this.currentState !== P2PClientState.SYNCHING) { return; } - const syncedFinalizedBlock = await this.getSyncedFinalizedBlockNum(); - const syncedProvenBlock = await this.getSyncedProvenBlockNum(); - const syncedLatestBlock = await this.getSyncedLatestBlockNum(); + const tips = await this.l2Tips.getL2Tips(); + const syncedFinalizedBlock = tips.finalized.block.number; + const syncedProvenBlock = tips.proven.block.number; + const syncedLatestBlock = tips.proposed.number; if ( syncedLatestBlock >= this.latestBlockNumberAtStart && diff --git a/yarn-project/p2p/src/client/test/p2p_client.integration_block_txs.test.ts b/yarn-project/p2p/src/client/test/p2p_client.integration_block_txs.test.ts index 912c5eab72f1..45e68be36511 100644 --- a/yarn-project/p2p/src/client/test/p2p_client.integration_block_txs.test.ts +++ b/yarn-project/p2p/src/client/test/p2p_client.integration_block_txs.test.ts @@ -58,6 +58,7 @@ describe('p2p client integration block txs protocol ', () => { epochCache.getEpochAndSlotInNextL1Slot.mockReturnValue({ ts: BigInt(0) }); epochCache.getRegisteredValidators.mockResolvedValue([]); + txPool.isEmpty.mockResolvedValue(true); txPool.hasTxs.mockResolvedValue([]); txPool.getAllTxs.mockImplementation(() => { return Promise.resolve([] as Tx[]); @@ -67,6 +68,8 @@ describe('p2p client integration block txs protocol ', () => { return Promise.resolve([] as Tx[]); }); + attestationPool.isEmpty.mockResolvedValue(true); + worldState.status.mockResolvedValue({ state: mock(), syncSummary: { diff --git a/yarn-project/p2p/src/client/test/p2p_client.integration_message_propagation.test.ts b/yarn-project/p2p/src/client/test/p2p_client.integration_message_propagation.test.ts index 2d0ede637df9..af49d3fd0ac1 100644 --- a/yarn-project/p2p/src/client/test/p2p_client.integration_message_propagation.test.ts +++ b/yarn-project/p2p/src/client/test/p2p_client.integration_message_propagation.test.ts @@ -59,6 +59,7 @@ describe('p2p client integration message propagation', () => { epochCache.getEpochAndSlotInNextL1Slot.mockReturnValue({ ts: BigInt(0) }); epochCache.getRegisteredValidators.mockResolvedValue([]); + txPool.isEmpty.mockResolvedValue(true); txPool.hasTxs.mockResolvedValue([]); txPool.getAllTxs.mockImplementation(() => { return Promise.resolve([] as Tx[]); @@ -68,6 +69,8 @@ describe('p2p client integration message propagation', () => { return Promise.resolve([] as Tx[]); }); + attestationPool.isEmpty.mockResolvedValue(true); + worldState.status.mockResolvedValue({ state: mock(), syncSummary: { diff --git a/yarn-project/p2p/src/client/test/p2p_client.integration_status_handshake.test.ts b/yarn-project/p2p/src/client/test/p2p_client.integration_status_handshake.test.ts index e30df6f65dd6..e88109d37ffb 100644 --- a/yarn-project/p2p/src/client/test/p2p_client.integration_status_handshake.test.ts +++ b/yarn-project/p2p/src/client/test/p2p_client.integration_status_handshake.test.ts @@ -48,6 +48,9 @@ describe('p2p client integration status handshake', () => { epochCache.getEpochAndSlotInNextL1Slot.mockReturnValue({ ts: BigInt(0) }); epochCache.getRegisteredValidators.mockResolvedValue([]); + txPool.isEmpty.mockResolvedValue(true); + attestationPool.isEmpty.mockResolvedValue(true); + worldState.status.mockResolvedValue({ state: mock(), syncSummary: { diff --git a/yarn-project/p2p/src/client/test/p2p_client.integration_txs.test.ts b/yarn-project/p2p/src/client/test/p2p_client.integration_txs.test.ts index 521b974663f3..7776173fd59a 100644 --- a/yarn-project/p2p/src/client/test/p2p_client.integration_txs.test.ts +++ b/yarn-project/p2p/src/client/test/p2p_client.integration_txs.test.ts @@ -50,6 +50,7 @@ describe('p2p client integration', () => { epochCache.getEpochAndSlotInNextL1Slot.mockReturnValue({ ts: BigInt(0) }); epochCache.getRegisteredValidators.mockResolvedValue([]); + txPool.isEmpty.mockResolvedValue(true); txPool.hasTxs.mockResolvedValue([]); txPool.getAllTxs.mockImplementation(() => { return Promise.resolve([] as Tx[]); @@ -59,6 +60,8 @@ describe('p2p client integration', () => { return Promise.resolve([] as Tx[]); }); + attestationPool.isEmpty.mockResolvedValue(true); + worldState.status.mockResolvedValue({ state: mock(), syncSummary: { @@ -297,7 +300,6 @@ describe('p2p client integration', () => { // Even though we got a response, the proof was deemed invalid expect(requestedTxs).toEqual([]); - // Low tolerance error is due to the invalid proof expect(penalizePeerSpy).toHaveBeenCalledWith(client2PeerId, PeerErrorSeverity.LowToleranceError); }); @@ -335,7 +337,6 @@ describe('p2p client integration', () => { // Even though we got a response, the proof was deemed invalid expect(requestedTxs).toEqual([]); - // Received wrong tx expect(penalizePeerSpy).toHaveBeenCalledWith(client2PeerId, PeerErrorSeverity.MidToleranceError); }); }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool/tx_pool_bench.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool/tx_pool_bench.test.ts index 3272f5eac168..7e67cde584cc 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool/tx_pool_bench.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool/tx_pool_bench.test.ts @@ -1,6 +1,6 @@ import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants'; import { insertIntoSortedArray, shuffle } from '@aztec/foundation/array'; -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { timesAsync } from '@aztec/foundation/collection'; import { getDefaultConfig } from '@aztec/foundation/config'; import { Fr } from '@aztec/foundation/curves/bn254'; @@ -157,12 +157,18 @@ describe('TxPool: Benchmarks', () => { syncImmediate: () => Promise.resolve(), getProvenBlockNumber: () => Promise.resolve(BlockNumber.ZERO), getBlockNumber: () => Promise.resolve(BlockNumber.ZERO), - getL2Tips: () => - Promise.resolve({ - latest: { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, - proven: { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, - finalized: { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, - }), + getL2Tips: () => { + const tipId = { + block: { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }; + return Promise.resolve({ + proposed: { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + checkpointed: tipId, + proven: tipId, + finalized: tipId, + }); + }, }); wsSync = new ServerWorldStateSynchronizer(ws, l2, getDefaultConfig(worldStateConfigMappings)); await wsSync.start(); diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts index 41b164ba63ae..bb13112575e3 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts @@ -472,17 +472,6 @@ export class LibP2PService extends } const announceTcpMultiaddr = convertToMultiaddr(p2pIp, p2pPort, 'tcp'); - await this.peerManager.initializePeers(); - if (!this.config.p2pDiscoveryDisabled) { - await this.peerDiscoveryService.start(); - } - await this.node.start(); - - // Subscribe to standard GossipSub topics by default - for (const topic of getTopicsForClientAndConfig(this.clientType, this.config.disableTransactions)) { - this.subscribeToTopic(this.topicStrings[topic]); - } - // Create request response protocol handlers const txHandler = reqRespTxHandler(this.mempools); const goodbyeHandler = reqGoodbyeHandler(this.peerManager); @@ -505,10 +494,32 @@ export class LibP2PService extends requestResponseHandlers[ReqRespSubProtocol.TX] = txHandler.bind(this); } + // Define the sub protocol validators - This is done within this start() method to gain a callback to the existing validateTx function + const reqrespSubProtocolValidators = { + ...DEFAULT_SUB_PROTOCOL_VALIDATORS, + [ReqRespSubProtocol.TX]: this.validateRequestedTxs.bind(this), + [ReqRespSubProtocol.BLOCK_TXS]: this.validateRequestedBlockTxs.bind(this), + [ReqRespSubProtocol.BLOCK]: this.validateRequestedBlock.bind(this), + }; + + await this.peerManager.initializePeers(); + + await this.reqresp.start(requestResponseHandlers, reqrespSubProtocolValidators); + + await this.node.start(); + + // Subscribe to standard GossipSub topics by default + for (const topic of getTopicsForClientAndConfig(this.clientType, this.config.disableTransactions)) { + this.subscribeToTopic(this.topicStrings[topic]); + } + // add GossipSub listener this.node.services.pubsub.addEventListener(GossipSubEvent.MESSAGE, this.gossipSubEventHandler); // Start running promise for peer discovery and metrics collection + if (!this.config.p2pDiscoveryDisabled) { + await this.peerDiscoveryService.start(); + } this.discoveryRunningPromise = new RunningPromise( async () => { await this.peerManager.heartbeat(); @@ -518,14 +529,6 @@ export class LibP2PService extends ); this.discoveryRunningPromise.start(); - // Define the sub protocol validators - This is done within this start() method to gain a callback to the existing validateTx function - const reqrespSubProtocolValidators = { - ...DEFAULT_SUB_PROTOCOL_VALIDATORS, - [ReqRespSubProtocol.TX]: this.validateRequestedTxs.bind(this), - [ReqRespSubProtocol.BLOCK_TXS]: this.validateRequestedBlockTxs.bind(this), - [ReqRespSubProtocol.BLOCK]: this.validateRequestedBlock.bind(this), - }; - await this.reqresp.start(requestResponseHandlers, reqrespSubProtocolValidators); this.logger.info(`Started P2P service`, { listen: this.config.listenAddress, port: this.config.p2pPort, diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index 1d9e83bfd70c..26d79b59ee70 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -149,14 +149,20 @@ describe('prover-node', () => { l2BlockSource.getL1Constants.mockResolvedValue({ ...EmptyL1RollupConstants, l1GenesisTime: BigInt(l1GenesisTime) }); l2BlockSource.getCheckpointsForEpoch.mockResolvedValue(checkpoints); l2BlockSource.getPublishedCheckpoints.mockResolvedValue([lastPublishedCheckpoint]); + const latestBlockNumber = BlockNumber.fromCheckpointNumber(checkpoints.at(-1)!.number); + const latestHash = checkpoints.at(-1)!.hash().toString(); + const genesisTipId = { + block: { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }; l2BlockSource.getL2Tips.mockResolvedValue({ - latest: { - number: BlockNumber.fromCheckpointNumber(checkpoints.at(-1)!.number), - // TODO: This should be the actual block hash - hash: checkpoints.at(-1)!.hash().toString(), + proposed: { number: latestBlockNumber, hash: latestHash }, + checkpointed: { + block: { number: latestBlockNumber, hash: latestHash }, + checkpoint: { number: checkpoints.at(-1)!.number, hash: latestHash }, }, - proven: { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, - finalized: { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + proven: genesisTipId, + finalized: genesisTipId, }); l2BlockSource.getBlockHeader.mockImplementation(number => Promise.resolve(number === checkpoints[0].blocks[0].number - 1 ? previousBlockHeader : undefined), diff --git a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts index fc241f8da2cb..9853e37f2c1b 100644 --- a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts +++ b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts @@ -1,10 +1,9 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { timesParallel } from '@aztec/foundation/collection'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { L2TipsKVStore } from '@aztec/kv-store/stores'; -import { L2Block, type L2BlockStream } from '@aztec/stdlib/block'; +import { L2Block, L2BlockNew, type L2BlockStream } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; -import { randomPublishedL2Block } from '@aztec/stdlib/testing'; import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -41,11 +40,11 @@ describe('BlockSynchronizer', () => { }); it('sets header from latest block', async () => { - const block = await randomPublishedL2Block(1); + const block = await L2BlockNew.random(BlockNumber(1)); await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); const obtainedHeader = await anchorBlockStore.getBlockHeader(); - expect(obtainedHeader).toEqual(block.block.getBlockHeader()); + expect(obtainedHeader.equals(block.header)).toBe(true); }); it('removes notes from db on a reorg', async () => { @@ -58,9 +57,14 @@ describe('BlockSynchronizer', () => { await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', - blocks: await timesParallel(5, randomPublishedL2Block), + blocks: await timesParallel(5, i => L2BlockNew.random(BlockNumber(i))), + }); + await synchronizer.handleBlockStreamEvent({ + type: 'chain-pruned', + block: { number: BlockNumber(3), hash: '0x3' }, + reason: 'unproven', + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, }); - await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', block: { number: BlockNumber(3), hash: '0x3' } }); expect(rollbackNotesAndNullifiers).toHaveBeenCalledWith(3, 4); }); @@ -75,9 +79,14 @@ describe('BlockSynchronizer', () => { await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', - blocks: await timesParallel(5, randomPublishedL2Block), + blocks: await timesParallel(5, i => L2BlockNew.random(BlockNumber(i))), + }); + await synchronizer.handleBlockStreamEvent({ + type: 'chain-pruned', + block: { number: BlockNumber(3), hash: '0x3' }, + reason: 'unproven', + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, }); - await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', block: { number: BlockNumber(3), hash: '0x3' } }); expect(rollbackEventsAfterBlock).toHaveBeenCalledWith(3, 4); }); diff --git a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts index c33911947d29..f8ed0d758e8e 100644 --- a/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts +++ b/yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts @@ -50,13 +50,13 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler { switch (event.type) { case 'blocks-added': { - const lastBlock = event.blocks.at(-1)!.block; + const lastBlock = event.blocks.at(-1)!; this.log.verbose(`Updated pxe last block to ${lastBlock.number}`, { blockHash: lastBlock.hash(), archive: lastBlock.archive.root.toString(), header: lastBlock.header.toInspect(), }); - await this.anchorBlockStore.setHeader(lastBlock.getBlockHeader()); + await this.anchorBlockStore.setHeader(lastBlock.header); break; } case 'chain-pruned': { diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts index 846afe6c7b5c..297f9474496a 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts @@ -48,7 +48,7 @@ import { computeAppNullifierSecretKey, deriveKeys } from '@aztec/stdlib/keys'; import type { SiloedTag } from '@aztec/stdlib/logs'; import { L1Actor, L1ToL2Message, L2Actor } from '@aztec/stdlib/messaging'; import { Note, NoteDao } from '@aztec/stdlib/note'; -import { makeBlockHeader } from '@aztec/stdlib/testing'; +import { makeBlockHeader, makeL2Tips } from '@aztec/stdlib/testing'; import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; import { BlockHeader, @@ -330,9 +330,7 @@ describe('Private Execution test suite', () => { aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => Promise.resolve(tags.map(() => []))); // Mock getL2Tips and getBlockHeader for loadPrivateLogsForSenderRecipientPair - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: anchorBlockHeader.globalVariables.blockNumber }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(anchorBlockHeader.globalVariables.blockNumber)); aztecNode.getBlockHeader.mockImplementation((blockNumber: BlockNumber | 'latest') => { if (blockNumber === 'latest') { return Promise.resolve(anchorBlockHeader); diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts index 3a3714156158..7ac113d1607a 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts @@ -11,6 +11,7 @@ import { CompleteAddress, type ContractInstanceWithAddress } from '@aztec/stdlib import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { deriveKeys } from '@aztec/stdlib/keys'; import { Note, NoteDao } from '@aztec/stdlib/note'; +import { makeL2Tips } from '@aztec/stdlib/testing'; import { BlockHeader, GlobalVariables, TxHash } from '@aztec/stdlib/tx'; import { mock } from 'jest-mock-extended'; @@ -74,9 +75,7 @@ describe('Utility Execution test suite', () => { senderAddressBookStore.getSenders.mockResolvedValue([]); // Mock getL2Tips and getBlockHeader for loadPrivateLogsForSenderRecipientPair - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: anchorBlockHeader.globalVariables.blockNumber }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(anchorBlockHeader.globalVariables.blockNumber)); aztecNode.getBlockHeader.mockImplementation((blockNumber: BlockNumber | 'latest') => { if (blockNumber === 'latest') { return Promise.resolve(anchorBlockHeader); diff --git a/yarn-project/pxe/src/pxe.test.ts b/yarn-project/pxe/src/pxe.test.ts index 1ead43423678..5b616b098b42 100644 --- a/yarn-project/pxe/src/pxe.test.ts +++ b/yarn-project/pxe/src/pxe.test.ts @@ -1,7 +1,7 @@ import { BBBundlePrivateKernelProver } from '@aztec/bb-prover/client/bundle'; import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants'; import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'; -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { omit } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; @@ -177,10 +177,15 @@ describe('PXE', () => { node.getBlockHeader.mockResolvedValue(blockHeader); // Mock getL2Tips which is needed for syncing tagged logs + const tipId = { + block: { number: lastKnownBlockNumber, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + checkpoint: { number: CheckpointNumber(lastKnownBlockNumber), hash: '' }, + }; node.getL2Tips.mockResolvedValue({ - latest: { number: lastKnownBlockNumber, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, - proven: { number: lastKnownBlockNumber, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, - finalized: { number: lastKnownBlockNumber, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + proposed: { number: lastKnownBlockNumber, hash: GENESIS_BLOCK_HEADER_HASH.toString() }, + checkpointed: tipId, + proven: tipId, + finalized: tipId, }); // This is read when PXE tries to resolve the diff --git a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts index a5fe72cb9ab5..37e28eed71a6 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.test.ts @@ -5,7 +5,7 @@ import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { DirectionalAppTaggingSecret, SiloedTag, Tag } from '@aztec/stdlib/logs'; -import { makeBlockHeader, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; +import { makeBlockHeader, makeL2Tips, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -49,9 +49,7 @@ describe('loadPrivateLogsForSenderRecipientPair', () => { }); it('returns empty array when no logs found', async () => { - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: BlockNumber(10) }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(10)); aztecNode.getBlockHeader.mockResolvedValue(makeBlockHeader(0, { timestamp: currentTimestamp })); @@ -80,9 +78,7 @@ describe('loadPrivateLogsForSenderRecipientPair', () => { const logIndex = 5; const logTag = await computeSiloedTagForIndex(logIndex); - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: BlockNumber(finalizedBlockNumber) }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(finalizedBlockNumber)); aztecNode.getBlockHeader.mockResolvedValue(makeBlockHeader(0, { timestamp: currentTimestamp })); @@ -115,9 +111,7 @@ describe('loadPrivateLogsForSenderRecipientPair', () => { const logIndex = 7; const logTag = await computeSiloedTagForIndex(logIndex); - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: BlockNumber(finalizedBlockNumber) }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(finalizedBlockNumber)); aztecNode.getBlockHeader.mockResolvedValue(makeBlockHeader(0, { timestamp: currentTimestamp })); @@ -159,9 +153,7 @@ describe('loadPrivateLogsForSenderRecipientPair', () => { await taggingStore.updateHighestAgedIndex(secret, highestAgedIndex); await taggingStore.updateHighestFinalizedIndex(secret, highestFinalizedIndex); - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: BlockNumber(finalizedBlockNumber) }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(finalizedBlockNumber)); aztecNode.getBlockHeader.mockResolvedValue(makeBlockHeader(0, { timestamp: currentTimestamp })); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts index 6b9fe982816e..fe12bc138010 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/load_private_logs_for_sender_recipient_pair.ts @@ -68,7 +68,10 @@ export async function loadPrivateLogsForSenderRecipientPair( throw new Error('Node failed to return latest block header when syncing logs'); } - [finalizedBlockNumber, currentTimestamp] = [l2Tips.finalized.number, latestBlockHeader.globalVariables.timestamp]; + [finalizedBlockNumber, currentTimestamp] = [ + l2Tips.finalized.block.number, + latestBlockHeader.globalVariables.timestamp, + ]; } let start: number, end: number; diff --git a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts index 43af9587edf5..ed4f2d5a4109 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts @@ -2,7 +2,7 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; -import { randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; +import { makeL2Tips, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; import { TxHash, TxStatus } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -82,9 +82,7 @@ describe('syncSenderTaggingIndexes', () => { } as any); // Mock getL2Tips to return a finalized block number >= the tx block number - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: finalizedBlockNumberStep1 }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(finalizedBlockNumberStep1)); await syncSenderTaggingIndexes(secret, contractAddress, aztecNode, taggingStore); @@ -111,9 +109,7 @@ describe('syncSenderTaggingIndexes', () => { blockNumber: finalizedBlockNumberStep1 + 1, } as any); - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: finalizedBlockNumberStep1 }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(finalizedBlockNumberStep1)); await syncSenderTaggingIndexes(secret, contractAddress, aztecNode, taggingStore); @@ -180,9 +176,7 @@ describe('syncSenderTaggingIndexes', () => { }); // Mock getL2Tips with the new finalized block number - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: newFinalizedBlockNumber }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(newFinalizedBlockNumber)); await syncSenderTaggingIndexes(secret, contractAddress, aztecNode, taggingStore); @@ -233,9 +227,7 @@ describe('syncSenderTaggingIndexes', () => { } }); - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: finalizedBlockNumber }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(finalizedBlockNumber)); // Sync tagged logs await syncSenderTaggingIndexes(secret, contractAddress, aztecNode, taggingStore); diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts index 953c12e3a9b0..6d15ac87a3a6 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.test.ts @@ -1,5 +1,6 @@ import { BlockNumber } from '@aztec/foundation/branded-types'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; +import { makeL2Tips } from '@aztec/stdlib/testing'; import { TxHash, TxStatus } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -55,9 +56,7 @@ describe('getStatusChangeOfPending', () => { } }); - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: BlockNumber(finalizedBlockNumber) }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(finalizedBlockNumber)); const result = await getStatusChangeOfPending( [ @@ -89,9 +88,7 @@ describe('getStatusChangeOfPending', () => { blockNumber: BlockNumber(finalizedBlockNumber), } as any); - aztecNode.getL2Tips.mockResolvedValue({ - finalized: { number: BlockNumber(finalizedBlockNumber) }, - } as any); + aztecNode.getL2Tips.mockResolvedValue(makeL2Tips(finalizedBlockNumber)); const result = await getStatusChangeOfPending([txHash], aztecNode); diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts index 6cf934ae1861..d16d8069fa1c 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/get_status_change_of_pending.ts @@ -10,7 +10,7 @@ export async function getStatusChangeOfPending( aztecNode: AztecNode, ): Promise<{ txHashesToFinalize: TxHash[]; txHashesToDrop: TxHash[] }> { // Get receipts for all pending tx hashes and the finalized block number. - const [receipts, { finalized }] = await Promise.all([ + const [receipts, tips] = await Promise.all([ Promise.all(pending.map(pendingTxHash => aztecNode.getTxReceipt(pendingTxHash))), aztecNode.getL2Tips(), ]); @@ -22,7 +22,11 @@ export async function getStatusChangeOfPending( const receipt = receipts[i]; const txHash = pending[i]; - if (receipt.status === TxStatus.SUCCESS && receipt.blockNumber && receipt.blockNumber <= finalized.number) { + if ( + receipt.status === TxStatus.SUCCESS && + receipt.blockNumber && + receipt.blockNumber <= tips.finalized.block.number + ) { // Tx has been included in a block and the corresponding block is finalized --> we mark the indexes as // finalized. txHashesToFinalize.push(txHash); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index 834f78154988..ac17febf222d 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -229,7 +229,15 @@ describe('sequencer', () => { l2BlockSource = mock({ getL2BlockNew: mockFn().mockResolvedValue(L2BlockNew.empty()), getBlockNumber: mockFn().mockResolvedValue(lastBlockNumber), - getL2Tips: mockFn().mockResolvedValue({ latest: { number: lastBlockNumber, hash } }), + getL2Tips: mockFn().mockResolvedValue({ + proposed: { number: lastBlockNumber, hash }, + checkpointed: { + block: { number: lastBlockNumber, hash }, + checkpoint: { number: CheckpointNumber(0), hash: '' }, + }, + proven: { block: { number: lastBlockNumber, hash }, checkpoint: { number: CheckpointNumber(0), hash: '' } }, + finalized: { block: { number: lastBlockNumber, hash }, checkpoint: { number: CheckpointNumber(0), hash: '' } }, + }), getL1Timestamp: mockFn().mockResolvedValue(1000n), isPendingChainInvalid: mockFn().mockResolvedValue(false), getPendingChainValidationStatus: mockFn().mockResolvedValue({ valid: true }), @@ -237,7 +245,15 @@ describe('sequencer', () => { l1ToL2MessageSource = mock({ getL1ToL2Messages: () => Promise.resolve(Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP).fill(Fr.ZERO)), - getL2Tips: mockFn().mockResolvedValue({ latest: { number: lastBlockNumber, hash } }), + getL2Tips: mockFn().mockResolvedValue({ + proposed: { number: lastBlockNumber, hash }, + checkpointed: { + block: { number: lastBlockNumber, hash }, + checkpoint: { number: CheckpointNumber(0), hash: '' }, + }, + proven: { block: { number: lastBlockNumber, hash }, checkpoint: { number: CheckpointNumber(0), hash: '' } }, + finalized: { block: { number: lastBlockNumber, hash }, checkpoint: { number: CheckpointNumber(0), hash: '' } }, + }), }); validatorClient = mock(); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index bd13f11c7035..47e255e7f913 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -471,9 +471,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter t.latest), + this.l2BlockSource.getL2Tips().then(t => t.proposed), this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block), - this.l1ToL2MessageSource.getL2Tips().then(t => t.latest), + this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed), this.l2BlockSource.getPendingChainValidationStatus(), ] as const); diff --git a/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts b/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts index 11f2045c5429..84f155ce0cbf 100644 --- a/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/epoch_prune_watcher.test.ts @@ -1,8 +1,8 @@ import type { EpochCache } from '@aztec/epoch-cache'; -import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { EthAddress } from '@aztec/foundation/eth-address'; import { sleep } from '@aztec/foundation/sleep'; -import { L2Block, type L2BlockSourceEventEmitter, L2BlockSourceEvents } from '@aztec/stdlib/block'; +import { L2BlockNew, type L2BlockSourceEventEmitter, L2BlockSourceEvents } from '@aztec/stdlib/block'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import type { BuildBlockResult, @@ -74,9 +74,12 @@ describe('EpochPruneWatcher', () => { const emitSpy = jest.spyOn(watcher, 'emit'); const epochNumber = EpochNumber(1); - const block = await L2Block.random( + const block = await L2BlockNew.random( BlockNumber(12), // block number - 4, // txs per block + { + txsPerBlock: 4, + slotNumber: SlotNumber(10), + }, ); txProvider.getAvailableTxs.mockResolvedValue({ txs: [], missingTxs: [block.body.txEffects[0].txHash] }); @@ -118,9 +121,12 @@ describe('EpochPruneWatcher', () => { it('should slash if the data is available and the epoch could have been proven', async () => { const emitSpy = jest.spyOn(watcher, 'emit'); - const block = await L2Block.random( + const block = await L2BlockNew.random( BlockNumber(12), // block number - 4, // txs per block + { + txsPerBlock: 4, + slotNumber: SlotNumber(10), + }, ); const tx = Tx.random(); txProvider.getAvailableTxs.mockResolvedValue({ txs: [tx], missingTxs: [] }); @@ -170,13 +176,20 @@ describe('EpochPruneWatcher', () => { it('should not slash if the data is available but the epoch could not have been proven', async () => { const emitSpy = jest.spyOn(watcher, 'emit'); - const blockFromL1 = await L2Block.random( + const blockFromL1 = await L2BlockNew.random( BlockNumber(12), // block number - 1, // txs per block + { + txsPerBlock: 1, + slotNumber: SlotNumber(10), + }, ); - const blockFromBuilder = await L2Block.random( + + const blockFromBuilder = await L2BlockNew.random( BlockNumber(13), // block number - 1, // txs per block + { + txsPerBlock: 1, + slotNumber: SlotNumber(10), + }, ); const tx = Tx.random(); txProvider.getAvailableTxs.mockResolvedValue({ txs: [tx], missingTxs: [] }); diff --git a/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts b/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts index cd05872eb1e4..d1db413ad1f7 100644 --- a/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts +++ b/yarn-project/slasher/src/watchers/epoch_prune_watcher.ts @@ -4,7 +4,7 @@ import { merge, pick } from '@aztec/foundation/collection'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { EthAddress, - L2Block, + L2BlockNew, type L2BlockPruneEvent, type L2BlockSourceEventEmitter, L2BlockSourceEvents, @@ -95,10 +95,10 @@ export class EpochPruneWatcher extends (EventEmitter as new () => WatcherEmitter this.emit(WANT_TO_SLASH_EVENT, args); } - private async processPruneL2Blocks(blocks: L2Block[], epochNumber: EpochNumber): Promise { + private async processPruneL2Blocks(blocks: L2BlockNew[], epochNumber: EpochNumber): Promise { try { const l1Constants = this.epochCache.getL1Constants(); - const epochBlocks = blocks.filter(b => getEpochAtSlot(b.slot, l1Constants) === epochNumber); + const epochBlocks = blocks.filter(b => getEpochAtSlot(b.header.getSlot(), l1Constants) === epochNumber); this.log.info( `Detected chain prune. Validating epoch ${epochNumber} with blocks ${epochBlocks[0]?.number} to ${epochBlocks[epochBlocks.length - 1]?.number}.`, { blocks: epochBlocks.map(b => b.toBlockInfo()) }, @@ -119,7 +119,7 @@ export class EpochPruneWatcher extends (EventEmitter as new () => WatcherEmitter } } - public async validateBlocks(blocks: L2Block[]): Promise { + public async validateBlocks(blocks: L2BlockNew[]): Promise { if (blocks.length === 0) { return; } @@ -133,7 +133,7 @@ export class EpochPruneWatcher extends (EventEmitter as new () => WatcherEmitter } } - public async validateBlock(blockFromL1: L2Block, fork: MerkleTreeWriteOperations): Promise { + public async validateBlock(blockFromL1: L2BlockNew, fork: MerkleTreeWriteOperations): Promise { this.log.debug(`Validating pruned block ${blockFromL1.header.globalVariables.blockNumber}`); const txHashes = blockFromL1.body.txEffects.map(txEffect => txEffect.txHash); // We load txs from the mempool directly, since the TxCollector running in the background has already been diff --git a/yarn-project/stdlib/src/block/attestation_info.ts b/yarn-project/stdlib/src/block/attestation_info.ts index 1ee0e54a976b..c95fa654e50a 100644 --- a/yarn-project/stdlib/src/block/attestation_info.ts +++ b/yarn-project/stdlib/src/block/attestation_info.ts @@ -1,9 +1,9 @@ import { recoverAddress } from '@aztec/foundation/crypto/secp256k1-signer'; import type { EthAddress } from '@aztec/foundation/eth-address'; +import { Checkpoint } from '../checkpoint/checkpoint.js'; import { ConsensusPayload } from '../p2p/consensus_payload.js'; import { SignatureDomainSeparator, getHashedSignaturePayloadEthSignedMessage } from '../p2p/signature_utils.js'; -import type { L2Block } from './l2_block.js'; import type { CommitteeAttestation } from './proposal/committee_attestation.js'; /** @@ -29,14 +29,14 @@ export type AttestationInfo = }; /** - * Extracts attestation information from a published L2 block. + * Extracts attestation information from a published checkpoint. * Returns info for each attestation, preserving array indices. */ -export function getAttestationInfoFromPublishedL2Block(block: { +export function getAttestationInfoFromPublishedCheckpoint(block: { attestations: CommitteeAttestation[]; - block: L2Block; + checkpoint: Checkpoint; }): AttestationInfo[] { - const payload = ConsensusPayload.fromBlock(block.block); + const payload = ConsensusPayload.fromCheckpoint(block.checkpoint); return getAttestationInfoFromPayload(payload, block.attestations); } diff --git a/yarn-project/stdlib/src/block/l2_block_source.ts b/yarn-project/stdlib/src/block/l2_block_source.ts index 64b40130e59f..e3c01aeccf4e 100644 --- a/yarn-project/stdlib/src/block/l2_block_source.ts +++ b/yarn-project/stdlib/src/block/l2_block_source.ts @@ -2,6 +2,7 @@ import { BlockNumber, BlockNumberSchema, CheckpointNumber, + CheckpointNumberSchema, type EpochNumber, type SlotNumber, } from '@aztec/foundation/branded-types'; @@ -66,6 +67,8 @@ export interface L2BlockSource { */ getCheckpointedBlock(number: BlockNumber): Promise; + getCheckpointedBlocks(from: BlockNumber, limit: number, proven?: boolean): Promise; + /** * Retrieves a collection of published checkpoints * @param checkpointNumber The first checkpoint to be retrieved @@ -179,6 +182,10 @@ export interface L2BlockSource { */ getBlock(number: BlockNumber): Promise; + getL2BlockNew(number: BlockNumber): Promise; + + getL2BlocksNew(from: BlockNumber, limit: number, proven?: boolean): Promise; + /** * Returns all blocks for a given epoch. * @dev Use this method only with recent epochs, since it walks the block list backwards. @@ -233,23 +240,41 @@ export type ArchiverEmitter = TypedEventEmitter<{ [L2BlockSourceEvents.L2PruneDetected]: (args: L2BlockPruneEvent) => void; [L2BlockSourceEvents.L2BlockProven]: (args: L2BlockProvenEvent) => void; [L2BlockSourceEvents.InvalidAttestationsBlockDetected]: (args: InvalidBlockDetectedEvent) => void; + [L2BlockSourceEvents.L2BlocksCheckpointed]: (args: L2CheckpointEvent) => void; }>; export interface L2BlockSourceEventEmitter extends L2BlockSource, ArchiverEmitter {} /** * Identifier for L2 block tags. - * - latest: Latest block pushed to L1. + * - proposed: Latest block proposed on L2. + * - checkpointed: Checkpointed block on L1. * - proven: Proven block on L1. * - finalized: Proven block on a finalized L1 block (not implemented, set to proven for now). */ -export type L2BlockTag = 'latest' | 'proven' | 'finalized'; +export type L2BlockTag = 'proposed' | 'checkpointed' | 'proven' | 'finalized'; + +/** + * Reason for L2 block prune. + * - uncheckpointed: L2 blocks were pruned due to a failure to checkpoint. + * - unproven: L2 blocks were pruned due to a failure to prove. + */ +export type L2BlockPruneReason = 'uncheckpointed' | 'unproven'; /** Tips of the L2 chain. */ -export type L2Tips = Record; +export type L2Tips = { + proposed: L2BlockId; + checkpointed: L2TipId; + proven: L2TipId; + finalized: L2TipId; +}; /** Identifies a block by number and hash. */ export type L2BlockId = { number: BlockNumber; hash: string }; +export type CheckpointId = { number: CheckpointNumber; hash: string }; + +export type L2TipId = { block: L2BlockId; checkpoint: CheckpointId }; + /** Creates an L2 block id */ export function makeL2BlockId(number: BlockNumber, hash?: string): L2BlockId { if (number !== 0 && !hash) { @@ -258,20 +283,37 @@ export function makeL2BlockId(number: BlockNumber, hash?: string): L2BlockId { return { number, hash: hash! }; } +/** Creates an L2 checkpoint id */ +export function makeL2CheckpointId(number: CheckpointNumber, hash: string): CheckpointId { + return { number, hash }; +} + const L2BlockIdSchema = z.object({ number: BlockNumberSchema, hash: z.string(), }); +const L2CheckpointIdSchema = z.object({ + number: CheckpointNumberSchema, + hash: z.string(), +}); + +const L2TipIdSchema = z.object({ + block: L2BlockIdSchema, + checkpoint: L2CheckpointIdSchema, +}); + export const L2TipsSchema = z.object({ - latest: L2BlockIdSchema, - proven: L2BlockIdSchema, - finalized: L2BlockIdSchema, + proposed: L2BlockIdSchema, + checkpointed: L2TipIdSchema, + proven: L2TipIdSchema, + finalized: L2TipIdSchema, }); export enum L2BlockSourceEvents { L2PruneDetected = 'l2PruneDetected', L2BlockProven = 'l2BlockProven', + L2BlocksCheckpointed = 'l2BlocksCheckpointed', InvalidAttestationsBlockDetected = 'invalidBlockDetected', } @@ -285,7 +327,12 @@ export type L2BlockProvenEvent = { export type L2BlockPruneEvent = { type: 'l2PruneDetected'; epochNumber: EpochNumber; - blocks: L2Block[]; + blocks: L2BlockNew[]; +}; + +export type L2CheckpointEvent = { + type: 'l2BlocksCheckpointed'; + checkpoint: PublishedCheckpoint; }; export type InvalidBlockDetectedEvent = { diff --git a/yarn-project/stdlib/src/block/l2_block_stream/interfaces.ts b/yarn-project/stdlib/src/block/l2_block_stream/interfaces.ts index 3cf2c61411ec..7a16689dcfb8 100644 --- a/yarn-project/stdlib/src/block/l2_block_stream/interfaces.ts +++ b/yarn-project/stdlib/src/block/l2_block_stream/interfaces.ts @@ -1,5 +1,6 @@ -import type { PublishedL2Block } from '../checkpointed_l2_block.js'; -import type { L2BlockId, L2Tips } from '../l2_block_source.js'; +import type { PublishedCheckpoint } from '../../checkpoint/published_checkpoint.js'; +import type { L2BlockNew } from '../l2_block_new.js'; +import type { CheckpointId, L2BlockId, L2BlockPruneReason, L2Tips } from '../l2_block_source.js'; /** Interface to the local view of the chain. Implemented by world-state and l2-tips-store. */ export interface L2BlockStreamLocalDataProvider { @@ -15,11 +16,18 @@ export interface L2BlockStreamEventHandler { export type L2BlockStreamEvent = | /** Emits blocks added to the chain. */ { type: 'blocks-added'; - blocks: PublishedL2Block[]; + blocks: L2BlockNew[]; } - | /** Reports last correct block (new tip of the unproven chain). */ { + | /** Emits checkpoints published to L1. */ { + type: 'chain-checkpointed'; + checkpoint: PublishedCheckpoint; + block: L2BlockId; + } + | /** Reports last correct block (new tip of the proposed chain). */ { type: 'chain-pruned'; + reason: L2BlockPruneReason; block: L2BlockId; + checkpoint: CheckpointId; } | /** Reports new proven block. */ { type: 'chain-proven'; diff --git a/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.test.ts b/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.test.ts index 203ece8532ce..2e7be5f6c59c 100644 --- a/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.test.ts +++ b/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.test.ts @@ -1,13 +1,14 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { compactArray } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { type MockProxy, mock } from 'jest-mock-extended'; import times from 'lodash.times'; +import type { PublishedCheckpoint } from '../../checkpoint/published_checkpoint.js'; import type { BlockHeader } from '../../tx/block_header.js'; -import type { PublishedL2Block } from '../checkpointed_l2_block.js'; -import type { L2Block } from '../l2_block.js'; +import type { CheckpointedL2Block } from '../checkpointed_l2_block.js'; +import type { L2BlockNew } from '../l2_block_new.js'; import type { L2BlockId, L2BlockSource, L2Tips } from '../l2_block_source.js'; import type { L2BlockStreamEvent, L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider } from './interfaces.js'; import { L2BlockStream } from './l2_block_stream.js'; @@ -17,24 +18,79 @@ describe('L2BlockStream', () => { let blockSource: MockProxy; let latest: number = 0; + let checkpointed: number = 0; const makeHash = (number: number) => new Fr(number).toString(); - const makeBlock = (number: number) => ({ block: { number: BlockNumber(number) } as L2Block }) as PublishedL2Block; + const makeBlock = (number: number) => + ({ + number: BlockNumber(number), + checkpointNumber: CheckpointNumber(number), + indexWithinCheckpoint: 0, + }) as L2BlockNew; + + /** Makes a block with hash method (for use in mocks that need hash) */ + const makeBlockWithHash = (number: number) => + ({ + number: BlockNumber(number), + checkpointNumber: CheckpointNumber(number), + indexWithinCheckpoint: 0, + hash: () => Promise.resolve(new Fr(number)), + }) as L2BlockNew; + + const makeCheckpointedBlock = (number: number, checkpointNum: number): CheckpointedL2Block => + ({ + block: makeBlock(number), + checkpointNumber: checkpointNum, + }) as CheckpointedL2Block; const makeHeader = (number: number) => ({ hash: () => Promise.resolve(new Fr(number)) }) as BlockHeader; const makeBlockId = (number: number): L2BlockId => ({ number: BlockNumber(number), hash: makeHash(number) }); - const setRemoteTips = (latest_: number, proven?: number, finalized?: number) => { + /** Helper to match a blocks-added event with blocks that may have extra properties like hash */ + const expectBlocksAdded = (blockNumbers: number[]) => + expect.objectContaining({ + type: 'blocks-added', + blocks: blockNumbers.map(n => + expect.objectContaining({ + number: BlockNumber(n), + }), + ), + }); + + /** Helper to match a chain-checkpointed event */ + const expectCheckpointed = (checkpointNumber?: number) => + checkpointNumber !== undefined + ? expect.objectContaining({ + type: 'chain-checkpointed', + checkpoint: expect.objectContaining({ + checkpoint: expect.objectContaining({ number: checkpointNumber }), + }), + block: expect.objectContaining({ number: expect.any(Number) }), + }) + : expect.objectContaining({ type: 'chain-checkpointed' }); + + const makeCheckpointId = (number: number) => ({ number: CheckpointNumber(number), hash: makeHash(number) }); + + const makeTipId = (number: number) => ({ + block: { number: BlockNumber(number), hash: makeHash(number) }, + checkpoint: { number: CheckpointNumber(number), hash: makeHash(number) }, + }); + + /** Sets the remote tips. All tips default to 0 except latest. */ + const setRemoteTips = (latest_: number, checkpointed_?: number, proven?: number, finalized?: number) => { + checkpointed_ = checkpointed_ ?? 0; proven = proven ?? 0; finalized = finalized ?? 0; latest = latest_; + checkpointed = checkpointed_; blockSource.getL2Tips.mockResolvedValue({ - latest: { number: BlockNumber(latest), hash: makeHash(latest) }, - proven: { number: BlockNumber(proven), hash: makeHash(proven) }, - finalized: { number: BlockNumber(finalized), hash: makeHash(finalized) }, + proposed: { number: BlockNumber(latest), hash: makeHash(latest) }, + checkpointed: makeTipId(checkpointed_), + proven: makeTipId(proven), + finalized: makeTipId(finalized), }); }; @@ -49,10 +105,32 @@ describe('L2BlockStream', () => { ), ); - // And returns blocks up until what was reported as the latest block - blockSource.getPublishedBlocks.mockImplementation((from, limit) => + // Returns blocks up until what was reported as the latest block (for uncheckpointed blocks) + blockSource.getL2BlocksNew.mockImplementation((from, limit) => Promise.resolve(compactArray(times(limit, i => (from + i > latest ? undefined : makeBlock(from + i))))), ); + + // Returns checkpointed blocks (for blocks up to checkpointed tip) + blockSource.getCheckpointedBlocks.mockImplementation((from, limit) => + Promise.resolve( + compactArray( + times(limit, i => (from + i > checkpointed ? undefined : makeCheckpointedBlock(from + i, from + i))), + ), + ), + ); + + // Returns published checkpoints - each checkpoint contains just the one block for simplicity + blockSource.getPublishedCheckpoints.mockImplementation((checkpointNumber: CheckpointNumber, _limit: number) => + Promise.resolve([ + { + checkpoint: { + number: checkpointNumber, + hash: () => new Fr(checkpointNumber), + blocks: [makeBlockWithHash(checkpointNumber)], + }, + } as unknown as PublishedCheckpoint, + ]), + ); }); describe('with mock local data provider', () => { @@ -77,10 +155,10 @@ describe('L2BlockStream', () => { it('pulls new blocks from offset', async () => { setRemoteTips(15); - localData.latest.number = BlockNumber(10); + localData.proposed.number = BlockNumber(10); await blockStream.work(); - expect(blockSource.getPublishedBlocks).toHaveBeenCalledWith(BlockNumber(11), 5, undefined); + expect(blockSource.getL2BlocksNew).toHaveBeenCalledWith(BlockNumber(11), 5, undefined); expect(handler.events).toEqual([ { type: 'blocks-added', blocks: times(5, i => makeBlock(i + 11)) }, ] satisfies L2BlockStreamEvent[]); @@ -90,7 +168,7 @@ describe('L2BlockStream', () => { setRemoteTips(45); await blockStream.work(); - expect(blockSource.getPublishedBlocks).toHaveBeenCalledTimes(5); + expect(blockSource.getL2BlocksNew).toHaveBeenCalledTimes(5); expect(handler.callCount).toEqual(5); expect(handler.events).toEqual([ { type: 'blocks-added', blocks: times(10, i => makeBlock(i + 1)) }, @@ -106,7 +184,7 @@ describe('L2BlockStream', () => { blockStream.running = false; await blockStream.work(); - expect(blockSource.getPublishedBlocks).toHaveBeenCalledTimes(1); + expect(blockSource.getL2BlocksNew).toHaveBeenCalledTimes(1); expect(handler.events).toEqual([ { type: 'blocks-added', blocks: times(10, i => makeBlock(i + 1)) }, ] satisfies L2BlockStreamEvent[]); @@ -127,7 +205,7 @@ describe('L2BlockStream', () => { it('handles a reorg and requests blocks from new tip', async () => { setRemoteTips(45); - localData.latest.number = BlockNumber(40); + localData.proposed.number = BlockNumber(40); for (const i of [37, 38, 39, 40]) { // Mess up the block hashes for a bunch of blocks @@ -136,16 +214,16 @@ describe('L2BlockStream', () => { await blockStream.work(); expect(handler.events).toEqual([ - { type: 'chain-pruned', block: makeBlockId(36) }, + { type: 'chain-pruned', block: makeBlockId(36), reason: 'unproven', checkpoint: makeCheckpointId(0) }, { type: 'blocks-added', blocks: times(9, i => makeBlock(i + 37)) }, ] satisfies L2BlockStreamEvent[]); }); it('emits events for chain proven and finalized', async () => { - setRemoteTips(45, 40, 35); - localData.latest.number = BlockNumber(40); - localData.proven.number = BlockNumber(10); - localData.finalized.number = BlockNumber(10); + setRemoteTips(45, 0, 40, 35); + localData.proposed.number = BlockNumber(40); + localData.proven.block.number = BlockNumber(10); + localData.finalized.block.number = BlockNumber(10); await blockStream.work(); expect(handler.events).toEqual([ @@ -154,6 +232,71 @@ describe('L2BlockStream', () => { { type: 'chain-finalized', block: makeBlockId(35) }, ] satisfies L2BlockStreamEvent[]); }); + + it('fetches checkpointed blocks and emits chain-checkpointed events', async () => { + // All blocks are checkpointed (checkpointed=5, proposed=5) + setRemoteTips(5, 5); + + await blockStream.work(); + + // Each checkpointed block triggers a blocks-added and chain-checkpointed event + // (since each checkpoint contains one block in our mock) + expect(handler.events).toEqual([ + expectBlocksAdded([1]), + expectCheckpointed(), + expectBlocksAdded([2]), + expectCheckpointed(), + expectBlocksAdded([3]), + expectCheckpointed(), + expectBlocksAdded([4]), + expectCheckpointed(), + expectBlocksAdded([5]), + expectCheckpointed(), + ]); + expect(blockSource.getCheckpointedBlocks).toHaveBeenCalledTimes(5); + expect(blockSource.getL2BlocksNew).not.toHaveBeenCalled(); + }); + + it('fetches checkpointed blocks first, then uncheckpointed blocks', async () => { + // Blocks 1-3 are checkpointed, blocks 4-5 are uncheckpointed + setRemoteTips(5, 3); + + await blockStream.work(); + + // First 3 blocks come via checkpoints, last 2 via getL2BlocksNew + expect(handler.events).toEqual([ + expectBlocksAdded([1]), + expectCheckpointed(), + expectBlocksAdded([2]), + expectCheckpointed(), + expectBlocksAdded([3]), + expectCheckpointed(), + expectBlocksAdded([4, 5]), + ]); + expect(blockSource.getCheckpointedBlocks).toHaveBeenCalledTimes(3); + expect(blockSource.getL2BlocksNew).toHaveBeenCalledWith(BlockNumber(4), 2, undefined); + }); + + it('handles reorg with uncheckpointed reason when pruned to checkpointed tip', async () => { + // Source: checkpointed=3, proposed=5 + setRemoteTips(5, 3); + localData.proposed.number = BlockNumber(5); + localData.checkpointed.block.number = BlockNumber(3); + + // Mess up hashes for blocks 4 and 5 (uncheckpointed blocks) + localData.blockHashes[4] = `0xaa4`; + localData.blockHashes[5] = `0xaa5`; + + await blockStream.work(); + + // Prune to block 3 (checkpointed tip), reason should be 'uncheckpointed' + expect(handler.events[0]).toEqual({ + type: 'chain-pruned', + block: makeBlockId(3), + reason: 'uncheckpointed', + checkpoint: makeCheckpointId(3), + }); + }); }); describe('with memory tips store', () => { @@ -169,25 +312,718 @@ describe('L2BlockStream', () => { // Regression test for https://github.com/AztecProtocol/aztec-packages/issues/13471 it('handles a prune to a block before start block', async () => { - setRemoteTips(35, 25, 10); + setRemoteTips(35, 30, 25, 10); blockStream = new TestL2BlockStream(blockSource, localData, handler, undefined, { batchSize: 10, startingBlock: 30, }); // We first seed a few blocks into the blockstream + // Block 30 comes via checkpoint, blocks 31-35 via uncheckpointed await blockStream.work(); expect(handler.events).toEqual([ - { type: 'blocks-added', blocks: times(6, i => makeBlock(i + 30)) }, + expectBlocksAdded([30]), + expectCheckpointed(30), + { type: 'blocks-added', blocks: times(5, i => makeBlock(i + 31)) }, { type: 'chain-proven', block: makeBlockId(25) }, { type: 'chain-finalized', block: makeBlockId(10) }, - ] satisfies L2BlockStreamEvent[]); + ]); handler.clearEvents(); // And then we reorg - setRemoteTips(25, 25, 10); + setRemoteTips(25, 25, 25, 10); + await blockStream.work(); + expect(handler.events).toEqual([ + { type: 'chain-pruned', block: makeBlockId(25), reason: 'unproven', checkpoint: makeCheckpointId(25) }, + ]); + }); + }); + + describe('multiple blocks per checkpoint', () => { + let localData: TestL2BlockStreamLocalDataProvider; + let handler: TestL2BlockStreamEventHandler; + let blockStream: TestL2BlockStream; + + // Configuration for checkpoint structure: each checkpoint contains 3 blocks + const blocksPerCheckpoint = 3; + + /** Gets the checkpoint number for a given block number */ + const getCheckpointForBlock = (blockNum: number) => Math.ceil(blockNum / blocksPerCheckpoint); + + /** Gets the first block number in a checkpoint */ + const getFirstBlockInCheckpoint = (checkpointNum: number) => (checkpointNum - 1) * blocksPerCheckpoint + 1; + + /** Gets the last block number in a checkpoint */ + const getLastBlockInCheckpoint = (checkpointNum: number) => checkpointNum * blocksPerCheckpoint; + + /** Makes a block with correct checkpoint info */ + const makeBlockInCheckpoint = (blockNum: number) => { + const checkpointNum = getCheckpointForBlock(blockNum); + const firstBlockInCheckpoint = getFirstBlockInCheckpoint(checkpointNum); + return { + number: BlockNumber(blockNum), + checkpointNumber: CheckpointNumber(checkpointNum), + indexWithinCheckpoint: blockNum - firstBlockInCheckpoint, + } as L2BlockNew; + }; + + /** Makes a block with hash method (for use in mocks that need hash) */ + const makeBlockInCheckpointWithHash = (blockNum: number) => { + const checkpointNum = getCheckpointForBlock(blockNum); + const firstBlockInCheckpoint = getFirstBlockInCheckpoint(checkpointNum); + return { + number: BlockNumber(blockNum), + checkpointNumber: CheckpointNumber(checkpointNum), + indexWithinCheckpoint: blockNum - firstBlockInCheckpoint, + hash: () => Promise.resolve(new Fr(blockNum)), + } as L2BlockNew; + }; + + /** Makes a checkpointed block */ + const makeCheckpointedBlockInCheckpoint = (blockNum: number): CheckpointedL2Block => + ({ + block: makeBlockInCheckpoint(blockNum), + checkpointNumber: getCheckpointForBlock(blockNum), + }) as CheckpointedL2Block; + + /** Sets the remote tips with correct checkpoint numbers for multi-block checkpoints. */ + const setRemoteTipsMultiBlock = ( + latest_: number, + checkpointedBlock?: number, + proven?: number, + finalized?: number, + ) => { + checkpointedBlock = checkpointedBlock ?? 0; + proven = proven ?? 0; + finalized = finalized ?? 0; + latest = latest_; + checkpointed = checkpointedBlock; + + const checkpointedCheckpointNum = checkpointedBlock > 0 ? getCheckpointForBlock(checkpointedBlock) : 0; + const provenCheckpointNum = proven > 0 ? getCheckpointForBlock(proven) : 0; + const finalizedCheckpointNum = finalized > 0 ? getCheckpointForBlock(finalized) : 0; + + blockSource.getL2Tips.mockResolvedValue({ + proposed: { number: BlockNumber(latest), hash: makeHash(latest) }, + checkpointed: { + block: { number: BlockNumber(checkpointedBlock), hash: makeHash(checkpointedBlock) }, + checkpoint: { + number: CheckpointNumber(checkpointedCheckpointNum), + hash: makeHash(checkpointedCheckpointNum), + }, + }, + proven: { + block: { number: BlockNumber(proven), hash: makeHash(proven) }, + checkpoint: { number: CheckpointNumber(provenCheckpointNum), hash: makeHash(provenCheckpointNum) }, + }, + finalized: { + block: { number: BlockNumber(finalized), hash: makeHash(finalized) }, + checkpoint: { number: CheckpointNumber(finalizedCheckpointNum), hash: makeHash(finalizedCheckpointNum) }, + }, + }); + }; + + beforeEach(() => { + localData = new TestL2BlockStreamLocalDataProvider(); + handler = new TestL2BlockStreamEventHandler(); + blockStream = new TestL2BlockStream(blockSource, localData, handler, undefined, { batchSize: 10 }); + + // Override the mocks to support multiple blocks per checkpoint + blockSource.getCheckpointedBlocks.mockImplementation((from, limit) => + Promise.resolve( + compactArray( + times(limit, i => (from + i > checkpointed ? undefined : makeCheckpointedBlockInCheckpoint(from + i))), + ), + ), + ); + + // Returns published checkpoints with multiple blocks each + blockSource.getPublishedCheckpoints.mockImplementation((checkpointNumber: CheckpointNumber, _limit: number) => { + const firstBlock = getFirstBlockInCheckpoint(checkpointNumber); + const lastBlock = Math.min(getLastBlockInCheckpoint(checkpointNumber), checkpointed); + const blocks = times(lastBlock - firstBlock + 1, i => makeBlockInCheckpointWithHash(firstBlock + i)); + return Promise.resolve([ + { + checkpoint: { + number: checkpointNumber, + hash: () => new Fr(checkpointNumber), + blocks, + }, + } as unknown as PublishedCheckpoint, + ]); + }); + }); + + it('emits all blocks in a checkpoint before chain-checkpointed event', async () => { + // Set up: 6 blocks in 2 checkpoints (blocks 1-3 in checkpoint 1, blocks 4-6 in checkpoint 2) + setRemoteTipsMultiBlock(6, 6); + + await blockStream.work(); + + // Should emit blocks 1-3, then checkpoint 1, then blocks 4-6, then checkpoint 2 + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + expectCheckpointed(2), + ]); + }); + + it('handles partial checkpoint at the end (uncheckpointed blocks)', async () => { + // Set up: 5 blocks total, but only first 3 are checkpointed (checkpoint 1 complete) + // Blocks 4-5 are uncheckpointed + setRemoteTipsMultiBlock(5, 3); + + await blockStream.work(); + + // Should emit checkpoint 1 blocks, then checkpoint event, then uncheckpointed blocks 4-5 + expect(handler.events).toEqual([expectBlocksAdded([1, 2, 3]), expectCheckpointed(1), expectBlocksAdded([4, 5])]); + }); + + it('handles starting from middle of a checkpoint', async () => { + // Set up: 9 blocks in 3 checkpoints, but we start from block 5 (middle of checkpoint 2) + // Local has blocks 1-4, local checkpointed = 0 + setRemoteTipsMultiBlock(9, 9); + localData.proposed.number = BlockNumber(4); + + await blockStream.work(); + + // Should first emit checkpoint 1 (blocks 1-4 already local) + // Then continue from block 5, which is in checkpoint 2 + // Blocks 5-6 complete checkpoint 2, then blocks 7-9 complete checkpoint 3 + expect(handler.events).toEqual([ + expectCheckpointed(1), // checkpoint 1 for already-local blocks 1-4 + expectBlocksAdded([5, 6]), + expectCheckpointed(2), // checkpoint 2 + expectBlocksAdded([7, 8, 9]), + expectCheckpointed(3), // checkpoint 3 + ]); + + // Verify checkpoint order + const checkpointEvents = handler.events.filter(e => e.type === 'chain-checkpointed'); + expect(checkpointEvents).toHaveLength(3); + expect((checkpointEvents[0] as any).checkpoint.checkpoint.number).toBe(CheckpointNumber(1)); + expect((checkpointEvents[1] as any).checkpoint.checkpoint.number).toBe(CheckpointNumber(2)); + expect((checkpointEvents[2] as any).checkpoint.checkpoint.number).toBe(CheckpointNumber(3)); + }); + + it('correctly identifies checkpoint number in chain-checkpointed events', async () => { + // Set up: 6 blocks in 2 checkpoints + setRemoteTipsMultiBlock(6, 6); + + await blockStream.work(); + + // Extract the chain-checkpointed events + const checkpointEvents = handler.events.filter(e => e.type === 'chain-checkpointed'); + expect(checkpointEvents).toHaveLength(2); + expect((checkpointEvents[0] as any).checkpoint.checkpoint.number).toBe(CheckpointNumber(1)); + expect((checkpointEvents[1] as any).checkpoint.checkpoint.number).toBe(CheckpointNumber(2)); + }); + + it('handles many checkpoints with batching', async () => { + // Set up: 12 blocks in 4 checkpoints (3 blocks each), with batch size of 5 + // Batch size doesn't align with checkpoint boundaries, so the stream must + // respect checkpoint boundaries and emit events correctly + blockStream = new TestL2BlockStream(blockSource, localData, handler, undefined, { batchSize: 5 }); + setRemoteTipsMultiBlock(12, 12); + + await blockStream.work(); + + // Even though batch size is 5, checkpoint boundaries (every 3 blocks) take precedence + // Expected sequence: + // - Blocks 1-3 (checkpoint 1), then checkpoint 1 event + // - Blocks 4-6 (checkpoint 2), then checkpoint 2 event + // - Blocks 7-9 (checkpoint 3), then checkpoint 3 event + // - Blocks 10-12 (checkpoint 4), then checkpoint 4 event + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + expectCheckpointed(2), + expectBlocksAdded([7, 8, 9]), + expectCheckpointed(3), + expectBlocksAdded([10, 11, 12]), + expectCheckpointed(4), + ]); + }); + + it('emits checkpoint event when blocks become checkpointed after being added as uncheckpointed', async () => { + // Phase 1: Start with 3 checkpointed blocks (checkpoint 1), then add blocks 4-6 as uncheckpointed + setRemoteTipsMultiBlock(6, 3); + + await blockStream.work(); + + // Expect: blocks 1-3 via checkpoint, then uncheckpointed blocks 4-6 + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + ]); + + handler.clearEvents(); + + // Update local state to reflect what the handler would have stored + localData.proposed.number = BlockNumber(6); + localData.checkpointed.block.number = BlockNumber(3); + localData.checkpointed.checkpoint.number = CheckpointNumber(1); + + // Phase 2: Now checkpoint 2 completes (blocks 4-6 become checkpointed) + setRemoteTipsMultiBlock(6, 6); + + await blockStream.work(); + + // Should emit a checkpoint event for checkpoint 2 (blocks 4-6), even though blocks were already added + const checkpointEvents = handler.events.filter(e => e.type === 'chain-checkpointed'); + expect(checkpointEvents).toHaveLength(1); + expect((checkpointEvents[0] as any).checkpoint.checkpoint.number).toBe(CheckpointNumber(2)); + }); + + it('emits checkpoint event BEFORE new uncheckpointed blocks when checkpoint completes', async () => { + // Phase 1: Start with 3 checkpointed blocks (checkpoint 1), then add blocks 4-6 as uncheckpointed + setRemoteTipsMultiBlock(6, 3); + + await blockStream.work(); + + // Expect: blocks 1-3 via checkpoint, then uncheckpointed blocks 4-6 + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + ]); + + handler.clearEvents(); + + // Update local state to reflect what the handler would have stored + localData.proposed.number = BlockNumber(6); + localData.checkpointed.block.number = BlockNumber(3); + localData.checkpointed.checkpoint.number = CheckpointNumber(1); + + // Phase 2: Checkpoint 2 completes (blocks 4-6) AND a new block 7 arrives + setRemoteTipsMultiBlock(7, 6); + + await blockStream.work(); + + // Should emit checkpoint 2 FIRST, then the new uncheckpointed block 7 + // NOT: block 7 first, then checkpoint 2 + expect(handler.events).toEqual([expectCheckpointed(2), expectBlocksAdded([7])]); + }); + + it('emits checkpoint as soon as last block in checkpoint arrives', async () => { + // This tests the realistic scenario where checkpoints are published as blocks arrive. + // Uncheckpointed blocks are always just a partial checkpoint (the current incomplete one). + + // Sync 1: Source has checkpointed=6 (checkpoint 2), proposed=9 + // Client gets blocks 1-6 via checkpoints, blocks 7-9 as uncheckpointed + setRemoteTipsMultiBlock(9, 6); + + await blockStream.work(); + + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + expectCheckpointed(2), + expectBlocksAdded([7, 8, 9]), // uncheckpointed + ]); + + handler.clearEvents(); + + // Update local state + localData.proposed.number = BlockNumber(9); + localData.checkpointed.block.number = BlockNumber(6); + localData.checkpointed.checkpoint.number = CheckpointNumber(2); + + // Sync 2: Checkpoint 3 is now published (blocks 7-9), new blocks 10-12 are uncheckpointed + setRemoteTipsMultiBlock(12, 9); + + await blockStream.work(); + + // Should emit checkpoint 3 for already-local blocks 7-9, then uncheckpointed blocks 10-12 + expect(handler.events).toEqual([ + expectCheckpointed(3), + expectBlocksAdded([10, 11, 12]), // uncheckpointed + ]); + + handler.clearEvents(); + + // Update local state + localData.proposed.number = BlockNumber(12); + localData.checkpointed.block.number = BlockNumber(9); + localData.checkpointed.checkpoint.number = CheckpointNumber(3); + + // Sync 3: Checkpoint 4 is now published (blocks 10-12), no new blocks + setRemoteTipsMultiBlock(12, 12); + + await blockStream.work(); + + // Should emit checkpoint 4 for already-local blocks 10-12 + expect(handler.events).toEqual([expectCheckpointed(4)]); + }); + + it('emits all checkpoints when source jumps ahead with multiple new checkpoints', async () => { + // Phase 1: Start with checkpoint 1 complete (blocks 1-3), blocks 4-6 uncheckpointed + setRemoteTipsMultiBlock(6, 3); + + await blockStream.work(); + + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + ]); + + handler.clearEvents(); + + // Update local state + localData.proposed.number = BlockNumber(6); + localData.checkpointed.block.number = BlockNumber(3); + localData.checkpointed.checkpoint.number = CheckpointNumber(1); + + // Phase 2: Source jumps to block 12 with checkpoints at 6, 9, and 12 + // - Checkpoint 2 (blocks 4-6) - blocks already local, needs checkpoint event + // - Checkpoint 3 (blocks 7-9) - new blocks + checkpoint event + // - Checkpoint 4 (blocks 10-12) - new blocks + checkpoint event + setRemoteTipsMultiBlock(12, 12); + await blockStream.work(); - expect(handler.events).toEqual([{ type: 'chain-pruned', block: makeBlockId(25) }] satisfies L2BlockStreamEvent[]); + + // Should emit: + // 1. Checkpoint 2 event (blocks 4-6 were already local) + // 2. Blocks 7-9 + checkpoint 3 event + // 3. Blocks 10-12 + checkpoint 4 event + expect(handler.events).toEqual([ + expectCheckpointed(2), + expectBlocksAdded([7, 8, 9]), + expectCheckpointed(3), + expectBlocksAdded([10, 11, 12]), + expectCheckpointed(4), + ]); + }); + + describe('prune scenarios', () => { + it('prunes proposed chain back to checkpointed tip, then continues', async () => { + // Phase 1: Sync blocks 1-9 with checkpoints 1-2, blocks 7-9 uncheckpointed + setRemoteTipsMultiBlock(9, 6); + + await blockStream.work(); + + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + expectCheckpointed(2), + expectBlocksAdded([7, 8, 9]), // uncheckpointed + ]); + + handler.clearEvents(); + + // Update local state to reflect what the handler stored + localData.proposed.number = BlockNumber(9); + localData.checkpointed.block.number = BlockNumber(6); + localData.checkpointed.checkpoint.number = CheckpointNumber(2); + + // Phase 2: Prune - proposed chain pruned back to checkpointed tip (block 6) + // This happens when uncheckpointed blocks (7-9) are invalid + // Mess up hashes for blocks 7-9 to simulate reorg + localData.blockHashes[7] = '0xbad7'; + localData.blockHashes[8] = '0xbad8'; + localData.blockHashes[9] = '0xbad9'; + + // Source now has proposed=6, checkpointed=6 + setRemoteTipsMultiBlock(6, 6); + + await blockStream.work(); + + // Should emit chain-pruned back to block 6, reason 'uncheckpointed' + expect(handler.events).toEqual([ + { + type: 'chain-pruned', + block: makeBlockId(6), + reason: 'uncheckpointed', + checkpoint: expect.objectContaining({ number: CheckpointNumber(2) }), + }, + ]); + + handler.clearEvents(); + + // Update local state after prune + localData.proposed.number = BlockNumber(6); + delete localData.blockHashes[7]; + delete localData.blockHashes[8]; + delete localData.blockHashes[9]; + + // Phase 3: Chain continues - new blocks 7-12 arrive with checkpoints 3-4 + setRemoteTipsMultiBlock(12, 12); + + await blockStream.work(); + + // Should continue normally: blocks 7-9 + checkpoint 3, blocks 10-12 + checkpoint 4 + expect(handler.events).toEqual([ + expectBlocksAdded([7, 8, 9]), + expectCheckpointed(3), + expectBlocksAdded([10, 11, 12]), + expectCheckpointed(4), + ]); + }); + + it('prunes proposed and checkpointed chains back to proven tip, then continues', async () => { + // Phase 1: Sync blocks 1-12 with checkpoints 1-3, proven at checkpoint 2 (block 6) + setRemoteTipsMultiBlock(12, 9, 6); + + await blockStream.work(); + + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + expectCheckpointed(2), + expectBlocksAdded([7, 8, 9]), + expectCheckpointed(3), + expectBlocksAdded([10, 11, 12]), + { type: 'chain-proven', block: makeBlockId(6) }, + ]); + + handler.clearEvents(); + + // Update local state + localData.proposed.number = BlockNumber(12); + localData.checkpointed.block.number = BlockNumber(9); + localData.checkpointed.checkpoint.number = CheckpointNumber(3); + localData.proven.block.number = BlockNumber(6); + localData.proven.checkpoint.number = CheckpointNumber(2); + + // Phase 2: Prune - checkpoint 3 failed to prove, prune back to proven tip (block 6) + // Mess up hashes for blocks 7-12 to simulate reorg + for (let i = 7; i <= 12; i++) { + localData.blockHashes[i] = `0xbad${i}`; + } + + // Source now has proposed=6, checkpointed=6, proven=6 + setRemoteTipsMultiBlock(6, 6, 6); + + await blockStream.work(); + + // Should emit chain-pruned back to block 6 + // Reason is 'unproven' because we're pruning beyond the local checkpointed tip (12) + expect(handler.events).toEqual([ + { + type: 'chain-pruned', + block: makeBlockId(6), + reason: 'unproven', + checkpoint: expect.objectContaining({ number: CheckpointNumber(2) }), + }, + ]); + + handler.clearEvents(); + + // Update local state after prune + localData.proposed.number = BlockNumber(6); + localData.checkpointed.block.number = BlockNumber(6); + localData.checkpointed.checkpoint.number = CheckpointNumber(2); + for (let i = 7; i <= 12; i++) { + delete localData.blockHashes[i]; + } + + // Phase 3: Chain continues with new blocks and checkpoints + // New blocks 7-15 arrive with checkpoints 3-5, proven advances to checkpoint 3 + setRemoteTipsMultiBlock(15, 15, 9); + + await blockStream.work(); + + // Should continue normally with new blocks and checkpoints + expect(handler.events).toEqual([ + expectBlocksAdded([7, 8, 9]), + expectCheckpointed(3), + expectBlocksAdded([10, 11, 12]), + expectCheckpointed(4), + expectBlocksAdded([13, 14, 15]), + expectCheckpointed(5), + { type: 'chain-proven', block: makeBlockId(9) }, + ]); + }); + + it('prunes uncheckpointed blocks and immediately receives new ones', async () => { + // Phase 1: Sync blocks 1-6 with checkpoint 1, blocks 4-6 uncheckpointed + setRemoteTipsMultiBlock(6, 3); + + await blockStream.work(); + + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + ]); + + handler.clearEvents(); + + // Update local state + localData.proposed.number = BlockNumber(6); + localData.checkpointed.block.number = BlockNumber(3); + localData.checkpointed.checkpoint.number = CheckpointNumber(1); + + // Phase 2: Prune blocks 4-6 due to bad hashes + localData.blockHashes[4] = '0xbad4'; + localData.blockHashes[5] = '0xbad5'; + localData.blockHashes[6] = '0xbad6'; + + // Source still at checkpointed=3 (no new checkpoints yet) + setRemoteTipsMultiBlock(3, 3); + + await blockStream.work(); + + // Should emit prune back to block 3 + expect(handler.events).toEqual([ + { + type: 'chain-pruned', + block: makeBlockId(3), + reason: 'uncheckpointed', + checkpoint: expect.objectContaining({ number: CheckpointNumber(1) }), + }, + ]); + + handler.clearEvents(); + + // Update local state after prune + localData.proposed.number = BlockNumber(3); + delete localData.blockHashes[4]; + delete localData.blockHashes[5]; + delete localData.blockHashes[6]; + + // Phase 3: New blocks 4-9 arrive with checkpoints 2-3 + setRemoteTipsMultiBlock(9, 9); + + await blockStream.work(); + + // Should continue normally with new blocks and checkpoints + expect(handler.events).toEqual([ + expectBlocksAdded([4, 5, 6]), + expectCheckpointed(2), + expectBlocksAdded([7, 8, 9]), + expectCheckpointed(3), + ]); + }); + + it('prunes proposed chain back to genesis when no checkpoints exist', async () => { + // Phase 1: Sync blocks 1-6, no checkpoints (checkpointed=0, proven=0, finalized=0) + setRemoteTipsMultiBlock(6, 0); + + await blockStream.work(); + + // All blocks come as uncheckpointed + expect(handler.events).toEqual([expectBlocksAdded([1, 2, 3, 4, 5, 6])]); + + handler.clearEvents(); + + // Update local state + localData.proposed.number = BlockNumber(6); + + // Phase 2: All blocks are invalid, prune back to genesis (block 0) + for (let i = 1; i <= 6; i++) { + localData.blockHashes[i] = `0xbad${i}`; + } + + // Source now has proposed=0, checkpointed=0 + setRemoteTipsMultiBlock(0, 0); + + await blockStream.work(); + + // Should emit chain-pruned back to block 0, reason 'uncheckpointed' (pruned to checkpointed tip which is 0) + expect(handler.events).toEqual([ + { + type: 'chain-pruned', + block: makeBlockId(0), + reason: 'uncheckpointed', + checkpoint: expect.objectContaining({ number: CheckpointNumber(0) }), + }, + ]); + + handler.clearEvents(); + + // Update local state after prune + localData.proposed.number = BlockNumber(0); + for (let i = 1; i <= 6; i++) { + delete localData.blockHashes[i]; + } + + // Phase 3: New blocks 1-6 arrive with checkpoints 1-2 + setRemoteTipsMultiBlock(6, 6); + + await blockStream.work(); + + // Should continue normally from genesis + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + expectCheckpointed(2), + ]); + }); + + it('prunes both proposed and checkpointed chains back to genesis', async () => { + // Phase 1: Sync blocks 1-6 with checkpoint 1 (blocks 1-3), blocks 4-6 uncheckpointed + setRemoteTipsMultiBlock(6, 3); + + await blockStream.work(); + + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + ]); + + handler.clearEvents(); + + // Update local state + localData.proposed.number = BlockNumber(6); + localData.checkpointed.block.number = BlockNumber(3); + localData.checkpointed.checkpoint.number = CheckpointNumber(1); + + // Phase 2: All blocks are invalid (even checkpointed ones), prune back to genesis + for (let i = 1; i <= 6; i++) { + localData.blockHashes[i] = `0xbad${i}`; + } + + // Source now has proposed=0, checkpointed=0 (full chain reset) + setRemoteTipsMultiBlock(0, 0); + + await blockStream.work(); + + // Should emit chain-pruned back to block 0 + // Reason is 'unproven' because we're pruning beyond the local checkpointed tip (3) + expect(handler.events).toEqual([ + { + type: 'chain-pruned', + block: makeBlockId(0), + reason: 'unproven', + checkpoint: expect.objectContaining({ number: CheckpointNumber(0) }), + }, + ]); + + handler.clearEvents(); + + // Update local state after prune + localData.proposed.number = BlockNumber(0); + localData.checkpointed.block.number = BlockNumber(0); + localData.checkpointed.checkpoint.number = CheckpointNumber(0); + for (let i = 1; i <= 6; i++) { + delete localData.blockHashes[i]; + } + + // Phase 3: New chain starts fresh with blocks 1-9 and checkpoints 1-3 + setRemoteTipsMultiBlock(9, 9); + + await blockStream.work(); + + // Should continue normally from genesis + expect(handler.events).toEqual([ + expectBlocksAdded([1, 2, 3]), + expectCheckpointed(1), + expectBlocksAdded([4, 5, 6]), + expectCheckpointed(2), + expectBlocksAdded([7, 8, 9]), + expectCheckpointed(3), + ]); + }); }); }); @@ -206,11 +1042,11 @@ describe('L2BlockStream', () => { }); it('skips ahead to the latest finalized block', async () => { - setRemoteTips(40, 38, 35); + setRemoteTips(40, 0, 38, 35); - localData.latest.number = BlockNumber(5); - localData.proven.number = BlockNumber(2); - localData.finalized.number = BlockNumber(2); + localData.proposed.number = BlockNumber(5); + localData.proven.block.number = BlockNumber(2); + localData.finalized.block.number = BlockNumber(2); await blockStream.work(); @@ -223,11 +1059,11 @@ describe('L2BlockStream', () => { }); it('does not skip if already ahead of finalized', async () => { - setRemoteTips(40, 38, 35); + setRemoteTips(40, 0, 38, 35); - localData.latest.number = BlockNumber(38); - localData.proven.number = BlockNumber(38); - localData.finalized.number = BlockNumber(35); + localData.proposed.number = BlockNumber(38); + localData.proven.block.number = BlockNumber(38); + localData.finalized.block.number = BlockNumber(35); await blockStream.work(); @@ -264,18 +1100,33 @@ class TestL2BlockStreamEventHandler implements L2BlockStreamEventHandler { class TestL2BlockStreamLocalDataProvider implements L2BlockStreamLocalDataProvider { public readonly blockHashes: Record = {}; - public latest = { number: BlockNumber.ZERO, hash: '' }; - public proven = { number: BlockNumber.ZERO, hash: '' }; - public finalized = { number: BlockNumber.ZERO, hash: '' }; + public proposed = { number: BlockNumber.ZERO, hash: '' }; + public checkpointed = { + block: { number: BlockNumber.ZERO, hash: '' }, + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }; + public proven = { + block: { number: BlockNumber.ZERO, hash: '' }, + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }; + public finalized = { + block: { number: BlockNumber.ZERO, hash: '' }, + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }; public getL2BlockHash(number: number): Promise { return Promise.resolve( - number > this.latest.number ? undefined : (this.blockHashes[number] ?? new Fr(number).toString()), + number > this.proposed.number ? undefined : (this.blockHashes[number] ?? new Fr(number).toString()), ); } public getL2Tips(): Promise { - return Promise.resolve(this); + return Promise.resolve({ + proposed: this.proposed, + checkpointed: this.checkpointed, + proven: this.proven, + finalized: this.finalized, + }); } } @@ -292,7 +1143,7 @@ class TestL2BlockStream extends L2BlockStream { } class TestL2TipsMemoryStore extends L2TipsMemoryStore { - protected override computeBlockHash(block: L2Block): Promise<`0x${string}`> { + protected override computeBlockHash(block: L2BlockNew): Promise<`0x${string}`> { return Promise.resolve(new Fr(block.number).toString()); } } diff --git a/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts b/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts index a3e94128c87f..261aa4b01372 100644 --- a/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts +++ b/yarn-project/stdlib/src/block/l2_block_stream/l2_block_stream.ts @@ -1,9 +1,10 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { INITIAL_CHECKPOINT_NUMBER } from '@aztec/constants'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { AbortError } from '@aztec/foundation/error'; import { createLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; -import { type L2BlockId, type L2BlockSource, makeL2BlockId } from '../l2_block_source.js'; +import { type L2BlockId, type L2BlockPruneReason, type L2BlockSource, makeL2BlockId } from '../l2_block_source.js'; import type { L2BlockStreamEvent, L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider } from './interfaces.js'; /** Creates a stream of events for new blocks, chain tips updates, and reorgs, out of polling an archiver or a node. */ @@ -13,7 +14,10 @@ export class L2BlockStream { private hasStarted = false; constructor( - private l2BlockSource: Pick, + private l2BlockSource: Pick< + L2BlockSource, + 'getL2BlocksNew' | 'getBlockHeader' | 'getL2Tips' | 'getPublishedCheckpoints' | 'getCheckpointedBlocks' + >, private localData: L2BlockStreamLocalDataProvider, private handler: L2BlockStreamEventHandler, private readonly log = createLogger('types:block_stream'), @@ -62,35 +66,50 @@ export class L2BlockStream { const sourceTips = await this.l2BlockSource.getL2Tips(); const localTips = await this.localData.getL2Tips(); this.log.trace(`Running L2 block stream`, { - sourceLatest: sourceTips.latest.number, - localLatest: localTips.latest.number, - sourceFinalized: sourceTips.finalized.number, - localFinalized: localTips.finalized.number, - sourceProven: sourceTips.proven.number, - localProven: localTips.proven.number, - sourceLatestHash: sourceTips.latest.hash, - localLatestHash: localTips.latest.hash, - sourceProvenHash: sourceTips.proven.hash, - localProvenHash: localTips.proven.hash, - sourceFinalizedHash: sourceTips.finalized.hash, - localFinalizedHash: localTips.finalized.hash, + sourceLatest: sourceTips.proposed.number, + localLatest: localTips.proposed.number, + sourceFinalized: sourceTips.finalized.block.number, + localFinalized: localTips.finalized.block.number, + sourceProven: sourceTips.proven.block.number, + localProven: localTips.proven.block.number, + sourceLatestHash: sourceTips.proposed.hash, + localLatestHash: localTips.proposed.hash, + sourceProvenHash: sourceTips.proven.block.hash, + localProvenHash: localTips.proven.block.hash, + sourceFinalizedHash: sourceTips.finalized.block.hash, + localFinalizedHash: localTips.finalized.block.hash, }); // Check if there was a reorg and emit a chain-pruned event if so. - let latestBlockNumber = localTips.latest.number; - const sourceCache = new BlockHashCache([sourceTips.latest]); + let latestBlockNumber = localTips.proposed.number; + const sourceCache = new BlockHashCache([sourceTips.proposed]); while (!(await this.areBlockHashesEqualAt(latestBlockNumber, { sourceCache }))) { latestBlockNumber--; } - if (latestBlockNumber < localTips.latest.number) { - latestBlockNumber = BlockNumber(Math.min(latestBlockNumber, sourceTips.latest.number)); // see #13471 + if (latestBlockNumber < localTips.proposed.number) { + latestBlockNumber = BlockNumber(Math.min(latestBlockNumber, sourceTips.proposed.number)); // see #13471 const hash = sourceCache.get(latestBlockNumber) ?? (await this.getBlockHashFromSource(latestBlockNumber)); if (latestBlockNumber !== 0 && !hash) { throw new Error(`Block hash not found in block source for block number ${latestBlockNumber}`); } - this.log.verbose(`Reorg detected. Pruning blocks from ${latestBlockNumber + 1} to ${localTips.latest.number}.`); - await this.emitEvent({ type: 'chain-pruned', block: makeL2BlockId(latestBlockNumber, hash) }); + this.log.verbose( + `Reorg detected. Pruning blocks from ${latestBlockNumber + 1} to ${localTips.proposed.number}.`, + ); + // This check is not 100% accurate + // If the local tips are sufficiently behind the source tips, such that we are missing at least one checkpoint + // that has now been re-orged due to a proof failure then this will indicate a failure to checkpoint rather than a failure to prove + // TODO: (mbps/PhilWindle): Improve re-org detection accuracy when we come to do re-orgs + let reason: L2BlockPruneReason = 'unproven'; + if (latestBlockNumber === localTips.checkpointed.block.number) { + reason = 'uncheckpointed'; + } + await this.emitEvent({ + type: 'chain-pruned', + block: makeL2BlockId(latestBlockNumber, hash), + reason, + checkpoint: sourceTips.checkpointed.checkpoint, + }); } // If we are just starting, use the starting block number from the options. @@ -111,34 +130,104 @@ export class L2BlockStream { // last finalized block however in order to guarantee that we will eventually find a block in which our local // store matches the source. // If the last finalized block is behind our local tip, there is nothing to skip. - nextBlockNumber = Math.max(sourceTips.finalized.number, nextBlockNumber); + nextBlockNumber = Math.max(sourceTips.finalized.block.number, nextBlockNumber); + } + + // First, emit checkpoint events for checkpoints whose blocks are already in local storage. + // As we should only ever have a single checkpoint's worth of uncheckpointed blocks locally, this + // should only iterate once + let nextCheckpointToEmit = CheckpointNumber(localTips.checkpointed.checkpoint.number + 1); + let iterations = 0; + while (nextCheckpointToEmit <= sourceTips.checkpointed.checkpoint.number) { + const checkpoints = await this.l2BlockSource.getPublishedCheckpoints(nextCheckpointToEmit, 1); + if (checkpoints.length === 0) { + break; + } + // Check if all blocks in this checkpoint are already in local storage + const lastBlockInCheckpoint = checkpoints[0].checkpoint.blocks.at(-1)!; + if (lastBlockInCheckpoint.number > localTips.proposed.number) { + // This checkpoint has blocks we haven't seen yet, stop here + break; + } + iterations++; + if (iterations > 1) { + this.log.warn(`Emitting multiple checkpoints (${iterations}) without new blocks being added.`); + } + const lastBlockHash = await lastBlockInCheckpoint.hash(); + await this.emitEvent({ + type: 'chain-checkpointed', + checkpoint: checkpoints[0], + block: makeL2BlockId(lastBlockInCheckpoint.number, lastBlockHash.toString()), + }); + nextCheckpointToEmit = CheckpointNumber(nextCheckpointToEmit + 1); } - // Request new blocks from the source. - while (nextBlockNumber <= sourceTips.latest.number) { - const limit = Math.min(this.opts.batchSize ?? 50, sourceTips.latest.number - nextBlockNumber + 1); + // We have now effectively checkpointed our view of the chain. As in there should be no checkpointed blocks + // that we have seen locally and not emitted checkpoints for. + + // Now fetch any new checkpointed blocks. If nextBlockNumber is below the source's checkpointed block number + // then we will retrieve it as a checkpointed block, retrieve the checkpoint and emit all blocks from that point forward + // that are part of the checkpoint, before emitting the checkpoint itself. + // We do this until all checkpointed blocks and checkpoints are emitted. + // This takes our local chain up to date with the source's checkpointed blocks. + let checkpointNumber = CheckpointNumber(INITIAL_CHECKPOINT_NUMBER - 1); + while (nextBlockNumber <= sourceTips.checkpointed.block.number) { + const limit = Math.min(this.opts.batchSize ?? 50, sourceTips.checkpointed.block.number - nextBlockNumber + 1); this.log.trace(`Requesting blocks from ${nextBlockNumber} limit ${limit} proven=${this.opts.proven}`); - const blocks = await this.l2BlockSource.getPublishedBlocks( + // Get this block as a checkpointed block + const blocks = await this.l2BlockSource.getCheckpointedBlocks( BlockNumber(nextBlockNumber), - limit, + 1, this.opts.proven, ); if (blocks.length === 0) { break; } + checkpointNumber = CheckpointNumber(blocks[0].checkpointNumber); + const checkpoints = await this.l2BlockSource.getPublishedCheckpoints(checkpointNumber, 1); + if (checkpoints.length === 0) { + break; + } + // we have the checkpoint for the next block number, get the remaining blocks in this checkpoint + const blocksforCheckpoint = checkpoints[0].checkpoint.blocks + .filter(b => b.number >= nextBlockNumber) + .slice(0, limit); + await this.emitEvent({ type: 'blocks-added', blocks: blocksforCheckpoint }); + nextBlockNumber = blocksforCheckpoint.at(-1)!.number + 1; + + // If we have reached the end of the checkpoint, signal as such + const lastBlockInCheckpoint = checkpoints[0].checkpoint.blocks.at(-1)!; + if (nextBlockNumber > lastBlockInCheckpoint.number) { + const lastBlockHash = await lastBlockInCheckpoint.hash(); + await this.emitEvent({ + type: 'chain-checkpointed', + checkpoint: checkpoints[0], + block: makeL2BlockId(lastBlockInCheckpoint.number, lastBlockHash.toString()), + }); + } + } + + // Now we pull any remaining, uncheckpointed block and emit them. + while (nextBlockNumber <= sourceTips.proposed.number) { + const limit = Math.min(this.opts.batchSize ?? 50, sourceTips.proposed.number - nextBlockNumber + 1); + this.log.trace(`Requesting blocks from ${nextBlockNumber} limit ${limit} proven=${this.opts.proven}`); + const blocks = await this.l2BlockSource.getL2BlocksNew(BlockNumber(nextBlockNumber), limit, this.opts.proven); + if (blocks.length === 0) { + break; + } await this.emitEvent({ type: 'blocks-added', blocks }); - nextBlockNumber = blocks.at(-1)!.block.number + 1; + nextBlockNumber = blocks.at(-1)!.number + 1; } // Update the proven and finalized tips. - if (localTips.proven !== undefined && sourceTips.proven.number !== localTips.proven.number) { + if (localTips.proven !== undefined && sourceTips.proven.block.number !== localTips.proven.block.number) { await this.emitEvent({ type: 'chain-proven', - block: sourceTips.proven, + block: sourceTips.proven.block, }); } - if (localTips.finalized !== undefined && sourceTips.finalized.number !== localTips.finalized.number) { - await this.emitEvent({ type: 'chain-finalized', block: sourceTips.finalized }); + if (localTips.finalized !== undefined && sourceTips.finalized.block.number !== localTips.finalized.block.number) { + await this.emitEvent({ type: 'chain-finalized', block: sourceTips.finalized.block }); } } catch (err: any) { if (err.name === 'AbortError') { @@ -186,7 +275,7 @@ export class L2BlockStream { private async emitEvent(event: L2BlockStreamEvent) { this.log.debug( - `Emitting ${event.type} (${event.type === 'blocks-added' ? event.blocks.length : event.block.number})`, + `Emitting ${event.type} (${event.type === 'blocks-added' ? event.blocks.length : event.type === 'chain-checkpointed' ? event.checkpoint.checkpoint.number : event.block.number})`, ); await this.handler.handleBlockStreamEvent(event); if (!this.isRunning() && !this.isSyncing) { diff --git a/yarn-project/stdlib/src/block/l2_block_stream/l2_tips_memory_store.ts b/yarn-project/stdlib/src/block/l2_block_stream/l2_tips_memory_store.ts index f393789e3d80..a39fc0cb45d1 100644 --- a/yarn-project/stdlib/src/block/l2_block_stream/l2_tips_memory_store.ts +++ b/yarn-project/stdlib/src/block/l2_block_stream/l2_tips_memory_store.ts @@ -1,31 +1,45 @@ import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants'; -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; -import type { L2Block } from '../l2_block.js'; -import type { L2BlockId, L2BlockTag, L2Tips } from '../l2_block_source.js'; +import type { PublishedCheckpoint } from '../../checkpoint/published_checkpoint.js'; +import type { L2BlockNew } from '../l2_block_new.js'; +import type { CheckpointId, L2BlockId, L2BlockTag, L2Tips } from '../l2_block_source.js'; import type { L2BlockStreamEvent, L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider } from './interfaces.js'; /** - * Stores currently synced L2 tips and unfinalized block hashes. + * Maintains and returns the current set of L2 Tips. Maintains stores of block hashes and checkpoints in order to do so. * @dev tests in kv-store/src/stores/l2_tips_memory_store.test.ts */ export class L2TipsMemoryStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider { protected readonly l2TipsStore: Map = new Map(); protected readonly l2BlockHashesStore: Map = new Map(); + protected readonly l2BlocktoCheckpointStore: Map = new Map(); + protected readonly checkpointStore: Map = new Map(); public getL2BlockHash(number: number): Promise { return Promise.resolve(this.l2BlockHashesStore.get(number)); } public getL2Tips(): Promise { - return Promise.resolve({ - latest: this.getL2Tip('latest'), - finalized: this.getL2Tip('finalized'), - proven: this.getL2Tip('proven'), - }); + const proposedBlockId = this.getBlockId('proposed'); + const finalizedBlockId = this.getBlockId('finalized'); + const provenBlockId = this.getBlockId('proven'); + const checkpointedBlockId = this.getBlockId('checkpointed'); + + const finalizedCheckpointId = this.getCheckpointId('finalized'); + const provenCheckpointId = this.getCheckpointId('proven'); + const checkpointedCheckpointId = this.getCheckpointId('checkpointed'); + + const l2Tips: L2Tips = { + proposed: proposedBlockId, + finalized: { block: finalizedBlockId, checkpoint: finalizedCheckpointId }, + proven: { block: provenBlockId, checkpoint: provenCheckpointId }, + checkpointed: { block: checkpointedBlockId, checkpoint: checkpointedCheckpointId }, + }; + return Promise.resolve(l2Tips); } - private getL2Tip(tag: L2BlockTag): L2BlockId { + private getBlockId(tag: L2BlockTag): L2BlockId { const blockNumber = this.l2TipsStore.get(tag); if (blockNumber === undefined || blockNumber === 0) { return { number: BlockNumber.ZERO, hash: GENESIS_BLOCK_HEADER_HASH.toString() }; @@ -38,33 +52,112 @@ export class L2TipsMemoryStore implements L2BlockStreamEventHandler, L2BlockStre return { number: blockNumber, hash: blockHash }; } + private getCheckpointId(tag: L2BlockTag): CheckpointId { + const blockNumber = this.l2TipsStore.get(tag); + if (blockNumber === undefined || blockNumber === 0) { + return { number: CheckpointNumber.ZERO, hash: '' }; + } + const checkpointNumber = this.l2BlocktoCheckpointStore.get(blockNumber); + if (checkpointNumber === undefined) { + // No checkpoint associated with this block yet + return { number: CheckpointNumber.ZERO, hash: '' }; + } + const checkpoint = this.checkpointStore.get(checkpointNumber); + if (!checkpoint) { + throw new Error(`Checkpoint not found for checkpoint number ${checkpointNumber}`); + } + + return { number: checkpointNumber, hash: checkpoint.checkpoint.hash().toString() }; + } + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { switch (event.type) { - case 'blocks-added': { - const blocks = event.blocks.map(b => b.block); - for (const block of blocks) { - this.l2BlockHashesStore.set(block.number, await this.computeBlockHash(block)); - } - this.l2TipsStore.set('latest', blocks.at(-1)!.number); + case 'blocks-added': + await this.handleBlocksAdded(event); + break; + case 'chain-checkpointed': + this.handleChainCheckpointed(event); break; - } case 'chain-pruned': - this.saveTag('latest', event.block); + this.handleChainPruned(event); break; case 'chain-proven': - this.saveTag('proven', event.block); + this.handleChainProven(event); break; case 'chain-finalized': - this.saveTag('finalized', event.block); - for (const key of this.l2BlockHashesStore.keys()) { - if (key < event.block.number) { - this.l2BlockHashesStore.delete(key); - } - } + this.handleChainFinalized(event); break; } } + private async handleBlocksAdded(event: L2BlockStreamEvent) { + if (event.type !== 'blocks-added') { + return; + } + // Simply add the new block hashes by the block number and update the proposed tip + const blocks = event.blocks; + for (const block of blocks) { + this.l2BlockHashesStore.set(block.number, await this.computeBlockHash(block)); + } + this.l2TipsStore.set('proposed', blocks.at(-1)!.number); + } + + private handleChainCheckpointed(event: L2BlockStreamEvent) { + if (event.type !== 'chain-checkpointed') { + return; + } + this.saveTag('checkpointed', event.block); + // Only store the mapping for the last block since tips only point to checkpoint boundaries + this.l2BlocktoCheckpointStore.set(event.block.number, event.checkpoint.checkpoint.number); + this.checkpointStore.set(event.checkpoint.checkpoint.number, event.checkpoint); + } + + private handleChainPruned(event: L2BlockStreamEvent) { + if (event.type !== 'chain-pruned') { + return; + } + // update the proposed and checkpointed chain tips + this.saveTag('proposed', event.block); + this.saveTag('checkpointed', event.block); + } + + private handleChainProven(event: L2BlockStreamEvent) { + if (event.type !== 'chain-proven') { + return; + } + // Updtae the proven chain tip + this.saveTag('proven', event.block); + } + + private handleChainFinalized(event: L2BlockStreamEvent) { + if (event.type !== 'chain-finalized') { + return; + } + this.saveTag('finalized', event.block); + // Get the checkpoint number for the finalized block before cleanup + const finalizedCheckpointNumber = this.l2BlocktoCheckpointStore.get(event.block.number); + // Clean up block hashes for blocks before finalized + for (const key of this.l2BlockHashesStore.keys()) { + if (key < event.block.number) { + this.l2BlockHashesStore.delete(key); + } + } + // Clean up block-to-checkpoint mappings for blocks before finalized + for (const key of this.l2BlocktoCheckpointStore.keys()) { + if (key < event.block.number) { + this.l2BlocktoCheckpointStore.delete(key); + } + } + // Clean up checkpoints older than the finalized checkpoint + if (finalizedCheckpointNumber !== undefined) { + for (const key of this.checkpointStore.keys()) { + if (key < finalizedCheckpointNumber) { + this.checkpointStore.delete(key); + } + } + } + } + protected saveTag(name: L2BlockTag, block: L2BlockId) { this.l2TipsStore.set(name, block.number); if (block.hash) { @@ -72,7 +165,7 @@ export class L2TipsMemoryStore implements L2BlockStreamEventHandler, L2BlockStre } } - protected computeBlockHash(block: L2Block) { + protected computeBlockHash(block: L2BlockNew) { return block.hash().then(hash => hash.toString()); } } diff --git a/yarn-project/stdlib/src/block/test/l2_tips_store_test_suite.ts b/yarn-project/stdlib/src/block/test/l2_tips_store_test_suite.ts index 2de7a5150c27..e5579a12bf16 100644 --- a/yarn-project/stdlib/src/block/test/l2_tips_store_test_suite.ts +++ b/yarn-project/stdlib/src/block/test/l2_tips_store_test_suite.ts @@ -1,9 +1,9 @@ import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants'; -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { times } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; -import { type L2Block, type L2BlockId, PublishedL2Block } from '@aztec/stdlib/block'; -import { L1PublishedData } from '@aztec/stdlib/checkpoint'; +import { type CheckpointId, type L2BlockId, L2BlockNew, type L2TipId } from '@aztec/stdlib/block'; +import { Checkpoint, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import { jestExpect as expect } from '@jest/expect'; @@ -11,83 +11,526 @@ import type { L2TipsStore } from '../l2_block_stream/index.js'; export function testL2TipsStore(makeTipsStore: () => Promise) { let tipsStore: L2TipsStore; + // Track blocks and their hashes for test assertions + const blockHashes: Map = new Map(); + // Track checkpoints and their hashes + const checkpointHashes: Map = new Map(); + // Track which blocks belong to which checkpoint + const blockToCheckpoint: Map = new Map(); beforeEach(async () => { tipsStore = await makeTipsStore(); + blockHashes.clear(); + checkpointHashes.clear(); + blockToCheckpoint.clear(); }); - const makeBlock = (number: number): PublishedL2Block => - PublishedL2Block.fromFields({ - block: { number: BlockNumber(number), hash: () => Promise.resolve(new Fr(number)) } as L2Block, - l1: new L1PublishedData(BigInt(number), BigInt(number), `0x${number}`), - attestations: [], - }); + const makeBlock = async (number: number): Promise => { + const block = await L2BlockNew.random(BlockNumber(number)); + blockHashes.set(number, (await block.hash()).toString()); + return block; + }; const makeBlockId = (number: number): L2BlockId => ({ number: BlockNumber(number), - hash: new Fr(number).toString(), + hash: blockHashes.get(number) ?? new Fr(number).toString(), }); const makeTip = (number: number): L2BlockId => ({ number: BlockNumber(number), - hash: number === 0 ? GENESIS_BLOCK_HEADER_HASH.toString() : new Fr(number).toString(), + hash: number === 0 ? GENESIS_BLOCK_HEADER_HASH.toString() : (blockHashes.get(number) ?? new Fr(number).toString()), + }); + + const makeCheckpointIdForBlock = (blockNumber: number): CheckpointId => { + if (blockNumber === 0) { + return { number: CheckpointNumber.ZERO, hash: '' }; + } + const checkpointNum = blockToCheckpoint.get(blockNumber); + if (checkpointNum === undefined) { + return { number: CheckpointNumber.ZERO, hash: '' }; + } + const hash = checkpointHashes.get(checkpointNum); + if (!hash) { + return { number: CheckpointNumber.ZERO, hash: '' }; + } + return { number: CheckpointNumber(checkpointNum), hash }; + }; + + const makeTipId = (blockNumber: number): L2TipId => ({ + block: makeTip(blockNumber), + checkpoint: makeCheckpointIdForBlock(blockNumber), }); - const makeTips = (latest: number, proven: number, finalized: number) => ({ - latest: makeTip(latest), - proven: makeTip(proven), - finalized: makeTip(finalized), + const makeTips = (proposed: number, proven: number, finalized: number, checkpointed: number = 0) => ({ + proposed: makeTip(proposed), + proven: makeTipId(proven), + finalized: makeTipId(finalized), + checkpointed: makeTipId(checkpointed), }); + const makeCheckpoint = async (checkpointNumber: number, blocks: L2BlockNew[]): Promise => { + const checkpoint = await Checkpoint.random(CheckpointNumber(checkpointNumber), { + numBlocks: blocks.length, + startBlockNumber: blocks[0].number, + }); + // Override the blocks with our actual blocks (to keep hashes consistent) + (checkpoint as any).blocks = blocks; + + const checkpointHash = checkpoint.hash().toString(); + checkpointHashes.set(checkpointNumber, checkpointHash); + + // Track which blocks belong to this checkpoint + for (const block of blocks) { + blockToCheckpoint.set(block.number, checkpointNumber); + } + + return new PublishedCheckpoint(checkpoint, L1PublishedData.random(), []); + }; + + /** Creates a chain-checkpointed event with the required block field */ + const makeCheckpointedEvent = async (checkpoint: PublishedCheckpoint) => { + const lastBlock = checkpoint.checkpoint.blocks.at(-1)!; + const blockId: L2BlockId = { + number: lastBlock.number, + hash: (await lastBlock.hash()).toString(), + }; + return { type: 'chain-checkpointed' as const, checkpoint, block: blockId }; + }; + it('returns zero if no tips are stored', async () => { const tips = await tipsStore.getL2Tips(); expect(tips).toEqual(makeTips(0, 0, 0)); }); - it('stores chain tips', async () => { - await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(20, i => makeBlock(i + 1)) }); + it('sets proposed tip from blocks added', async () => { + await tipsStore.handleBlockStreamEvent({ + type: 'blocks-added', + blocks: await Promise.all(times(3, i => makeBlock(i + 1))), + }); - await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', block: makeBlockId(5) }); - await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', block: makeBlockId(8) }); - await tipsStore.handleBlockStreamEvent({ type: 'chain-pruned', block: makeBlockId(10) }); + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(3, 0, 0)); + + expect(await tipsStore.getL2BlockHash(1)).toEqual(blockHashes.get(1)); + expect(await tipsStore.getL2BlockHash(2)).toEqual(blockHashes.get(2)); + expect(await tipsStore.getL2BlockHash(3)).toEqual(blockHashes.get(3)); + }); + + it('checkpoints all proposed blocks', async () => { + // Propose blocks 1-5 + const blocks = await Promise.all(times(5, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks }); + + // Checkpoint all proposed blocks (1-5) + const checkpoint1 = await makeCheckpoint(1, blocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); const tips = await tipsStore.getL2Tips(); - expect(tips).toEqual(makeTips(10, 8, 5)); + // Proposed and checkpointed should be the same + expect(tips.proposed).toEqual(makeTip(5)); + expect(tips.checkpointed.block).toEqual(makeTip(5)); + expect(tips.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1)); }); - it('sets latest tip from blocks added', async () => { - await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(3, i => makeBlock(i + 1)) }); + it('advances proven chain with checkpoint info', async () => { + // Propose and checkpoint blocks 1-5 + const blocks = await Promise.all(times(5, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks }); + const checkpoint1 = await makeCheckpoint(1, blocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); + + // Prove up to block 5 + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', block: makeBlockId(5) }); const tips = await tipsStore.getL2Tips(); - expect(tips).toEqual(makeTips(3, 0, 0)); + expect(tips.proposed).toEqual(makeTip(5)); + expect(tips.checkpointed.block).toEqual(makeTip(5)); + expect(tips.proven.block).toEqual(makeTip(5)); + + // Proven tip should have the checkpoint info + expect(tips.proven.checkpoint.number).toEqual(CheckpointNumber(1)); + expect(tips.proven.checkpoint.hash).toEqual(checkpointHashes.get(1)); + }); + + it('advances finalized chain with checkpoint info', async () => { + // Propose and checkpoint blocks 1-5 + const blocks = await Promise.all(times(5, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks }); + const checkpoint1 = await makeCheckpoint(1, blocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); + + // Prove and finalize + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', block: makeBlockId(5) }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', block: makeBlockId(5) }); - expect(await tipsStore.getL2BlockHash(1)).toEqual(new Fr(1).toString()); - expect(await tipsStore.getL2BlockHash(2)).toEqual(new Fr(2).toString()); - expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + const tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(5)); + expect(tips.checkpointed.block).toEqual(makeTip(5)); + expect(tips.proven.block).toEqual(makeTip(5)); + expect(tips.finalized.block).toEqual(makeTip(5)); + + // Finalized tip should have checkpoint info + expect(tips.finalized.checkpoint.number).toEqual(CheckpointNumber(1)); + expect(tips.finalized.checkpoint.hash).toEqual(checkpointHashes.get(1)); + }); + + it('handles multiple checkpoints advancing the chain', async () => { + // Propose blocks 1-5 + const blocks1 = await Promise.all(times(5, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: blocks1 }); + + // Checkpoint 1: all proposed blocks 1-5 + const checkpoint1 = await makeCheckpoint(1, blocks1); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); + + // Propose more blocks 6-10 + const blocks2 = await Promise.all(times(5, i => makeBlock(i + 6))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: blocks2 }); + + // Checkpoint 2: all remaining proposed blocks 6-10 + const checkpoint2 = await makeCheckpoint(2, blocks2); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint2)); + + const tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(10)); + expect(tips.checkpointed.block).toEqual(makeTip(10)); + expect(tips.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2)); + expect(tips.checkpointed.checkpoint.hash).toEqual(checkpointHashes.get(2)); }); it('clears block hashes when setting finalized chain', async () => { - await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, i => makeBlock(i + 1)) }); + // Propose blocks 1-3 + const blocks1to3 = await Promise.all(times(3, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: blocks1to3 }); + + // Checkpoint all proposed blocks (1-3) + const checkpoint1 = await makeCheckpoint(1, blocks1to3); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); + + // Propose more blocks 4-5 + const blocks4to5 = await Promise.all(times(2, i => makeBlock(i + 4))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: blocks4to5 }); + + // Checkpoint all remaining proposed blocks (4-5) + const checkpoint2 = await makeCheckpoint(2, blocks4to5); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint2)); + + // Prove and finalize up to block 3 (checkpoint 1) await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', block: makeBlockId(3) }); await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', block: makeBlockId(3) }); - const tips = await tipsStore.getL2Tips(); - expect(tips).toEqual(makeTips(5, 3, 3)); - + // Blocks before finalized should be cleared expect(await tipsStore.getL2BlockHash(1)).toBeUndefined(); expect(await tipsStore.getL2BlockHash(2)).toBeUndefined(); - expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); - expect(await tipsStore.getL2BlockHash(4)).toEqual(new Fr(4).toString()); - expect(await tipsStore.getL2BlockHash(5)).toEqual(new Fr(5).toString()); + // Finalized and later blocks should remain + expect(await tipsStore.getL2BlockHash(3)).toEqual(blockHashes.get(3)); + expect(await tipsStore.getL2BlockHash(4)).toEqual(blockHashes.get(4)); + expect(await tipsStore.getL2BlockHash(5)).toEqual(blockHashes.get(5)); + }); + + it('handles chain pruning by updating proposed tip', async () => { + const blocks = await Promise.all(times(10, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks }); + + // Prune to block 5 + await tipsStore.handleBlockStreamEvent({ + type: 'chain-pruned', + block: makeBlockId(5), + reason: 'unproven', + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }); + + const tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(5)); + }); + + it('handles pruning proposed chain to genesis, re-proposing, and checkpointing', async () => { + // Propose blocks 1-3 + const firstBlocks = await Promise.all(times(3, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: firstBlocks }); + + let tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(3)); + + // Store original hashes + const originalHash1 = blockHashes.get(1); + const originalHash2 = blockHashes.get(2); + const originalHash3 = blockHashes.get(3); + + // Prune back to genesis (block 0) + await tipsStore.handleBlockStreamEvent({ + type: 'chain-pruned', + block: makeTip(0), + reason: 'unproven', + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(0)); + expect(tips.checkpointed.block).toEqual(makeTip(0)); + + // Clear hashes and propose new blocks 1-3 (different from original) + blockHashes.delete(1); + blockHashes.delete(2); + blockHashes.delete(3); + const newBlocks = await Promise.all(times(3, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: newBlocks }); + + // Verify new blocks have different hashes + expect(blockHashes.get(1)).not.toEqual(originalHash1); + expect(blockHashes.get(2)).not.toEqual(originalHash2); + expect(blockHashes.get(3)).not.toEqual(originalHash3); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(3)); + expect(tips.checkpointed.block).toEqual(makeTip(0)); // Not yet checkpointed + + // Checkpoint all the new proposed blocks (1-3) + const checkpoint1 = await makeCheckpoint(1, newBlocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(3)); + expect(tips.checkpointed.block).toEqual(makeTip(3)); + expect(tips.checkpointed.checkpoint.number).toEqual(CheckpointNumber(1)); + + // Verify block hashes in store are the new ones + expect(await tipsStore.getL2BlockHash(1)).toEqual(blockHashes.get(1)); + expect(await tipsStore.getL2BlockHash(2)).toEqual(blockHashes.get(2)); + expect(await tipsStore.getL2BlockHash(3)).toEqual(blockHashes.get(3)); + }); + + it('handles reorg: prune proposed blocks back to checkpoint, then re-propose with different blocks', async () => { + // Propose blocks 1-5 + const firstBlocks = await Promise.all(times(5, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: firstBlocks }); + + // Checkpoint all proposed blocks (1-5) - these are now committed + const checkpoint1 = await makeCheckpoint(1, firstBlocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); + + // Propose more blocks 6-10 (not yet checkpointed, can be pruned) + const originalBlocks6to10 = await Promise.all(times(5, i => makeBlock(i + 6))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: originalBlocks6to10 }); + + let tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(10)); + expect(tips.checkpointed.block).toEqual(makeTip(5)); // Only blocks 1-5 are checkpointed + + // Store original hashes for proposed (non-checkpointed) blocks 6-8 + const originalHash6 = blockHashes.get(6); + const originalHash7 = blockHashes.get(7); + const originalHash8 = blockHashes.get(8); + + // Prune proposed blocks back to checkpoint (block 5) + // This removes proposed blocks 6-10, but checkpoint remains at 5 + await tipsStore.handleBlockStreamEvent({ + type: 'chain-pruned', + block: makeBlockId(5), + reason: 'unproven', + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(5)); + expect(tips.checkpointed.block).toEqual(makeTip(5)); // Checkpoint unchanged + + // Propose new blocks 6-8 (different from original 6-10) + blockHashes.delete(6); + blockHashes.delete(7); + blockHashes.delete(8); + const newBlocks = await Promise.all(times(3, i => makeBlock(i + 6))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: newBlocks }); + + // Verify the new blocks have different hashes than the original ones + expect(blockHashes.get(6)).not.toEqual(originalHash6); + expect(blockHashes.get(7)).not.toEqual(originalHash7); + expect(blockHashes.get(8)).not.toEqual(originalHash8); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(8)); + expect(tips.checkpointed.block).toEqual(makeTip(5)); // Still at checkpoint 1 + + // Checkpoint all the new proposed blocks (6-8) + const checkpoint2 = await makeCheckpoint(2, newBlocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint2)); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(8)); + expect(tips.checkpointed.block).toEqual(makeTip(8)); + expect(tips.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2)); + + // Block hashes in the store should reflect the new blocks + expect(await tipsStore.getL2BlockHash(6)).toEqual(blockHashes.get(6)); + expect(await tipsStore.getL2BlockHash(7)).toEqual(blockHashes.get(7)); + expect(await tipsStore.getL2BlockHash(8)).toEqual(blockHashes.get(8)); + + // And should NOT equal the original hashes + expect(await tipsStore.getL2BlockHash(6)).not.toEqual(originalHash6); + expect(await tipsStore.getL2BlockHash(7)).not.toEqual(originalHash7); + expect(await tipsStore.getL2BlockHash(8)).not.toEqual(originalHash8); + }); + + it('handles reorg with different chain length after prune', async () => { + // Propose blocks 1-3 + const firstBlocks = await Promise.all(times(3, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: firstBlocks }); + + // Checkpoint all proposed blocks (1-3) - these are now committed + const checkpoint1 = await makeCheckpoint(1, firstBlocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); + + // Propose more blocks 4-10 (not yet checkpointed, can be pruned) + const originalBlocks4to10 = await Promise.all(times(7, i => makeBlock(i + 4))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: originalBlocks4to10 }); + + let tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(10)); + expect(tips.checkpointed.block).toEqual(makeTip(3)); // Only blocks 1-3 are checkpointed + + // Prune proposed blocks back to checkpoint (block 3) + await tipsStore.handleBlockStreamEvent({ + type: 'chain-pruned', + block: makeBlockId(3), + reason: 'unproven', + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(3)); + expect(tips.checkpointed.block).toEqual(makeTip(3)); // Checkpoint unchanged + + // Now propose only 2 new blocks (4-5) instead of the original 7 blocks (4-10) + blockHashes.delete(4); + blockHashes.delete(5); + const newBlocks = await Promise.all(times(2, i => makeBlock(i + 4))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: newBlocks }); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(5)); + expect(tips.checkpointed.block).toEqual(makeTip(3)); // Still at checkpoint 1 + + // Checkpoint all the new proposed blocks (4-5) + const checkpoint2 = await makeCheckpoint(2, newBlocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint2)); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(5)); + expect(tips.checkpointed.block).toEqual(makeTip(5)); + expect(tips.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2)); + }); + + it('handles reorg: prune back to proven tip (including checkpointed blocks), then re-propose and checkpoint', async () => { + // Propose blocks 1-3 + const firstBlocks = await Promise.all(times(3, i => makeBlock(i + 1))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: firstBlocks }); + + // Checkpoint all proposed blocks (1-3) + const checkpoint1 = await makeCheckpoint(1, firstBlocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint1)); + + // Prove up to block 3 + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', block: makeBlockId(3) }); + + let tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(3)); + expect(tips.checkpointed.block).toEqual(makeTip(3)); + expect(tips.proven.block).toEqual(makeTip(3)); + + // Propose more blocks 4-6 + const blocks4to6 = await Promise.all(times(3, i => makeBlock(i + 4))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: blocks4to6 }); + + // Checkpoint blocks 4-6 (now checkpointed is ahead of proven) + const checkpoint2 = await makeCheckpoint(2, blocks4to6); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint2)); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(6)); + expect(tips.checkpointed.block).toEqual(makeTip(6)); + expect(tips.proven.block).toEqual(makeTip(3)); // Proven is behind checkpointed + + // Propose even more blocks 7-10 (proposed is now ahead of checkpointed) + const originalBlocks7to10 = await Promise.all(times(4, i => makeBlock(i + 7))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: originalBlocks7to10 }); + + tips = await tipsStore.getL2Tips(); + // Now all three tips are different: proposed=10, checkpointed=6, proven=3 + expect(tips.proposed).toEqual(makeTip(10)); + expect(tips.checkpointed.block).toEqual(makeTip(6)); + expect(tips.proven.block).toEqual(makeTip(3)); + + // Store original hashes for blocks 4-7 + const originalHash4 = blockHashes.get(4); + const originalHash5 = blockHashes.get(5); + const originalHash6 = blockHashes.get(6); + const originalHash7 = blockHashes.get(7); + + // Prune all the way back to proven tip (block 3) + // This prunes both proposed blocks (7-10) AND checkpointed blocks (4-6) + await tipsStore.handleBlockStreamEvent({ + type: 'chain-pruned', + block: makeBlockId(3), + reason: 'unproven', + checkpoint: { number: CheckpointNumber.ZERO, hash: '' }, + }); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(3)); + expect(tips.checkpointed.block).toEqual(makeTip(3)); // Checkpointed also pruned back + expect(tips.proven.block).toEqual(makeTip(3)); + + // Propose new blocks 4-7 (different from original) + blockHashes.delete(4); + blockHashes.delete(5); + blockHashes.delete(6); + blockHashes.delete(7); + const newBlocks = await Promise.all(times(4, i => makeBlock(i + 4))); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: newBlocks }); + + // Verify the new blocks have different hashes than the original ones + expect(blockHashes.get(4)).not.toEqual(originalHash4); + expect(blockHashes.get(5)).not.toEqual(originalHash5); + expect(blockHashes.get(6)).not.toEqual(originalHash6); + expect(blockHashes.get(7)).not.toEqual(originalHash7); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(7)); + expect(tips.proven.block).toEqual(makeTip(3)); + + // Checkpoint all the new proposed blocks (4-7) + const checkpoint3 = await makeCheckpoint(3, newBlocks); + await tipsStore.handleBlockStreamEvent(await makeCheckpointedEvent(checkpoint3)); + + tips = await tipsStore.getL2Tips(); + expect(tips.proposed).toEqual(makeTip(7)); + expect(tips.checkpointed.block).toEqual(makeTip(7)); + expect(tips.checkpointed.checkpoint.number).toEqual(CheckpointNumber(3)); + expect(tips.proven.block).toEqual(makeTip(3)); // Proven hasn't moved yet + + // Block hashes in the store should reflect the new blocks + expect(await tipsStore.getL2BlockHash(4)).toEqual(blockHashes.get(4)); + expect(await tipsStore.getL2BlockHash(5)).toEqual(blockHashes.get(5)); + expect(await tipsStore.getL2BlockHash(6)).toEqual(blockHashes.get(6)); + expect(await tipsStore.getL2BlockHash(7)).toEqual(blockHashes.get(7)); + + // And should NOT equal the original hashes + expect(await tipsStore.getL2BlockHash(4)).not.toEqual(originalHash4); + expect(await tipsStore.getL2BlockHash(5)).not.toEqual(originalHash5); + expect(await tipsStore.getL2BlockHash(6)).not.toEqual(originalHash6); + expect(await tipsStore.getL2BlockHash(7)).not.toEqual(originalHash7); }); // Regression test for #13142 it('does not blow up when setting proven chain on an unseen block number', async () => { - await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: [makeBlock(5)] }); + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: [await makeBlock(5)] }); await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', block: makeBlockId(3) }); const tips = await tipsStore.getL2Tips(); - expect(tips).toEqual(makeTips(5, 3, 0)); + expect(tips.proposed).toEqual(makeTip(5)); + expect(tips.proven.block).toEqual(makeTip(3)); + // No checkpoint for block 3 since it wasn't checkpointed + expect(tips.proven.checkpoint.number).toEqual(CheckpointNumber.ZERO); }); } diff --git a/yarn-project/stdlib/src/interfaces/api_limit.ts b/yarn-project/stdlib/src/interfaces/api_limit.ts index 5f7754bdfe34..81956427c57c 100644 --- a/yarn-project/stdlib/src/interfaces/api_limit.ts +++ b/yarn-project/stdlib/src/interfaces/api_limit.ts @@ -1,3 +1,4 @@ export const MAX_RPC_LEN = 100; export const MAX_RPC_TXS_LEN = 50; export const MAX_RPC_BLOCKS_LEN = 50; +export const MAX_RPC_CHECKPOINTS_LEN = 50; diff --git a/yarn-project/stdlib/src/interfaces/archiver.test.ts b/yarn-project/stdlib/src/interfaces/archiver.test.ts index 1033989195af..86fc1b089b4d 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.test.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.test.ts @@ -181,6 +181,14 @@ describe('ArchiverApiSchema', () => { expect(result!.l1).toBeDefined(); }); + it('getCheckpointedBlocks', async () => { + const result = await context.client.getCheckpointedBlocks(BlockNumber(1), 10); + expect(result).toHaveLength(1); + expect(result[0].block.constructor.name).toEqual('L2BlockNew'); + expect(result[0].attestations[0]).toBeInstanceOf(CommitteeAttestation); + expect(result[0].l1).toBeDefined(); + }); + it('getBlocksForEpoch', async () => { const result = await context.client.getBlocksForEpoch(EpochNumber(1)); expect(result).toEqual([expect.any(L2Block)]); @@ -198,10 +206,15 @@ describe('ArchiverApiSchema', () => { it('getL2Tips', async () => { const result = await context.client.getL2Tips(); + const expectedTipId = { + block: { number: 1, hash: `0x01` }, + checkpoint: { number: 1, hash: `0x01` }, + }; expect(result).toEqual({ - latest: { number: 1, hash: `0x01` }, - proven: { number: 1, hash: `0x01` }, - finalized: { number: 1, hash: `0x01` }, + proposed: { number: 1, hash: `0x01` }, + checkpointed: expectedTipId, + proven: expectedTipId, + finalized: expectedTipId, }); }); @@ -319,6 +332,16 @@ describe('ArchiverApiSchema', () => { const result = await context.client.getGenesisValues(); expect(result).toEqual({ genesisArchiveRoot: expect.any(Fr) }); }); + + it('getL2BlockNew', async () => { + const result = await context.client.getL2BlockNew(BlockNumber(1)); + expect(result).toEqual(expect.any(L2BlockNew)); + }); + + it('getL2BlocksNew', async () => { + const result = await context.client.getL2BlocksNew(BlockNumber(1), 1); + expect(result).toEqual([expect.any(L2BlockNew)]); + }); }); class MockArchiver implements ArchiverApi { @@ -364,6 +387,16 @@ class MockArchiver implements ArchiverApi { }), ); } + async getCheckpointedBlocks(from: BlockNumber, _limit: number, _proven?: boolean): Promise { + return [ + CheckpointedL2Block.fromFields({ + checkpointNumber: CheckpointNumber(1), + block: await L2BlockNew.random(from), + attestations: [CommitteeAttestation.random()], + l1: new L1PublishedData(1n, 0n, `0x`), + }), + ]; + } async getBlocks(from: BlockNumber, _limit: number, _proven?: boolean): Promise { return [await L2Block.random(from)]; } @@ -388,6 +421,12 @@ class MockArchiver implements ArchiverApi { }), ]; } + + async getL2BlocksNew(from: BlockNumber, _1: number, _2?: boolean): Promise { + const block = await L2BlockNew.random(from); + return [block]; + } + async getPublishedBlockByHash(_blockHash: Fr): Promise { return PublishedL2Block.fromFields({ block: await L2Block.random(BlockNumber(1)), @@ -448,10 +487,15 @@ class MockArchiver implements ArchiverApi { return Promise.resolve(true); } getL2Tips(): Promise { + const tipId = { + block: { number: BlockNumber(1), hash: `0x01` }, + checkpoint: { number: CheckpointNumber(1), hash: `0x01` }, + }; return Promise.resolve({ - latest: { number: BlockNumber(1), hash: `0x01` }, - proven: { number: BlockNumber(1), hash: `0x01` }, - finalized: { number: BlockNumber(1), hash: `0x01` }, + proposed: { number: BlockNumber(1), hash: `0x01` }, + checkpointed: tipId, + proven: tipId, + finalized: tipId, }); } getL2BlockHash(blockNumber: BlockNumber): Promise { diff --git a/yarn-project/stdlib/src/interfaces/archiver.ts b/yarn-project/stdlib/src/interfaces/archiver.ts index 503ec3d3f0e9..d248732efe80 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.ts @@ -89,6 +89,10 @@ export const ArchiverApiSchema: ApiSchemaFor = { .args(z.union([BlockNumberSchema, z.literal('latest')])) .returns(BlockHeader.schema.optional()), getCheckpointedBlock: z.function().args(BlockNumberSchema).returns(CheckpointedL2Block.schema.optional()), + getCheckpointedBlocks: z + .function() + .args(BlockNumberSchema, schemas.Integer, optional(z.boolean())) + .returns(z.array(CheckpointedL2Block.schema)), getBlocks: z .function() .args(BlockNumberSchema, schemas.Integer, optional(z.boolean())) @@ -101,6 +105,10 @@ export const ArchiverApiSchema: ApiSchemaFor = { .function() .args(BlockNumberSchema, schemas.Integer, optional(z.boolean())) .returns(z.array(PublishedL2Block.schema)), + getL2BlocksNew: z + .function() + .args(BlockNumberSchema, schemas.Integer, optional(z.boolean())) + .returns(z.array(L2BlockNew.schema)), getPublishedBlockByHash: z.function().args(schemas.Fr).returns(PublishedL2Block.schema.optional()), getPublishedBlockByArchive: z.function().args(schemas.Fr).returns(PublishedL2Block.schema.optional()), getBlockHeaderByHash: z.function().args(schemas.Fr).returns(BlockHeader.schema.optional()), diff --git a/yarn-project/stdlib/src/interfaces/aztec-node.test.ts b/yarn-project/stdlib/src/interfaces/aztec-node.test.ts index ed92b4589612..3ec16addd69c 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node.test.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node.test.ts @@ -6,7 +6,7 @@ import { PUBLIC_DATA_TREE_HEIGHT, } from '@aztec/constants'; import { type L1ContractAddresses, L1ContractsNames } from '@aztec/ethereum/l1-contract-addresses'; -import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { Buffer32 } from '@aztec/foundation/buffer'; import { timesAsync } from '@aztec/foundation/collection'; import { randomInt } from '@aztec/foundation/crypto/random'; @@ -23,10 +23,11 @@ import type { ContractArtifact } from '../abi/abi.js'; import { AztecAddress } from '../aztec-address/index.js'; import { PublishedL2Block } from '../block/checkpointed_l2_block.js'; import type { DataInBlock } from '../block/in_block.js'; -import { type BlockParameter, CommitteeAttestation, L2BlockHash } from '../block/index.js'; +import { type BlockParameter, CommitteeAttestation, L2BlockHash, L2BlockNew } from '../block/index.js'; import { L2Block } from '../block/l2_block.js'; import type { L2Tips } from '../block/l2_block_source.js'; -import { L1PublishedData } from '../checkpoint/published_checkpoint.js'; +import { Checkpoint } from '../checkpoint/checkpoint.js'; +import { L1PublishedData, PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import { type ContractClassPublic, type ContractInstanceWithAddress, @@ -93,10 +94,15 @@ describe('AztecNodeApiSchema', () => { it('getL2Tips', async () => { const result = await context.client.getL2Tips(); + const expectedTipId = { + block: { number: 1, hash: `0x01` }, + checkpoint: { number: 1, hash: `0x01` }, + }; expect(result).toEqual({ - latest: { number: 1, hash: `0x01` }, - proven: { number: 1, hash: `0x01` }, - finalized: { number: 1, hash: `0x01` }, + proposed: { number: 1, hash: `0x01` }, + checkpointed: expectedTipId, + proven: expectedTipId, + finalized: expectedTipId, }); }); @@ -258,6 +264,16 @@ describe('AztecNodeApiSchema', () => { await expect(context.client.getBlocks(BlockNumber(1), MAX_RPC_LEN + 1)).rejects.toThrow(); }); + it('getL2BlocksNew', async () => { + const response = await context.client.getL2BlocksNew(BlockNumber(1), BlockNumber(1)); + expect(response).toHaveLength(1); + expect(response[0]).toBeInstanceOf(L2BlockNew); + + await expect(context.client.getBlocks(BlockNumber.ZERO, BlockNumber(1))).rejects.toThrow(); + await expect(context.client.getBlocks(BlockNumber(1), BlockNumber.ZERO)).rejects.toThrow(); + await expect(context.client.getBlocks(BlockNumber(1), MAX_RPC_LEN + 1)).rejects.toThrow(); + }); + it('getPublishedBlocks', async () => { const response = await context.client.getPublishedBlocks(BlockNumber(1), BlockNumber(1)); expect(response).toHaveLength(1); @@ -266,6 +282,17 @@ describe('AztecNodeApiSchema', () => { expect(response[0].l1).toBeDefined(); }); + it('getPublishedCheckpoints', async () => { + const response = await context.client.getPublishedCheckpoints(CheckpointNumber(1), 1); + expect(response).toHaveLength(1); + expect(response[0]).toBeInstanceOf(PublishedCheckpoint); + }); + + it('getCheckpointedBlocks', async () => { + const response = await context.client.getCheckpointedBlocks(BlockNumber(1), 1); + expect(response).toEqual([]); + }); + it('getNodeVersion', async () => { const response = await context.client.getNodeVersion(); expect(response).toBe('1.0.0'); @@ -533,13 +560,27 @@ class MockAztecNode implements AztecNode { } getL2Tips(): Promise { + const tipId = { + block: { number: BlockNumber(1), hash: `0x01` }, + checkpoint: { number: CheckpointNumber(1), hash: `0x01` }, + }; return Promise.resolve({ - latest: { number: BlockNumber(1), hash: `0x01` }, - proven: { number: BlockNumber(1), hash: `0x01` }, - finalized: { number: BlockNumber(1), hash: `0x01` }, + proposed: { number: BlockNumber(1), hash: `0x01` }, + checkpointed: tipId, + proven: tipId, + finalized: tipId, }); } + async getL2BlocksNew(from: BlockNumber, _1: number, _2?: boolean): Promise { + const block = await L2BlockNew.random(from); + return [block]; + } + + getCheckpointedBlocks(_from: BlockNumber, _limit: number, _proven?: boolean) { + return Promise.resolve([]); + } + findLeavesIndexes( blockNumber: number | 'latest', treeId: MerkleTreeId, @@ -701,6 +742,15 @@ class MockAztecNode implements AztecNode { }), ); } + getPublishedCheckpoints(from: CheckpointNumber, limit: number): Promise { + return timesAsync(limit, async i => + PublishedCheckpoint.from({ + checkpoint: await Checkpoint.random(CheckpointNumber(from + i)), + attestations: [CommitteeAttestation.random()], + l1: new L1PublishedData(1n, 1n, Buffer32.random().toString()), + }), + ); + } getNodeVersion(): Promise { return Promise.resolve('1.0.0'); } diff --git a/yarn-project/stdlib/src/interfaces/aztec-node.ts b/yarn-project/stdlib/src/interfaces/aztec-node.ts index 66720ffbffb1..c5423374b9d3 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node.ts @@ -10,6 +10,7 @@ import { BlockNumber, BlockNumberPositiveSchema, BlockNumberSchema, + CheckpointNumberPositiveSchema, EpochNumber, EpochNumberSchema, type SlotNumber, @@ -23,10 +24,12 @@ import { z } from 'zod'; import type { AztecAddress } from '../aztec-address/index.js'; import { type BlockParameter, BlockParameterSchema } from '../block/block_parameter.js'; -import { PublishedL2Block } from '../block/checkpointed_l2_block.js'; +import { CheckpointedL2Block, PublishedL2Block } from '../block/checkpointed_l2_block.js'; import { type DataInBlock, dataInBlockSchemaFor } from '../block/in_block.js'; import { L2Block } from '../block/l2_block.js'; +import { L2BlockNew } from '../block/l2_block_new.js'; import { type L2BlockSource, type L2Tips, L2TipsSchema } from '../block/l2_block_source.js'; +import { PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import { type ContractClassPublic, ContractClassPublicSchema, @@ -59,7 +62,7 @@ import { SingleValidatorStatsSchema, ValidatorsStatsSchema } from '../validators import type { SingleValidatorStats, ValidatorsStats } from '../validators/types.js'; import { type ComponentsVersions, getVersioningResponseHandler } from '../versioning/index.js'; import { type AllowedElement, AllowedElementSchema } from './allowed_element.js'; -import { MAX_RPC_BLOCKS_LEN, MAX_RPC_LEN, MAX_RPC_TXS_LEN } from './api_limit.js'; +import { MAX_RPC_BLOCKS_LEN, MAX_RPC_CHECKPOINTS_LEN, MAX_RPC_LEN, MAX_RPC_TXS_LEN } from './api_limit.js'; import { type GetContractClassLogsResponse, GetContractClassLogsResponseSchema, @@ -73,7 +76,16 @@ import { type WorldStateSyncStatus, WorldStateSyncStatusSchema } from './world_s * We will probably implement the additional interfaces by means other than Aztec Node as it's currently a privacy leak */ export interface AztecNode - extends Pick { + extends Pick< + L2BlockSource, + | 'getBlocks' + | 'getL2BlocksNew' + | 'getPublishedBlocks' + | 'getPublishedCheckpoints' + | 'getBlockHeader' + | 'getL2Tips' + | 'getCheckpointedBlocks' + > { /** * Returns the tips of the L2 chain. */ @@ -582,6 +594,21 @@ export const AztecNodeApiSchema: ApiSchemaFor = { .args(BlockNumberPositiveSchema, z.number().gt(0).lte(MAX_RPC_BLOCKS_LEN)) .returns(z.array(PublishedL2Block.schema)), + getPublishedCheckpoints: z + .function() + .args(CheckpointNumberPositiveSchema, z.number().gt(0).lte(MAX_RPC_CHECKPOINTS_LEN)) + .returns(z.array(PublishedCheckpoint.schema)), + + getL2BlocksNew: z + .function() + .args(BlockNumberPositiveSchema, z.number().gt(0).lte(MAX_RPC_BLOCKS_LEN)) + .returns(z.array(L2BlockNew.schema)), + + getCheckpointedBlocks: z + .function() + .args(BlockNumberPositiveSchema, z.number().gt(0).lte(MAX_RPC_BLOCKS_LEN), optional(z.boolean())) + .returns(z.array(CheckpointedL2Block.schema)), + getCurrentMinFees: z.function().returns(GasFees.schema), getMaxPriorityFees: z.function().returns(GasFees.schema), diff --git a/yarn-project/stdlib/src/interfaces/prover-node.test.ts b/yarn-project/stdlib/src/interfaces/prover-node.test.ts index 539d32059926..884d5110cb7d 100644 --- a/yarn-project/stdlib/src/interfaces/prover-node.test.ts +++ b/yarn-project/stdlib/src/interfaces/prover-node.test.ts @@ -1,4 +1,4 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { type JsonRpcTestContext, createJsonRpcTestSetup } from '@aztec/foundation/json-rpc/test'; import type { L2Tips } from '../block/l2_block_source.js'; @@ -38,10 +38,15 @@ describe('ProvingNodeApiSchema', () => { it('getL2Tips', async () => { const result = await context.client.getL2Tips(); + const expectedTipId = { + block: { number: 1, hash: `0x01` }, + checkpoint: { number: 1, hash: `0x01` }, + }; expect(result).toEqual({ - latest: { number: 1, hash: `0x01` }, - proven: { number: 1, hash: `0x01` }, - finalized: { number: 1, hash: `0x01` }, + proposed: { number: 1, hash: `0x01` }, + checkpointed: expectedTipId, + proven: expectedTipId, + finalized: expectedTipId, }); }); @@ -63,10 +68,15 @@ class MockProverNode implements ProverNodeApi { } getL2Tips(): Promise { + const tipId = { + block: { number: BlockNumber(1), hash: `0x01` }, + checkpoint: { number: CheckpointNumber(1), hash: `0x01` }, + }; return Promise.resolve({ - latest: { number: BlockNumber(1), hash: `0x01` }, - proven: { number: BlockNumber(1), hash: `0x01` }, - finalized: { number: BlockNumber(1), hash: `0x01` }, + proposed: { number: BlockNumber(1), hash: `0x01` }, + checkpointed: tipId, + proven: tipId, + finalized: tipId, }); } diff --git a/yarn-project/stdlib/src/tests/factories.ts b/yarn-project/stdlib/src/tests/factories.ts index 9759e6ba6947..fea2067844aa 100644 --- a/yarn-project/stdlib/src/tests/factories.ts +++ b/yarn-project/stdlib/src/tests/factories.ts @@ -43,7 +43,7 @@ import { VK_TREE_HEIGHT, } from '@aztec/constants'; import { type FieldsOf, makeTuple } from '@aztec/foundation/array'; -import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { compact } from '@aztec/foundation/collection'; import { Grumpkin } from '@aztec/foundation/crypto/grumpkin'; import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto/poseidon'; @@ -88,6 +88,7 @@ import { PublicDataRead } from '../avm/public_data_read.js'; import { PublicDataWrite } from '../avm/public_data_write.js'; import { AztecAddress } from '../aztec-address/index.js'; import { L2BlockHeader } from '../block/l2_block_header.js'; +import type { L2Tips } from '../block/l2_block_source.js'; import { type ContractClassPublic, ContractDeploymentData, @@ -1736,3 +1737,43 @@ export async function randomTxScopedPublicL2Log(opts?: { opts?.firstNullifier ?? Fr.random(), ); } + +/** + * Creates L2Tips with all tips pointing to the same block number. + * Useful for mocking aztecNode.getL2Tips() in tests. + * @param blockNumber - The block number to use for all tips. + * @param hash - Optional hash for the block (defaults to empty string). + * @param checkpointNumber - Optional checkpoint number (defaults to blockNumber). + * @param checkpointHash - Optional checkpoint hash (defaults to block hash). + * @returns L2Tips object with all tips at the same block. + */ +export function makeL2Tips( + blockNumber: number | BlockNumber, + hash = '', + checkpointNumber?: number | CheckpointNumber, + checkpointHash?: string, +): L2Tips { + const bn = typeof blockNumber === 'number' ? BlockNumber(blockNumber) : blockNumber; + const cpn = + checkpointNumber !== undefined + ? typeof checkpointNumber === 'number' + ? CheckpointNumber(checkpointNumber) + : checkpointNumber + : CheckpointNumber(bn); + const cph = checkpointHash ?? hash; + return { + proposed: { number: bn, hash }, + checkpointed: { + block: { number: bn, hash }, + checkpoint: { number: cpn, hash: cph }, + }, + proven: { + block: { number: bn, hash }, + checkpoint: { number: cpn, hash: cph }, + }, + finalized: { + block: { number: bn, hash }, + checkpoint: { number: cpn, hash: cph }, + }, + }; +} diff --git a/yarn-project/telemetry-client/src/wrappers/l2_block_stream.ts b/yarn-project/telemetry-client/src/wrappers/l2_block_stream.ts index 9a32e82e9e2f..8dc87154b445 100644 --- a/yarn-project/telemetry-client/src/wrappers/l2_block_stream.ts +++ b/yarn-project/telemetry-client/src/wrappers/l2_block_stream.ts @@ -11,7 +11,10 @@ import { type Traceable, type Tracer, trackSpan } from '@aztec/telemetry-client' /** Extends an L2BlockStream with a tracer to create a new trace per iteration. */ export class TraceableL2BlockStream extends L2BlockStream implements Traceable { constructor( - l2BlockSource: Pick, + l2BlockSource: Pick< + L2BlockSource, + 'getL2BlocksNew' | 'getBlockHeader' | 'getL2Tips' | 'getPublishedCheckpoints' | 'getCheckpointedBlocks' + >, localData: L2BlockStreamLocalDataProvider, handler: L2BlockStreamEventHandler, public readonly tracer: Tracer, diff --git a/yarn-project/txe/src/state_machine/archiver.ts b/yarn-project/txe/src/state_machine/archiver.ts index 9b355b56a66c..f93b408b2706 100644 --- a/yarn-project/txe/src/state_machine/archiver.ts +++ b/yarn-project/txe/src/state_machine/archiver.ts @@ -7,11 +7,13 @@ import { isDefined } from '@aztec/foundation/types'; import type { AztecAsyncKVStore } from '@aztec/kv-store'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; import { + type CheckpointId, CommitteeAttestation, L2Block, type L2BlockId, type L2BlockNew, type L2BlockSource, + type L2TipId, type L2Tips, PublishedL2Block, type ValidateBlockResult, @@ -93,6 +95,16 @@ export class TXEArchiver extends ArchiverStoreHelper implements L2BlockSource { return this.retrievePublishedBlocks(from, limit, proven); } + async getL2BlocksNew(from: BlockNumber, limit: number, proven?: boolean): Promise { + const blocks = await this.store.getBlocks(from, limit); + + if (proven === true) { + const provenBlockNumber = await this.store.getProvenBlockNumber(); + return blocks.filter(b => b.number <= provenBlockNumber); + } + return blocks; + } + private async retrievePublishedBlocks( from: BlockNumber, limit: number, @@ -211,10 +223,25 @@ export class TXEArchiver extends ArchiverStoreHelper implements L2BlockSource { const number = blockHeader.globalVariables.blockNumber; const hash = (await blockHeader.hash()).toString(); + const checkpointedBlock = await this.getCheckpointedBlock(number); + if (!checkpointedBlock) { + throw new Error(`L2Tips requested from TXE Archiver but no checkpointed block found for block number ${number}`); + } + const checkpoint = await this.store.getRangeOfCheckpoints(CheckpointNumber(number), 1); + if (checkpoint.length === 0) { + throw new Error(`L2Tips requested from TXE Archiver but no checkpoint found for block number ${number}`); + } + const blockId: L2BlockId = { number, hash }; + const checkpointId: CheckpointId = { + number: checkpoint[0].checkpointNumber, + hash: checkpoint[0].header.hash().toString(), + }; + const tipId: L2TipId = { block: blockId, checkpoint: checkpointId }; return { - latest: { number, hash } as L2BlockId, - proven: { number, hash } as L2BlockId, - finalized: { number, hash } as L2BlockId, + proposed: blockId, + proven: tipId, + finalized: tipId, + checkpointed: tipId, }; } @@ -260,4 +287,8 @@ export class TXEArchiver extends ArchiverStoreHelper implements L2BlockSource { getPublishedBlockByArchive(_archive: Fr): Promise { throw new Error('Method not implemented.'); } + + getCheckpointedBlocks(_from: BlockNumber, _limit: number, _proven?: boolean): Promise { + throw new Error('TXE Archiver does not implement "getCheckpointedBlocks"'); + } } diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index 091d873c92c7..89f22f108069 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -2,7 +2,7 @@ import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; import { timesParallel } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { type Logger, createLogger } from '@aztec/foundation/log'; -import { L2BlockNew, type L2BlockSource, type L2BlockStream, type PublishedL2Block } from '@aztec/stdlib/block'; +import { L2BlockNew, type L2BlockSource, type L2BlockStream } from '@aztec/stdlib/block'; import type { Checkpoint } from '@aztec/stdlib/checkpoint'; import { type MerkleTreeReadOperations, WorldStateRunningState } from '@aztec/stdlib/interfaces/server'; import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; @@ -93,15 +93,7 @@ describe('ServerWorldStateSynchronizer', () => { }); const pushBlocks = async (from: number, to: number) => { - const blocks = checkpoints - .flatMap(c => c.checkpoint.blocks) - .filter(b => b.number >= from && b.number <= to) - .map( - b => - ({ - block: { toL2Block: () => b }, - }) as any as PublishedL2Block, - ); + const blocks = checkpoints.flatMap(c => c.checkpoint.blocks).filter(b => b.number >= from && b.number <= to); await server.handleBlockStreamEvent({ type: 'blocks-added', blocks, @@ -270,6 +262,15 @@ class TestWorldStateSynchronizer extends ServerWorldStateSynchronizer { } public override getL2Tips() { - return Promise.resolve({ latest: this.latest, proven: this.proven, finalized: this.finalized }); + const makeTipId = (blockId: typeof this.latest) => ({ + block: blockId, + checkpoint: { number: CheckpointNumber(blockId.number), hash: blockId.hash }, + }); + return Promise.resolve({ + proposed: this.latest, + checkpointed: makeTipId(this.latest), + proven: makeTipId(this.proven), + finalized: makeTipId(this.finalized), + }); } } diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index 3240981d986a..cfe45d9dc9fc 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -1,3 +1,4 @@ +import { INITIAL_L2_BLOCK_NUM, INITIAL_L2_CHECKPOINT_NUM } from '@aztec/constants'; import { BlockNumber } from '@aztec/foundation/branded-types'; import type { Fr } from '@aztec/foundation/curves/bn254'; import { type Logger, createLogger } from '@aztec/foundation/log'; @@ -159,7 +160,7 @@ export class ServerWorldStateSynchronizer } public async getLatestBlockNumber() { - return (await this.getL2Tips()).latest.number; + return (await this.getL2Tips()).proposed.number; } public async stopSync() { @@ -255,10 +256,24 @@ export class ServerWorldStateSynchronizer const unfinalizedBlockHash = await this.getL2BlockHash(status.unfinalizedBlockNumber); const latestBlockId: L2BlockId = { number: status.unfinalizedBlockNumber, hash: unfinalizedBlockHash! }; + // This is all a bit ugly. World state knows nothing of checkpoints and I can't think of any reason + // why anyone depending on world state would need to know if the world state was 'at a checkpoint' or anything similar. + // so we just default a load of stuff here to initial values and empty hashes. + // We could implement a store to track the values but it would be redundant. return { - latest: latestBlockId, - finalized: { number: status.finalizedBlockNumber, hash: '' }, - proven: { number: this.provenBlockNumber ?? status.finalizedBlockNumber, hash: '' }, // TODO(palla/reorg): Using finalized as proven for now + proposed: latestBlockId, + checkpointed: { + block: { number: INITIAL_L2_BLOCK_NUM, hash: '' }, + checkpoint: { number: INITIAL_L2_CHECKPOINT_NUM, hash: '' }, + }, + finalized: { + block: { number: status.finalizedBlockNumber, hash: '' }, + checkpoint: { number: INITIAL_L2_CHECKPOINT_NUM, hash: '' }, + }, + proven: { + block: { number: this.provenBlockNumber ?? status.finalizedBlockNumber, hash: '' }, + checkpoint: { number: INITIAL_L2_CHECKPOINT_NUM, hash: '' }, + }, // TODO(palla/reorg): Using finalized as proven for now }; } @@ -266,7 +281,7 @@ export class ServerWorldStateSynchronizer public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { switch (event.type) { case 'blocks-added': - await this.handleL2Blocks(event.blocks.map(b => b.block.toL2Block())); + await this.handleL2Blocks(event.blocks); break; case 'chain-pruned': await this.handleChainPruned(event.block.number); diff --git a/yarn-project/world-state/src/test/integration.test.ts b/yarn-project/world-state/src/test/integration.test.ts index 93696e48e378..59d491ad84c7 100644 --- a/yarn-project/world-state/src/test/integration.test.ts +++ b/yarn-project/world-state/src/test/integration.test.ts @@ -80,13 +80,13 @@ describe('world-state integration', () => { return finalized > tipFinalized; }; - while (tips.latest.number < blockToSyncTo && sleepTime < maxTimeoutMS) { + while (tips.proposed.number < blockToSyncTo && sleepTime < maxTimeoutMS) { await sleep(100); sleepTime = Date.now() - startTime; tips = await synchronizer.getL2Tips(); } - while (waitForFinalized(tips.finalized.number) && sleepTime < maxTimeoutMS) { + while (waitForFinalized(tips.finalized.block.number) && sleepTime < maxTimeoutMS) { await sleep(100); sleepTime = Date.now() - startTime; tips = await synchronizer.getL2Tips(); @@ -101,11 +101,11 @@ describe('world-state integration', () => { const expectSynchedToBlock = async (latest: number, finalized?: number) => { const tips = await synchronizer.getL2Tips(); - expect(tips.latest.number).toEqual(latest); + expect(tips.proposed.number).toEqual(latest); await expectSynchedBlockHashMatches(latest); if (finalized !== undefined) { - expect(tips.finalized.number).toEqual(finalized); + expect(tips.finalized.block.number).toEqual(finalized); await expectSynchedBlockHashMatches(finalized); } }; @@ -162,7 +162,6 @@ describe('world-state integration', () => { await awaitSync(12); await expectSynchedToBlock(12); - expect(getBlocksSpy).toHaveBeenCalledTimes(3); expect(getBlocksSpy).toHaveBeenCalledWith(1, 5, false); expect(getBlocksSpy).toHaveBeenCalledWith(6, 3, false); expect(getBlocksSpy).toHaveBeenCalledWith(9, 4, false);