chore: Update L2 block stream with new block format and checkpoints

yarn-project/archiver/src/archiver/archiver.ts

@@ -1,5 +1,5 @@
 import type { BlobClientInterface } from '@aztec/blob-client/client';
-import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/constants';
+import { GENESIS_BLOCK_HEADER_HASH, INITIAL_L2_BLOCK_NUM, INITIAL_L2_CHECKPOINT_NUM } from '@aztec/constants';
 import { EpochCache } from '@aztec/epoch-cache';
 import { createEthereumChain } from '@aztec/ethereum/chain';
 import { BlockTagTooOldError, InboxContract, RollupContract } from '@aztec/ethereum/contracts';
@@ -593,14 +593,11 @@ export class Archiver
       );
       const newBlocks = blockPromises.filter(isDefined).flat();
 
-      // TODO(pw/mbps): Don't convert to legacy blocks here
-      const blocks: L2Block[] = (await Promise.all(newBlocks.map(x => this.getBlock(x.number)))).filter(isDefined);
-
       // Emit an event for listening services to react to the chain prune
       this.emit(L2BlockSourceEvents.L2PruneDetected, {
         type: L2BlockSourceEvents.L2PruneDetected,
         epochNumber: pruneFromEpochNumber,
-        blocks,
+        blocks: newBlocks,
       });
 
       this.log.debug(
@@ -1392,6 +1389,16 @@ export class Archiver
     return publishedBlock;
   }
 
+  public async getL2BlocksNew(from: BlockNumber, limit: number, proven?: boolean): Promise<L2BlockNew[]> {
+    const blocks = await this.store.store.getBlocks(from, limit);
+
+    if (proven === true) {
+      const provenBlockNumber = await this.store.getProvenBlockNumber();
+      return blocks.filter(b => b.number <= provenBlockNumber);
+    }
+    return blocks;
+  }
+
   public async getBlockHeader(number: BlockNumber | 'latest'): Promise<BlockHeader | undefined> {
     if (number === 'latest') {
       number = await this.store.getSynchedL2BlockNumber();
@@ -1407,13 +1414,30 @@ export class Archiver
     return this.store.getCheckpointedBlock(number);
   }
 
+  public async getCheckpointedBlocks(
+    from: BlockNumber,
+    limit: number,
+    proven?: boolean,
+  ): Promise<CheckpointedL2Block[]> {
+    const blocks = await this.store.store.getCheckpointedBlocks(from, limit);
+
+    if (proven === true) {
+      const provenBlockNumber = await this.store.getProvenBlockNumber();
+      return blocks.filter(b => b.block.number <= provenBlockNumber);
+    }
+    return blocks;
+  }
+
   getCheckpointedBlockByHash(blockHash: Fr): Promise<CheckpointedL2Block | undefined> {
     return this.store.getCheckpointedBlockByHash(blockHash);
   }
 
   getProvenBlockNumber(): Promise<BlockNumber> {
     return this.store.getProvenBlockNumber();
   }
+  getCheckpointedBlockNumber(): Promise<BlockNumber> {
+    return this.store.getCheckpointedL2BlockNumber();
+  }
   getCheckpointedBlockByArchive(archive: Fr): Promise<CheckpointedL2Block | undefined> {
     return this.store.getCheckpointedBlockByArchive(archive);
   }
@@ -1524,54 +1548,125 @@ export class Archiver
   }
 
   async getL2Tips(): Promise<L2Tips> {
-    const [latestBlockNumber, provenBlockNumber] = await Promise.all([
+    const [latestBlockNumber, provenBlockNumber, checkpointedBlockNumber] = await Promise.all([
       this.getBlockNumber(),
       this.getProvenBlockNumber(),
+      this.getCheckpointedBlockNumber(),
     ] as const);
 
     // TODO(#13569): Compute proper finalized block number based on L1 finalized block.
     // We just force it 2 epochs worth of proven data for now.
     // NOTE: update end-to-end/src/e2e_epochs/epochs_empty_blocks.test.ts as that uses finalized blocks in computations
     const finalizedBlockNumber = BlockNumber(Math.max(provenBlockNumber - this.l1constants.epochDuration * 2, 0));
 
-    const [latestBlockHeader, provenBlockHeader, finalizedBlockHeader] = await Promise.all([
-      latestBlockNumber > 0 ? this.getBlockHeader(latestBlockNumber) : undefined,
-      provenBlockNumber > 0 ? this.getBlockHeader(provenBlockNumber) : undefined,
-      finalizedBlockNumber > 0 ? this.getBlockHeader(finalizedBlockNumber) : undefined,
-    ] as const);
+    const beforeInitialblockNumber = BlockNumber(INITIAL_L2_BLOCK_NUM - 1);
 
-    if (latestBlockNumber > 0 && !latestBlockHeader) {
+    // Get the latest block header and checkpointed blocks for proven, finalised and checkpointed blocks
+    const [latestBlockHeader, provenCheckpointedBlock, finalizedCheckpointedBlock, checkpointedBlock] =
+      await Promise.all([
+        latestBlockNumber > beforeInitialblockNumber ? this.getBlockHeader(latestBlockNumber) : undefined,
+        provenBlockNumber > beforeInitialblockNumber ? this.getCheckpointedBlock(provenBlockNumber) : undefined,
+        finalizedBlockNumber > beforeInitialblockNumber ? this.getCheckpointedBlock(finalizedBlockNumber) : undefined,
+        checkpointedBlockNumber > beforeInitialblockNumber
+          ? this.getCheckpointedBlock(checkpointedBlockNumber)
+          : undefined,
+      ] as const);
+
+    const beforeInitialCheckpointNumber = CheckpointNumber(INITIAL_L2_CHECKPOINT_NUM - 1);
+
+    if (latestBlockNumber > beforeInitialblockNumber && !latestBlockHeader) {
       throw new Error(`Failed to retrieve latest block header for block ${latestBlockNumber}`);
     }
 
-    if (provenBlockNumber > 0 && !provenBlockHeader) {
+    // Checkpointed blocks must exist for proven, finalized and checkpointed tips if they are beyond the initial block number.
+    if (checkpointedBlockNumber > beforeInitialblockNumber && !checkpointedBlock?.block.header) {
       throw new Error(
-        `Failed to retrieve proven block header for block ${provenBlockNumber} (latest block is ${latestBlockNumber})`,
+        `Failed to retrieve checkpointed block header for block ${checkpointedBlockNumber} (latest block is ${latestBlockNumber})`,
       );
     }
 
-    if (finalizedBlockNumber > 0 && !finalizedBlockHeader) {
+    if (provenBlockNumber > beforeInitialblockNumber && !provenCheckpointedBlock?.block.header) {
+      throw new Error(
+        `Failed to retrieve proven checkpointed for block ${provenBlockNumber} (latest block is ${latestBlockNumber})`,
+      );
+    }
+
+    if (finalizedBlockNumber > beforeInitialblockNumber && !finalizedCheckpointedBlock?.block.header) {
       throw new Error(
         `Failed to retrieve finalized block header for block ${finalizedBlockNumber} (latest block is ${latestBlockNumber})`,
       );
     }
 
     const latestBlockHeaderHash = (await latestBlockHeader?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
-    const provenBlockHeaderHash = (await provenBlockHeader?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
-    const finalizedBlockHeaderHash = (await finalizedBlockHeader?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
+    const provenBlockHeaderHash = (await provenCheckpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
+    const finalizedBlockHeaderHash =
+      (await finalizedCheckpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
+    const checkpointedBlockHeaderHash = (await checkpointedBlock?.block.header?.hash()) ?? GENESIS_BLOCK_HEADER_HASH;
+
+    // Now attempt to retrieve checkpoints for proven, finalised and checkpointed blocks
+    const [[provenBlockCheckpoint], [finalizedBlockCheckpoint], [checkpointedBlockCheckpoint]] = await Promise.all([
+      provenCheckpointedBlock !== undefined
+        ? await this.getPublishedCheckpoints(provenCheckpointedBlock?.checkpointNumber, 1)
+        : [undefined],
+      finalizedCheckpointedBlock !== undefined
+        ? await this.getPublishedCheckpoints(finalizedCheckpointedBlock?.checkpointNumber, 1)
+        : [undefined],
+      checkpointedBlock !== undefined
+        ? await this.getPublishedCheckpoints(checkpointedBlock?.checkpointNumber, 1)
+        : [undefined],
+    ]);
 
-    return {
-      latest: { number: latestBlockNumber, hash: latestBlockHeaderHash.toString() },
-      proven: { number: provenBlockNumber, hash: provenBlockHeaderHash.toString() },
-      finalized: { number: finalizedBlockNumber, hash: finalizedBlockHeaderHash.toString() },
+    const initialcheckpointId = {
+      number: beforeInitialCheckpointNumber,
+      hash: '',
     };
+
+    const makeCheckpointId = (checkpoint: PublishedCheckpoint | undefined) => {
+      if (checkpoint === undefined) {
+        return initialcheckpointId;
+      }
+      return {
+        number: checkpoint.checkpoint.number,
+        hash: checkpoint.checkpoint.hash().toString(),
+      };
+    };
+
+    const l2Tips: L2Tips = {
+      proposed: {
+        number: latestBlockNumber,
+        hash: latestBlockHeaderHash.toString(),
+      },
+      proven: {
+        block: {
+          number: provenBlockNumber,
+          hash: provenBlockHeaderHash.toString(),
+        },
+        checkpoint: makeCheckpointId(provenBlockCheckpoint),
+      },
+      finalized: {
+        block: {
+          number: finalizedBlockNumber,
+          hash: finalizedBlockHeaderHash.toString(),
+        },
+        checkpoint: makeCheckpointId(finalizedBlockCheckpoint),
+      },
+      checkpointed: {
+        block: {
+          number: checkpointedBlockNumber,
+          hash: checkpointedBlockHeaderHash.toString(),
+        },
+        checkpoint: makeCheckpointId(checkpointedBlockCheckpoint),
+      },
+    };
+
+    return l2Tips;
   }
 
   public async rollbackTo(targetL2BlockNumber: BlockNumber): Promise<void> {
     // TODO(pw/mbps): This still assumes 1 block per checkpoint
     const currentBlocks = await this.getL2Tips();
-    const currentL2Block = currentBlocks.latest.number;
-    const currentProvenBlock = currentBlocks.proven.number;
+    const currentL2Block = currentBlocks.proposed.number;
+    const currentProvenBlock = currentBlocks.proven.block.number;
 
     if (targetL2BlockNumber >= currentL2Block) {
       throw new Error(`Target L2 block ${targetL2BlockNumber} must be less than current L2 block ${currentL2Block}`);
@@ -1777,6 +1872,7 @@ export class ArchiverStoreHelper
       | 'addBlocks'
       | 'getBlock'
       | 'getBlocks'
+      | 'getCheckpointedBlocks'
     >
 {
   #log = createLogger('archiver:block-helper');

yarn-project/archiver/src/archiver/archiver_store.ts

@@ -85,6 +85,14 @@ export interface ArchiverDataStore {
    */
   getCheckpointedBlock(number: number): Promise<CheckpointedL2Block | undefined>;
 
+  /**
+   * Gets up to `limit` amount of checkpointed L2 blocks starting from `from`.
+   * @param from - Number of the first block to return (inclusive).
+   * @param limit - The number of blocks to return.
+   * @returns The requested checkpointed L2 blocks.
+   */
+  getCheckpointedBlocks(from: number, limit: number): Promise<CheckpointedL2Block[]>;
+
   /**
    * Returns the block for the given hash, or undefined if not exists.
    * @param blockHash - The block hash to return.

yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts

@@ -503,6 +503,34 @@ export class BlockStore {
     );
   }
 
+  /**
+   * Gets up to `limit` amount of Checkpointed L2 blocks starting from `from`.
+   * @param start - Number of the first block to return (inclusive).
+   * @param limit - The number of blocks to return.
+   * @returns The requested L2 blocks
+   */
+  async *getCheckpointedBlocks(start: BlockNumber, limit: number): AsyncIterableIterator<CheckpointedL2Block> {
+    const checkpointCache = new Map<CheckpointNumber, CheckpointStorage>();
+    for await (const [blockNumber, blockStorage] of this.getBlockStorages(start, limit)) {
+      const block = await this.getBlockFromBlockStorage(blockNumber, blockStorage);
+      if (block) {
+        const checkpoint =
+          checkpointCache.get(CheckpointNumber(blockStorage.checkpointNumber)) ??
+          (await this.#checkpoints.getAsync(blockStorage.checkpointNumber));
+        if (checkpoint) {
+          checkpointCache.set(CheckpointNumber(blockStorage.checkpointNumber), checkpoint);
+          const checkpointedBlock = new CheckpointedL2Block(
+            CheckpointNumber(checkpoint.checkpointNumber),
+            block,
+            L1PublishedData.fromBuffer(checkpoint.l1),
+            checkpoint.attestations.map(buf => CommitteeAttestation.fromBuffer(buf)),
+          );
+          yield checkpointedBlock;
+        }
+      }
+    }
+  }
+
   async getCheckpointedBlockByHash(blockHash: Fr): Promise<CheckpointedL2Block | undefined> {
     const blockNumber = await this.#blockHashIndex.getAsync(blockHash.toString());
     if (blockNumber === undefined) {

yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts

@@ -234,6 +234,10 @@ export class KVArchiverDataStore implements ArchiverDataStore, ContractDataSourc
     return toArray(this.#blockStore.getBlocks(from, limit));
   }
 
+  getCheckpointedBlocks(from: BlockNumber, limit: number): Promise<CheckpointedL2Block[]> {
+    return toArray(this.#blockStore.getCheckpointedBlocks(from, limit));
+  }
+
   /**
    * Gets up to `limit` amount of L2 blocks headers starting from `from`.
    *

yarn-project/archiver/src/test/mock_l1_to_l2_message_source.ts

@@ -1,6 +1,6 @@
-import { BlockNumber, type CheckpointNumber } from '@aztec/foundation/branded-types';
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
 import { Fr } from '@aztec/foundation/curves/bn254';
-import type { L2Tips } from '@aztec/stdlib/block';
+import type { CheckpointId, L2BlockId, L2TipId, L2Tips } from '@aztec/stdlib/block';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
 
 /**
@@ -33,9 +33,15 @@ export class MockL1ToL2MessageSource implements L1ToL2MessageSource {
 
   getL2Tips(): Promise<L2Tips> {
     const number = this.blockNumber;
-    const tip = { number: BlockNumber(number), hash: new Fr(number).toString() };
+    const blockId: L2BlockId = { number: BlockNumber(number), hash: new Fr(number).toString() };
+    const checkpointId: CheckpointId = {
+      number: CheckpointNumber(number),
+      hash: new Fr(number + 1).toString(),
+    };
+    const tip: L2TipId = { block: blockId, checkpoint: checkpointId };
     return Promise.resolve({
-      latest: tip,
+      proposed: blockId,
+      checkpointed: tip,
       proven: tip,
       finalized: tip,
     });