feat: Provisional blocks can be added to archiver (#18949)

This PR makes a number of changes to the archiver.

1. Blocks are now stored separately from checkpoints within the archiver
store.
2. Blocks can be added directly to the archiver as well as being synced
via checkpoints from L1.
3. Blocks added independently are then deemed to be 'checkpointed' once
their checkpoint is received from L1.
4. The archiver now supports multiple blocks per checkpoint.

Some APIs are now considered legacy and will be removed as part of
further work. Previously we had `L2Block` and `PublishedL2Block`. We now
have `L2BlockNew` and `CheckpointedL2Block`. There are currently APIs
for each and this will be updated.

Some operations such manually rolling back blocks still assume 1 block
per checkpoint. This will be fixed when we tackle re-orgs of the
provisional chain more generally.

Author: PhilWindle

yarn-project/archiver/src/archiver/archiver.test.ts

@@ -4,7 +4,8 @@ import type { Fr } from '@aztec/foundation/curves/bn254';
 import type { CustomRange } from '@aztec/kv-store';
 import type { FunctionSelector } from '@aztec/stdlib/abi';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
-import type { L2Block, ValidateBlockResult } from '@aztec/stdlib/block';
+import type { CheckpointedL2Block, L2BlockNew, ValidateBlockResult } from '@aztec/stdlib/block';
+import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
 import type {
   ContractClassPublic,
   ContractInstanceUpdateWithAddress,
@@ -17,14 +18,14 @@ import type { LogFilter, TxScopedL2Log } from '@aztec/stdlib/logs';
 import { BlockHeader, type IndexedTxEffect, type TxHash, type TxReceipt } from '@aztec/stdlib/tx';
 import type { UInt64 } from '@aztec/stdlib/types';
 
+import type { CheckpointData } from './kv_archiver_store/block_store.js';
 import type { InboxMessage } from './structs/inbox_message.js';
-import type { PublishedL2Block } from './structs/published.js';
 
 /**
  * Represents the latest L1 block processed by the archiver for various objects in L2.
  */
 export type ArchiverL1SynchPoint = {
-  /** Number of the last L1 block that added a new L2 block metadata.  */
+  /** Number of the last L1 block that added a new L2 checkpoint metadata.  */
   blocksSynchedTo?: bigint;
   /** Last L1 block checked for L1 to L2 messages. */
   messagesSynchedTo?: L1BlockId;
@@ -45,42 +46,95 @@ export interface ArchiverDataStore {
    * @param opts.force - If true, the blocks will be added even if they have gaps.
    * @returns True if the operation is successful.
    */
-  addBlocks(blocks: PublishedL2Block[], opts?: { force?: boolean }): Promise<boolean>;
+  addBlocks(blocks: L2BlockNew[], opts?: { force?: boolean }): Promise<boolean>;
 
   /**
-   * Unwinds blocks from the database
+   * Appends new checkpoints, and their blocks to the store's collection
+   * @param checkpoints The collectionn of checkpoints to be added
+   * @returns True if the operation is successful
+   */
+  addCheckpoints(checkpoints: PublishedCheckpoint[]): Promise<boolean>;
+
+  /**
+   * Retrieves all blocks for the requested chackpoint
+   * @param checkpointNumber Retreieves all blocks for the given checkpoint
+   * @returns The collection of blocks for the requested checkpoint if available (undefined otherwise)
+   */
+  getBlocksForCheckpoint(checkpointNumber: CheckpointNumber): Promise<L2BlockNew[] | undefined>;
+
+  /**
+   * Returns an array of checkpoint objects
+   * @param from The first checkpoint number to be retrieved
+   * @param limit The maximum number of chackpoints to retrieve
+   * @returns The array of requested checkpoint data objects
+   */
+  getRangeOfCheckpoints(from: CheckpointNumber, limit: number): Promise<CheckpointData[]>;
+
+  /**
+   * Unwinds checkpoints from the database
    * @param from -  The tip of the chain, passed for verification purposes,
    *                ensuring that we don't end up deleting something we did not intend
-   * @param blocksToUnwind - The number of blocks we are to unwind
+   * @param checkpointsToUnwind - The number of checkpoints we are to unwind
    * @returns True if the operation is successful
    */
-  unwindBlocks(from: BlockNumber, blocksToUnwind: number): Promise<boolean>;
+  unwindCheckpoints(from: CheckpointNumber, checkpointsToUnwind: number): Promise<boolean>;
 
   /**
    * Returns the block for the given number, or undefined if not exists.
    * @param number - The block number to return.
    */
-  getPublishedBlock(number: BlockNumber): Promise<PublishedL2Block | undefined>;
+  getCheckpointedBlock(number: number): Promise<CheckpointedL2Block | undefined>;
 
   /**
    * Returns the block for the given hash, or undefined if not exists.
    * @param blockHash - The block hash to return.
    */
-  getPublishedBlockByHash(blockHash: Fr): Promise<PublishedL2Block | undefined>;
+  getCheckpointedBlockByHash(blockHash: Fr): Promise<CheckpointedL2Block | undefined>;
 
   /**
    * Returns the block for the given archive root, or undefined if not exists.
    * @param archive - The archive root to return.
    */
-  getPublishedBlockByArchive(archive: Fr): Promise<PublishedL2Block | undefined>;
+  getCheckpointedBlockByArchive(archive: Fr): Promise<CheckpointedL2Block | undefined>;
+
+  /**
+   * Returns checkpoint data for the requested checkpoint number
+   * @param checkpointNumber - The checkpoint requested
+   * @returns The checkpoint data or undefined if not found
+   */
+  getCheckpointData(checkpointNumber: CheckpointNumber): Promise<CheckpointData | undefined>;
+
+  /**
+   * Returns the number of the latest block
+   * @returns The number of the latest block
+   */
+  getLatestBlockNumber(): Promise<BlockNumber>;
+
+  /**
+   * Returns the block for the given number, or undefined if not exists.
+   * @param number - The block number to return.
+   */
+  getBlock(number: number): Promise<L2BlockNew | undefined>;
+
+  /**
+   * Returns the block for the given hash, or undefined if not exists.
+   * @param blockHash - The block hash to return.
+   */
+  getBlockByHash(blockHash: Fr): Promise<L2BlockNew | undefined>;
+
+  /**
+   * Returns the block for the given archive root, or undefined if not exists.
+   * @param archive - The archive root to return.
+   */
+  getBlockByArchive(archive: Fr): Promise<L2BlockNew | undefined>;
 
   /**
    * Gets up to `limit` amount of published L2 blocks starting from `from`.
    * @param from - Number of the first block to return (inclusive).
    * @param limit - The number of blocks to return.
    * @returns The requested L2 blocks.
    */
-  getPublishedBlocks(from: BlockNumber, limit: number): Promise<PublishedL2Block[]>;
+  getBlocks(from: number, limit: number): Promise<L2BlockNew[]>;
 
   /**
    * Gets up to `limit` amount of L2 block headers starting from `from`.
@@ -121,8 +175,8 @@ export interface ArchiverDataStore {
    * @param blocks - The blocks for which to add the logs.
    * @returns True if the operation is successful.
    */
-  addLogs(blocks: L2Block[]): Promise<boolean>;
-  deleteLogs(blocks: L2Block[]): Promise<boolean>;
+  addLogs(blocks: L2BlockNew[]): Promise<boolean>;
+  deleteLogs(blocks: L2BlockNew[]): Promise<boolean>;
 
   /**
    * Append L1 to L2 messages to the store.
@@ -178,25 +232,37 @@ export interface ArchiverDataStore {
    * Gets the number of the latest L2 block processed.
    * @returns The number of the latest L2 block processed.
    */
-  getSynchedL2BlockNumber(): Promise<BlockNumber>;
+  getCheckpointedL2BlockNumber(): Promise<BlockNumber>;
+
+  /**
+   * Gets the number of the latest published checkpoint processed.
+   * @returns The number of the latest published checkpoint processed
+   */
+  getSynchedCheckpointNumber(): Promise<CheckpointNumber>;
+
+  /**
+   * Gets the number of the latest proven checkpoint processed.
+   * @returns The number of the latest proven checkpoint processed.
+   */
+  getProvenCheckpointNumber(): Promise<CheckpointNumber>;
 
   /**
-   * Gets the number of the latest proven L2 block processed.
-   * @returns The number of the latest proven L2 block processed.
+   * Returns the number of the most recent proven block
+   * @returns The number of the most recent proven block
    */
-  getProvenL2BlockNumber(): Promise<BlockNumber>;
+  getProvenBlockNumber(): Promise<BlockNumber>;
 
   /**
-   * Stores the number of the latest proven L2 block processed.
-   * @param l2BlockNumber - The number of the latest proven L2 block processed.
+   * Stores the number of the latest proven checkpoint processed.
+   * @param checkpointNumber - The number of the latest proven checkpoint processed.
    */
-  setProvenL2BlockNumber(l2BlockNumber: BlockNumber): Promise<void>;
+  setProvenCheckpointNumber(checkpointNumber: CheckpointNumber): Promise<void>;
 
   /**
-   * Stores the l1 block number that blocks have been synched until
+   * Stores the l1 block number that checkpoints have been synched until
    * @param l1BlockNumber  - The l1 block number
    */
-  setBlockSynchedL1BlockNumber(l1BlockNumber: bigint): Promise<void>;
+  setCheckpointSynchedL1BlockNumber(l1BlockNumber: bigint): Promise<void>;
 
   /**
    * Stores the l1 block that messages have been synched until