feat: integrating new log sync (#19125)

Fixes #17775

In this PR I integrate the new recipient log sync algo and the old one
gets dropped. The file structure is still a bit messy but that I will
handle in a followup PR to make this PR not insane to review.

I Introduced `SenderAddressBook` that was originally part of
`RecipientTaggingDataProvider`. I decided to do this separation because
I want the new log sync algo to not be aware of sender addresses etc. -
I wanted it to deal only with `DirectionalAppTaggingSecret`s as it feels
like a nice separation.

Author: benesjan

noir-projects/noir-contracts/contracts/test/pending_note_hashes_contract/src/main.nr

@@ -294,50 +294,66 @@ pub contract PendingNoteHashes {
     //}
 
     #[external("private")]
-    fn test_recursively_create_notes(
-        owner: AztecAddress,
-        sender: AztecAddress,
-        how_many_recursions: u64,
-    ) {
-        self.internal.create_max_notes(owner);
-
-        self.call_self.recursively_destroy_and_create_notes(owner, sender, how_many_recursions);
+    fn test_recursively_create_notes(recipients: [AztecAddress; 10], how_many_recursions: u64) {
+        let initial_offset: u64 = 0;
+        self.internal.create_max_notes(recipients, initial_offset);
+
+        let max_notes = self.internal.max_notes_per_call() as u64;
+        self.call_self.recursively_destroy_and_create_notes(
+            recipients,
+            how_many_recursions,
+            max_notes,
+        );
     }
 
     #[external("private")]
     fn recursively_destroy_and_create_notes(
-        owner: AztecAddress,
-        sender: AztecAddress,
+        recipients: [AztecAddress; 10],
         executions_left: u64,
+        current_offset: u64,
     ) {
         assert(executions_left > 0);
 
-        self.internal.destroy_max_notes(owner);
-        self.internal.create_max_notes(owner);
+        self.internal.destroy_max_notes(recipients);
+        self.internal.create_max_notes(recipients, current_offset);
 
         let executions_left = executions_left - 1;
 
         if executions_left > 0 {
-            self.call_self.recursively_destroy_and_create_notes(owner, sender, executions_left);
+            let max_notes = self.internal.max_notes_per_call() as u64;
+            self.call_self.recursively_destroy_and_create_notes(
+                recipients,
+                executions_left,
+                current_offset + max_notes,
+            );
         }
     }
 
     #[internal("private")]
-    fn create_max_notes(owner: AztecAddress) {
-        let owner_balance = self.storage.balances.at(owner);
-
+    fn create_max_notes(recipients: [AztecAddress; 10], offset: u64) {
+        // Distribute notes across recipients using global offset to ensure
+        // no recipient receives more than 10 notes (UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN)
         for i in 0..self.internal.max_notes_per_call() {
+            let global_index = offset + i as u64;
+            let recipient_index = (global_index % 10) as u32;
+            let recipient = recipients[recipient_index];
+            let recipient_balance = self.storage.balances.at(recipient);
+
             let note = FieldNote { value: i as Field };
-            owner_balance.insert(note).deliver(MessageDelivery.CONSTRAINED_ONCHAIN);
+            recipient_balance.insert(note).deliver(MessageDelivery.CONSTRAINED_ONCHAIN);
         }
     }
 
     #[internal("private")]
-    fn destroy_max_notes(owner: AztecAddress) {
-        let owner_balance = self.storage.balances.at(owner);
-        // Note that we're relying on PXE actually returning the notes, we're not constraining that any specific
-        // number of notes are deleted.
-        let _ = owner_balance.pop_notes(NoteGetterOptions::new());
+    fn destroy_max_notes(recipients: [AztecAddress; 10]) {
+        // Pop notes from all recipients
+        for i in 0..10 {
+            let recipient = recipients[i];
+            let recipient_balance = self.storage.balances.at(recipient);
+            // Note that we're relying on PXE actually returning the notes, we're not constraining that any specific
+            // number of notes are deleted.
+            let _ = recipient_balance.pop_notes(NoteGetterOptions::new());
+        }
     }
 
     #[internal("private")]

yarn-project/end-to-end/src/bench/bench_build_block.test.ts

@@ -35,14 +35,14 @@ describe('benchmarks/build_block', () => {
   const TX_COUNT = 32;
   it(`builds a block with ${TX_COUNT} standard txs`, async () => {
     sequencer.updateConfig({ minTxsPerBlock: TX_COUNT });
-    const sentTxs = sendTxs(TX_COUNT, context, contract);
+    const sentTxs = await sendTxs(TX_COUNT, context, contract);
     await waitTxs(sentTxs, context);
   });
 
   const TX_COUNT_HEAVY_COMPUTE = 8;
   it(`builds a block with ${TX_COUNT_HEAVY_COMPUTE} compute-heavy txs`, async () => {
     sequencer.updateConfig({ minTxsPerBlock: TX_COUNT_HEAVY_COMPUTE });
-    const sentTxs = sendTxs(TX_COUNT_HEAVY_COMPUTE, context, contract, /*heavyPublicComput=*/ true);
+    const sentTxs = await sendTxs(TX_COUNT_HEAVY_COMPUTE, context, contract, /*heavyPublicComput=*/ true);
     await waitTxs(sentTxs, context);
   });
 });

yarn-project/end-to-end/src/bench/utils.ts

@@ -1,4 +1,5 @@
 import type { AztecNodeService } from '@aztec/aztec-node';
+import { AztecAddress } from '@aztec/aztec.js/addresses';
 import { BatchCall, type SentTx, type WaitOpts } from '@aztec/aztec.js/contracts';
 import { mean, stdDev, times } from '@aztec/foundation/collection';
 import { BenchmarkingContract } from '@aztec/noir-test-contracts.js/Benchmarking';
@@ -103,19 +104,22 @@ function getMetricValues(points: BenchmarkDataPoint[]) {
  * @param heavyPublicCompute - Whether the transactions include heavy public compute (like a big sha256).
  * @returns A BatchCall instance.
  */
-function makeCall(
+async function makeCall(
   index: number,
   context: EndToEndContext,
   contract: BenchmarkingContract,
   heavyPublicCompute: boolean,
 ) {
-  const [owner] = context.accounts;
   if (heavyPublicCompute) {
     return new BatchCall(context.wallet, [contract.methods.sha256_hash_1024(randomBytesAsBigInts(1024))]);
   } else {
+    // We use random address for the new note owner because we can emit at most UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN
+    // logs for a given sender-recipient-contract tuple.
+    const ownerOfNewNote = await AztecAddress.random();
+    const [ownerOfBalance] = context.accounts;
     return new BatchCall(context.wallet, [
-      contract.methods.create_note(owner, index + 1),
-      contract.methods.increment_balance(owner, index + 1),
+      contract.methods.create_note(ownerOfNewNote, index + 1),
+      contract.methods.increment_balance(ownerOfBalance, index + 1),
     ]);
   }
 }
@@ -129,13 +133,13 @@ function makeCall(
  * @param heavyPublicCompute - Whether the transactions include heavy public compute (like a big sha256).
  * @returns Array of sent txs.
  */
-export function sendTxs(
+export async function sendTxs(
   txCount: number,
   context: EndToEndContext,
   contract: BenchmarkingContract,
   heavyPublicCompute: boolean = false,
-): SentTx[] {
-  const calls = times(txCount, index => makeCall(index, context, contract, heavyPublicCompute));
+): Promise<SentTx[]> {
+  const calls = await Promise.all(times(txCount, index => makeCall(index, context, contract, heavyPublicCompute)));
   context.logger.info(`Creating ${txCount} txs`);
   const [from] = context.accounts;
   context.logger.info(`Sending ${txCount} txs`);

yarn-project/end-to-end/src/e2e_pending_note_hashes_contract.test.ts

@@ -1,5 +1,5 @@
-import type { AztecAddress } from '@aztec/aztec.js/addresses';
-import { Fr } from '@aztec/aztec.js/fields';
+import { AztecAddress } from '@aztec/aztec.js/addresses';
+import { Fr, GrumpkinScalar } from '@aztec/aztec.js/fields';
 import type { Logger } from '@aztec/aztec.js/log';
 import type { AztecNode } from '@aztec/aztec.js/node';
 import {
@@ -289,12 +289,23 @@ describe('e2e_pending_note_hashes_contract', () => {
   it('Should handle overflowing the kernel data structures in nested calls', async () => {
     // This test verifies that a transaction can emit more notes than MAX_NOTE_HASHES_PER_TX without failing, since
     // the notes are nullified and will be squashed by the kernel reset circuit.
-    const sender = owner;
+
     const notesPerIteration = Math.min(MAX_NOTE_HASHES_PER_CALL, MAX_NOTE_HASH_READ_REQUESTS_PER_CALL);
     const minToNeedReset = Math.min(MAX_NOTE_HASHES_PER_TX, MAX_NOTE_HASH_READ_REQUESTS_PER_TX) + 1;
     const deployedContract = await deployContract();
+
+    // We use 10 different recipients to send private logs to in order to avoid exceeding
+    // UNFINALIZED_TAGGING_INDEXES_WINDOW_LEN logs emitted for any single sender-recipient pair.
+    const recipients = (
+      await Promise.all(
+        Array.from({ length: 10 }, () =>
+          wallet.createSchnorrAccount(Fr.random(), Fr.random(), GrumpkinScalar.random()),
+        ),
+      )
+    ).map(a => a.address);
+
     await deployedContract.methods
-      .test_recursively_create_notes(owner, sender, Math.ceil(minToNeedReset / notesPerIteration))
+      .test_recursively_create_notes(recipients, Math.ceil(minToNeedReset / notesPerIteration))
       .send({ from: owner })
       .wait();
   });

yarn-project/pxe/src/block_synchronizer/block_synchronizer.test.ts

@@ -11,15 +11,13 @@ import { type MockProxy, mock } from 'jest-mock-extended';
 
 import { AnchorBlockDataProvider } from '../storage/anchor_block_data_provider/anchor_block_data_provider.js';
 import { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js';
-import { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js';
 import { BlockSynchronizer } from './block_synchronizer.js';
 
 describe('BlockSynchronizer', () => {
   let synchronizer: BlockSynchronizer;
   let tipsStore: L2TipsKVStore;
   let anchorBlockDataProvider: AnchorBlockDataProvider;
   let noteDataProvider: NoteDataProvider;
-  let recipientTaggingDataProvider: RecipientTaggingDataProvider;
   let aztecNode: MockProxy<AztecNode>;
   let blockStream: MockProxy<L2BlockStream>;
 
@@ -36,14 +34,7 @@ describe('BlockSynchronizer', () => {
     tipsStore = new L2TipsKVStore(store, 'pxe');
     anchorBlockDataProvider = new AnchorBlockDataProvider(store);
     noteDataProvider = await NoteDataProvider.create(store);
-    recipientTaggingDataProvider = new RecipientTaggingDataProvider(store);
-    synchronizer = new TestSynchronizer(
-      aztecNode,
-      anchorBlockDataProvider,
-      noteDataProvider,
-      recipientTaggingDataProvider,
-      tipsStore,
-    );
+    synchronizer = new TestSynchronizer(aztecNode, anchorBlockDataProvider, noteDataProvider, tipsStore);
   });
 
   it('sets header from latest block', async () => {
@@ -58,9 +49,6 @@ describe('BlockSynchronizer', () => {
     const rollbackNotesAndNullifiers = jest
       .spyOn(noteDataProvider, 'rollbackNotesAndNullifiers')
       .mockImplementation(() => Promise.resolve());
-    const resetNoteSyncData = jest
-      .spyOn(recipientTaggingDataProvider, 'resetNoteSyncData')
-      .mockImplementation(() => Promise.resolve());
     aztecNode.getBlockHeader.mockImplementation(async blockNumber =>
       (await L2Block.random(BlockNumber(blockNumber as number))).getBlockHeader(),
     );
@@ -72,6 +60,5 @@ describe('BlockSynchronizer', () => {
     await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', block: { number: BlockNumber(3), hash: '0x3' } });
 
     expect(rollbackNotesAndNullifiers).toHaveBeenCalledWith(3, 4);
-    expect(resetNoteSyncData).toHaveBeenCalled();
   });
 });

yarn-project/pxe/src/block_synchronizer/block_synchronizer.ts

@@ -7,7 +7,6 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/client';
 import type { PXEConfig } from '../config/index.js';
 import type { AnchorBlockDataProvider } from '../storage/anchor_block_data_provider/anchor_block_data_provider.js';
 import type { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js';
-import type { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js';
 
 /**
  * The BlockSynchronizer class orchestrates synchronization between PXE and Aztec node, maintaining an up-to-date
@@ -23,7 +22,6 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler {
     private node: AztecNode,
     private anchorBlockDataProvider: AnchorBlockDataProvider,
     private noteDataProvider: NoteDataProvider,
-    private recipientTaggingDataProvider: RecipientTaggingDataProvider,
     private l2TipsStore: L2TipsKVStore,
     config: Partial<Pick<PXEConfig, 'l2BlockBatchSize'>> = {},
     loggerOrSuffix?: string | Logger,
@@ -64,12 +62,6 @@ export class BlockSynchronizer implements L2BlockStreamEventHandler {
         // We first unnullify and then remove so that unnullified notes that were created after the block number end up deleted.
         const lastSynchedBlockNumber = (await this.anchorBlockDataProvider.getBlockHeader()).getBlockNumber();
         await this.noteDataProvider.rollbackNotesAndNullifiers(event.block.number, lastSynchedBlockNumber);
-        // Remove all note tagging indexes to force a full resync. This is suboptimal, but unless we track the
-        // block number in which each index is used it's all we can do.
-        // Note: This is now unnecessary for the sender tagging data provider because the new algorithm handles reorgs.
-        // TODO(#17775): Once this issue is implemented we will have the index-block number mapping, so we can
-        // implement this more intelligently.
-        await this.recipientTaggingDataProvider.resetNoteSyncData();
         // Update the header to the last block.
         const newHeader = await this.node.getBlockHeader(event.block.number);
         if (!newHeader) {

yarn-project/pxe/src/contract_function_simulator/contract_function_simulator.ts

@@ -77,8 +77,9 @@ import type { CapsuleDataProvider } from '../storage/capsule_data_provider/capsu
 import type { ContractDataProvider } from '../storage/contract_data_provider/contract_data_provider.js';
 import type { NoteDataProvider } from '../storage/note_data_provider/note_data_provider.js';
 import type { PrivateEventDataProvider } from '../storage/private_event_data_provider/private_event_data_provider.js';
-import type { RecipientTaggingDataProvider } from '../storage/tagging_data_provider/recipient_tagging_data_provider.js';
+import type { SenderAddressBook } from '../storage/tagging_data_provider/sender_address_book.js';
 import type { SenderTaggingDataProvider } from '../storage/tagging_data_provider/sender_tagging_data_provider.js';
+import type { RecipientTaggingDataProvider } from '../tagging/recipient_sync/recipient_tagging_data_provider.js';
 import { ExecutionNoteCache } from './execution_note_cache.js';
 import { ExecutionTaggingIndexCache } from './execution_tagging_index_cache.js';
 import { HashedValuesCache } from './hashed_values_cache.js';
@@ -103,6 +104,7 @@ export class ContractFunctionSimulator {
     private anchorBlockDataProvider: AnchorBlockDataProvider,
     private senderTaggingDataProvider: SenderTaggingDataProvider,
     private recipientTaggingDataProvider: RecipientTaggingDataProvider,
+    private senderAddressBook: SenderAddressBook,
     private capsuleDataProvider: CapsuleDataProvider,
     private privateEventDataProvider: PrivateEventDataProvider,
     private simulator: CircuitSimulator,
@@ -183,6 +185,7 @@ export class ContractFunctionSimulator {
       this.anchorBlockDataProvider,
       this.senderTaggingDataProvider,
       this.recipientTaggingDataProvider,
+      this.senderAddressBook,
       this.capsuleDataProvider,
       this.privateEventDataProvider,
       0, // totalPublicArgsCount
@@ -278,8 +281,8 @@ export class ContractFunctionSimulator {
       this.addressDataProvider,
       this.aztecNode,
       this.anchorBlockDataProvider,
-      this.senderTaggingDataProvider,
       this.recipientTaggingDataProvider,
+      this.senderAddressBook,
       this.capsuleDataProvider,
       this.privateEventDataProvider,
       undefined,

yarn-project/pxe/src/contract_function_simulator/oracle/oracle_version_is_checked.test.ts

@@ -18,8 +18,9 @@ import type { CapsuleDataProvider } from '../../storage/capsule_data_provider/ca
 import type { ContractDataProvider } from '../../storage/contract_data_provider/contract_data_provider.js';
 import type { NoteDataProvider } from '../../storage/note_data_provider/note_data_provider.js';
 import type { PrivateEventDataProvider } from '../../storage/private_event_data_provider/private_event_data_provider.js';
-import type { RecipientTaggingDataProvider } from '../../storage/tagging_data_provider/recipient_tagging_data_provider.js';
+import type { SenderAddressBook } from '../../storage/tagging_data_provider/sender_address_book.js';
 import type { SenderTaggingDataProvider } from '../../storage/tagging_data_provider/sender_tagging_data_provider.js';
+import type { RecipientTaggingDataProvider } from '../../tagging/recipient_sync/recipient_tagging_data_provider.js';
 import { ContractFunctionSimulator } from '../contract_function_simulator.js';
 import { UtilityExecutionOracle } from './utility_execution_oracle.js';
 
@@ -34,6 +35,7 @@ describe('Oracle Version Check test suite', () => {
   let anchorBlockDataProvider: ReturnType<typeof mock<AnchorBlockDataProvider>>;
   let senderTaggingDataProvider: ReturnType<typeof mock<SenderTaggingDataProvider>>;
   let recipientTaggingDataProvider: ReturnType<typeof mock<RecipientTaggingDataProvider>>;
+  let senderAddressBook: ReturnType<typeof mock<SenderAddressBook>>;
   let capsuleDataProvider: ReturnType<typeof mock<CapsuleDataProvider>>;
   let privateEventDataProvider: ReturnType<typeof mock<PrivateEventDataProvider>>;
   let acirSimulator: ContractFunctionSimulator;
@@ -52,6 +54,7 @@ describe('Oracle Version Check test suite', () => {
     anchorBlockDataProvider = mock<AnchorBlockDataProvider>();
     senderTaggingDataProvider = mock<SenderTaggingDataProvider>();
     recipientTaggingDataProvider = mock<RecipientTaggingDataProvider>();
+    senderAddressBook = mock<SenderAddressBook>();
     capsuleDataProvider = mock<CapsuleDataProvider>();
     privateEventDataProvider = mock<PrivateEventDataProvider>();
     utilityAssertCompatibleOracleVersionSpy = jest.spyOn(
@@ -69,10 +72,7 @@ describe('Oracle Version Check test suite', () => {
     senderTaggingDataProvider.getLastUsedIndex.mockResolvedValue(undefined);
     senderTaggingDataProvider.getTxHashesOfPendingIndexes.mockResolvedValue([]);
     senderTaggingDataProvider.storePendingIndexes.mockResolvedValue();
-    recipientTaggingDataProvider.getSenderAddresses.mockResolvedValue([]);
-    recipientTaggingDataProvider.getLastUsedIndexes.mockImplementation(secrets =>
-      Promise.resolve(secrets.map(() => undefined)),
-    );
+
     noteDataProvider.getNotes.mockResolvedValue([]);
     keyStore.getAccounts.mockResolvedValue([]);
 
@@ -100,6 +100,7 @@ describe('Oracle Version Check test suite', () => {
       anchorBlockDataProvider,
       senderTaggingDataProvider,
       recipientTaggingDataProvider,
+      senderAddressBook,
       capsuleDataProvider,
       privateEventDataProvider,
       simulator,