feat: merge-train/avm (#16991)

BEGIN_COMMIT_OVERRIDE
chore(avm): always print bulk bench metrics
chore: betas optimization - 2^d multiplications (#16963)
docs: Add migration notes for gas costs (#16925)
feat(avm)!: protocol contracts (#16949)
END_COMMIT_OVERRIDE

Author: dbanks12

barretenberg/cpp/.clang-format-ignore

@@ -1,7 +1,8 @@
 include "bc_hashing.pil";
 include "address_derivation.pil";
 include "update_check.pil";
-include "../precomputed.pil";
+include "../ff_gt.pil";
+include "../protocol_contract.pil";
 
 /**
  * Contract Instance Retrieval gadget.
@@ -84,13 +85,22 @@ namespace contract_instance_retrieval;
     pol commit deployer_protocol_contract_address;
     sel * (constants.CONTRACT_INSTANCE_REGISTRY_CONTRACT_ADDRESS - deployer_protocol_contract_address) = 0;
 
-    // Indicates if the instance belongs to a protocol contract. Is hinted by the prover.
-    // If this is set to true for non-protocol contracts the lookup #[PROTOCOL_CONTRACT_DERIVED_ADDRESS] will fail.
-    // If this is set to false for protocol contracts #[ADDRESS_DERIVATION] will fail since it will use the canonical address.
+    // Indicates if the instance belongs to a protocol contract
     pol commit is_protocol_contract;
-    is_protocol_contract * (1 - is_protocol_contract) = 0;
 
-    // If is_protocol_contract = 1, then exists = 1. But if have is_protocol_contract = 0, we don't constrain exists  here.
+    // Canonical Addresses can be in the range of 1 <= address <= MAX_PROTOCOL_CONTRACT_ADDRESS
+    // currently MAX_PROTOCOL_CONTRACT_ADDRESS = ROUTER_ADDRESS (but it is bound to change)
+    pol commit max_protocol_contract_address;
+    max_protocol_contract_address = sel * constants.MAX_PROTOCOL_CONTRACT_ADDRESS;
+
+    pol commit address_sub_one;
+    address_sub_one = sel * (address - 1);
+    #[CHECK_PROTOCOL_ADDRESS_RANGE]
+    sel { max_protocol_contract_address, address_sub_one, is_protocol_contract }
+    in
+    ff_gt.sel { ff_gt.a, ff_gt.b, ff_gt.result};
+
+    // If is_protocol_contract = 1, then exists = 1. But if have is_protocol_contract = 0, we don't constrain exists here.
     // It is later constrained by the nullifier check 
     sel * is_protocol_contract * (1 - exists) = 0 ;
 
@@ -128,13 +138,13 @@ namespace contract_instance_retrieval;
 
     // The address that is checked in address derivation.
     pol commit derived_address;
-    // For protocol contracts we retrieve the derived address from the precomputed trace, else use the input address
+    // For protocol contracts we retrieve the derived address from the protocol contract trace, else use the input address
     #[UNCHANGED_ADDRESS_NON_PROTOCOL]
     sel * (1 - is_protocol_contract) * (derived_address - address) = 0;
 
     #[PROTOCOL_CONTRACT_DERIVED_ADDRESS]
     is_protocol_contract { address, derived_address } in
-    precomputed.sel_protocol_contract { precomputed.clk, precomputed.protocol_contract_derived_address };
+    protocol_contract.sel { protocol_contract.canonical_address, protocol_contract.derived_address };
 
     // Address derivation lookup (only if the nullifier exists or for protocol contract instances)
     #[ADDRESS_DERIVATION]

barretenberg/cpp/pil/vm2/bytecode/contract_instance_retrieval.pil

@@ -1,5 +1,6 @@
 // GENERATED FILE - DO NOT EDIT, RUN yarn remake-constants in yarn-project/constants
 namespace constants;
+    pol PROTOCOL_CONTRACT_TREE_HEIGHT = 3;
     pol NOTE_HASH_TREE_HEIGHT = 40;
     pol PUBLIC_DATA_TREE_HEIGHT = 40;
     pol NULLIFIER_TREE_HEIGHT = 40;
@@ -14,8 +15,13 @@ namespace constants;
     pol MAX_L2_TO_L1_MSGS_PER_TX = 8;
     pol MAX_PUBLIC_LOGS_PER_TX = 8;
     pol MAX_PACKED_PUBLIC_BYTECODE_SIZE_IN_FIELDS = 3000;
+    pol CANONICAL_AUTH_REGISTRY_ADDRESS = 1;
     pol CONTRACT_INSTANCE_REGISTRY_CONTRACT_ADDRESS = 2;
+    pol CONTRACT_CLASS_REGISTRY_CONTRACT_ADDRESS = 3;
+    pol MULTI_CALL_ENTRYPOINT_ADDRESS = 4;
     pol FEE_JUICE_ADDRESS = 5;
+    pol ROUTER_ADDRESS = 6;
+    pol MAX_PROTOCOL_CONTRACT_ADDRESS = 6;
     pol FEE_JUICE_BALANCES_SLOT = 1;
     pol UPDATED_CLASS_IDS_SLOT = 1;
     pol PUBLIC_LOG_SIZE_IN_FIELDS = 13;

barretenberg/cpp/pil/vm2/constants_gen.pil

@@ -0,0 +1,31 @@
+include "./trees/merkle_check.pil";
+include "precomputed.pil";
+include "constants_gen.pil";
+include "poseidon2_hash.pil";
+
+namespace protocol_contract;
+
+    pol commit sel;
+    sel * (1 - sel) = 0;
+
+    #[skippable_if]
+    sel = 0;
+
+    pol commit canonical_address;
+    pol commit derived_address;
+
+    pol commit root; // todo(ilyas): this needs to be constrained against public inputs
+    pol commit tree_depth; // todo: while we don't support constants in lookups
+    sel * (constants.PROTOCOL_CONTRACT_TREE_HEIGHT - tree_depth) = 0;
+
+    pol commit next_derived_address; // Is this ok to be purely hinted?
+    pol commit leaf_hash;
+    #[LEAF_HASH]
+    sel { sel, derived_address, next_derived_address, precomputed.zero, leaf_hash }
+    in
+    poseidon2_hash.end { poseidon2_hash.start, poseidon2_hash.input_0, poseidon2_hash.input_1, poseidon2_hash.input_2, poseidon2_hash.output };
+
+    #[MERKLE_CHECK]
+    sel { leaf_hash, canonical_address, tree_depth, root }
+    in 
+    merkle_check.start { merkle_check.read_node, merkle_check.index, merkle_check.path_len, merkle_check.read_root };

barretenberg/cpp/pil/vm2/protocol_contract.pil

@@ -8,6 +8,7 @@
 #include "barretenberg/common/bb_bench.hpp"
 #include "barretenberg/common/compiler_hints.hpp"
 #include "barretenberg/common/thread.hpp"
+#include "barretenberg/numeric/bitop/get_msb.hpp"
 #include "barretenberg/stdlib/primitives/bool/bool.hpp"
 
 #include <cstddef>
@@ -157,22 +158,62 @@ template <typename FF> struct GateSeparatorPolynomial {
         size_t num_threads = std::min(desired_num_threads, max_num_threads); // fewer than max if justified
         num_threads = num_threads > 0 ? num_threads : 1;                     // ensure num threads is >= 1
         size_t iterations_per_thread = pow_size / num_threads;               // actual iterations per thread
+        const size_t num_betas_per_thread = numeric::get_msb(iterations_per_thread);
+
+        // Explanations of the algorithm:
+        // The product of the betas at index i (beta_products[i]) contains the multiplicative factor betas[j] if and
+        // only if the jth bit of i is 1 (j starting with 0 for the least significant bit). For instance, i = 13 = 1101
+        // in binary, so the product is betas[0] * betas[2] * betas[3]. Observe that if we toggle the kth bit of i (0 to
+        // 1), i.e., we add 2^k to i, then the product is multiplied by betas[k]: beta_products[i + 2^k] =
+        // beta_products[i] * betas[k]. If we know the products for the interval of indices [0, 2^k), we can compute all
+        // the products for the interval of indices [2^k, 2^(k+1)) by multiplying each element by betas[k]. Iteratively,
+        // starting with beta_products[0] = 1, we can double the number of computed products at each iteration by
+        // multiplying the previous products by betas[k]. We first multiply beta_products[0] = 1 by betas[0], then we
+        // multiply beta_products[0] and beta_products[1] by betas[1], etc...
+        //
+        // We distribute the computation of the beta_products evenly across threads, i.e., thread number
+        // thread_idx will handle the interval of indices [thread_idx * iterations_per_thread, (thread_idx + 1) *
+        // iterations_per_thread). Note that for a given thread, all the processed indices have the same
+        // prefix in binary. Therefore, each beta_product of the thread is a multiple of this "prefix product". The
+        // successive products are then populated by the above algorithm whereby we double the interval at each
+        // iteration and multiply by the new beta to process the suffix bits. The difference is that we initialize the
+        // first product with this "prefix product" instead of 1.
+
+        // Compute the prefix products for each thread
+        std::vector<FF> thread_prefix_beta_products(num_threads);
+        thread_prefix_beta_products.at(0) = 1;
+
+        // Same algorithm applies for the prefix products. The difference is that we start at a beta which is not the
+        // first one (index 0), but the one at index num_betas_per_thread. We process the high bits only.
+        // Example: If num_betas_per_thread = 3, we compute after the first iteration:
+        //          (1, beta_3)
+        // 2nd iteration: (1, beta_3, beta_4, beta_3 * beta_4)
+        // 3nd iteration: (1, beta_3, beta_4, beta_3 * beta_4, beta_5, beta_3 * beta_5, beta_4 * beta_5, beta_3 * beta_4
+        // * beta_5)
+        // etc ....
+        for (size_t beta_idx = num_betas_per_thread, window_size = 1; beta_idx < log_num_monomials;
+             beta_idx++, window_size <<= 1) {
+            const auto& beta = betas.at(beta_idx);
+            for (size_t j = 0; j < window_size; j++) {
+                thread_prefix_beta_products.at(window_size + j) = beta * thread_prefix_beta_products.at(j);
+            }
+        }
 
-        // TODO(https://github.com/AztecProtocol/barretenberg/issues/864): This computation is asymtotically slow as it
-        // does pow_size * log(pow_size) work. However, in practice, its super efficient because its trivially
-        // parallelizable and only takes 45ms for the whole 6 iter IVC benchmark. Its also very readable, so we're
-        // leaving it unoptimized for now.
         parallel_for(num_threads, [&](size_t thread_idx) {
             size_t start = thread_idx * iterations_per_thread;
-            size_t end = (thread_idx + 1) * iterations_per_thread;
-            for (size_t i = start; i < end; i++) {
-                auto res = FF(1);
-                for (size_t j = i, beta_idx = 0; j > 0; j >>= 1, beta_idx++) {
-                    if ((j & 1) == 1) {
-                        res *= betas[beta_idx];
-                    }
+            beta_products.at(start) = thread_prefix_beta_products.at(thread_idx);
+
+            // Compute the suffix products for each thread
+            // Example: Assume we start with the prefix product = beta_3 * beta_5
+            // After the first iteration, we get: (beta_3 * beta_5, beta_0 * beta_3 * beta_5)
+            // 2nd iteration: (beta_3 * beta_5, beta_0 * beta_3 * beta_5, beta_1 * beta_3 * beta_5, beta_0 * beta_1 *
+            // beta_3 * beta_5)
+            // etc ...
+            for (size_t beta_idx = 0, window_size = 1; beta_idx < num_betas_per_thread; beta_idx++, window_size <<= 1) {
+                const auto& beta = betas.at(beta_idx);
+                for (size_t j = 0; j < window_size; j++) {
+                    beta_products.at(start + window_size + j) = beta * beta_products.at(start + j);
                 }
-                beta_products[i] = res;
             }
         });
 

barretenberg/cpp/src/barretenberg/polynomials/gate_separator.hpp

@@ -519,7 +519,7 @@ template <typename Flavor> class SumcheckProver {
             // virtually zeroize any leftover values beyond the limit (in-place computation).
             // This is important to zeroize leftover values to not mess up with compute_univariate().
             // Note that the virtual size of pep_view[j] remains unchanged.
-            pep_view[j].shrink_end_index(limit / 2 + limit % 2);
+            pep_view[j].shrink_end_index((limit / 2) + (limit % 2));
         });
     };
     /**
@@ -544,7 +544,7 @@ template <typename Flavor> class SumcheckProver {
             // virtually zeroize any leftover values beyond the limit (in-place computation).
             // This is important to zeroize leftover values to not mess up with compute_univariate().
             // Note that the virtual size of pep_view[j] remains unchanged.
-            pep_view[j].shrink_end_index(limit / 2 + limit % 2);
+            pep_view[j].shrink_end_index((limit / 2) + (limit % 2));
         });
     };
 

barretenberg/cpp/src/barretenberg/sumcheck/sumcheck.hpp

@@ -159,6 +159,15 @@ struct BytecodeCommitmentHint {
     MSGPACK_FIELDS(classId, commitment);
 };
 
+struct ProtocolContractAddressHint {
+    AztecAddress canonicalAddress;
+    AztecAddress derivedAddress;
+
+    bool operator==(const ProtocolContractAddressHint& other) const = default;
+
+    MSGPACK_FIELDS(canonicalAddress, derivedAddress);
+};
+
 ////////////////////////////////////////////////////////////////////////////
 // Hints (merkle db)
 ////////////////////////////////////////////////////////////////////////////
@@ -328,6 +337,8 @@ struct Tx {
 struct ExecutionHints {
     GlobalVariables globalVariables;
     Tx tx;
+    // Protocol Contract Hints
+    std::vector<ProtocolContractAddressHint> protocolContractDerivedAddresses;
     // Contracts.
     std::vector<ContractInstanceHint> contractInstances;
     std::vector<ContractClassHint> contractClasses;
@@ -352,6 +363,7 @@ struct ExecutionHints {
 
     MSGPACK_FIELDS(globalVariables,
                    tx,
+                   protocolContractDerivedAddresses,
                    contractInstances,
                    contractClasses,
                    bytecodeCommitments,

barretenberg/cpp/src/barretenberg/vm2/common/avm_inputs.hpp

@@ -1,6 +1,7 @@
 // GENERATED FILE - DO NOT EDIT, RUN yarn remake-constants in yarn-project/constants
 #pragma once
 
+#define PROTOCOL_CONTRACT_TREE_HEIGHT 3
 #define NOTE_HASH_TREE_HEIGHT 40
 #define PUBLIC_DATA_TREE_HEIGHT 40
 #define NULLIFIER_TREE_HEIGHT 40
@@ -17,12 +18,14 @@
 #define GENESIS_BLOCK_HEADER_HASH "0x0e40440ea6abb9b58877d3aea6628cc6b14b6bc2148ce2be6f46db23360a6aba"
 #define GENESIS_ARCHIVE_ROOT "0x1f9c798be7975bb34c3e605a4c92c75796eae7b9a08644bc9a6a55354ed470be"
 #define MAX_PACKED_PUBLIC_BYTECODE_SIZE_IN_FIELDS 3000
+#define MAX_PROTOCOL_CONTRACTS 7
 #define CANONICAL_AUTH_REGISTRY_ADDRESS 1
 #define CONTRACT_INSTANCE_REGISTRY_CONTRACT_ADDRESS 2
 #define CONTRACT_CLASS_REGISTRY_CONTRACT_ADDRESS 3
 #define MULTI_CALL_ENTRYPOINT_ADDRESS 4
 #define FEE_JUICE_ADDRESS 5
 #define ROUTER_ADDRESS 6
+#define MAX_PROTOCOL_CONTRACT_ADDRESS 6
 #define FEE_JUICE_BALANCES_SLOT 1
 #define UPDATED_CLASS_IDS_SLOT 1
 #define PUBLIC_LOG_SIZE_IN_FIELDS 13