feat: merge-train/barretenberg (#17971)

BEGIN_COMMIT_OVERRIDE
fix: make vm2_sim sources mutually exclusive and conditional linking
(#17969)
fix(build/avm): vm2_sim as dependency
chore: improved prover polynomial allocation (#17956)
END_COMMIT_OVERRIDE

Author: AztecBot

barretenberg/cpp/src/barretenberg/client_ivc/sumcheck_client_ivc.test.cpp

@@ -98,7 +98,7 @@ TEST_F(SumcheckClientIVCTests, TestCircuitSizes)
  */
 TEST_F(SumcheckClientIVCTests, Basic)
 {
-    const size_t NUM_APP_CIRCUITS = 5;
+    const size_t NUM_APP_CIRCUITS = 2;
     auto [proof, vk] = accumulate_and_prove_ivc(NUM_APP_CIRCUITS);
 
     EXPECT_TRUE(SumcheckClientIVC::verify(proof, vk));

barretenberg/cpp/src/barretenberg/constants.hpp

@@ -29,8 +29,6 @@ static constexpr uint32_t MEGA_AVM_LOG_N = 21;
 
 static constexpr uint32_t CONST_ECCVM_LOG_N = 16;
 
-static constexpr uint32_t MAX_DATABUS_SIZE = 10000;
-
 // The number of last rows in ProverPolynomials that are randomized to mask
 // 1) witness commitments,
 // 2) multilinear evaluations of witness polynomials in Sumcheck

barretenberg/cpp/src/barretenberg/polynomials/polynomial.hpp

@@ -108,6 +108,13 @@ template <typename Fr> class Polynomial {
     {
         return Polynomial(/*actual size*/ virtual_size - 1, virtual_size, /*shiftable offset*/ 1);
     }
+    /**
+     * @brief Utility to create a shiftable polynomial of given size and virtual size.
+     */
+    static Polynomial shiftable(size_t size, size_t virtual_size)
+    {
+        return Polynomial(/*actual size*/ size - 1, virtual_size, /*shiftable offset*/ 1);
+    }
     // Allow polynomials to be entirely reset/dormant
     Polynomial() = default;
 

barretenberg/cpp/src/barretenberg/ultra_honk/prover_instance.cpp

@@ -40,41 +40,35 @@ template <IsUltraOrMegaHonk Flavor> void ProverInstance_<Flavor>::allocate_wires
 {
     BB_BENCH_NAME("allocate_wires");
 
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1555):Wires can be allocated based on final active row
-    // rather than dyadic size.
+    // Allocate only enough memory for the active range; wires are zero outside this range
     for (auto& wire : polynomials.get_wires()) {
-        wire = Polynomial::shiftable(dyadic_size());
+        wire = Polynomial::shiftable(final_active_wire_idx + 1, dyadic_size());
     }
 }
 
 template <IsUltraOrMegaHonk Flavor> void ProverInstance_<Flavor>::allocate_permutation_argument_polynomials()
 {
     BB_BENCH_NAME("allocate_permutation_argument_polynomials");
 
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1555): Sigma and id polynomials can be allocated based
-    // on final active row rather than dyadic size.
+    // Allocate only enough memory for the active range; permutation polynomials are zero outside this range
     for (auto& sigma : polynomials.get_sigmas()) {
-        sigma = Polynomial(dyadic_size());
+        sigma = Polynomial::shiftable(final_active_wire_idx + 1, dyadic_size());
     }
     for (auto& id : polynomials.get_ids()) {
-        id = Polynomial(dyadic_size());
+        id = Polynomial::shiftable(final_active_wire_idx + 1, dyadic_size());
     }
-    polynomials.z_perm = Polynomial::shiftable(dyadic_size());
+    polynomials.z_perm = Polynomial::shiftable(final_active_wire_idx + 1, dyadic_size());
 }
 
 template <IsUltraOrMegaHonk Flavor> void ProverInstance_<Flavor>::allocate_lagrange_polynomials()
 {
     BB_BENCH_NAME("allocate_lagrange_polynomials");
 
-    // First and last lagrange polynomials (in the full circuit size)
     polynomials.lagrange_first = Polynomial(
         /* size=*/1, /*virtual size=*/dyadic_size(), /*start_index=*/0);
 
-    // Even though lagrange_last has a single non-zero element, we cannot set its size to 0 as different
-    // instances being folded might have lagrange_last set at different indexes and folding does not work
-    // correctly unless the polynomial is allocated in the correct range to accomodate this
     polynomials.lagrange_last = Polynomial(
-        /* size=*/dyadic_size(), /*virtual size=*/dyadic_size(), /*start_index=*/0);
+        /* size=*/1, /*virtual size=*/dyadic_size(), /*start_index=*/final_active_wire_idx);
 }
 
 template <IsUltraOrMegaHonk Flavor> void ProverInstance_<Flavor>::allocate_selectors(const Circuit& circuit)
@@ -98,28 +92,25 @@ void ProverInstance_<Flavor>::allocate_table_lookup_polynomials(const Circuit& c
     BB_BENCH_NAME("allocate_table_lookup_and_lookup_read_polynomials");
 
     size_t table_offset = circuit.blocks.lookup.trace_offset();
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1555): Can allocate table polynomials based on genuine
-    // lookup table sizes in all cases. Same applies to read_counts/tags, except for ZK case.
-    const size_t max_tables_size = dyadic_size() - table_offset;
-    BB_ASSERT_GT(dyadic_size(), max_tables_size);
+    const size_t tables_size = circuit.get_tables_size(); // cumulative size of all lookup tables used in the circuit
+    BB_ASSERT_GT(dyadic_size(), tables_size);
 
     // Allocate the polynomials containing the actual table data
-    if constexpr (IsUltraOrMegaHonk<Flavor>) {
-        for (auto& poly : polynomials.get_tables()) {
-            poly = Polynomial(max_tables_size, dyadic_size(), table_offset);
-        }
+    for (auto& poly : polynomials.get_tables()) {
+        poly = Polynomial(tables_size, dyadic_size(), table_offset);
     }
 
     // Allocate the read counts and tags polynomials
-    polynomials.lookup_read_counts = Polynomial(max_tables_size, dyadic_size(), table_offset);
-    polynomials.lookup_read_tags = Polynomial(max_tables_size, dyadic_size(), table_offset);
+    polynomials.lookup_read_counts = Polynomial(tables_size, dyadic_size(), table_offset);
+    polynomials.lookup_read_tags = Polynomial(tables_size, dyadic_size(), table_offset);
 
+    // Determine end index for the lookup block and the tables themselves
+    // Note that the start of the tables is aligned with the start of the lookup block in the trace
     const size_t lookup_block_end =
         static_cast<size_t>(circuit.blocks.lookup.trace_offset()) + circuit.blocks.lookup.size();
-    const auto tables_end = circuit.blocks.lookup.trace_offset() + max_tables_size;
+    const auto tables_end = circuit.blocks.lookup.trace_offset() + tables_size;
 
     // Allocate the lookup_inverses polynomial
-
     const size_t lookup_inverses_start = table_offset;
     const size_t lookup_inverses_end = std::max(lookup_block_end, tables_end);
 
@@ -146,25 +137,26 @@ void ProverInstance_<Flavor>::allocate_databus_polynomials(const Circuit& circui
     requires HasDataBus<Flavor>
 {
     BB_BENCH_NAME("allocate_databus_and_lookup_inverse_polynomials");
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1555): Each triple of databus polynomials can be
-    // allocated based on the size of the corresponding column (except for ZK case).
-    const size_t poly_size = std::min(static_cast<size_t>(MAX_DATABUS_SIZE), dyadic_size());
-    polynomials.calldata = Polynomial(poly_size, dyadic_size());
-    polynomials.calldata_read_counts = Polynomial(poly_size, dyadic_size());
-    polynomials.calldata_read_tags = Polynomial(poly_size, dyadic_size());
-    polynomials.secondary_calldata = Polynomial(poly_size, dyadic_size());
-    polynomials.secondary_calldata_read_counts = Polynomial(poly_size, dyadic_size());
-    polynomials.secondary_calldata_read_tags = Polynomial(poly_size, dyadic_size());
-    polynomials.return_data = Polynomial(poly_size, dyadic_size());
-    polynomials.return_data_read_counts = Polynomial(poly_size, dyadic_size());
-    polynomials.return_data_read_tags = Polynomial(poly_size, dyadic_size());
 
-    // Allocate log derivative lookup argument inverse polynomials
-    const size_t q_busread_end = circuit.blocks.busread.trace_offset() + circuit.blocks.busread.size();
     const size_t calldata_size = circuit.get_calldata().size();
     const size_t secondary_calldata_size = circuit.get_secondary_calldata().size();
     const size_t return_data_size = circuit.get_return_data().size();
 
+    polynomials.calldata = Polynomial(calldata_size, dyadic_size());
+    polynomials.calldata_read_counts = Polynomial(calldata_size, dyadic_size());
+    polynomials.calldata_read_tags = Polynomial(calldata_size, dyadic_size());
+
+    polynomials.secondary_calldata = Polynomial(secondary_calldata_size, dyadic_size());
+    polynomials.secondary_calldata_read_counts = Polynomial(secondary_calldata_size, dyadic_size());
+    polynomials.secondary_calldata_read_tags = Polynomial(secondary_calldata_size, dyadic_size());
+
+    polynomials.return_data = Polynomial(return_data_size, dyadic_size());
+    polynomials.return_data_read_counts = Polynomial(return_data_size, dyadic_size());
+    polynomials.return_data_read_tags = Polynomial(return_data_size, dyadic_size());
+
+    // Allocate log derivative lookup argument inverse polynomials
+    const size_t q_busread_end = circuit.blocks.busread.trace_offset() + circuit.blocks.busread.size();
+
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1555): Size of databus_id can always be set to max size
     // between the three databus columns. It currently uses dyadic_size because its values are later set based on its
     // size(). This means when we naively construct all ProverPolynomials with dyadic size (e.g. for ZK), we get a

barretenberg/cpp/src/barretenberg/ultra_honk/prover_instance.hpp

@@ -79,8 +79,9 @@ template <IsUltraOrMegaHonk Flavor_> class ProverInstance_ {
     using Polynomial = typename Flavor::Polynomial;
     using SubrelationSeparator = typename Flavor::SubrelationSeparator;
 
-    MetaData metadata;                 // circuit size and public inputs metadata
-    size_t final_active_wire_idx{ 0 }; // idx of last non-trivial wire value in the trace
+    MetaData metadata; // circuit size and public inputs metadata
+    // index of the last constrained wire in the execution trace; initialize to size_t::max to indicate uninitialized
+    size_t final_active_wire_idx{ std::numeric_limits<size_t>::max() };
 
   public:
     using Trace = TraceToPolynomials<Flavor>;
@@ -115,7 +116,12 @@ template <IsUltraOrMegaHonk Flavor_> class ProverInstance_ {
         return metadata.num_public_inputs;
     }
     MetaData get_metadata() const { return metadata; }
-    size_t get_final_active_wire_idx() const { return final_active_wire_idx; }
+    size_t get_final_active_wire_idx() const
+    {
+        BB_ASSERT(final_active_wire_idx != std::numeric_limits<size_t>::max(),
+                  "final_active_wire_idx has not been initialized");
+        return final_active_wire_idx;
+    }
 
     Flavor::PrecomputedData get_precomputed()
     {
@@ -133,17 +139,10 @@ template <IsUltraOrMegaHonk Flavor_> class ProverInstance_ {
         if (!circuit.circuit_finalized) {
             circuit.finalize_circuit(/* ensure_nonzero = */ true);
         }
-
-        // If using a structured trace, set fixed block sizes, check their validity, and set the dyadic circuit size
-        if constexpr (std::same_as<Circuit, UltraCircuitBuilder>) {
-            metadata.dyadic_size = compute_dyadic_size(circuit); // set dyadic size directly from circuit block sizes
-        } else if (std::same_as<Circuit, MegaCircuitBuilder>) {
-            metadata.dyadic_size = compute_dyadic_size(circuit); // set dyadic based on circuit block sizes
-        }
-
-        circuit.blocks.compute_offsets(); // compute offset of each block within the trace
+        metadata.dyadic_size = compute_dyadic_size(circuit);
 
         // Find index of last non-trivial wire value in the trace
+        circuit.blocks.compute_offsets(); // compute offset of each block within the trace
         for (auto& block : circuit.blocks.get()) {
             if (block.size() > 0) {
                 final_active_wire_idx = block.trace_offset() + block.size() - 1;

barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt

@@ -1,23 +1,49 @@
-if(NOT DISABLE_AZTEC_VM)
-  barretenberg_module(vm2 sumcheck stdlib_honk_verifier stdlib_goblin_verifier world_state)
-endif()
-
-# Include only simulation-related sources from vm2
+# Collect simulation-related sources (for vm2_sim module)
 file(GLOB_RECURSE VM2_SIM_SOURCE_FILES
   simulation/*.cpp
   common/*.cpp
   tooling/*.cpp
 )
 
+# Add top-level sim-related files
+list(APPEND VM2_SIM_SOURCE_FILES
+  ${CMAKE_CURRENT_SOURCE_DIR}/simulation_helper.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/avm_sim_api.cpp
+)
+
 # Exclude test, bench, and fuzzer files
 list(FILTER VM2_SIM_SOURCE_FILES EXCLUDE REGEX ".*\\.(test|bench|fuzzer)\\.cpp$")
 
-# TODO: this isn't a proper module. We should look into creating something we actually link.
-# This can be done by using barretenberg_module_with_sources above, linking this, but with the these files explicitly excluded.
+# Build vm2_sim module (for nodejs bindings)
 barretenberg_module_with_sources(
   vm2_sim
   SOURCE_FILES ${VM2_SIM_SOURCE_FILES}
-    simulation_helper.cpp
-    avm_sim_api.cpp
   DEPENDENCIES world_state
 )
+
+# Build vm2 module with sources mutually exclusive from vm2_sim
+if(NOT DISABLE_AZTEC_VM)
+  # Collect all source files
+  file(GLOB_RECURSE ALL_SOURCE_FILES *.cpp)
+  file(GLOB_RECURSE TEST_SOURCE_FILES *.test.cpp)
+  file(GLOB_RECURSE BENCH_SOURCE_FILES *.bench.cpp)
+  file(GLOB_RECURSE FUZZERS_SOURCE_FILES *.fuzzer.cpp)
+
+  # Start with all sources, then exclude test/bench/fuzzer files
+  set(VM2_SOURCE_FILES ${ALL_SOURCE_FILES})
+  list(FILTER VM2_SOURCE_FILES EXCLUDE REGEX ".*\\.(test|bench|fuzzer)\\.cpp$")
+
+  # Make vm2 and vm2_sim sources mutually exclusive
+  foreach(SIM_FILE ${VM2_SIM_SOURCE_FILES})
+    list(REMOVE_ITEM VM2_SOURCE_FILES ${SIM_FILE})
+  endforeach()
+
+  barretenberg_module_with_sources(
+    vm2
+    SOURCE_FILES ${VM2_SOURCE_FILES}
+    TEST_SOURCE_FILES ${TEST_SOURCE_FILES}
+    BENCH_SOURCE_FILES ${BENCH_SOURCE_FILES}
+    FUZZERS_SOURCE_FILES ${FUZZERS_SOURCE_FILES}
+    DEPENDENCIES sumcheck stdlib_honk_verifier stdlib_goblin_verifier world_state vm2_sim
+  )
+endif()