feat: merge-train/barretenberg (#16519)

BEGIN_COMMIT_OVERRIDE
feat!: Introduce proof type for recursive verification of ClientIVC
proofs in noir (#16400)
chore!: Remove recovery byte from ecdsa signature in stdlib + add
documentation (#16498)
chore: redo stdlib sha256 without packed_byte_array  (#16481)
chore: audit ECCVM transcript relation (#16380)
chore: remove databus non-issue (#16471)
feat: a minimal fixed offset append merge (#16517)
new civc inputs
END_COMMIT_OVERRIDE

Author: lucasxia01

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -11,7 +11,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="7298f572"
+pinned_short_hash="05a86d89"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp

@@ -10,7 +10,7 @@
 #include "barretenberg/common/throw_or_abort.hpp"
 #include "barretenberg/common/try_catch_shim.hpp"
 #include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp"
-#include "barretenberg/dsl/acir_format/ivc_recursion_constraint.hpp"
+#include "barretenberg/dsl/acir_format/pg_recursion_constraint.hpp"
 #include "barretenberg/serialize/msgpack.hpp"
 #include "barretenberg/serialize/msgpack_check_eq.hpp"
 #include <algorithm>

barretenberg/cpp/src/barretenberg/api/prove_tube.cpp

@@ -27,11 +27,10 @@ void prove_tube(const std::string& output_path, const std::string& vk_path)
     auto proof = ClientIVC::Proof::from_file_msgpack(proof_path);
     auto vk = from_buffer<ClientIVC::VerificationKey>(read_file(vk_path));
 
-    auto builder = std::make_shared<Builder>();
+    Builder builder;
+    ClientIVCRecursiveVerifier verifier{ &builder, vk.mega };
 
-    ClientIVCRecursiveVerifier verifier{ builder, vk };
-
-    StdlibProof stdlib_proof(*builder, proof);
+    StdlibProof stdlib_proof(builder, proof);
     ClientIVCRecursiveVerifier::Output client_ivc_rec_verifier_output = verifier.verify(stdlib_proof);
 
     // The public inputs in the proof are propagated to the base rollup by making them public inputs of this circuit.
@@ -49,12 +48,12 @@ void prove_tube(const std::string& output_path, const std::string& vk_path)
     inputs.set_public();
 
     // The tube only calls an IPA recursive verifier once, so we can just add this IPA proof
-    builder->ipa_proof = client_ivc_rec_verifier_output.ipa_proof.get_value();
-    BB_ASSERT_EQ(builder->ipa_proof.size(), IPA_PROOF_LENGTH, "IPA proof should be set.");
+    builder.ipa_proof = client_ivc_rec_verifier_output.ipa_proof.get_value();
+    BB_ASSERT_EQ(builder.ipa_proof.size(), IPA_PROOF_LENGTH, "IPA proof should be set.");
 
     using Prover = UltraProver_<UltraRollupFlavor>;
     using Verifier = UltraVerifier_<UltraRollupFlavor>;
-    auto proving_key = std::make_shared<DeciderProvingKey_<UltraRollupFlavor>>(*builder);
+    auto proving_key = std::make_shared<DeciderProvingKey_<UltraRollupFlavor>>(builder);
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1201): Precompute tube vk and pass it in.
     info("WARNING: computing tube vk in prove_tube, but a precomputed vk should be passed in.");
     auto tube_verification_key = std::make_shared<UltraRollupFlavor::VerificationKey>(proving_key->get_precomputed());

barretenberg/cpp/src/barretenberg/bbapi/bbapi_client_ivc.cpp

@@ -5,7 +5,7 @@
 #include "barretenberg/common/throw_or_abort.hpp"
 #include "barretenberg/dsl/acir_format/acir_format.hpp"
 #include "barretenberg/dsl/acir_format/acir_to_constraint_buf.hpp"
-#include "barretenberg/dsl/acir_format/ivc_recursion_constraint.hpp"
+#include "barretenberg/dsl/acir_format/pg_recursion_constraint.hpp"
 #include "barretenberg/dsl/acir_format/serde/witness_stack.hpp"
 #include "barretenberg/honk/execution_trace/execution_trace_usage_tracker.hpp"
 #include "barretenberg/serialize/msgpack_check_eq.hpp"
@@ -181,7 +181,7 @@ ClientIvcStats::Response ClientIvcStats::execute(BBApiRequest& request) &&
     acir_format::AcirProgram program{ constraint_system };
 
     // Get IVC constraints if any
-    const auto& ivc_constraints = constraint_system.ivc_recursion_constraints;
+    const auto& ivc_constraints = constraint_system.pg_recursion_constraints;
 
     // Create metadata with appropriate IVC context
     acir_format::ProgramMetadata metadata{

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_sha256.test.cpp

@@ -4,7 +4,6 @@
 #include "barretenberg/crypto/sha256/sha256.hpp"
 #include "barretenberg/stdlib/hash/sha256/sha256.hpp"
 #include "barretenberg/stdlib/primitives/byte_array/byte_array.hpp"
-#include "barretenberg/stdlib/primitives/packed_byte_array/packed_byte_array.hpp"
 #include "barretenberg/stdlib_circuit_builders/plookup_tables/plookup_tables.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
 
@@ -17,28 +16,46 @@ using namespace bb::stdlib;
 using namespace cdg;
 
 using Builder = UltraCircuitBuilder;
-using byte_array_pt = byte_array<Builder>;
-using packed_byte_array_pt = packed_byte_array<Builder>;
-using field_pt = field_t<Builder>;
+using byte_array_ct = byte_array<Builder>;
+using field_ct = field_t<Builder>;
+
+/**
+ * @brief Given a `byte_array` object, slice it into chunks of size `num_bytes_in_chunk` and compute field elements
+ * reconstructed from these chunks.
+ */
+
+std::vector<field_ct> pack_bytes_into_field_elements(const byte_array_ct& input, size_t num_bytes_in_chunk = 4)
+{
+    std::vector<field_t<Builder>> result;
+    const size_t byte_len = input.size();
+
+    for (size_t i = 0; i < byte_len; i += num_bytes_in_chunk) {
+        byte_array_ct chunk = input.slice(i, std::min(num_bytes_in_chunk, byte_len - i));
+        result.emplace_back(static_cast<field_ct>(chunk));
+    }
+
+    return result;
+}
 
 /**
  static analyzer usually prints input and output variables as variables in one gate. In tests these variables
  are not dangerous and usually we can filter them by adding gate for fixing witness. Then these variables will be
  in 2 gates, and static analyzer won't print them. functions fix_vector and fix_byte_array do it
- for vector of variables and packed_byte_array respectively
+ for vector of variables and byte_array respectively
 */
 
-void fix_vector(std::vector<field_pt>& vector)
+void fix_vector(std::vector<field_ct>& vector)
 {
     for (auto& elem : vector) {
         elem.fix_witness();
     }
 }
 
-void fix_byte_array(packed_byte_array_pt& input)
+void fix_byte_array(byte_array_ct& input)
 {
-    std::vector<field_pt> limbs = input.get_limbs();
-    fix_vector(limbs);
+    for (size_t idx = 0; idx < input.size(); idx++) {
+        input[idx].fix_witness();
+    }
 }
 
 /**
@@ -55,12 +72,12 @@ TEST(boomerang_stdlib_sha256, test_graph_for_sha256_55_bytes)
     // 55 bytes is the largest number of bytes that can be hashed in a single block,
     // accounting for the single padding bit, and the 64 size bits required by the SHA-256 standard.
     auto builder = Builder();
-    packed_byte_array_pt input(&builder, "An 8 character password? Snow White and the 7 Dwarves..");
+    byte_array_ct input(&builder, "An 8 character password? Snow White and the 7 Dwarves..");
     fix_byte_array(input);
 
-    packed_byte_array_pt output_bits = stdlib::SHA256<Builder>::hash(input);
+    byte_array_ct output_bytes = stdlib::SHA256<Builder>::hash(input);
 
-    std::vector<field_pt> output = output_bits.to_unverified_byte_slices(4);
+    std::vector<field_ct> output = pack_bytes_into_field_elements(output_bytes);
     fix_vector(output);
 
     StaticAnalyzer graph = StaticAnalyzer(builder);
@@ -87,7 +104,7 @@ HEAVY_TEST(boomerang_stdlib_sha256, test_graph_for_sha256_NIST_vector_five)
 {
     auto builder = Builder();
 
-    packed_byte_array_pt input(
+    byte_array_ct input(
         &builder,
         "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
         "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
@@ -101,9 +118,9 @@ HEAVY_TEST(boomerang_stdlib_sha256, test_graph_for_sha256_NIST_vector_five)
         "AAAAAAAAAA");
 
     fix_byte_array(input);
-    packed_byte_array_pt output_bits = stdlib::SHA256<bb::UltraCircuitBuilder>::hash(input);
+    byte_array_ct output_bytes = stdlib::SHA256<bb::UltraCircuitBuilder>::hash(input);
 
-    std::vector<field_pt> output = output_bits.to_unverified_byte_slices(4);
+    std::vector<field_ct> output = pack_bytes_into_field_elements(output_bytes);
     fix_vector(output);
 
     StaticAnalyzer graph = StaticAnalyzer(builder);
@@ -126,10 +143,10 @@ HEAVY_TEST(boomerang_stdlib_sha256, test_graph_for_sha256_NIST_vector_five)
 TEST(boomerang_stdlib_sha256, test_graph_for_sha256_NIST_vector_one)
 {
     auto builder = Builder();
-    packed_byte_array_pt input(&builder, "abc");
+    byte_array_ct input(&builder, "abc");
     fix_byte_array(input);
-    packed_byte_array_pt output_bits = stdlib::SHA256<Builder>::hash(input);
-    fix_byte_array(output_bits);
+    byte_array_ct output_bytes = stdlib::SHA256<Builder>::hash(input);
+    fix_byte_array(output_bytes);
     StaticAnalyzer graph = StaticAnalyzer(builder);
     auto connected_components = graph.find_connected_components();
     EXPECT_EQ(connected_components.size(), 1);
@@ -146,10 +163,10 @@ TEST(boomerang_stdlib_sha256, test_graph_for_sha256_NIST_vector_one)
 TEST(boomerang_stdlib_sha256, test_graph_for_sha256_NIST_vector_two)
 {
     auto builder = Builder();
-    packed_byte_array_pt input(&builder, "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq");
+    byte_array_ct input(&builder, "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq");
     fix_byte_array(input);
-    packed_byte_array_pt output_bits = stdlib::SHA256<Builder>::hash(input);
-    fix_byte_array(output_bits);
+    byte_array_ct output_bytes = stdlib::SHA256<Builder>::hash(input);
+    fix_byte_array(output_bytes);
     StaticAnalyzer graph = StaticAnalyzer(builder);
     auto connected_components = graph.find_connected_components();
     EXPECT_EQ(connected_components.size(), 1);
@@ -168,10 +185,10 @@ TEST(boomerang_stdlib_sha256, test_graph_for_sha256_NIST_vector_three)
     auto builder = Builder();
 
     // one byte, 0xbd
-    packed_byte_array_pt input(&builder, std::vector<uint8_t>{ 0xbd });
+    byte_array_ct input(&builder, std::vector<uint8_t>{ 0xbd });
     fix_byte_array(input);
-    packed_byte_array_pt output_bits = stdlib::SHA256<Builder>::hash(input);
-    fix_byte_array(output_bits);
+    byte_array_ct output_bytes = stdlib::SHA256<Builder>::hash(input);
+    fix_byte_array(output_bytes);
     StaticAnalyzer graph = StaticAnalyzer(builder);
     auto connected_components = graph.find_connected_components();
     EXPECT_EQ(connected_components.size(), 1);
@@ -190,10 +207,10 @@ TEST(boomerang_stdlib_sha256, test_graph_for_sha256_NIST_vector_four)
     auto builder = Builder();
 
     // 4 bytes, 0xc98c8e55
-    packed_byte_array_pt input(&builder, std::vector<uint8_t>{ 0xc9, 0x8c, 0x8e, 0x55 });
+    byte_array_ct input(&builder, std::vector<uint8_t>{ 0xc9, 0x8c, 0x8e, 0x55 });
     fix_byte_array(input);
-    packed_byte_array_pt output_bits = stdlib::SHA256<Builder>::hash(input);
-    fix_byte_array(output_bits);
+    byte_array_ct output_bytes = stdlib::SHA256<Builder>::hash(input);
+    fix_byte_array(output_bytes);
     StaticAnalyzer graph = StaticAnalyzer(builder);
     auto connected_components = graph.find_connected_components();
     EXPECT_EQ(connected_components.size(), 1);

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -611,6 +611,20 @@ size_t ClientIVC::Proof::size() const
     return mega_proof.size() + goblin_proof.size();
 }
 
+std::vector<ClientIVC::FF> ClientIVC::Proof::to_field_elements() const
+{
+    HonkProof proof;
+
+    proof.insert(proof.end(), mega_proof.begin(), mega_proof.end());
+    proof.insert(proof.end(), goblin_proof.merge_proof.begin(), goblin_proof.merge_proof.end());
+    proof.insert(
+        proof.end(), goblin_proof.eccvm_proof.pre_ipa_proof.begin(), goblin_proof.eccvm_proof.pre_ipa_proof.end());
+    proof.insert(proof.end(), goblin_proof.eccvm_proof.ipa_proof.begin(), goblin_proof.eccvm_proof.ipa_proof.end());
+    proof.insert(proof.end(), goblin_proof.translator_proof.begin(), goblin_proof.translator_proof.end());
+
+    return proof;
+};
+
 msgpack::sbuffer ClientIVC::Proof::to_msgpack_buffer() const
 {
     msgpack::sbuffer buffer;

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp

@@ -97,8 +97,42 @@ class ClientIVC {
         HonkProof mega_proof;
         GoblinProof goblin_proof;
 
+        /**
+         * @brief The size of a ClientIVC proof without backend-added public inputs
+         *
+         * @param virtual_log_n
+         * @return constexpr size_t
+         */
+        static constexpr size_t PROOF_LENGTH_WITHOUT_PUB_INPUTS(size_t virtual_log_n = MegaZKFlavor::VIRTUAL_LOG_N)
+        {
+            return /*mega_proof*/ MegaZKFlavor::PROOF_LENGTH_WITHOUT_PUB_INPUTS(virtual_log_n) +
+                   /*merge_proof*/ MERGE_PROOF_SIZE +
+                   /*eccvm pre-ipa proof*/ (ECCVMFlavor::PROOF_LENGTH_WITHOUT_PUB_INPUTS - IPA_PROOF_LENGTH) +
+                   /*eccvm ipa proof*/ IPA_PROOF_LENGTH +
+                   /*translator*/ TranslatorFlavor::PROOF_LENGTH_WITHOUT_PUB_INPUTS;
+        }
+
+        /**
+         * @brief The size of a ClientIVC proof with backend-added public inputs: HidingKernelIO
+         *
+         * @param virtual_log_n
+         * @return constexpr size_t
+         */
+        static constexpr size_t PROOF_LENGTH(size_t virtual_log_n = MegaZKFlavor::VIRTUAL_LOG_N)
+        {
+            return PROOF_LENGTH_WITHOUT_PUB_INPUTS(virtual_log_n) +
+                   /*public_inputs*/ bb::HidingKernelIO::PUBLIC_INPUTS_SIZE;
+        }
+
         size_t size() const;
 
+        /**
+         * @brief Serialize proof to field elements
+         *
+         * @return std::vector<FF>
+         */
+        std::vector<FF> to_field_elements() const;
+
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/1299): The following msgpack methods are generic
         // and should leverage some kind of shared msgpack utility.
         msgpack::sbuffer to_msgpack_buffer() const;

barretenberg/cpp/src/barretenberg/dsl/CMakeLists.txt

@@ -6,7 +6,8 @@ set(DSL_DEPENDENCIES
     stdlib_keccak
     stdlib_poseidon2
     stdlib_schnorr
-    stdlib_honk_verifier)
+    stdlib_honk_verifier
+    stdlib_client_ivc_verifier)
 
 if (NOT DISABLE_AZTEC_VM)
     list(APPEND DSL_DEPENDENCIES vm2)