feat: merge-train/avm (#19422)

BEGIN_COMMIT_OVERRIDE
feat(avm)!: dont expose/require VK (#19414)
chore(avm): params should be references
chore: defensive asserts for map/vector retrievals and optional values
in cpp (#19439)
END_COMMIT_OVERRIDE

Author: AztecBot

barretenberg/cpp/src/barretenberg/api/api_avm.hpp

@@ -7,13 +7,13 @@ namespace bb {
 extern const bool avm_enabled;
 
 /**
- * @brief Writes an avm proof and corresponding (incomplete) verification key to files.
+ * @brief Writes an avm proof to a file.
  *
  * Communication:
- * - Filesystem: The proof and vk are written to the paths output_path/proof and output_path/vk
+ * - Filesystem: The proof is written to the path output_path/proof
  *
  * @param inputs_path Path to the file containing the serialised avm public inputs and hints
- * @param output_path Path (directory) to write the output proof and verification keys
+ * @param output_path Path (directory) to write the output proof
  */
 void avm_prove(const std::filesystem::path& inputs_path, const std::filesystem::path& output_path);
 
@@ -27,14 +27,12 @@ void avm_check_circuit(const std::filesystem::path& inputs_path);
  *   an exit code of 0 will be returned for success and 1 for failure.
  *
  * @param proof_path Path to the file containing the serialized proof
- * @param vk_path Path to the file containing the serialized verification key
+ * @param public_inputs_path Path to the file containing the serialized public inputs
  * @return true If the proof is valid
  * @return false If the proof is invalid
  */
 // NOTE: The proof should NOT include the public inputs.
-bool avm_verify(const std::filesystem::path& proof_path,
-                const std::filesystem::path& public_inputs_path,
-                const std::filesystem::path& vk_path);
+bool avm_verify(const std::filesystem::path& proof_path, const std::filesystem::path& public_inputs_path);
 
 /**
  * @brief Simulates an public transaction

barretenberg/cpp/src/barretenberg/bb/cli.cpp

@@ -585,7 +585,6 @@ int parse_and_run_cli_command(int argc, char* argv[])
     add_crs_path_option(avm_verify_command);
     add_avm_public_inputs_option(avm_verify_command);
     add_proof_path_option(avm_verify_command);
-    add_vk_path_option(avm_verify_command);
 
     /***************************************************************************************************************
      * Subcommand: aztec_process_artifact
@@ -894,7 +893,7 @@ int parse_and_run_cli_command(int argc, char* argv[])
         } else if (avm_check_circuit_command->parsed()) {
             avm_check_circuit(avm_inputs_path);
         } else if (avm_verify_command->parsed()) {
-            return avm_verify(proof_path, avm_public_inputs_path, vk_path) ? 0 : 1;
+            return avm_verify(proof_path, avm_public_inputs_path) ? 0 : 1;
         } else if (avm_simulate_command->parsed()) {
             avm_simulate(avm_inputs_path);
         } else if (avm_write_vk_command->parsed()) {

barretenberg/cpp/src/barretenberg/vm2/api_avm.cpp

@@ -31,19 +31,18 @@ void avm_prove(const std::filesystem::path& inputs_path, const std::filesystem::
 {
     avm2::AvmAPI avm;
     auto inputs = avm2::AvmAPI::ProvingInputs::from(read_file(inputs_path));
-    auto [proof, vk] = avm.prove(inputs);
+    auto proof = avm.prove(inputs);
 
-    // NOTE: As opposed to Avm1 and other proof systems, the public inputs are NOT part of the proof.
+    // NOTE: As opposed to other proof systems, the public inputs are NOT part of the proof.
     write_file(output_path / "proof", to_buffer(proof));
-    write_file(output_path / "vk", vk);
 
     print_avm_stats();
 
     // NOTE: Temporarily we also verify after proving.
     // The reasoning is that proving will always pass unless it crashes.
     // We want to return an exit code != 0 if the proof is invalid so that the prover client saves the inputs.
     info("verifying...");
-    bool res = avm.verify(proof, inputs.public_inputs, vk);
+    bool res = avm.verify(proof, inputs.public_inputs);
     info("verification: ", res ? "success" : "failure");
     if (!res) {
         throw std::runtime_error("Generated proof is invalid!!!!!");
@@ -62,16 +61,13 @@ void avm_check_circuit(const std::filesystem::path& inputs_path)
 }
 
 // NOTE: The proof should NOT include the public inputs.
-bool avm_verify(const std::filesystem::path& proof_path,
-                const std::filesystem::path& public_inputs_path,
-                const std::filesystem::path& vk_path)
+bool avm_verify(const std::filesystem::path& proof_path, const std::filesystem::path& public_inputs_path)
 {
     const auto proof = many_from_buffer<fr>(read_file(proof_path));
-    std::vector<uint8_t> vk_bytes = read_file(vk_path);
     auto public_inputs = avm2::PublicInputs::from(read_file(public_inputs_path));
 
     avm2::AvmAPI avm;
-    bool res = avm.verify(proof, public_inputs, vk_bytes);
+    bool res = avm.verify(proof, public_inputs);
     info("verification: ", res ? "success" : "failure");
 
     print_avm_stats();

barretenberg/cpp/src/barretenberg/vm2/api_avm.hpp

@@ -7,13 +7,13 @@ namespace bb {
 extern const bool avm_enabled;
 
 /**
- * @brief Writes an avm proof and corresponding (incomplete) verification key to files.
+ * @brief Writes an avm proof to a file.
  *
  * Communication:
- * - Filesystem: The proof and vk are written to the paths output_path/proof and output_path/vk
+ * - Filesystem: The proof is written to the path output_path/proof
  *
  * @param inputs_path Path to the file containing the serialised avm public inputs and hints
- * @param output_path Path (directory) to write the output proof and verification keys
+ * @param output_path Path (directory) to write the output proof
  */
 void avm_prove(const std::filesystem::path& inputs_path, const std::filesystem::path& output_path);
 
@@ -27,14 +27,12 @@ void avm_check_circuit(const std::filesystem::path& inputs_path);
  *   an exit code of 0 will be returned for success and 1 for failure.
  *
  * @param proof_path Path to the file containing the serialized proof
- * @param vk_path Path to the file containing the serialized verification key
+ * @param public_inputs_path Path to the file containing the serialized public inputs
  * @return true If the proof is valid
  * @return false If the proof is invalid
  */
 // NOTE: The proof should NOT include the public inputs.
-bool avm_verify(const std::filesystem::path& proof_path,
-                const std::filesystem::path& public_inputs_path,
-                const std::filesystem::path& vk_path);
+bool avm_verify(const std::filesystem::path& proof_path, const std::filesystem::path& public_inputs_path);
 
 /**
  * @brief Simulates an public transaction

barretenberg/cpp/src/barretenberg/vm2/avm_api.cpp

@@ -10,7 +10,7 @@ namespace bb::avm2 {
 
 using namespace bb::avm2::simulation;
 
-std::pair<AvmAPI::AvmProof, AvmAPI::AvmVerificationKey> AvmAPI::prove(const AvmAPI::ProvingInputs& inputs)
+AvmAPI::AvmProof AvmAPI::prove(const AvmAPI::ProvingInputs& inputs)
 {
     // Simulate.
     vinfo("Simulating...");
@@ -27,10 +27,10 @@ std::pair<AvmAPI::AvmProof, AvmAPI::AvmVerificationKey> AvmAPI::prove(const AvmA
     // Prove.
     vinfo("Proving...");
     AvmProvingHelper proving_helper;
-    auto [proof, vk] = AVM_TRACK_TIME_V("proving/all", proving_helper.prove(std::move(trace)));
+    auto proof = AVM_TRACK_TIME_V("proving/all", proving_helper.prove(std::move(trace)));
 
     vinfo("Done!");
-    return { std::move(proof), std::move(vk) };
+    return proof;
 }
 
 bool AvmAPI::check_circuit(const AvmAPI::ProvingInputs& inputs)
@@ -63,11 +63,11 @@ bool AvmAPI::check_circuit(const AvmAPI::ProvingInputs& inputs)
     return proving_helper.check_circuit(std::move(trace));
 }
 
-bool AvmAPI::verify(const AvmProof& proof, const PublicInputs& pi, const AvmVerificationKey& vk_data)
+bool AvmAPI::verify(const AvmProof& proof, const PublicInputs& pi)
 {
     vinfo("Verifying...");
     AvmProvingHelper proving_helper;
-    return AVM_TRACK_TIME_V("verifing/all", proving_helper.verify(proof, pi, vk_data));
+    return AVM_TRACK_TIME_V("verifing/all", proving_helper.verify(proof, pi));
 }
 
 AvmAPI::AvmVerificationKey AvmAPI::get_verification_key()

barretenberg/cpp/src/barretenberg/vm2/avm_api.hpp

@@ -15,9 +15,9 @@ class AvmAPI : public AvmSimAPI {
     AvmAPI() = default;
 
     // NOTE: The public inputs are NOT part of the proof.
-    std::pair<AvmProof, AvmVerificationKey> prove(const ProvingInputs& inputs);
+    AvmProof prove(const ProvingInputs& inputs);
     bool check_circuit(const ProvingInputs& inputs);
-    bool verify(const AvmProof& proof, const PublicInputs& pi, const AvmVerificationKey& vk_data);
+    bool verify(const AvmProof& proof, const PublicInputs& pi);
     AvmVerificationKey get_verification_key();
 };
 

barretenberg/cpp/src/barretenberg/vm2/constraining/recursion/recursive_verifier.test.cpp

@@ -44,8 +44,8 @@ class AvmRecursiveTests : public ::testing::Test {
             const auto public_inputs_cols = public_inputs.to_columns();
 
             InnerProver prover;
-            const auto [proof, vk_data] = prover.prove(std::move(trace));
-            const auto verification_key = InnerProver::create_verification_key(vk_data);
+            const auto proof = prover.prove(std::move(trace));
+            const auto verification_key = InnerProver::create_verification_key(InnerProver().get_verification_key());
             InnerVerifier verifier(verification_key);
 
             const bool verified = verifier.verify_proof(proof, public_inputs_cols);

barretenberg/cpp/src/barretenberg/vm2/constraining/verifier.test.cpp

@@ -24,16 +24,16 @@ class AvmVerifierTests : public ::testing::Test {
     };
 
     // Helper function to create proof.
-    static NativeProofResult create_proof_and_vk()
+    static NativeProofResult create_proof()
     {
         auto [trace, public_inputs] = testing::get_minimal_trace_with_pi();
 
         Prover prover;
         auto public_inputs_cols = public_inputs.to_columns();
-        const auto [proof, vk_data] = prover.prove(std::move(trace));
-        const auto verification_key = prover.create_verification_key(vk_data);
+        const auto proof = prover.prove(std::move(trace));
+        auto vk = AvmProvingHelper::create_verification_key(AvmProvingHelper().get_verification_key());
 
-        return { proof, verification_key, public_inputs_cols };
+        return { proof, vk, public_inputs_cols };
     }
 };
 
@@ -43,11 +43,9 @@ TEST_F(AvmVerifierTests, GoodPublicInputs)
         GTEST_SKIP() << "Skipping slow test";
     }
 
-    NativeProofResult proof_result = create_proof_and_vk();
-    auto [proof, verification_key, public_inputs_cols] = proof_result;
-
-    Verifier verifier(verification_key);
+    auto [proof, vk, public_inputs_cols] = create_proof();
 
+    Verifier verifier(vk);
     const bool verified = verifier.verify_proof(proof, public_inputs_cols);
 
     ASSERT_TRUE(verified) << "native proof verification failed";
@@ -59,11 +57,10 @@ TEST_F(AvmVerifierTests, NegativeBadPublicInputs)
         GTEST_SKIP() << "Skipping slow test";
     }
 
-    NativeProofResult proof_result = create_proof_and_vk();
-    auto [proof, verification_key, public_inputs_cols] = proof_result;
+    auto [proof, vk, public_inputs_cols] = create_proof();
     auto verify_with_corrupt_pi_col = [&](size_t col_idx) {
         public_inputs_cols[col_idx][5] += FF::one();
-        Verifier verifier(verification_key);
+        Verifier verifier(vk);
         const bool verified = verifier.verify_proof(proof, public_inputs_cols);
         ASSERT_FALSE(verified)
             << "native proof verification succeeded, but should have failed due to corruption of public inputs col "
@@ -73,17 +70,17 @@ TEST_F(AvmVerifierTests, NegativeBadPublicInputs)
     for (size_t col_idx = 0; col_idx < 4; col_idx++) {
         verify_with_corrupt_pi_col(col_idx);
     }
-    Verifier verifier(verification_key);
+    Verifier verifier(vk);
     const bool verified = verifier.verify_proof(proof, public_inputs_cols);
     ASSERT_TRUE(verified) << "native proof verification failed, but should have succeeded";
 }
 
 // Verify that the actual proof size matches COMPUTED_AVM_PROOF_LENGTH_IN_FIELDS
 TEST_F(AvmVerifierTests, ProofSizeMatchesComputedConstant)
 {
-    NativeProofResult proof_result = create_proof_and_vk();
+    auto [proof, vk, public_inputs_cols] = create_proof();
 
-    const size_t actual_proof_size = proof_result.proof.size();
+    const size_t actual_proof_size = proof.size();
     const size_t computed_proof_size = AvmFlavor::COMPUTED_AVM_PROOF_LENGTH_IN_FIELDS;
 
     EXPECT_EQ(actual_proof_size, computed_proof_size)

barretenberg/cpp/src/barretenberg/vm2/dsl/avm2_recursion_constraint.test.cpp

@@ -52,10 +52,10 @@ class AvmRecursionConstraintTestingFunctions {
         auto [trace, public_inputs] = avm2::testing::get_minimal_trace_with_pi();
 
         AvmProver prover;
-        auto [proof, vk_data] = prover.prove(std::move(trace));
+        auto proof = prover.prove(std::move(trace));
         proof.resize(AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED, FF::zero()); // Pad proof
 
-        const bool verified = prover.verify(proof, public_inputs, vk_data);
+        const bool verified = prover.verify(proof, public_inputs);
         EXPECT_TRUE(verified) << "native proof verification failed";
 
         auto public_inputs_flat = PublicInputs::columns_to_flat(public_inputs.to_columns());

barretenberg/cpp/src/barretenberg/vm2/proving_helper.cpp

@@ -51,7 +51,7 @@ AvmProvingHelper::VkData AvmProvingHelper::get_verification_key()
     return serialized_vk;
 }
 
-std::pair<AvmProvingHelper::Proof, AvmProvingHelper::VkData> AvmProvingHelper::prove(tracegen::TraceContainer&& trace)
+AvmProvingHelper::Proof AvmProvingHelper::prove(tracegen::TraceContainer&& trace)
 {
     auto polynomials = AVM_TRACK_TIME_V("proving/prove:compute_polynomials", constraining::compute_polynomials(trace));
     auto proving_key =
@@ -66,7 +66,7 @@ std::pair<AvmProvingHelper::Proof, AvmProvingHelper::VkData> AvmProvingHelper::p
     auto proof = AVM_TRACK_TIME_V("proving/construct_proof", prover.construct_proof());
     auto serialized_vk = to_buffer(verification_key->to_field_elements());
 
-    return { std::move(proof), std::move(serialized_vk) };
+    return proof;
 }
 
 bool AvmProvingHelper::check_circuit(tracegen::TraceContainer&& trace)
@@ -97,8 +97,9 @@ bool AvmProvingHelper::check_circuit(tracegen::TraceContainer&& trace)
     return true;
 }
 
-bool AvmProvingHelper::verify(const AvmProvingHelper::Proof& proof, const PublicInputs& pi, const VkData& vk_data)
+bool AvmProvingHelper::verify(const AvmProvingHelper::Proof& proof, const PublicInputs& pi)
 {
+    auto vk_data = AVM_TRACK_TIME_V("proving/verify:get_verification_key", get_verification_key());
     auto vk = AVM_TRACK_TIME_V("proving/verify:create_verification_key", create_verification_key(vk_data));
     auto verifier = AVM_TRACK_TIME_V("proving/verify:construct_verifier", AvmVerifier(std::move(vk)));
     return AVM_TRACK_TIME_V("proving/verify_proof", verifier.verify_proof(proof, pi.to_columns()));