refactor(bb): remove --output_format, have one-size-fits-all format (#16201)

This changes vks to be serialized as bytes using to_field_elements. Proofs were already byte-compatible, this allows having bb not worry about how to split fields into JSON

The changes are holistic as we now need to split the fields from the binary vk/proof - but this is now trivial with the new deserialization format.

Author: ludamad

barretenberg/acir_tests/bbjs-test/src/index.ts

@@ -9,9 +9,7 @@ const logger = pino({
 });
 
 const proofPath = (dir: string) => path.join(dir, "proof");
-const proofAsFieldsPath = (dir: string) => path.join(dir, "proof_fields.json");
-const publicInputsAsFieldsPath = (dir: string) =>
-  path.join(dir, "public_inputs_fields.json");
+const publicInputsPath = (dir: string) => path.join(dir, "public_inputs");
 const vkeyPath = (dir: string) => path.join(dir, "vk");
 
 async function generateProof({
@@ -27,7 +25,7 @@ async function generateProof({
   oracleHash?: string;
   multiThreaded?: boolean;
 }) {
-  const { UltraHonkBackend, deflattenFields } = await import("@aztec/bb.js");
+  const { UltraHonkBackend } = await import("@aztec/bb.js");
 
   logger.debug(`Generating proof for ${bytecodePath}...`);
   const circuitArtifact = await fs.readFile(bytecodePath);
@@ -45,17 +43,16 @@ async function generateProof({
   await fs.writeFile(proofPath(outputDirectory), Buffer.from(proof.proof));
   logger.debug("Proof written to " + proofPath(outputDirectory));
 
-  await fs.writeFile(
-    publicInputsAsFieldsPath(outputDirectory),
-    JSON.stringify(proof.publicInputs)
+  // Convert public inputs from field strings to binary
+  const publicInputsBuffer = Buffer.concat(
+    proof.publicInputs.map((field: string) => {
+      const hex = field.startsWith('0x') ? field.slice(2) : field;
+      return Buffer.from(hex.padStart(64, '0'), 'hex');
+    })
   );
+  await fs.writeFile(publicInputsPath(outputDirectory), publicInputsBuffer);
   logger.debug(
-    "Public inputs written to " + publicInputsAsFieldsPath(outputDirectory)
-  );
-
-  await fs.writeFile(
-    proofAsFieldsPath(outputDirectory),
-    JSON.stringify(deflattenFields(proof.proof))
+    "Public inputs written to " + publicInputsPath(outputDirectory)
   );
 
   const verificationKey = await backend.getVerificationKey({
@@ -69,21 +66,24 @@ async function generateProof({
 }
 
 async function verifyProof({ directory }: { directory: string }) {
-  const { BarretenbergVerifier } = await import("@aztec/bb.js");
+  const { UltraHonkVerifierBackend } = await import("@aztec/bb.js");
 
-  const verifier = new BarretenbergVerifier();
+  const verifier = new UltraHonkVerifierBackend();
 
   const proof = await fs.readFile(proofPath(directory));
 
-  const publicInputs = JSON.parse(
-    await fs.readFile(publicInputsAsFieldsPath(directory), "utf8")
-  );
+  // Read binary public inputs and convert to field strings
+  const publicInputsBinary = await fs.readFile(publicInputsPath(directory));
+  const publicInputs = [];
+  for (let i = 0; i < publicInputsBinary.length; i += 32) {
+    const chunk = publicInputsBinary.slice(i, Math.min(i + 32, publicInputsBinary.length));
+    publicInputs.push('0x' + chunk.toString('hex'));
+  }
   logger.debug(`publicInputs: ${JSON.stringify(publicInputs)}`);
-  const vkey = await fs.readFile(vkeyPath(directory));
+  const verificationKey = await fs.readFile(vkeyPath(directory));
 
-  const verified = await verifier.verifyUltraHonkProof(
-    { proof: new Uint8Array(proof), publicInputs },
-    new Uint8Array(vkey)
+  const verified = await verifier.verifyProof(
+    { proof: new Uint8Array(proof), publicInputs, verificationKey},
   );
 
   await verifier.destroy();

barretenberg/acir_tests/bootstrap.sh

@@ -21,14 +21,18 @@ tests_hash=$(hash_str \
     ../ts/.rebuild_patterns \
     ../noir/))
 
+function hex_to_fields_json {
+  # 1. split encoded hex into 64-character lines 3. encode as JSON array of hex strings
+  fold -w64 | jq -R -s -c 'split("\n") | map(select(length > 0)) | map("0x" + .)'
+}
+
 # Generate inputs for a given recursively verifying program.
 function run_proof_generation {
   local program=$1
   local native_build_dir=$(../cpp/scripts/native-preset-build-dir)
   local bb=$(realpath ../cpp/$native_build_dir/bin/bb)
   local outdir=$(mktemp -d)
   trap "rm -rf $outdir" EXIT
-  local adjustment=16
   local ipa_accumulation_flag=""
 
   cd ./acir_tests/assert_statement
@@ -37,35 +41,37 @@ function run_proof_generation {
 
   # Adjust settings based on program type
   if [[ $program == *"rollup"* ]]; then
-      adjustment=26
       ipa_accumulation_flag="--ipa_accumulation"
   fi
   # If the test program has zk in it's name would like to use the zk prover, so we empty the flag in this case.
   if [[ $program == *"zk"* ]]; then
     disable_zk=""
   fi
-  local prove_cmd="$bb prove --scheme ultra_honk $disable_zk $ipa_accumulation_flag --output_format fields --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz"
+  local prove_cmd="$bb prove --scheme ultra_honk $disable_zk $ipa_accumulation_flag --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz"
   echo_stderr "$prove_cmd"
   dump_fail "$prove_cmd"
 
-  local vk_fields=$(cat "$outdir/vk_fields.json")
-  local vk_hash_fields=$(cat "$outdir/vk_hash_fields.json")
-  local public_inputs_fields=$(cat "$outdir/public_inputs_fields.json")
-  local proof_fields=$(cat "$outdir/proof_fields.json")
 
-  generate_toml "$program" "$vk_fields" "$vk_hash_fields" "$proof_fields" "$public_inputs_fields"
+  # Split the hex-encoded vk bytes into fields boundaries (but still hex-encoded), first making 64-character lines and then encoding as JSON.
+  # This used to be done by barretenberg itself, but with serialization now always being in field elements we can do it outside of bb.
+  local vk_fields=$(cat "$outdir/vk" | xxd -p -c 0 | hex_to_fields_json)
+  local vk_hash_field="\"0x$(cat "$outdir/vk_hash" | xxd -p -c 0)\""
+  local public_inputs_fields=$(cat "$outdir/public_inputs" | xxd -p -c 0 | hex_to_fields_json)
+  local proof_fields=$(cat "$outdir/proof" | xxd -p -c 0 | hex_to_fields_json)
+
+  generate_toml "$program" "$vk_fields" "$vk_hash_field" "$proof_fields" "$public_inputs_fields"
 }
 
 function generate_toml {
   local program=$1
   local vk_fields=$2
-  local vk_hash_fields=$3
+  local vk_hash_field=$3
   local proof_fields=$4
-  local num_inner_public_inputs=$5
+  local public_inputs_fields=$5
   local output_file="../$program/Prover.toml"
 
   jq -nr \
-      --arg key_hash "$vk_hash_fields" \
+      --arg key_hash "$vk_hash_field" \
       --argjson vk_f "$vk_fields" \
       --argjson public_inputs_f "$public_inputs_fields" \
       --argjson proof_f "$proof_fields" \
@@ -79,7 +85,6 @@ function generate_toml {
 }
 
 function regenerate_recursive_inputs {
-  local program=$1
   # Compile the assert_statement test as it's used for the recursive tests.
   cd ./acir_tests/assert_statement
   local nargo=$(realpath ../../../../noir/noir-repo/target/release/nargo)
@@ -91,7 +96,7 @@ function regenerate_recursive_inputs {
   parallel 'run_proof_generation {}' ::: $(ls internal_test_programs)
 }
 
-export -f regenerate_recursive_inputs run_proof_generation generate_toml
+export -f hex_to_fields_json regenerate_recursive_inputs run_proof_generation generate_toml
 
 function compile {
   echo_header "Compiling acir_tests"
@@ -160,10 +165,10 @@ function test_cmds {
 
   # bb.js browser tests. Isolate because server.
   local browser_prefix="$tests_hash:ISOLATE=1:NET=1:CPUS=8"
-  echo "$browser_prefix:NAME=chrome_verify_honk_proof $scripts/browser_prove.sh verify_honk_proof chrome"
-  echo "$browser_prefix:NAME=chrome_a_1_mul $scripts/browser_prove.sh a_1_mul chrome"
-  echo "$browser_prefix:NAME=webkit_verify_honk_proof $scripts/browser_prove.sh verify_honk_proof webkit"
-  echo "$browser_prefix:NAME=webkit_a_1_mul $scripts/browser_prove.sh a_1_mul webkit"
+  echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof chrome"
+  echo "$browser_prefix $scripts/browser_prove.sh a_1_mul chrome"
+  echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof webkit"
+  echo "$browser_prefix $scripts/browser_prove.sh a_1_mul webkit"
 
   # bb.js tests.
   # ecdsa_secp256r1_3x through bb.js on node to check 256k support.

barretenberg/acir_tests/browser-test-app/src/index.ts

@@ -30,17 +30,16 @@ function installUltraHonkGlobals() {
   }
 
   async function verify(proofData: ProofData, verificationKey: Uint8Array) {
-    const { BarretenbergVerifier } = await import("@aztec/bb.js");
+    const { UltraHonkVerifierBackend } = await import("@aztec/bb.js");
 
     logger.debug(`verifying...`);
-    const verifier = new BarretenbergVerifier();
-    const verified = await verifier.verifyUltraHonkProof(
-      proofData,
-      verificationKey
+    const backend = new UltraHonkVerifierBackend();
+    const verified = await backend.verifyProof(
+      {...proofData, verificationKey}
     );
     logger.debug(`verified: ${verified}`);
 
-    await verifier.destroy();
+    await backend.destroy();
 
     logger.debug("test complete.");
     return verified;

barretenberg/acir_tests/scripts/bb_prove_bbjs_verify.sh

@@ -22,7 +22,6 @@ $bb prove \
   -b target/program.json \
   -w target/witness.gz \
   -k output-$$/vk \
-  --output_format bytes_and_fields \
   -o output-$$
 
 # Verify the proof with bb.js classes

barretenberg/acir_tests/scripts/bb_prove_sol_verify.sh

@@ -34,16 +34,15 @@ mkdir -p output-$$
 trap "rm -rf output-$$" EXIT
 
 # Create a proof, write the solidity contract, write the proof as fields in order to extract the public inputs
-$bb prove $flags -b target/program.json --oracle_hash keccak --output_format bytes_and_fields --write_vk -o output-$$
+$bb prove $flags -b target/program.json --oracle_hash keccak --write_vk -o output-$$
 $bb verify $flags --oracle_hash keccak -i output-$$/public_inputs -k output-$$/vk -p output-$$/proof
 $bb write_solidity_verifier $write_contract_flags -k output-$$/vk -o output-$$/Verifier.sol
 
 # Use solcjs to compile the generated key contract with the template verifier and test contract
 # index.js will start an anvil, on a random port
 # Deploy the verifier then send a test transaction
 PROOF="output-$$/proof" \
-PROOF_AS_FIELDS="output-$$/proof_fields.json" \
-PUBLIC_INPUTS_AS_FIELDS="output-$$/public_inputs_fields.json" \
+PUBLIC_INPUTS="output-$$/public_inputs" \
 VERIFIER_PATH="output-$$/Verifier.sol" \
 TEST_PATH="../../sol-test/HonkTest.sol" \
 HAS_ZK="$has_zk" \

barretenberg/acir_tests/scripts/bbjs_prove_bb_verify.sh

@@ -14,17 +14,7 @@ node ../../bbjs-test prove \
   -w target/witness.gz \
   -o output-$$
 
-proof_bytes=$(cat output-$$/proof | xxd -p)
-public_inputs=$(cat output-$$/public_inputs_fields.json | jq -r '.[]')
-
-public_inputs_bytes=""
-for input in $public_inputs; do
-  public_inputs_bytes+=$input
-done
-
-# Combine proof header and the proof to a single file
-echo -n $proof_bytes | xxd -r -p > output-$$/proof
-echo -n $public_inputs_bytes | xxd -r -p > output-$$/public_inputs
+# The proof and public_inputs are already in binary format from bbjs-test
 
 bb=$(../../../cpp/scripts/find-bb)
 # Verify the proof with bb cli

barretenberg/acir_tests/scripts/bbjs_prove_sol_verify.sh

@@ -35,8 +35,7 @@ $bb write_solidity_verifier --scheme ultra_honk -k output-$$/vk -o output-$$/Ver
 
 # Verify the proof using the solidity verifier
 PROOF="output-$$/proof" \
-PROOF_AS_FIELDS="output-$$/proof_fields.json" \
-PUBLIC_INPUTS_AS_FIELDS="output-$$/public_inputs_fields.json" \
+PUBLIC_INPUTS="output-$$/public_inputs" \
 VERIFIER_PATH="output-$$/Verifier.sol" \
 TEST_PATH="../../sol-test/HonkTest.sol" \
 HAS_ZK="$has_zk" \

barretenberg/acir_tests/sol-test/src/index.js

@@ -165,6 +165,20 @@ const linkLibrary = (bytecode, libraryName, libraryAddress) => {
   return bytecode.replace(regex, address);
 };
 
+/**
+ * Converts binary data to array of field elements (32-byte chunks as hex strings)
+ * @param {Buffer} buffer - Binary data
+ * @return {Array<String>} Array of hex strings with 0x prefix
+ */
+const binaryToFields = (buffer) => {
+  const fields = [];
+  for (let i = 0; i < buffer.length; i += 32) {
+    const chunk = buffer.slice(i, i + 32);
+    fields.push('0x' + chunk.toString('hex'));
+  }
+  return fields;
+};
+
 /**
  * Takes in a proof as fields, and returns the public inputs, as well as the number of public inputs
  * @param {Array<String>} proofAsFields
@@ -225,26 +239,22 @@ try {
   const proof = readFileSync(proofPath);
   proofStr = proof.toString("hex");
 
-  let publicInputsAsFieldsPath = getEnvVarCanBeUndefined(
-    "PUBLIC_INPUTS_AS_FIELDS"
-  ); // PUBLIC_INPUTS_AS_FIELDS is not defined for bb plonk, but is for bb honk and bbjs honk.
-  var publicInputs;
-  let proofAsFieldsPath = getEnvVarCanBeUndefined("PROOF_AS_FIELDS"); // PROOF_AS_FIELDS is not defined for bbjs, but is for bb plonk and bb honk.
+  let publicInputsPath = getEnvVarCanBeUndefined("PUBLIC_INPUTS");
+  var publicInputs = [];
   let numExtraPublicInputs = 0;
   let extraPublicInputs = [];
-  if (proofAsFieldsPath) {
-    const proofAsFields = readFileSync(proofAsFieldsPath);
+
+  // For flows that use binary proof format, extract public inputs from the proof
+  const proofAsFields = binaryToFields(proof);
+  if (proofAsFields.length > NUMBER_OF_FIELDS_IN_PROOF) {
     // We need to extract the public inputs from the proof. This might be empty, or just the pairing point object, or be the entire public inputs...
-    [numExtraPublicInputs, extraPublicInputs] = readPublicInputs(
-      JSON.parse(proofAsFields.toString())
-    );
+    [numExtraPublicInputs, extraPublicInputs] = readPublicInputs(proofAsFields);
   }
-  // We need to do this because plonk doesn't define this path
-  if (publicInputsAsFieldsPath) {
-    const innerPublicInputs = JSON.parse(
-      readFileSync(publicInputsAsFieldsPath).toString()
-    ); // assumes JSON array of PI hex strings
 
+  // Read public inputs from binary file if available
+  if (publicInputsPath) {
+    const publicInputsBinary = readFileSync(publicInputsPath);
+    const innerPublicInputs = binaryToFields(publicInputsBinary);
     publicInputs = innerPublicInputs.concat(extraPublicInputs);
   } else {
     // for plonk, the extraPublicInputs are all of the public inputs

barretenberg/bootstrap.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 source $(git rev-parse --show-toplevel)/ci3/source
 
-
 function bootstrap_all {
   # To run bb we need a crs.
   # Download ignition up front to ensure no race conditions at runtime.
@@ -32,20 +31,6 @@ case "$cmd" in
   "release-preview")
     ./docs/bootstrap.sh release-preview
     ;;
-  bootstrap_e2e_hack)
-    echo "WARNING: This assumes your PR only changes barretenberg and the rest of the repository is unchanged from master."
-    echo "WARNING: This is only sound if you have not changed VK generation! (or noir-projects VKs will be incorrect)."
-    echo "WARNING: It builds up until yarn-project and allows end-to-end tests (not boxes/playground/release image etc)."
-    merge_base=$(git merge-base HEAD origin/master)
-    for project in noir barretenberg avm-transpiler noir-projects l1-contracts yarn-project ; do
-      if [ $project == barretenberg ]; then
-        ../$project/bootstrap.sh # i.e. this script
-      else
-        AZTEC_CACHE_COMMIT=$merge_base ../$project/bootstrap.sh
-      fi
-    done
-    ;;
-
   *)
     echo "Unknown command: $cmd"
     exit 1

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 source $(git rev-parse --show-toplevel)/ci3/source
 
+# export bb as it is needed when using exported functions
+export bb=$(./find-bb)
 cd ..
 
 # NOTE: We pin the captured IVC inputs to a known master commit, exploiting that there won't be frequent changes.
@@ -11,7 +13,8 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="e5081516"
+
+pinned_short_hash="9c83acbc"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {
@@ -44,7 +47,7 @@ if [[ "${1:-}" == "--update_inputs" ]]; then
     # Generate new inputs
     echo "Running bootstrap to generate new IVC inputs..."
 
-    ../../bootstrap.sh # bootstrap aztec-packages from root
+    BOOTSTRAP_TO=yarn-project ../../bootstrap.sh # bootstrap aztec-packages from root
     ../../yarn-project/end-to-end/bootstrap.sh build_bench # build bench to generate IVC inputs
 
     compress_and_upload ../../yarn-project/end-to-end/example-app-ivc-inputs-out
@@ -62,9 +65,9 @@ function check_circuit_vks {
   local flow_folder="$inputs_tmp_dir/$1"
 
   if [[ "${2:-}" == "--update_inputs" ]]; then
-    ./build/bin/bb check --update_inputs --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder! Updating inputs."; exit 1; }
+    $bb check --update_inputs --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder! Updating inputs."; exit 1; }
   else
-    ./build/bin/bb check --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder!"; exit 1; }
+    $bb check --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder!"; exit 1; }
   fi
 }