feat: merge-train/barretenberg (#16715)

BEGIN_COMMIT_OVERRIDE
feat(bb): --print_bench (#16673)
refactor(bb): remove --output_format, have one-size-fits-all format
(#16201)
fix(bb): fix vk simplified buffer (#16720)
fix(avm): avm vk after serialization unification (#16721)
END_COMMIT_OVERRIDE

Author: ludamad

.vscode/launch.json

@@ -36,8 +36,8 @@
       "name": "Debug CIVC transaction",
       "type": "lldb",
       "request": "launch",
-      "program": "${workspaceFolder}/barretenberg/cpp/build-debug-no-avm/bin/bb",
-      "args": ["prove", "--scheme", "client_ivc", "--output_path", ".", "--ivc_inputs_path", "ivc-inputs.msgpack"],
+      "program": "${workspaceFolder}/barretenberg/cpp/build-debug-fast-no-avm/bin/bb",
+      "args": ["prove", "--scheme", "client_ivc", "--output_path", ".", "--ivc_inputs_path", "ivc-inputs.msgpack", "--print_bench"],
       "cwd": "${workspaceFolder}/yarn-project/end-to-end/example-app-ivc-inputs-out/ecdsar1+transfer_0_recursions+sponsored_fpc",
       "initCommands": [
         "command script import ${workspaceFolder}/barretenberg/cpp/scripts/lldb_format.py"

barretenberg/README.md

@@ -450,7 +450,7 @@ In terms of general usage, you should be able to use scrolling or the WASD keys
 
 ##### Adding Zones
 
-Zones are how you can keep track of where you are relative in the code and how you can bucket allocations together. All of the colored blocks in the Main Thread row and other threads' rows refer to zones. You can nest zones in deeper and deeper scopes, which leads to stacks of these zones. To add a named zone, all you have to do is add PROFILE_THIS() or PROFILE_THIS_NAME(<name>) to a scope and it will create a zone. Note that you can't create multiple zones in the same scope.
+Zones are how you can keep track of where you are relative in the code and how you can bucket allocations together. All of the colored blocks in the Main Thread row and other threads' rows refer to zones. You can nest zones in deeper and deeper scopes, which leads to stacks of these zones. To add a named zone, all you have to do is add BB_BENCH_TRACY() or BB_BENCH_TRACY_NAME(<name>) to a scope and it will create a zone. Note that you can't create multiple zones in the same scope.
 
 ##### Analyzing Fragmentation
 

barretenberg/acir_tests/bbjs-test/src/index.ts

@@ -9,9 +9,7 @@ const logger = pino({
 });
 
 const proofPath = (dir: string) => path.join(dir, "proof");
-const proofAsFieldsPath = (dir: string) => path.join(dir, "proof_fields.json");
-const publicInputsAsFieldsPath = (dir: string) =>
-  path.join(dir, "public_inputs_fields.json");
+const publicInputsPath = (dir: string) => path.join(dir, "public_inputs");
 const vkeyPath = (dir: string) => path.join(dir, "vk");
 
 async function generateProof({
@@ -27,7 +25,7 @@ async function generateProof({
   oracleHash?: string;
   multiThreaded?: boolean;
 }) {
-  const { UltraHonkBackend, deflattenFields } = await import("@aztec/bb.js");
+  const { UltraHonkBackend } = await import("@aztec/bb.js");
 
   logger.debug(`Generating proof for ${bytecodePath}...`);
   const circuitArtifact = await fs.readFile(bytecodePath);
@@ -45,17 +43,16 @@ async function generateProof({
   await fs.writeFile(proofPath(outputDirectory), Buffer.from(proof.proof));
   logger.debug("Proof written to " + proofPath(outputDirectory));
 
-  await fs.writeFile(
-    publicInputsAsFieldsPath(outputDirectory),
-    JSON.stringify(proof.publicInputs)
+  // Convert public inputs from field strings to binary
+  const publicInputsBuffer = Buffer.concat(
+    proof.publicInputs.map((field: string) => {
+      const hex = field.startsWith('0x') ? field.slice(2) : field;
+      return Buffer.from(hex.padStart(64, '0'), 'hex');
+    })
   );
+  await fs.writeFile(publicInputsPath(outputDirectory), publicInputsBuffer);
   logger.debug(
-    "Public inputs written to " + publicInputsAsFieldsPath(outputDirectory)
-  );
-
-  await fs.writeFile(
-    proofAsFieldsPath(outputDirectory),
-    JSON.stringify(deflattenFields(proof.proof))
+    "Public inputs written to " + publicInputsPath(outputDirectory)
   );
 
   const verificationKey = await backend.getVerificationKey({
@@ -69,21 +66,24 @@ async function generateProof({
 }
 
 async function verifyProof({ directory }: { directory: string }) {
-  const { BarretenbergVerifier } = await import("@aztec/bb.js");
+  const { UltraHonkVerifierBackend } = await import("@aztec/bb.js");
 
-  const verifier = new BarretenbergVerifier();
+  const verifier = new UltraHonkVerifierBackend();
 
   const proof = await fs.readFile(proofPath(directory));
 
-  const publicInputs = JSON.parse(
-    await fs.readFile(publicInputsAsFieldsPath(directory), "utf8")
-  );
+  // Read binary public inputs and convert to field strings
+  const publicInputsBinary = await fs.readFile(publicInputsPath(directory));
+  const publicInputs = [];
+  for (let i = 0; i < publicInputsBinary.length; i += 32) {
+    const chunk = publicInputsBinary.slice(i, Math.min(i + 32, publicInputsBinary.length));
+    publicInputs.push('0x' + chunk.toString('hex'));
+  }
   logger.debug(`publicInputs: ${JSON.stringify(publicInputs)}`);
-  const vkey = await fs.readFile(vkeyPath(directory));
+  const verificationKey = await fs.readFile(vkeyPath(directory));
 
-  const verified = await verifier.verifyUltraHonkProof(
-    { proof: new Uint8Array(proof), publicInputs },
-    new Uint8Array(vkey)
+  const verified = await verifier.verifyProof(
+    { proof: new Uint8Array(proof), publicInputs, verificationKey},
   );
 
   await verifier.destroy();

barretenberg/acir_tests/bootstrap.sh

@@ -21,14 +21,18 @@ tests_hash=$(hash_str \
     ../ts/.rebuild_patterns \
     ../noir/))
 
+function hex_to_fields_json {
+  # 1. split encoded hex into 64-character lines 3. encode as JSON array of hex strings
+  fold -w64 | jq -R -s -c 'split("\n") | map(select(length > 0)) | map("0x" + .)'
+}
+
 # Generate inputs for a given recursively verifying program.
 function run_proof_generation {
   local program=$1
   local native_build_dir=$(../cpp/scripts/native-preset-build-dir)
   local bb=$(realpath ../cpp/$native_build_dir/bin/bb)
   local outdir=$(mktemp -d)
   trap "rm -rf $outdir" EXIT
-  local adjustment=16
   local ipa_accumulation_flag=""
 
   cd ./acir_tests/assert_statement
@@ -37,35 +41,37 @@ function run_proof_generation {
 
   # Adjust settings based on program type
   if [[ $program == *"rollup"* ]]; then
-      adjustment=26
       ipa_accumulation_flag="--ipa_accumulation"
   fi
   # If the test program has zk in it's name would like to use the zk prover, so we empty the flag in this case.
   if [[ $program == *"zk"* ]]; then
     disable_zk=""
   fi
-  local prove_cmd="$bb prove --scheme ultra_honk $disable_zk $ipa_accumulation_flag --output_format fields --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz"
+  local prove_cmd="$bb prove --scheme ultra_honk $disable_zk $ipa_accumulation_flag --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz"
   echo_stderr "$prove_cmd"
   dump_fail "$prove_cmd"
 
-  local vk_fields=$(cat "$outdir/vk_fields.json")
-  local vk_hash_fields=$(cat "$outdir/vk_hash_fields.json")
-  local public_inputs_fields=$(cat "$outdir/public_inputs_fields.json")
-  local proof_fields=$(cat "$outdir/proof_fields.json")
 
-  generate_toml "$program" "$vk_fields" "$vk_hash_fields" "$proof_fields" "$public_inputs_fields"
+  # Split the hex-encoded vk bytes into fields boundaries (but still hex-encoded), first making 64-character lines and then encoding as JSON.
+  # This used to be done by barretenberg itself, but with serialization now always being in field elements we can do it outside of bb.
+  local vk_fields=$(cat "$outdir/vk" | xxd -p -c 0 | hex_to_fields_json)
+  local vk_hash_field="\"0x$(cat "$outdir/vk_hash" | xxd -p -c 0)\""
+  local public_inputs_fields=$(cat "$outdir/public_inputs" | xxd -p -c 0 | hex_to_fields_json)
+  local proof_fields=$(cat "$outdir/proof" | xxd -p -c 0 | hex_to_fields_json)
+
+  generate_toml "$program" "$vk_fields" "$vk_hash_field" "$proof_fields" "$public_inputs_fields"
 }
 
 function generate_toml {
   local program=$1
   local vk_fields=$2
-  local vk_hash_fields=$3
+  local vk_hash_field=$3
   local proof_fields=$4
-  local num_inner_public_inputs=$5
+  local public_inputs_fields=$5
   local output_file="../$program/Prover.toml"
 
   jq -nr \
-      --arg key_hash "$vk_hash_fields" \
+      --arg key_hash "$vk_hash_field" \
       --argjson vk_f "$vk_fields" \
       --argjson public_inputs_f "$public_inputs_fields" \
       --argjson proof_f "$proof_fields" \
@@ -79,7 +85,6 @@ function generate_toml {
 }
 
 function regenerate_recursive_inputs {
-  local program=$1
   # Compile the assert_statement test as it's used for the recursive tests.
   cd ./acir_tests/assert_statement
   local nargo=$(realpath ../../../../noir/noir-repo/target/release/nargo)
@@ -91,7 +96,7 @@ function regenerate_recursive_inputs {
   parallel 'run_proof_generation {}' ::: $(ls internal_test_programs)
 }
 
-export -f regenerate_recursive_inputs run_proof_generation generate_toml
+export -f hex_to_fields_json regenerate_recursive_inputs run_proof_generation generate_toml
 
 function compile {
   echo_header "Compiling acir_tests"
@@ -160,10 +165,10 @@ function test_cmds {
 
   # bb.js browser tests. Isolate because server.
   local browser_prefix="$tests_hash:ISOLATE=1:NET=1:CPUS=8"
-  echo "$browser_prefix:NAME=chrome_verify_honk_proof $scripts/browser_prove.sh verify_honk_proof chrome"
-  echo "$browser_prefix:NAME=chrome_a_1_mul $scripts/browser_prove.sh a_1_mul chrome"
-  echo "$browser_prefix:NAME=webkit_verify_honk_proof $scripts/browser_prove.sh verify_honk_proof webkit"
-  echo "$browser_prefix:NAME=webkit_a_1_mul $scripts/browser_prove.sh a_1_mul webkit"
+  echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof chrome"
+  echo "$browser_prefix $scripts/browser_prove.sh a_1_mul chrome"
+  echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof webkit"
+  echo "$browser_prefix $scripts/browser_prove.sh a_1_mul webkit"
 
   # bb.js tests.
   # ecdsa_secp256r1_3x through bb.js on node to check 256k support.

barretenberg/acir_tests/browser-test-app/src/index.ts

@@ -30,17 +30,16 @@ function installUltraHonkGlobals() {
   }
 
   async function verify(proofData: ProofData, verificationKey: Uint8Array) {
-    const { BarretenbergVerifier } = await import("@aztec/bb.js");
+    const { UltraHonkVerifierBackend } = await import("@aztec/bb.js");
 
     logger.debug(`verifying...`);
-    const verifier = new BarretenbergVerifier();
-    const verified = await verifier.verifyUltraHonkProof(
-      proofData,
-      verificationKey
+    const backend = new UltraHonkVerifierBackend();
+    const verified = await backend.verifyProof(
+      {...proofData, verificationKey}
     );
     logger.debug(`verified: ${verified}`);
 
-    await verifier.destroy();
+    await backend.destroy();
 
     logger.debug("test complete.");
     return verified;

barretenberg/acir_tests/scripts/bb_prove_bbjs_verify.sh

@@ -22,7 +22,6 @@ $bb prove \
   -b target/program.json \
   -w target/witness.gz \
   -k output-$$/vk \
-  --output_format bytes_and_fields \
   -o output-$$
 
 # Verify the proof with bb.js classes

barretenberg/acir_tests/scripts/bb_prove_sol_verify.sh

@@ -34,16 +34,15 @@ mkdir -p output-$$
 trap "rm -rf output-$$" EXIT
 
 # Create a proof, write the solidity contract, write the proof as fields in order to extract the public inputs
-$bb prove $flags -b target/program.json --oracle_hash keccak --output_format bytes_and_fields --write_vk -o output-$$
+$bb prove $flags -b target/program.json --oracle_hash keccak --write_vk -o output-$$
 $bb verify $flags --oracle_hash keccak -i output-$$/public_inputs -k output-$$/vk -p output-$$/proof
 $bb write_solidity_verifier $write_contract_flags -k output-$$/vk -o output-$$/Verifier.sol
 
 # Use solcjs to compile the generated key contract with the template verifier and test contract
 # index.js will start an anvil, on a random port
 # Deploy the verifier then send a test transaction
 PROOF="output-$$/proof" \
-PROOF_AS_FIELDS="output-$$/proof_fields.json" \
-PUBLIC_INPUTS_AS_FIELDS="output-$$/public_inputs_fields.json" \
+PUBLIC_INPUTS="output-$$/public_inputs" \
 VERIFIER_PATH="output-$$/Verifier.sol" \
 TEST_PATH="../../sol-test/HonkTest.sol" \
 HAS_ZK="$has_zk" \

barretenberg/acir_tests/scripts/bbjs_prove_bb_verify.sh

@@ -14,17 +14,7 @@ node ../../bbjs-test prove \
   -w target/witness.gz \
   -o output-$$
 
-proof_bytes=$(cat output-$$/proof | xxd -p)
-public_inputs=$(cat output-$$/public_inputs_fields.json | jq -r '.[]')
-
-public_inputs_bytes=""
-for input in $public_inputs; do
-  public_inputs_bytes+=$input
-done
-
-# Combine proof header and the proof to a single file
-echo -n $proof_bytes | xxd -r -p > output-$$/proof
-echo -n $public_inputs_bytes | xxd -r -p > output-$$/public_inputs
+# The proof and public_inputs are already in binary format from bbjs-test
 
 bb=$(../../../cpp/scripts/find-bb)
 # Verify the proof with bb cli

barretenberg/acir_tests/scripts/bbjs_prove_sol_verify.sh

@@ -35,8 +35,7 @@ $bb write_solidity_verifier --scheme ultra_honk -k output-$$/vk -o output-$$/Ver
 
 # Verify the proof using the solidity verifier
 PROOF="output-$$/proof" \
-PROOF_AS_FIELDS="output-$$/proof_fields.json" \
-PUBLIC_INPUTS_AS_FIELDS="output-$$/public_inputs_fields.json" \
+PUBLIC_INPUTS="output-$$/public_inputs" \
 VERIFIER_PATH="output-$$/Verifier.sol" \
 TEST_PATH="../../sol-test/HonkTest.sol" \
 HAS_ZK="$has_zk" \

barretenberg/acir_tests/sol-test/src/index.js

@@ -165,6 +165,20 @@ const linkLibrary = (bytecode, libraryName, libraryAddress) => {
   return bytecode.replace(regex, address);
 };
 
+/**
+ * Converts binary data to array of field elements (32-byte chunks as hex strings)
+ * @param {Buffer} buffer - Binary data
+ * @return {Array<String>} Array of hex strings with 0x prefix
+ */
+const binaryToFields = (buffer) => {
+  const fields = [];
+  for (let i = 0; i < buffer.length; i += 32) {
+    const chunk = buffer.slice(i, i + 32);
+    fields.push('0x' + chunk.toString('hex'));
+  }
+  return fields;
+};
+
 /**
  * Takes in a proof as fields, and returns the public inputs, as well as the number of public inputs
  * @param {Array<String>} proofAsFields
@@ -225,26 +239,22 @@ try {
   const proof = readFileSync(proofPath);
   proofStr = proof.toString("hex");
 
-  let publicInputsAsFieldsPath = getEnvVarCanBeUndefined(
-    "PUBLIC_INPUTS_AS_FIELDS"
-  ); // PUBLIC_INPUTS_AS_FIELDS is not defined for bb plonk, but is for bb honk and bbjs honk.
-  var publicInputs;
-  let proofAsFieldsPath = getEnvVarCanBeUndefined("PROOF_AS_FIELDS"); // PROOF_AS_FIELDS is not defined for bbjs, but is for bb plonk and bb honk.
+  let publicInputsPath = getEnvVarCanBeUndefined("PUBLIC_INPUTS");
+  var publicInputs = [];
   let numExtraPublicInputs = 0;
   let extraPublicInputs = [];
-  if (proofAsFieldsPath) {
-    const proofAsFields = readFileSync(proofAsFieldsPath);
+
+  // For flows that use binary proof format, extract public inputs from the proof
+  const proofAsFields = binaryToFields(proof);
+  if (proofAsFields.length > NUMBER_OF_FIELDS_IN_PROOF) {
     // We need to extract the public inputs from the proof. This might be empty, or just the pairing point object, or be the entire public inputs...
-    [numExtraPublicInputs, extraPublicInputs] = readPublicInputs(
-      JSON.parse(proofAsFields.toString())
-    );
+    [numExtraPublicInputs, extraPublicInputs] = readPublicInputs(proofAsFields);
   }
-  // We need to do this because plonk doesn't define this path
-  if (publicInputsAsFieldsPath) {
-    const innerPublicInputs = JSON.parse(
-      readFileSync(publicInputsAsFieldsPath).toString()
-    ); // assumes JSON array of PI hex strings
 
+  // Read public inputs from binary file if available
+  if (publicInputsPath) {
+    const publicInputsBinary = readFileSync(publicInputsPath);
+    const innerPublicInputs = binaryToFields(publicInputsBinary);
     publicInputs = innerPublicInputs.concat(extraPublicInputs);
   } else {
     // for plonk, the extraPublicInputs are all of the public inputs