Merge branch 'merge-train/barretenberg' into lde/civc-tidy-3

Author: ledwards2225

.github/workflows/deploy-aztec-infra.yml

@@ -100,6 +100,10 @@ on:
         description: Whether to use an external ingress for the rpc
         required: true
         type: boolean
+      otel_collector_url:
+        description: The OpenTelemetry collector that will receive metrics from this deployment. Optional
+        required: false
+        type: string
     secrets:
       GCP_SA_KEY:
         description: The GCP service account key
@@ -207,6 +211,10 @@ on:
         description: Whether to use an external ingress for the rpc
         required: true
         type: boolean
+      otel_collector_url:
+        description: The OpenTelemetry collector that will receive metrics from this deployment. Optional
+        required: false
+        type: string
 
 jobs:
   deploy_aztec_infra:
@@ -237,6 +245,7 @@ jobs:
       TF_VAR_PROVER_RESOURCE_PROFILE: ${{ inputs.prover_resource_profile }}
       TF_VAR_RPC_RESOURCE_PROFILE: ${{ inputs.rpc_resource_profile }}
       TF_VAR_RPC_EXTERNAL_INGRESS: ${{ inputs.rpc_external_ingress }}
+      TF_VAR_OTEL_COLLECTOR_URL: ${{ inputs.otel_collector_url }}
 
     steps:
       - name: Debug inputs

.vscode/launch.json

@@ -36,8 +36,8 @@
       "name": "Debug CIVC transaction",
       "type": "lldb",
       "request": "launch",
-      "program": "${workspaceFolder}/barretenberg/cpp/build-debug-no-avm/bin/bb",
-      "args": ["prove", "--scheme", "client_ivc", "--output_path", ".", "--ivc_inputs_path", "ivc-inputs.msgpack"],
+      "program": "${workspaceFolder}/barretenberg/cpp/build-debug-fast-no-avm/bin/bb",
+      "args": ["prove", "--scheme", "client_ivc", "--output_path", ".", "--ivc_inputs_path", "ivc-inputs.msgpack", "--print_bench"],
       "cwd": "${workspaceFolder}/yarn-project/end-to-end/example-app-ivc-inputs-out/ecdsar1+transfer_0_recursions+sponsored_fpc",
       "initCommands": [
         "command script import ${workspaceFolder}/barretenberg/cpp/scripts/lldb_format.py"

avm-transpiler/Cargo.lock

@@ -450,7 +450,7 @@ In terms of general usage, you should be able to use scrolling or the WASD keys
 
 ##### Adding Zones
 
-Zones are how you can keep track of where you are relative in the code and how you can bucket allocations together. All of the colored blocks in the Main Thread row and other threads' rows refer to zones. You can nest zones in deeper and deeper scopes, which leads to stacks of these zones. To add a named zone, all you have to do is add PROFILE_THIS() or PROFILE_THIS_NAME(<name>) to a scope and it will create a zone. Note that you can't create multiple zones in the same scope.
+Zones are how you can keep track of where you are relative in the code and how you can bucket allocations together. All of the colored blocks in the Main Thread row and other threads' rows refer to zones. You can nest zones in deeper and deeper scopes, which leads to stacks of these zones. To add a named zone, all you have to do is add BB_BENCH_TRACY() or BB_BENCH_TRACY_NAME(<name>) to a scope and it will create a zone. Note that you can't create multiple zones in the same scope.
 
 ##### Analyzing Fragmentation
 

barretenberg/README.md

@@ -9,9 +9,7 @@ const logger = pino({
 });
 
 const proofPath = (dir: string) => path.join(dir, "proof");
-const proofAsFieldsPath = (dir: string) => path.join(dir, "proof_fields.json");
-const publicInputsAsFieldsPath = (dir: string) =>
-  path.join(dir, "public_inputs_fields.json");
+const publicInputsPath = (dir: string) => path.join(dir, "public_inputs");
 const vkeyPath = (dir: string) => path.join(dir, "vk");
 
 async function generateProof({
@@ -27,7 +25,7 @@ async function generateProof({
   oracleHash?: string;
   multiThreaded?: boolean;
 }) {
-  const { UltraHonkBackend, deflattenFields } = await import("@aztec/bb.js");
+  const { UltraHonkBackend } = await import("@aztec/bb.js");
 
   logger.debug(`Generating proof for ${bytecodePath}...`);
   const circuitArtifact = await fs.readFile(bytecodePath);
@@ -45,17 +43,16 @@ async function generateProof({
   await fs.writeFile(proofPath(outputDirectory), Buffer.from(proof.proof));
   logger.debug("Proof written to " + proofPath(outputDirectory));
 
-  await fs.writeFile(
-    publicInputsAsFieldsPath(outputDirectory),
-    JSON.stringify(proof.publicInputs)
+  // Convert public inputs from field strings to binary
+  const publicInputsBuffer = Buffer.concat(
+    proof.publicInputs.map((field: string) => {
+      const hex = field.startsWith('0x') ? field.slice(2) : field;
+      return Buffer.from(hex.padStart(64, '0'), 'hex');
+    })
   );
+  await fs.writeFile(publicInputsPath(outputDirectory), publicInputsBuffer);
   logger.debug(
-    "Public inputs written to " + publicInputsAsFieldsPath(outputDirectory)
-  );
-
-  await fs.writeFile(
-    proofAsFieldsPath(outputDirectory),
-    JSON.stringify(deflattenFields(proof.proof))
+    "Public inputs written to " + publicInputsPath(outputDirectory)
   );
 
   const verificationKey = await backend.getVerificationKey({
@@ -69,21 +66,24 @@ async function generateProof({
 }
 
 async function verifyProof({ directory }: { directory: string }) {
-  const { BarretenbergVerifier } = await import("@aztec/bb.js");
+  const { UltraHonkVerifierBackend } = await import("@aztec/bb.js");
 
-  const verifier = new BarretenbergVerifier();
+  const verifier = new UltraHonkVerifierBackend();
 
   const proof = await fs.readFile(proofPath(directory));
 
-  const publicInputs = JSON.parse(
-    await fs.readFile(publicInputsAsFieldsPath(directory), "utf8")
-  );
+  // Read binary public inputs and convert to field strings
+  const publicInputsBinary = await fs.readFile(publicInputsPath(directory));
+  const publicInputs = [];
+  for (let i = 0; i < publicInputsBinary.length; i += 32) {
+    const chunk = publicInputsBinary.slice(i, Math.min(i + 32, publicInputsBinary.length));
+    publicInputs.push('0x' + chunk.toString('hex'));
+  }
   logger.debug(`publicInputs: ${JSON.stringify(publicInputs)}`);
-  const vkey = await fs.readFile(vkeyPath(directory));
+  const verificationKey = await fs.readFile(vkeyPath(directory));
 
-  const verified = await verifier.verifyUltraHonkProof(
-    { proof: new Uint8Array(proof), publicInputs },
-    new Uint8Array(vkey)
+  const verified = await verifier.verifyProof(
+    { proof: new Uint8Array(proof), publicInputs, verificationKey},
   );
 
   await verifier.destroy();

barretenberg/acir_tests/bbjs-test/src/index.ts

@@ -21,14 +21,18 @@ tests_hash=$(hash_str \
     ../ts/.rebuild_patterns \
     ../noir/))
 
+function hex_to_fields_json {
+  # 1. split encoded hex into 64-character lines 3. encode as JSON array of hex strings
+  fold -w64 | jq -R -s -c 'split("\n") | map(select(length > 0)) | map("0x" + .)'
+}
+
 # Generate inputs for a given recursively verifying program.
 function run_proof_generation {
   local program=$1
   local native_build_dir=$(../cpp/scripts/native-preset-build-dir)
   local bb=$(realpath ../cpp/$native_build_dir/bin/bb)
   local outdir=$(mktemp -d)
   trap "rm -rf $outdir" EXIT
-  local adjustment=16
   local ipa_accumulation_flag=""
 
   cd ./acir_tests/assert_statement
@@ -37,35 +41,37 @@ function run_proof_generation {
 
   # Adjust settings based on program type
   if [[ $program == *"rollup"* ]]; then
-      adjustment=26
       ipa_accumulation_flag="--ipa_accumulation"
   fi
   # If the test program has zk in it's name would like to use the zk prover, so we empty the flag in this case.
   if [[ $program == *"zk"* ]]; then
     disable_zk=""
   fi
-  local prove_cmd="$bb prove --scheme ultra_honk $disable_zk $ipa_accumulation_flag --output_format fields --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz"
+  local prove_cmd="$bb prove --scheme ultra_honk $disable_zk $ipa_accumulation_flag --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz"
   echo_stderr "$prove_cmd"
   dump_fail "$prove_cmd"
 
-  local vk_fields=$(cat "$outdir/vk_fields.json")
-  local vk_hash_fields=$(cat "$outdir/vk_hash_fields.json")
-  local public_inputs_fields=$(cat "$outdir/public_inputs_fields.json")
-  local proof_fields=$(cat "$outdir/proof_fields.json")
 
-  generate_toml "$program" "$vk_fields" "$vk_hash_fields" "$proof_fields" "$public_inputs_fields"
+  # Split the hex-encoded vk bytes into fields boundaries (but still hex-encoded), first making 64-character lines and then encoding as JSON.
+  # This used to be done by barretenberg itself, but with serialization now always being in field elements we can do it outside of bb.
+  local vk_fields=$(cat "$outdir/vk" | xxd -p -c 0 | hex_to_fields_json)
+  local vk_hash_field="\"0x$(cat "$outdir/vk_hash" | xxd -p -c 0)\""
+  local public_inputs_fields=$(cat "$outdir/public_inputs" | xxd -p -c 0 | hex_to_fields_json)
+  local proof_fields=$(cat "$outdir/proof" | xxd -p -c 0 | hex_to_fields_json)
+
+  generate_toml "$program" "$vk_fields" "$vk_hash_field" "$proof_fields" "$public_inputs_fields"
 }
 
 function generate_toml {
   local program=$1
   local vk_fields=$2
-  local vk_hash_fields=$3
+  local vk_hash_field=$3
   local proof_fields=$4
-  local num_inner_public_inputs=$5
+  local public_inputs_fields=$5
   local output_file="../$program/Prover.toml"
 
   jq -nr \
-      --arg key_hash "$vk_hash_fields" \
+      --arg key_hash "$vk_hash_field" \
       --argjson vk_f "$vk_fields" \
       --argjson public_inputs_f "$public_inputs_fields" \
       --argjson proof_f "$proof_fields" \
@@ -79,7 +85,6 @@ function generate_toml {
 }
 
 function regenerate_recursive_inputs {
-  local program=$1
   # Compile the assert_statement test as it's used for the recursive tests.
   cd ./acir_tests/assert_statement
   local nargo=$(realpath ../../../../noir/noir-repo/target/release/nargo)
@@ -91,7 +96,7 @@ function regenerate_recursive_inputs {
   parallel 'run_proof_generation {}' ::: $(ls internal_test_programs)
 }
 
-export -f regenerate_recursive_inputs run_proof_generation generate_toml
+export -f hex_to_fields_json regenerate_recursive_inputs run_proof_generation generate_toml
 
 function compile {
   echo_header "Compiling acir_tests"
@@ -152,17 +157,18 @@ function test_cmds {
   for t in assert_statement a_1_mul slices verify_honk_proof; do
     echo "$sol_prefix $scripts/bb_prove_sol_verify.sh $t --disable_zk"
     echo "$sol_prefix $scripts/bb_prove_sol_verify.sh $t"
+    echo "$sol_prefix USE_OPTIMIZED_CONTRACT=true $scripts/bb_prove_sol_verify.sh $t --disable_zk"
   done
   # prove with bb cli and verify with bb.js classes
   echo "$sol_prefix $scripts/bb_prove_bbjs_verify.sh a_1_mul"
   echo "$sol_prefix $scripts/bb_prove_bbjs_verify.sh assert_statement"
 
   # bb.js browser tests. Isolate because server.
   local browser_prefix="$tests_hash:ISOLATE=1:NET=1:CPUS=8"
-  echo "$browser_prefix:NAME=chrome_verify_honk_proof $scripts/browser_prove.sh verify_honk_proof chrome"
-  echo "$browser_prefix:NAME=chrome_a_1_mul $scripts/browser_prove.sh a_1_mul chrome"
-  echo "$browser_prefix:NAME=webkit_verify_honk_proof $scripts/browser_prove.sh verify_honk_proof webkit"
-  echo "$browser_prefix:NAME=webkit_a_1_mul $scripts/browser_prove.sh a_1_mul webkit"
+  echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof chrome"
+  echo "$browser_prefix $scripts/browser_prove.sh a_1_mul chrome"
+  echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof webkit"
+  echo "$browser_prefix $scripts/browser_prove.sh a_1_mul webkit"
 
   # bb.js tests.
   # ecdsa_secp256r1_3x through bb.js on node to check 256k support.

barretenberg/acir_tests/bootstrap.sh

@@ -30,17 +30,16 @@ function installUltraHonkGlobals() {
   }
 
   async function verify(proofData: ProofData, verificationKey: Uint8Array) {
-    const { BarretenbergVerifier } = await import("@aztec/bb.js");
+    const { UltraHonkVerifierBackend } = await import("@aztec/bb.js");
 
     logger.debug(`verifying...`);
-    const verifier = new BarretenbergVerifier();
-    const verified = await verifier.verifyUltraHonkProof(
-      proofData,
-      verificationKey
+    const backend = new UltraHonkVerifierBackend();
+    const verified = await backend.verifyProof(
+      {...proofData, verificationKey}
     );
     logger.debug(`verified: ${verified}`);
 
-    await verifier.destroy();
+    await backend.destroy();
 
     logger.debug("test complete.");
     return verified;

barretenberg/acir_tests/browser-test-app/src/index.ts

@@ -22,7 +22,6 @@ $bb prove \
   -b target/program.json \
   -w target/witness.gz \
   -k output-$$/vk \
-  --output_format bytes_and_fields \
   -o output-$$
 
 # Verify the proof with bb.js classes

barretenberg/acir_tests/scripts/bb_prove_bbjs_verify.sh

@@ -16,6 +16,13 @@ for arg in "$@"; do
     flags+=" $arg"
 done
 
+USE_OPTIMIZED_CONTRACT=${USE_OPTIMIZED_CONTRACT:-false}
+
+write_contract_flags=$flags
+if [[ -z "$USE_OPTIMIZED_CONTRACT" ]]; then
+    write_contract_flags+=" --optimized"
+fi
+
 # Check if --disable_zk is in the flags to determine HAS_ZK
 if [[ "$flags" == *"--disable_zk"* ]]; then
     has_zk="false"
@@ -27,16 +34,15 @@ mkdir -p output-$$
 trap "rm -rf output-$$" EXIT
 
 # Create a proof, write the solidity contract, write the proof as fields in order to extract the public inputs
-$bb prove $flags -b target/program.json --oracle_hash keccak --output_format bytes_and_fields --write_vk -o output-$$
+$bb prove $flags -b target/program.json --oracle_hash keccak --write_vk -o output-$$
 $bb verify $flags --oracle_hash keccak -i output-$$/public_inputs -k output-$$/vk -p output-$$/proof
-$bb write_solidity_verifier $flags -k output-$$/vk -o output-$$/Verifier.sol
+$bb write_solidity_verifier $write_contract_flags -k output-$$/vk -o output-$$/Verifier.sol
 
 # Use solcjs to compile the generated key contract with the template verifier and test contract
 # index.js will start an anvil, on a random port
 # Deploy the verifier then send a test transaction
 PROOF="output-$$/proof" \
-PROOF_AS_FIELDS="output-$$/proof_fields.json" \
-PUBLIC_INPUTS_AS_FIELDS="output-$$/public_inputs_fields.json" \
+PUBLIC_INPUTS="output-$$/public_inputs" \
 VERIFIER_PATH="output-$$/Verifier.sol" \
 TEST_PATH="../../sol-test/HonkTest.sol" \
 HAS_ZK="$has_zk" \

barretenberg/acir_tests/scripts/bb_prove_sol_verify.sh

@@ -14,17 +14,7 @@ node ../../bbjs-test prove \
   -w target/witness.gz \
   -o output-$$
 
-proof_bytes=$(cat output-$$/proof | xxd -p)
-public_inputs=$(cat output-$$/public_inputs_fields.json | jq -r '.[]')
-
-public_inputs_bytes=""
-for input in $public_inputs; do
-  public_inputs_bytes+=$input
-done
-
-# Combine proof header and the proof to a single file
-echo -n $proof_bytes | xxd -r -p > output-$$/proof
-echo -n $public_inputs_bytes | xxd -r -p > output-$$/public_inputs
+# The proof and public_inputs are already in binary format from bbjs-test
 
 bb=$(../../../cpp/scripts/find-bb)
 # Verify the proof with bb cli