fix: Cl/ci3.1 (#10888)

Please read [contributing guidelines](CONTRIBUTING.md) and remove this
line.

avm-transpiler/bootstrap.sh

@@ -17,8 +17,8 @@ function build {
 }
 
 function test {
-  cargo fmt --check
-  cargo clippy
+  denoise cargo fmt --check
+  denoise cargo clippy
 }
 
 case "$cmd" in

barretenberg/acir_tests/browser-test-app/yarn.lock

@@ -6,8 +6,8 @@ __metadata:
   cacheKey: 10c0
 
 "@aztec/bb.js@file:../../ts::locator=browser-test-app%40workspace%3A.":
-  version: 0.67.0
-  resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=e236e3&locator=browser-test-app%40workspace%3A."
+  version: 0.67.1
+  resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=29e47a&locator=browser-test-app%40workspace%3A."
   dependencies:
     comlink: "npm:^4.4.1"
     commander: "npm:^12.1.0"
@@ -17,7 +17,7 @@ __metadata:
     tslib: "npm:^2.4.0"
   bin:
     bb.js: ./dest/node/main.js
-  checksum: 10c0/4ca2234f9d1a4b59a60733979f97c5ca09f508e4d3091bf33755ee6a80f03f733e393ea848b812bc73bc1f0673a3d92f96462a1cbd5aa01eeca4e9d5a3be367f
+  checksum: 10c0/c01128ff74f29b6bbc5c46362792525ef5612c5fc8787341551bcf457ba9816a971e24a74292ab230c47b0b9efe8d7e0d1cabd44247e1b6e718727d0b6372400
   languageName: node
   linkType: hard
 

barretenberg/cpp/bootstrap.sh

@@ -115,7 +115,7 @@ case "$cmd" in
     test
     ;;
   "ci")
-    build_native
+    build
     test
     ;;
   "hash")

build-images/bootstrap.sh

@@ -3,7 +3,7 @@
 source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 
 cmd=${1:-}
-hash=$(REBUILD_PATTERNS="^build-images/Earthfile" cache_content_hash)
+hash=$(cache_content_hash "^build-images/Earthfile")
 
 function build {
   github_group "build-images build"

ci3/bootstrap_local

@@ -9,30 +9,24 @@
 
 root=$(git rev-parse --show-toplevel)
 source $root/ci3/source
-current_commit=$(git rev-parse HEAD)
-cmd=${1:-"CI=1 ./bootstrap.sh fast || exec bash"}
+cmd=${1:-"./bootstrap.sh ci || exec bash"}
 
 if ! git diff --quiet; then
-  echo "There are local changes to tracked files."
-  exit 1
+  echo -e "${yellow}WARNING: There are local changes to tracked files.${reset}"
 fi
 
-interactive_args=""
-# Only add -ti and a name if we are not in CI.
-[ "${CI:-0}" = "0" ] && interactive_args="--name aztec_build -ti"
-docker run $interactive_args --rm \
+docker run --name aztec_build -ti --rm \
   --privileged \
   -v bootstrap_ci_local_docker:/var/lib/docker \
   -v $root:/aztec-packages-host:ro \
   -v $HOME/.aws:/root/.aws \
   aztecprotocol/ci:2.0 bash -c "
+  set -e
   /usr/local/share/docker-init.sh &> /dev/null
   git config --global --add safe.directory /aztec-packages-host/.git
-  mkdir -p /root/aztec-packages && cd /root/aztec-packages
+  cd /root
   # Ensure we get a clean clone of the repo.
-  git init &>/dev/null
-  git remote add origin https://github.com/aztecprotocol/aztec-packages
-  git fetch --depth 1 origin $current_commit 2>/dev/null || (echo 'The commit was not pushed, run aborted.' && exit 1)
-  git checkout FETCH_HEAD &>/dev/null
+  git clone --depth 1 file:///aztec-packages-host aztec-packages
+  cd aztec-packages
   $cmd
 "

ci3/cache_content_hash

@@ -2,24 +2,29 @@
 set -euo pipefail
 [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x
 
-# Ensure REBUILD_PATTERNS or args are set
-if [[ -z "${REBUILD_PATTERNS:-}" &&  "$#" = 0 ]]; then
-    echo "Error: No arguments provided and REBUILD_PATTERNS environment variable is not set."
+# Ensure args are set
+if [[ "$#" = 0 ]]; then
+    echo "Error: No arguments provided."
     exit 1
 fi
 
 # If too many spurious cache misses: can be customized to pin artifacts to a specific version
 AZTEC_CACHE_COMMIT=${AZTEC_CACHE_COMMIT:-HEAD}
 PLATFORM_TAG="${PLATFORM_TAG:-${OSTYPE:-unknown}-$(uname -m)}"
-# Literal patterns can be provided with REBUILD_PATTERNS
-REBUILD_PATTERNS=${REBUILD_PATTERNS:-}
-if [ "$#" != 0 ]; then
-  REBUILD_PATTERNS+=$'\n'$(cat "$@")
-fi
+
+rebuild_patterns=()
+for arg in "$@"; do
+  if [[ -f "$arg" ]]; then
+    rebuild_patterns+=$(cat "$arg")
+    rebuild_patterns+=$'\n'
+  else
+    rebuild_patterns+="$arg"$'\n'
+  fi
+done
 
 # Concatenate patterns with '|' and double escape backslashes for AWK
 # filter empty lines
-AWK_PATTERN=$(echo "$REBUILD_PATTERNS" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//')
+AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//')
 
 # use git repo root because that is where our patterns are focused
 cd $(git rev-parse --show-toplevel)
@@ -30,7 +35,7 @@ CONTENT_HASH=$(git ls-tree -r $AZTEC_CACHE_COMMIT | awk -v pattern="($AWK_PATTER
 # Check if file list was empty by comparing against the result of 'echo '' | git hash-object --stdin | cut -c1-16'
 ECHO_BLANK_HASH="8b137891791fe969"
 if [ "$CONTENT_HASH" = "$ECHO_BLANK_HASH" ]; then
-    echo "No files matched the rebuild patterns $REBUILD_PATTERNS."
+    echo "No files matched the rebuild patterns $rebuild_patterns."
     echo "Awk pattern expanded: $AWK_PATTERN."
     exit 1
 fi

ci3/dump_fail

@@ -30,7 +30,7 @@ if [ "$status" -ne 0 ]; then
     cat "$stderr"
   } >&2
 else
-  echo "$output"
+  echo -n "$output"
 fi
 
 exit $status

ci3/source

@@ -18,6 +18,11 @@ function hash_str {
   set -eu
   echo $1 | git hash-object --stdin | cut -c1-16 | tr -d '\n'
 }
-export -f hash_str
+
+function echo_stderr {
+  echo $@ >&2
+}
+
+export -f hash_str echo_stderr
 
 source $ci3/source_color

ci3/source_bootstrap

@@ -10,7 +10,7 @@ case "${1:-}" in
     export DENOISE=${DENOISE:-1}
     ;;
   ""|"fast")
-    export USE_CACHE=1
+    export USE_CACHE=${USE_CACHE:-1}
     ;;
   "test")
     export TEST=1

noir-projects/noir-contracts/bootstrap.sh

@@ -7,7 +7,7 @@
 # - The exported functions called by parallel must enable their own flags at the start e.g. set -euo pipefail
 # - The exported functions are using stdin/stdout, so be very careful about what's printed where.
 # - The exported functions need to have external variables they require, to have been exported first.
-# - If you want to echo something, send it to stderr e.g. echo "My debug" >&2
+# - If you want to echo something, send it to stderr e.g. echo_stderr "My debug"
 # - If you call another script, be sure it also doesn't output something you don't want.
 # - Note calls to cache scripts swallow everything with &> /dev/null.
 # - Local assignments with subshells don't propagate errors e.g. local capture=$(false). Declare locals separately.
@@ -58,7 +58,7 @@ function process_function() {
   set +e
   make_vk=$(echo "$func" | jq -e '(.custom_attributes | index("public") == null) and (.is_unconstrained == false)')
   if [ $? -ne 0 ] && [ "$make_vk" != "false" ]; then
-    echo "Failed to check function $name is neither public nor unconstrained." >&2
+    echo_stderr "Failed to check function $name is neither public nor unconstrained."
     exit 1
   fi
   set -e
@@ -70,7 +70,7 @@ function process_function() {
     hash=$((echo "$BB_HASH"; echo "$bytecode_b64") | sha256sum | tr -d ' -')
     if ! cache_download vk-$hash.tar.gz &> /dev/null; then
       # It's not in the cache. Generate the vk file and upload it to the cache.
-      echo "Generating vk for function: $name..." >&2
+      echo_stderr "Generating vk for function: $name..."
       echo "$bytecode_b64" | base64 -d | gunzip | $BB write_vk_for_ivc -h -b - -o $tmp_dir/$hash 2>/dev/null
       cache_upload vk-$hash.tar.gz $tmp_dir/$hash &> /dev/null
     fi
@@ -97,8 +97,12 @@ function compile {
   contract_name=$(cat contracts/$1/src/main.nr | awk '/^contract / { print $2 }')
   local filename="$contract-$contract_name.json"
   local json_path="./target/$filename"
-  export REBUILD_PATTERNS="^noir-projects/noir-contracts/contracts/$contract/"
-  contract_hash="$(cache_content_hash ../../noir/.rebuild_patterns ../../avm-transpiler/.rebuild_patterns)"
+  contract_hash="$(cache_content_hash \
+    ../../noir/.rebuild_patterns \
+    ../../avm-transpiler/.rebuild_patterns \
+    "^noir-projects/noir-contracts/contracts/$contract/" \
+    "^noir-projects/aztec-nr/" \
+  )"
   if ! cache_download contract-$contract_hash.tar.gz &> /dev/null; then
     $NARGO compile --package $contract --silence-warnings --inliner-aggressiveness 0
     $TRANSPILER $json_path $json_path
@@ -125,7 +129,7 @@ function build {
     compile $1
   else
     set +e
-    echo "Compiling contracts (bb-hash: $BB_HASH)..."
+    echo_stderr "Compiling contracts (bb-hash: $BB_HASH)..."
     grep -oP '(?<=contracts/)[^"]+' Nargo.toml | \
       parallel -j16 --joblog joblog.txt -v --line-buffer --tag --halt now,fail=1 compile {}
     code=$?
@@ -143,15 +147,12 @@ case "$cmd" in
     ;;
   "clean-keys")
     for artifact in target/*.json; do
-      echo "Scrubbing vk from $artifact..."
+      echo_stderr "Scrubbing vk from $artifact..."
       jq '.functions |= map(del(.verification_key))' "$artifact" > "${artifact}.tmp"
       mv "${artifact}.tmp" "$artifact"
     done
     ;;
-  ""|"fast"|"ci")
-    USE_CACHE=1 build
-    ;;
-  "full")
+  ""|"fast"|"full"|"ci")
     build
     ;;
   "compile")
@@ -163,6 +164,6 @@ case "$cmd" in
     exit 0
     ;;
   *)
-    echo "Unknown command: $cmd"
+    echo_stderr "Unknown command: $cmd"
     exit 1
 esac

noir-projects/noir-protocol-circuits/bootstrap.sh

@@ -33,7 +33,7 @@ function on_exit() {
 }
 trap on_exit EXIT
 
-[ -f package.json ] && yarn && node ./scripts/generate_variants.js
+[ -f package.json ] && denoise "yarn && node ./scripts/generate_variants.js"
 
 mkdir -p $tmp_dir
 mkdir -p $key_dir
@@ -48,13 +48,19 @@ function compile {
   local filename="$name.json"
   local json_path="./target/$filename"
   local program_hash hash bytecode_hash vk vk_fields
-  program_hash=$($NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2)
-  hash=$(echo "$NARGO_HASH-$program_hash" | sha256sum | tr -d ' -')
-  if ! cache_download circuit-$hash.tar.gz &> /dev/null; then
+  local program_hash_cmd="$NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2"
+  # echo_stderr $program_hash_cmd
+  program_hash=$(dump_fail "$program_hash_cmd")
+  echo_stderr "Hash preimage: $NARGO_HASH-$program_hash"
+  hash=$(hash_str "$NARGO_HASH-$program_hash")
+  if ! cache_download circuit-$hash.tar.gz 1>&2; then
     SECONDS=0
+    rm -f $json_path
     # TODO: --skip-brillig-constraints-check added temporarily for blobs build time.
-    $NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check
-    echo "Compilation complete for: $name (${SECONDS}s)"
+    local compile_cmd="$NARGO compile --package $name --silence-warnings --skip-brillig-constraints-check"
+    echo_stderr "$compile_cmd"
+    dump_fail "$compile_cmd"
+    echo_stderr "Compilation complete for: $name (${SECONDS}s)"
     cache_upload circuit-$hash.tar.gz $json_path &> /dev/null
   fi
 
@@ -74,19 +80,19 @@ function compile {
   # Change this to add verification_key to original json, like contracts does.
   # Will require changing TS code downstream.
   bytecode_hash=$(jq -r '.bytecode' $json_path | sha256sum | tr -d ' -')
-  hash=$(echo "$BB_HASH-$bytecode_hash-$proto" | sha256sum | tr -d ' -')
-  if ! cache_download vk-$hash.tar.gz &> /dev/null; then
+  hash=$(hash_str "$BB_HASH-$bytecode_hash-$proto")
+  if ! cache_download vk-$hash.tar.gz 1>&2; then
     local key_path="$key_dir/$name.vk.data.json"
-    echo "Generating vk for function: $name..." >&2
+    echo_stderr "Generating vk for function: $name..."
     SECONDS=0
     local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -h -b - -o - --recursive | xxd -p -c 0"
-    echo $vk_cmd >&2
+    echo_stderr $vk_cmd
     vk=$(dump_fail "$vk_cmd")
     local vkf_cmd="echo '$vk' | xxd -r -p | $BB $vk_as_fields_cmd -k - -o -"
-    # echo $vkf_cmd >&2
+    # echo_stderrr $vkf_cmd
     vk_fields=$(dump_fail "$vkf_cmd")
     jq -n --arg vk "$vk" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path
-    echo "Key output at: $key_path (${SECONDS}s)"
+    echo_stderr "Key output at: $key_path (${SECONDS}s)"
     cache_upload vk-$hash.tar.gz $key_path &> /dev/null
   fi
 }
@@ -111,7 +117,7 @@ function test {
   set -eu
   # Whether we run the tests or not is coarse grained.
   name=$(basename "$PWD")
-  CIRCUITS_HASH=$(REBUILD_PATTERNS="^noir-projects/$name" cache_content_hash ../../noir/.rebuild_patterns)
+  CIRCUITS_HASH=$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/$name")
   if ! test_should_run $name-tests-$CIRCUITS_HASH; then
     return
   fi
@@ -144,6 +150,6 @@ case "$CMD" in
     parallel --line-buffered bash -c {} ::: build test
     ;;
   *)
-    echo "Unknown command: $CMD"
+    echo_stderr "Unknown command: $CMD"
     exit 1
 esac