Merge branch 'main' into feat/eugene/new-serializations

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ on:
 env:
   CARGO_TERM_COLOR: always
   CLICOLOR: 1
-  RUST_NIGHTLY_VERSION: nightly-2024-10-18
+  RUST_NIGHTLY_VERSION: nightly-2024-11-06
   CI: true
 
 concurrency:
@@ -174,7 +174,8 @@ jobs:
     env:
       NATS_URL: nats://127.0.0.1:4222
       NATS_ADMIN_PASS: secret
-      NATS_PUBLIC_PASS: temp-public-pass
+      NATS_PUBLIC_PASS: secret
+
     steps:
       - uses: actions/checkout@v4
 
@@ -270,15 +271,15 @@ jobs:
             os: ubuntu-latest
             target: x86_64-unknown-linux-musl
 
-            # linux aarch64
+          # linux aarch64
           - os_name: Linux-aarch64-gnu
             os: ubuntu-latest
             target: aarch64-unknown-linux-gnu
           - os_name: Linux-aarch64-musl
             os: ubuntu-latest
             target: aarch64-unknown-linux-musl
 
-            # macOS
+          # macOS
           - os_name: macOS-x86_64
             os: macOS-latest
             target: x86_64-apple-darwin

.github/workflows/prepare_release.yaml

@@ -15,7 +15,7 @@ env:
   CARGO_TERM_COLOR: always
   CLICOLOR: 1
   RUST_VERSION: 1.79.0
-  RUST_NIGHTLY_VERSION: nightly-2024-10-18
+  RUST_NIGHTLY_VERSION: nightly-2024-11-06
 
 jobs:
   setup:

CONTRIBUTING.md

@@ -7,7 +7,7 @@ This guide will show you how to run this project locally if you want to test or
 Most projects under the umbrella of data systems are written in Rust, so we prefer using Rust tooling and community standards. Ensure you have the following tools installed:
 
 -   [Rust](https://www.rust-lang.org/tools/install) (latest stable version recommended)
--   [Rust Nightly](https://rust-lang.github.io/rustup/concepts/channels.html) (version nightly-2024-10-18)
+-   [Rust Nightly](https://rust-lang.github.io/rustup/concepts/channels.html) (version nightly-2024-11-06)
 -   [Make](https://www.gnu.org/software/make/)
 -   [Pre-commit](https://pre-commit.com/#install)
 -   [NodeJS](https://nodejs.org/en/download/)
@@ -129,7 +129,7 @@ If you encounter any issues while setting up or contributing to the project, her
 
 1. **Pre-commit hooks failing**: Ensure you've installed all the required dependencies and run `make setup`. If issues persist, try running `pre-commit run --all-files` to see detailed error messages.
 
-2. **Build failures**: Make sure you're using the latest stable Rust version and the correct nightly version. You can update Rust using `rustup update stable` and `rustup update nightly-2024-10-18`.
+2. **Build failures**: Make sure you're using the latest stable Rust version and the correct nightly version. You can update Rust using `rustup update stable` and `rustup update nightly-2024-11-06`.
 
 3. **Test failures**: If specific tests are failing, try running them in isolation to see if it's a concurrency issue. Use `RUST_BACKTRACE=1` to get more detailed error information.
 

Makefile

@@ -136,7 +136,7 @@ NETWORK ?= testnet
 NETWORKS = mainnet testnet
 PROFILE ?= all
 PROFILES = all dev nats fuel monitoring indexer logging
-DOCKER_COMPOSE = set -a && source ./scripts/set_envs.sh && set +a && docker compose -f docker/docker-compose.yml
+DOCKER_COMPOSE = ./scripts/set_envs.sh && docker compose -f docker/docker-compose.yml --env-file .env
 
 # Helper functions to validate Docker environment and execute commands
 define check_docker_env
@@ -208,7 +208,7 @@ EXTRA_ARGS ?=
 
 # This defines how to run the publisher script with network and mode parameters
 define run_publisher
-	NETWORK=$(1) $(PUBLISHER_SCRIPT) --network $(1) --mode $(2) --port $(PORT) $(if $(EXTRA_ARGS),--extra-args "$(EXTRA_ARGS)")
+	@$(PUBLISHER_SCRIPT) --network $(1) --mode $(2) --port $(PORT) $(if $(EXTRA_ARGS),--extra-args "$(EXTRA_ARGS)")
 endef
 
 run-mainnet-dev: check-network

package.json

@@ -12,7 +12,7 @@
   "devDependencies": {
     "@commitlint/config-conventional": "^19.5.0",
     "commitlint": "^19.5.0",
-    "markdownlint": "^0.35.0",
+    "markdownlint": "^0.36.1",
     "markdownlint-cli": "^0.42.0",
     "prettier": "^3.3.3",
     "prettier-plugin-sh": "^0.14.0",

scripts/set_envs.sh

@@ -4,34 +4,53 @@
 NETWORK=${NETWORK:-"testnet"}
 NETWORK_UPPER="$(echo "$NETWORK" | tr '[:lower:]' '[:upper:]')"
 
-# Load .env file
-source .env
+# ------------------------------
+# Function to Load Environment Variables
+# ------------------------------
+load_env() {
+    if [ -f .env ]; then
+        # Read the .env file line by line, ignoring comments and empty lines
+        while IFS= read -r line || [ -n "$line" ]; do
+            # Skip comments and empty lines
+            [[ $line =~ ^[[:space:]]*# ]] && continue
+            [[ -z "$line" ]] && continue
+            # Export each variable
+            export "${line?}"
+        done < .env
+    else
+        echo "Error: .env file not found. Please create a .env file with the necessary variables."
+        exit 1
+    fi
+}
 
-# Map network-specific variables
-RESERVED_NODES="$(eval echo "\$${NETWORK_UPPER}_RESERVED_NODES")"
-RELAYER_V2_LISTENING_CONTRACTS="$(eval echo "\$${NETWORK_UPPER}_RELAYER_V2_LISTENING_CONTRACTS")"
-RELAYER_DA_DEPLOY_HEIGHT="$(eval echo "\$${NETWORK_UPPER}_RELAYER_DA_DEPLOY_HEIGHT")"
-RELAYER="$(eval echo "\$${NETWORK_UPPER}_RELAYER")"
-SYNC_HEADER_BATCH_SIZE="$(eval echo "\$${NETWORK_UPPER}_SYNC_HEADER_BATCH_SIZE")"
-RELAYER_LOG_PAGE_SIZE="$(eval echo "\$${NETWORK_UPPER}_RELAYER_LOG_PAGE_SIZE")"
+load_env
 
-# Validate required variables
-REQUIRED_VARS=(
-    "KEYPAIR"
-    "RELAYER"
-    "RESERVED_NODES"
-    "RELAYER_V2_LISTENING_CONTRACTS"
-    "RELAYER_DA_DEPLOY_HEIGHT"
-    "SYNC_HEADER_BATCH_SIZE"
-    "RELAYER_LOG_PAGE_SIZE"
-)
+# Remove existing network-specific variables if they exist
+# Create a temporary file and use it to rebuild the .env file
+awk '/\n# WARNING: Everything below this line is auto-generated by set_envs.sh/{exit} {print}' .env > .env.tmp
+mv .env.tmp .env
 
-for var in "${REQUIRED_VARS[@]}"; do
-    if [ -z "${!var}" ]; then
-        echo "Error: ${var} is not set"
-        exit 1
-    fi
-done
+# Set and export network-specific variables
+export RESERVED_NODES=$(eval echo "\$${NETWORK_UPPER}_RESERVED_NODES")
+export RELAYER_V2_LISTENING_CONTRACTS=$(eval echo "\$${NETWORK_UPPER}_RELAYER_V2_LISTENING_CONTRACTS")
+export RELAYER_DA_DEPLOY_HEIGHT=$(eval echo "\$${NETWORK_UPPER}_RELAYER_DA_DEPLOY_HEIGHT")
+export RELAYER=$(eval echo "\$${NETWORK_UPPER}_RELAYER")
+export SYNC_HEADER_BATCH_SIZE=$(eval echo "\$${NETWORK_UPPER}_SYNC_HEADER_BATCH_SIZE")
+export RELAYER_LOG_PAGE_SIZE=$(eval echo "\$${NETWORK_UPPER}_RELAYER_LOG_PAGE_SIZE")
+export CHAIN_CONFIG=$NETWORK
+export NETWORK=$NETWORK
 
-# Export chain config
-export CHAIN_CONFIG="$NETWORK"
+# Append network-specific variables to .env file
+{
+    echo -e "\n# WARNING: Everything below this line is auto-generated by set_envs.sh"
+    echo "# Network-specific variables for $NETWORK"
+    echo "# Last generated: $(date)"
+    echo "NETWORK=$NETWORK"
+    echo "RESERVED_NODES=$RESERVED_NODES"
+    echo "RELAYER_V2_LISTENING_CONTRACTS=$RELAYER_V2_LISTENING_CONTRACTS"
+    echo "RELAYER_DA_DEPLOY_HEIGHT=$RELAYER_DA_DEPLOY_HEIGHT"
+    echo "RELAYER=$RELAYER"
+    echo "SYNC_HEADER_BATCH_SIZE=$SYNC_HEADER_BATCH_SIZE"
+    echo "RELAYER_LOG_PAGE_SIZE=$RELAYER_LOG_PAGE_SIZE"
+    echo "CHAIN_CONFIG=$CHAIN_CONFIG"
+} >> .env

scripts/setup.sh

@@ -11,7 +11,7 @@ fi
 pnpm install
 
 # Install fixed nightly toolchain
-rustup toolchain install nightly-2024-10-18 -c rustfmt
+rustup toolchain install nightly-2024-11-06 -c rustfmt
 
 # Install cargo global crates
 cargo install cargo-binstall