From 0700c0ea88009598fe0708fc0c6592f9d0c042f5 Mon Sep 17 00:00:00 2001 From: Pedro Nauck Date: Sat, 9 Nov 2024 12:26:11 +0700 Subject: [PATCH 1/2] ci(repo): Fix test action (#303) * ci(repo): Fix github action * ci(repo): Fix set env script * ci(repo): Put actions back again --- .github/workflows/ci.yaml | 4 ++-- Makefile | 3 ++- scripts/set_envs.sh | 40 +++++++++++++++++++-------------------- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d3e21606..d124ae14 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -264,7 +264,7 @@ jobs: os: ubuntu-latest target: x86_64-unknown-linux-musl - # linux aarch64 + # linux aarch64 - os_name: Linux-aarch64-gnu os: ubuntu-latest target: aarch64-unknown-linux-gnu @@ -272,7 +272,7 @@ jobs: os: ubuntu-latest target: aarch64-unknown-linux-musl - # macOS + # macOS - os_name: macOS-x86_64 os: macOS-latest target: x86_64-apple-darwin diff --git a/Makefile b/Makefile index edbce2b6..baea8b1b 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ NETWORK ?= testnet NETWORKS = mainnet testnet PROFILE ?= all PROFILES = all dev nats fuel monitoring indexer logging -DOCKER_COMPOSE = set -a && source ./scripts/set_envs.sh && set +a && docker compose -f docker/docker-compose.yml +DOCKER_COMPOSE = docker compose -f docker/docker-compose.yml # Helper functions to validate Docker environment and execute commands define check_docker_env @@ -153,6 +153,7 @@ endef # Helper function to execute docker commands with consistent parameters define docker_cmd $(call check_docker_env) + ./scripts/set_envs.sh NETWORK=$(1) PORT=$(2) $(DOCKER_COMPOSE) --profile $(3) $(4) endef diff --git a/scripts/set_envs.sh b/scripts/set_envs.sh index a77ec061..df6c0c24 100755 --- a/scripts/set_envs.sh +++ b/scripts/set_envs.sh @@ -4,8 +4,26 @@ NETWORK=${NETWORK:-"testnet"} NETWORK_UPPER="$(echo "$NETWORK" | tr '[:lower:]' '[:upper:]')" -# Load .env file -source .env +# ------------------------------ +# Function to Load Environment Variables +# ------------------------------ +load_env() { + if [ -f .env ]; then + # Read the .env file line by line, ignoring comments and empty lines + while IFS= read -r line || [ -n "$line" ]; do + # Skip comments and empty lines + [[ $line =~ ^[[:space:]]*# ]] && continue + [[ -z "$line" ]] && continue + # Export each variable + export "$line" + done < .env + else + echo "Error: .env file not found. Please create a .env file with the necessary variables." + exit 1 + fi +} + +load_env # Map network-specific variables RESERVED_NODES="$(eval echo "\$${NETWORK_UPPER}_RESERVED_NODES")" @@ -15,23 +33,5 @@ RELAYER="$(eval echo "\$${NETWORK_UPPER}_RELAYER")" SYNC_HEADER_BATCH_SIZE="$(eval echo "\$${NETWORK_UPPER}_SYNC_HEADER_BATCH_SIZE")" RELAYER_LOG_PAGE_SIZE="$(eval echo "\$${NETWORK_UPPER}_RELAYER_LOG_PAGE_SIZE")" -# Validate required variables -REQUIRED_VARS=( - "KEYPAIR" - "RELAYER" - "RESERVED_NODES" - "RELAYER_V2_LISTENING_CONTRACTS" - "RELAYER_DA_DEPLOY_HEIGHT" - "SYNC_HEADER_BATCH_SIZE" - "RELAYER_LOG_PAGE_SIZE" -) - -for var in "${REQUIRED_VARS[@]}"; do - if [ -z "${!var}" ]; then - echo "Error: ${var} is not set" - exit 1 - fi -done - # Export chain config export CHAIN_CONFIG="$NETWORK" From 890d739e0fb302ea991ca5febdab9f7efa263fb4 Mon Sep 17 00:00:00 2001 From: 0xterminator Date: Sat, 9 Nov 2024 08:14:02 +0200 Subject: [PATCH 2/2] feat(repo): Upgraded to latest nightly rust version (#304) * feat(repo): Upgraded to latest nightly rust version * Revert "feat(repo): Upgraded to latest nightly rust version" This reverts commit 4aab03f7d1c8bc72ebf728ec73c5f6946dd589ec. * feat(repo): Upgraded to latest nightly rust version * build(repo): Fix makefile --------- Co-authored-by: Pedro Nauck --- .github/workflows/ci.yaml | 5 ++-- .github/workflows/prepare_release.yaml | 2 +- CONTRIBUTING.md | 4 +-- Makefile | 5 ++-- package.json | 2 +- pnpm-lock.yaml | 19 +++++++++++-- scripts/set_envs.sh | 39 +++++++++++++++++++------- scripts/setup.sh | 2 +- 8 files changed, 56 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d124ae14..0d263d0a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,7 +15,7 @@ on: env: CARGO_TERM_COLOR: always CLICOLOR: 1 - RUST_NIGHTLY_VERSION: nightly-2024-10-18 + RUST_NIGHTLY_VERSION: nightly-2024-11-06 CI: true concurrency: @@ -171,7 +171,8 @@ jobs: env: NATS_URL: nats://127.0.0.1:4222 NATS_ADMIN_PASS: secret - NATS_PUBLIC_PASS: temp-public-pass + NATS_PUBLIC_PASS: secret + steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/prepare_release.yaml b/.github/workflows/prepare_release.yaml index b831ee32..927f72d2 100644 --- a/.github/workflows/prepare_release.yaml +++ b/.github/workflows/prepare_release.yaml @@ -15,7 +15,7 @@ env: CARGO_TERM_COLOR: always CLICOLOR: 1 RUST_VERSION: 1.79.0 - RUST_NIGHTLY_VERSION: nightly-2024-10-18 + RUST_NIGHTLY_VERSION: nightly-2024-11-06 jobs: setup: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f7191e4..66c61059 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ This guide will show you how to run this project locally if you want to test or Most projects under the umbrella of data systems are written in Rust, so we prefer using Rust tooling and community standards. Ensure you have the following tools installed: - [Rust](https://www.rust-lang.org/tools/install) (latest stable version recommended) -- [Rust Nightly](https://rust-lang.github.io/rustup/concepts/channels.html) (version nightly-2024-10-18) +- [Rust Nightly](https://rust-lang.github.io/rustup/concepts/channels.html) (version nightly-2024-11-06) - [Make](https://www.gnu.org/software/make/) - [Pre-commit](https://pre-commit.com/#install) - [NodeJS](https://nodejs.org/en/download/) @@ -129,7 +129,7 @@ If you encounter any issues while setting up or contributing to the project, her 1. **Pre-commit hooks failing**: Ensure you've installed all the required dependencies and run `make setup`. If issues persist, try running `pre-commit run --all-files` to see detailed error messages. -2. **Build failures**: Make sure you're using the latest stable Rust version and the correct nightly version. You can update Rust using `rustup update stable` and `rustup update nightly-2024-10-18`. +2. **Build failures**: Make sure you're using the latest stable Rust version and the correct nightly version. You can update Rust using `rustup update stable` and `rustup update nightly-2024-11-06`. 3. **Test failures**: If specific tests are failing, try running them in isolation to see if it's a concurrency issue. Use `RUST_BACKTRACE=1` to get more detailed error information. diff --git a/Makefile b/Makefile index baea8b1b..862d657a 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ NETWORK ?= testnet NETWORKS = mainnet testnet PROFILE ?= all PROFILES = all dev nats fuel monitoring indexer logging -DOCKER_COMPOSE = docker compose -f docker/docker-compose.yml +DOCKER_COMPOSE = ./scripts/set_envs.sh && docker compose -f docker/docker-compose.yml --env-file .env # Helper functions to validate Docker environment and execute commands define check_docker_env @@ -153,7 +153,6 @@ endef # Helper function to execute docker commands with consistent parameters define docker_cmd $(call check_docker_env) - ./scripts/set_envs.sh NETWORK=$(1) PORT=$(2) $(DOCKER_COMPOSE) --profile $(3) $(4) endef @@ -209,7 +208,7 @@ EXTRA_ARGS ?= # This defines how to run the publisher script with network and mode parameters define run_publisher - NETWORK=$(1) $(PUBLISHER_SCRIPT) --network $(1) --mode $(2) --port $(PORT) $(if $(EXTRA_ARGS),--extra-args "$(EXTRA_ARGS)") + @$(PUBLISHER_SCRIPT) --network $(1) --mode $(2) --port $(PORT) $(if $(EXTRA_ARGS),--extra-args "$(EXTRA_ARGS)") endef run-mainnet-dev: check-network diff --git a/package.json b/package.json index 7b3d2f65..7d3d3396 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "devDependencies": { "@commitlint/config-conventional": "^19.5.0", "commitlint": "^19.5.0", - "markdownlint": "^0.35.0", + "markdownlint": "^0.36.1", "markdownlint-cli": "^0.42.0", "prettier": "^3.3.3", "prettier-plugin-sh": "^0.14.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 93eaef81..7fbed557 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,8 +15,8 @@ importers: specifier: ^19.5.0 version: 19.5.0(@types/node@22.5.0)(typescript@5.5.4) markdownlint: - specifier: ^0.35.0 - version: 0.35.0 + specifier: ^0.36.1 + version: 0.36.1 markdownlint-cli: specifier: ^0.42.0 version: 0.42.0 @@ -442,10 +442,18 @@ packages: resolution: {integrity: sha512-no5ZfdqAdWGxftCLlySHSgddEjyW4kui4z7amQcGsSKfYC5v/ou+8mIQVyg9KQMeEZLNtz9OPDTj7nnTnoR4FQ==} engines: {node: '>=18'} + markdownlint-micromark@0.1.12: + resolution: {integrity: sha512-RlB6EwMGgc0sxcIhOQ2+aq7Zw1V2fBnzbXKGgYK/mVWdT7cz34fteKSwfYeo4rL6+L/q2tyC9QtD/PgZbkdyJQ==} + engines: {node: '>=18'} + markdownlint@0.35.0: resolution: {integrity: sha512-wgp8yesWjFBL7bycA3hxwHRdsZGJhjhyP1dSxKVKrza0EPFYtn+mHtkVy6dvP1kGSjovyG5B8yNP6Frj0UFUJg==} engines: {node: '>=18'} + markdownlint@0.36.1: + resolution: {integrity: sha512-s73fU2CQN7WCgjhaQUQ8wYESQNzGRNOKDd+3xgVqu8kuTEhmwepd/mxOv1LR2oV046ONrTLBFsM7IoKWNvmy5g==} + engines: {node: '>=18'} + mdurl@2.0.0: resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} @@ -1083,11 +1091,18 @@ snapshots: markdownlint-micromark@0.1.10: {} + markdownlint-micromark@0.1.12: {} + markdownlint@0.35.0: dependencies: markdown-it: 14.1.0 markdownlint-micromark: 0.1.10 + markdownlint@0.36.1: + dependencies: + markdown-it: 14.1.0 + markdownlint-micromark: 0.1.12 + mdurl@2.0.0: {} meow@12.1.1: {} diff --git a/scripts/set_envs.sh b/scripts/set_envs.sh index df6c0c24..74c16e07 100755 --- a/scripts/set_envs.sh +++ b/scripts/set_envs.sh @@ -15,7 +15,7 @@ load_env() { [[ $line =~ ^[[:space:]]*# ]] && continue [[ -z "$line" ]] && continue # Export each variable - export "$line" + export "${line?}" done < .env else echo "Error: .env file not found. Please create a .env file with the necessary variables." @@ -25,13 +25,32 @@ load_env() { load_env -# Map network-specific variables -RESERVED_NODES="$(eval echo "\$${NETWORK_UPPER}_RESERVED_NODES")" -RELAYER_V2_LISTENING_CONTRACTS="$(eval echo "\$${NETWORK_UPPER}_RELAYER_V2_LISTENING_CONTRACTS")" -RELAYER_DA_DEPLOY_HEIGHT="$(eval echo "\$${NETWORK_UPPER}_RELAYER_DA_DEPLOY_HEIGHT")" -RELAYER="$(eval echo "\$${NETWORK_UPPER}_RELAYER")" -SYNC_HEADER_BATCH_SIZE="$(eval echo "\$${NETWORK_UPPER}_SYNC_HEADER_BATCH_SIZE")" -RELAYER_LOG_PAGE_SIZE="$(eval echo "\$${NETWORK_UPPER}_RELAYER_LOG_PAGE_SIZE")" +# Remove existing network-specific variables if they exist +# Create a temporary file and use it to rebuild the .env file +awk '/\n# WARNING: Everything below this line is auto-generated by set_envs.sh/{exit} {print}' .env > .env.tmp +mv .env.tmp .env -# Export chain config -export CHAIN_CONFIG="$NETWORK" +# Set and export network-specific variables +export RESERVED_NODES=$(eval echo "\$${NETWORK_UPPER}_RESERVED_NODES") +export RELAYER_V2_LISTENING_CONTRACTS=$(eval echo "\$${NETWORK_UPPER}_RELAYER_V2_LISTENING_CONTRACTS") +export RELAYER_DA_DEPLOY_HEIGHT=$(eval echo "\$${NETWORK_UPPER}_RELAYER_DA_DEPLOY_HEIGHT") +export RELAYER=$(eval echo "\$${NETWORK_UPPER}_RELAYER") +export SYNC_HEADER_BATCH_SIZE=$(eval echo "\$${NETWORK_UPPER}_SYNC_HEADER_BATCH_SIZE") +export RELAYER_LOG_PAGE_SIZE=$(eval echo "\$${NETWORK_UPPER}_RELAYER_LOG_PAGE_SIZE") +export CHAIN_CONFIG=$NETWORK +export NETWORK=$NETWORK + +# Append network-specific variables to .env file +{ + echo -e "\n# WARNING: Everything below this line is auto-generated by set_envs.sh" + echo "# Network-specific variables for $NETWORK" + echo "# Last generated: $(date)" + echo "NETWORK=$NETWORK" + echo "RESERVED_NODES=$RESERVED_NODES" + echo "RELAYER_V2_LISTENING_CONTRACTS=$RELAYER_V2_LISTENING_CONTRACTS" + echo "RELAYER_DA_DEPLOY_HEIGHT=$RELAYER_DA_DEPLOY_HEIGHT" + echo "RELAYER=$RELAYER" + echo "SYNC_HEADER_BATCH_SIZE=$SYNC_HEADER_BATCH_SIZE" + echo "RELAYER_LOG_PAGE_SIZE=$RELAYER_LOG_PAGE_SIZE" + echo "CHAIN_CONFIG=$CHAIN_CONFIG" +} >> .env diff --git a/scripts/setup.sh b/scripts/setup.sh index 6c1e8d09..6d6e3759 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -11,7 +11,7 @@ fi pnpm install # Install fixed nightly toolchain -rustup toolchain install nightly-2024-10-18 -c rustfmt +rustup toolchain install nightly-2024-11-06 -c rustfmt # Install cargo global crates cargo install cargo-binstall