Merge pull request #425 from valory-xyz/staging

staging to main

.github/workflows/backend.yml

@@ -14,6 +14,13 @@ jobs:
         with:
           node-version: "20.x"
 
+      # Configure Yarn network settings for timeout, retries, and reduced concurrency
+      - name: Configure Yarn network settings
+        run: |
+          yarn config set network-timeout 60000         # Set network timeout to 1 minute
+          yarn config set network-retries 10             # Retry up to 10 times
+          yarn config set network-concurrency 2          # Reduce concurrency to 2 connections
+
       # Python
       - uses: actions/setup-python@v4
         with:

.github/workflows/frontend.yml

@@ -15,6 +15,13 @@ jobs:
         with:
           node-version: "20.x"
 
+      # Configure Yarn network settings for timeout, retries, and reduced concurrency
+      - name: Configure Yarn network settings
+        run: |
+          yarn config set network-timeout 60000         # Set network timeout to 1 minute
+          yarn config set network-retries 10             # Retry up to 10 times
+          yarn config set network-concurrency 2          # Reduce concurrency to 2 connections
+
       # Install dependencies
       - name: Install dependencies
         run: yarn install:frontend

.github/workflows/release.yml

@@ -12,13 +12,15 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-14, macos-14-large]
+        os: [macos-14]
+        arch: [arm64, x64]
 
     steps:
       - uses: actions/checkout@v3
 
-      # Setup Python
-      - uses: actions/setup-python@v4
+      # Set up Python with setup-python action and add it to PATH
+      - uses: actions/setup-python@v5
+        id: setup-python
         with:
           python-version: "3.10"
 
@@ -32,10 +34,10 @@ jobs:
           virtualenvs-path: ~/my-custom-path
           installer-parallel: true
 
-      # Set architecture for macos-latest-large as x64 and arm64 for other versions
+      # Set OS_ARCH env
       - name: Set architecture environment variable
-        run: |
-          if [ "${{ matrix.os }}" == "macos-14-large" ]; then echo "OS_ARCH=x64" >> $GITHUB_ENV; else echo "OS_ARCH=arm64" >> $GITHUB_ENV; fi
+        run: 
+            echo "OS_ARCH=${{ matrix.arch }}" >> $GITHUB_ENV; 
 
       # Cache Poetry dependencies with unique key for each environment and architecture
       - name: Cache Poetry dependencies
@@ -45,9 +47,9 @@ jobs:
             ~/.cache/pypoetry
             ~/.cache/pip
             ~/.venv
-          key: poetry-${{ env.OS_ARCH }}-${{ hashFiles('**/poetry.lock') }}
+          key: poetry-${{ matrix.arch }}-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
-            poetry-${{ env.OS_ARCH }}-
+            poetry-${{ matrix.arch }}-
 
       - name: Install dependencies
         run: poetry install
@@ -57,22 +59,29 @@ jobs:
         run: |
           trader_version=$(poetry run python -c "import yaml; config = yaml.safe_load(open('templates/trader.yaml')); print(config['service_version'])")
           echo $trader_version
-          mkdir dist && curl -L -o dist/aea_bin "https://github.com/valory-xyz/trader/releases/download/${trader_version}/trader_bin_${{ env.OS_ARCH }}"
+          make ./dist/aea_bin
+        #instead of this one mwe use make; mkdir dist && curl -L -o dist/aea_bin "https://github.com/valory-xyz/trader/releases/download/${trader_version}/trader_bin_${{ env.OS_ARCH }}"
 
       - name: Build with PyInstaller
         run: |
           poetry run pyinstaller operate/services/utils/tendermint.py --onefile
+
+          # patch open aea in place
+          rm -fr ./open-aea
+          git clone https://github.com/valory-xyz/open-aea.git -b fix/1.5.2_encoding
+          poetry run pip install ./open-aea/
+
           poetry run pyinstaller --collect-data eth_account --collect-all aea --collect-all autonomy --collect-all operate --collect-all aea_ledger_ethereum --collect-all aea_ledger_cosmos --collect-all aea_ledger_ethereum_flashbots --hidden-import aea_ledger_ethereum --hidden-import aea_ledger_cosmos --hidden-import aea_ledger_ethereum_flashbots operate/pearl.py --add-binary dist/aea_bin:. --add-binary dist/tendermint:. --onefile --name pearl_${{ env.OS_ARCH }}
 
       - name: Upload Release Assets
         uses: actions/upload-artifact@v4
         with:
-          name: pearl_${{ env.OS_ARCH }}
-          path: dist/pearl_${{ env.OS_ARCH }}
+          name: pearl_${{ matrix.arch }}
+          path: dist/pearl_${{ matrix.arch }}
 
   # Jobs for production and development, running separately for x64 and arm64
   build-release:
-    runs-on: macos-latest
+    runs-on: macos-14
     strategy:
       matrix:
         env: [production, development]
@@ -86,7 +95,15 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: "20.x"
-          registry-url: "https://registry.npmjs.org"   
+          registry-url: "https://registry.npmjs.org"
+
+      # Configure Yarn network settings for timeout, retries, and reduced concurrency
+      - name: Configure Yarn network settings
+        run: |
+              yarn config set network-timeout 60000         # Set network timeout to 1 minute
+              yarn config set network-retries 10             # Retry up to 10 times
+              yarn config set network-concurrency 2          # Reduce concurrency to 2 connections
+    
 
       # Download the appropriate architecture artifact
       - name: Download Pearl binary for architecture
@@ -151,7 +168,7 @@ jobs:
         env:
           APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLEIDPASS }}
           APPLE_ID: ${{ secrets.APPLEID }}
-          APPLETEAMID: ${{ secrets.APPLETEAMID }}
+          APPLE_TEAM_ID: ${{ secrets.APPLETEAMID }}
           CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
           CSC_LINK: ${{ secrets.CSC_LINK }}
           GH_TOKEN: ${{ secrets.github_token }}
@@ -167,7 +184,7 @@ jobs:
         env:
           APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLEIDPASS }}
           APPLE_ID: ${{ secrets.APPLEID }}
-          APPLETEAMID: ${{ secrets.APPLETEAMID }}
+          APPLE_TEAM_ID: ${{ secrets.APPLETEAMID }}
           CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }}
           CSC_LINK: ${{ secrets.CSC_LINK }}
           GH_TOKEN: ${{ secrets.github_token }}

.github/workflows/release_win.yml

@@ -23,15 +23,27 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v4
+      # Set up Python with setup-python action and add it to PATH
+      - uses: actions/setup-python@v5
         id: setup-python
         with:
           python-version: "3.10"
 
+      - name: Add Python to PATH
+        run: |
+          echo "${{ steps.setup-python.outputs.python-path }}" >> $GITHUB_PATH
+
       - uses: actions/setup-node@v4
         with:
           node-version: "20.x"
-          registry-url: "https://registry.npmjs.org"     
+          registry-url: "https://registry.npmjs.org"
+
+      # Configure Yarn network settings for timeout, retries, and reduced concurrency
+      - name: Configure Yarn network settings
+        run: |
+          yarn config set network-timeout 60000         # Set network timeout to 1 minute
+          yarn config set network-retries 10             # Retry up to 10 times
+          yarn config set network-concurrency 2          # Reduce concurrency to 2 connections     
           
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1

Makefile

@@ -13,19 +13,26 @@ endef
 
 ./dist/aea_win.exe: ./trader/
 	mkdir -p dist
-	cd trader && poetry install && poetry run pyinstaller --collect-data eth_account --collect-all aea --collect-all autonomy --collect-all operate --collect-all aea_ledger_ethereum --collect-all aea_ledger_cosmos --collect-all aea_ledger_ethereum_flashbots --hidden-import aea_ledger_ethereum --hidden-import aea_ledger_cosmos --hidden-import aea_ledger_ethereum_flashbots --hidden-import grpc --hidden-import openapi_core --collect-all google.protobuf --collect-all openapi_core --collect-all openapi_spec_validator --collect-all asn1crypto --hidden-import py_ecc --hidden-import pytz --onefile pyinstaller/trader_bin.py --name trader_win
+	cd trader && poetry install && rm -fr ./open-aea  && git clone https://github.com/valory-xyz/open-aea.git -b fix/1.5.2_encoding && poetry run pip install ./open-aea/ && poetry run pyinstaller --collect-data eth_account --collect-all aea --collect-all autonomy --collect-all operate --collect-all aea_ledger_ethereum --collect-all aea_ledger_cosmos --collect-all aea_ledger_ethereum_flashbots --hidden-import aea_ledger_ethereum --hidden-import aea_ledger_cosmos --hidden-import aea_ledger_ethereum_flashbots --hidden-import grpc --hidden-import openapi_core --collect-all google.protobuf --collect-all openapi_core --collect-all openapi_spec_validator --collect-all asn1crypto --hidden-import py_ecc --hidden-import pytz --onefile pyinstaller/trader_bin.py --name trader_win
 	cp -f trader/dist/trader_win.exe ./dist/aea_win.exe
 	pwd
 
 
+./dist/aea_bin: ./trader/
+	mkdir -p dist
+	cd trader && poetry install && rm -fr ./open-aea  && git clone https://github.com/valory-xyz/open-aea.git -b fix/1.5.2_encoding && poetry run pip install ./open-aea/ && poetry run pyinstaller --collect-data eth_account --collect-all aea --collect-all autonomy --collect-all operate --collect-all aea_ledger_ethereum --collect-all aea_ledger_cosmos --collect-all aea_ledger_ethereum_flashbots --hidden-import aea_ledger_ethereum --hidden-import aea_ledger_cosmos --hidden-import aea_ledger_ethereum_flashbots --hidden-import grpc --hidden-import openapi_core --collect-all google.protobuf --collect-all openapi_core --collect-all openapi_spec_validator --collect-all asn1crypto --hidden-import py_ecc --hidden-import pytz --onefile pyinstaller/trader_bin.py --name trader_bin
+	cp -f trader/dist/trader_bin ./dist/aea_bin
+	pwd
+
+
 ./dist/tendermint_win.exe: ./operate/
 	pwd
 	poetry install && poetry run pyinstaller operate/services/utils/tendermint.py --onefile --name tendermint_win
 
 
 ./dist/pearl_win.exe: ./operate/ ./dist/aea_win.exe ./dist/tendermint_win.exe
 	pwd
-	poetry install && poetry run pyinstaller --collect-data eth_account --collect-all aea --collect-all coincurve --collect-all autonomy --collect-all operate --collect-all aea_ledger_ethereum --collect-all aea_ledger_cosmos --collect-all aea_ledger_ethereum_flashbots --hidden-import aea_ledger_ethereum --hidden-import aea_ledger_cosmos --hidden-import aea_ledger_ethereum_flashbots operate/pearl.py --add-binary dist/aea_win.exe:.  --add-binary dist/tendermint_win.exe:. --onefile --name pearl_win
+	poetry install && rm -fr ./open-aea  && git clone https://github.com/valory-xyz/open-aea.git -b fix/1.5.2_encoding && poetry run pip install ./open-aea/ && poetry run pyinstaller --collect-data eth_account --collect-all aea --collect-all coincurve --collect-all autonomy --collect-all operate --collect-all aea_ledger_ethereum --collect-all aea_ledger_cosmos --collect-all aea_ledger_ethereum_flashbots --hidden-import aea_ledger_ethereum --hidden-import aea_ledger_cosmos --hidden-import aea_ledger_ethereum_flashbots operate/pearl.py --add-binary dist/aea_win.exe:.  --add-binary dist/tendermint_win.exe:. --onefile --name pearl_win
 
 
 ./electron/bins/:

build-win-tenderly.js

@@ -35,4 +35,8 @@ const main = async () => {
   });
 };
 
-main().then((response) => { console.log('Build & Notarize complete'); }).catch((e) => console.error(e));
+main().then(() => {
+  console.log('Build & Notarize complete');
+}).catch(() => {
+  throw new Error('Failed to build and notarize.');
+});

build-win.js

@@ -46,4 +46,8 @@ const main = async () => {
   });
 };
 
-main().then((response) => { console.log('Build & Notarize complete'); }).catch((e) => console.error(e));
+main().then(() => {
+  console.log('Build & Notarize complete');
+}).catch(() => {
+  throw new Error('Failed to build and notarize.');
+});

build.js

@@ -57,12 +57,13 @@ const main = async () => {
         gatekeeperAssess: false,
         entitlements: 'electron/entitlements.mac.plist',
         entitlementsInherit: 'electron/entitlements.mac.plist',
-        notarize: {
-          teamId: process.env.APPLETEAMID,
-        },
       },
     },
   });
 };
 
-main().then((response) => { console.log('Build & Notarize complete'); }).catch((e) => console.error(e));
+main().then(() => {
+  console.log('Build & Notarize complete');
+}).catch(() => {
+  throw new Error('Failed to build and notarize.');
+});

build.tester.js

@@ -49,4 +49,8 @@ const main = async () => {
   });
 };
 
-main().then((response) => { console.log('Build & Notarize complete'); }).catch((e) => console.error(e));
+main().then(() => {
+  console.log('Build & Notarize complete');
+}).catch(() => {
+  throw new Error('Failed to build and notarize.');
+});

electron/install.js

@@ -25,7 +25,7 @@ const Env = {
   PATH: `${process.env.PATH}:/opt/homebrew/bin:/usr/local/bin`,
   HOMEBREW_NO_AUTO_UPDATE: '1',
   PYTHONUTF8: '1',
-  PYTHONIOENCODING: 'utf8'
+  PYTHONIOENCODING: 'utf-8'
 };
 
 const SudoOptions = {

electron/main.js

@@ -70,11 +70,20 @@ let splashWindow = null;
 /** @type {Electron.Tray | null} */
 let tray = null;
 
-let operateDaemon, operateDaemonPid, nextAppProcess, nextAppProcessPid;
+// Used in production and development
+let operateDaemon;
+let operateDaemonPid;
 
+// Child processes for running next app are only used in development
+// required for hot reloads and other dev features
+let devNextApp;
+let devNextAppPid;
+
+// Next.js app instance for production
+// requires http server wrap to work; assign port, receive requests, deliver responses
 // @ts-ignore - Workaround for the missing type definitions
 const nextApp = next({
-  dev: false, // this instance is only used for production
+  dev: false, // DO NOT SET TO TRUE
   dir: path.join(__dirname),
 });
 
@@ -85,28 +94,73 @@ function showNotification(title, body) {
 }
 
 async function beforeQuit() {
-  if (operateDaemonPid) {
+  // destroy all ui components for immediate feedback
+  tray?.destroy();
+  splashWindow?.destroy();
+  mainWindow?.destroy();
+
+  if (operateDaemon || operateDaemonPid) {
+    // gracefully stop running services
     try {
       await fetch(
         `http://localhost:${appConfig.ports.prod.operate}/stop_all_services`,
       );
-      await killProcesses(operateDaemonPid);
     } catch (e) {
-      logger.electron(e);
+      logger.electron("Couldn't stop_all_services gracefully:");
+      logger.electron(JSON.stringify(e, null, 2));
+    }
+
+    // clean-up via pid first*
+    // may have dangling subprocesses
+    try {
+      operateDaemonPid && (await killProcesses(operateDaemonPid));
+    } catch (e) {
+      logger.electron("Couldn't kill daemon processes via pid:");
+      logger.electron(JSON.stringify(e, null, 2));
+    }
+
+    // attempt to kill the daemon process via kill
+    // if the pid-based cleanup fails
+    try {
+      const dead = operateDaemon?.kill();
+      if (!dead) {
+        logger.electron('Daemon process still alive after kill');
+      }
+    } catch (e) {
+      logger.electron("Couldn't kill operate daemon process via kill:");
+      logger.electron(JSON.stringify(e, null, 2));
     }
   }
 
-  if (nextAppProcessPid) {
+  if (devNextApp || devNextAppPid) {
+    // attempt graceful kill first with next app
+    try {
+      const dead = devNextApp?.kill();
+      if (!dead) {
+        logger.electron('Dev NextApp process still alive after kill');
+      }
+    } catch (e) {
+      logger.electron("Couldn't kill devNextApp process via kill:");
+      logger.electron(JSON.stringify(e, null, 2));
+    }
+
+    // attempt to kill the dev next app process via pid
     try {
-      await killProcesses(nextAppProcessPid);
+      devNextAppPid && (await killProcesses(devNextAppPid));
     } catch (e) {
-      logger.electron(e);
+      logger.electron("Couldn't kill devNextApp processes via pid:");
+      logger.electron(JSON.stringify(e, null, 2));
     }
   }
 
-  tray?.destroy();
-  splashWindow?.destroy();
-  mainWindow?.destroy();
+  if (nextApp) {
+    // attempt graceful close of prod next app
+    await nextApp.close().catch((e) => {
+      logger.electron("Couldn't close NextApp gracefully:");
+      logger.electron(JSON.stringify(e, null, 2));
+    });
+    // electron will kill next service on exit
+  }
 }
 
 const APP_WIDTH = 460;
@@ -340,7 +394,7 @@ async function launchNextApp() {
 async function launchNextAppDev() {
   await new Promise(function (resolve, _reject) {
     process.env.NEXT_PUBLIC_BACKEND_PORT = appConfig.ports.dev.operate; // must set next env var to connect to backend
-    nextAppProcess = spawn(
+    devNextApp = spawn(
       'yarn',
       ['dev:frontend', '--port', appConfig.ports.dev.next],
       {
@@ -352,8 +406,8 @@ async function launchNextAppDev() {
         },
       },
     );
-    nextAppProcessPid = nextAppProcess.pid;
-    nextAppProcess.stdout.on('data', (data) => {
+    devNextAppPid = devNextApp.pid;
+    devNextApp.stdout.on('data', (data) => {
       logger.next(data.toString().trim());
       resolve();
     });