diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml new file mode 100644 index 00000000..979cfa4b --- /dev/null +++ b/.github/workflows/gitleaks.yml @@ -0,0 +1,22 @@ +name: Gitleaks +on: + pull_request: + push: + branches: + - main +jobs: + scan: + name: gitleaks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-go@v3 + with: + go-version: "1.17.7" + - run: | + wget https://github.com/zricethezav/gitleaks/releases/download/v8.10.1/gitleaks_8.10.1_linux_x64.tar.gz && \ + tar -xzf gitleaks_8.10.1_linux_x64.tar.gz && \ + sudo install gitleaks /usr/bin && \ + gitleaks detect --report-format json --report-path leak_report -v \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e68ec817..88431c92 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,16 +1,27 @@ -name: Build, Notarize, Release +name: Release # This workflow is triggered on pushing a tag BE CAREFUL this application AUTO UPDATES !!! # git tag vX.Y.Z # git push origin tag vX.Y.Z -on: + +on: push: tags: - 'v*.*.*' + jobs: + verify-main-branch: # ensures we only release from main + runs-on: ubuntu-latest + steps: + - name: Exit if not on main branch + if: github.ref != 'refs/heads/main' + run: echo "Not on main branch, exiting" && exit -1 + release-middleware: + needs: + - "verify-main-branch" runs-on: "ubuntu-latest" - steps: + steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 with: @@ -36,10 +47,12 @@ jobs: password: ${{ secrets.PYPI_TOKEN }} skip-existing: true packages-dir: dist/ + release-operate: runs-on: macos-latest needs: - "release-middleware" + - "verify-main-branch" steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v4 @@ -68,12 +81,10 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLEIDPASS }} APPLE_ID: ${{ secrets.APPLEID }} APPLETEAMID: ${{ secrets.APPLETEAMID }} - #CSC_FOR_PULL_REQUEST: true #required during testing CSC_KEY_PASSWORD: ${{ secrets.CSC_KEY_PASSWORD }} CSC_LINK: ${{ secrets.CSC_LINK }} GH_TOKEN: ${{ secrets.github_token}} NODE_ENV: production DEV_RPC: https://rpc-gate.autonolas.tech/gnosis-rpc/ FORK_URL: https://rpc-gate.autonolas.tech/gnosis-rpc/ - #PUBLISH_FOR_PULL_REQUEST: true #required during testing - run: node build.js + run: node build.js \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0aa98c9c..c7eba539 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,5 @@ temp/ dist/ electron/.next -cache \ No newline at end of file +cache +leak_report diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 00000000..3458d63f --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,19 @@ +ada5590acaa13a35afb62c368b13c3601e658c0c:operate/services/manage.py:generic-api-key:400 +ada5590acaa13a35afb62c368b13c3601e658c0c:operate/services/manage.py:generic-api-key:401 +ada5590acaa13a35afb62c368b13c3601e658c0c:operate/services/manage.py:generic-api-key:448 +ada5590acaa13a35afb62c368b13c3601e658c0c:operate/services/manage.py:generic-api-key:449 +ef9ec7a111816282b6185e8268a460d02329fbe4:api.md:generic-api-key:13 +ef9ec7a111816282b6185e8268a460d02329fbe4:api.md:generic-api-key:37 +44388a82d29ce4d96e554c828c3c2c12d6ee3b8a:operate/data/contracts/service_staking_token/contract.yaml:generic-api-key:10 +43bb67ace89a4a6e0eee84d3ee6495088288c528:backend/operate/data/contracts/service_staking_token/contract.yaml:generic-api-key:10 +19ecb1e59813c632971658183a9f2d9d88e0614b:backend/operate/data/contracts/service_staking_token/contract.yaml:generic-api-key:10 +37847b0c322a0dbc8987df526a49df70301e44d4:backend/operate/ledger/profiles.py:generic-api-key:29 +6834023917760bf7875cc7c107e0c59ad7925ef4:backend/operate/ledger/profiles.py:generic-api-key:32 +4e8c1c21dffd9283195052117ad4c371f770e0b2:backend/operate/ledger/profiles.py:generic-api-key:28 +88115a38d3843d0f233f234816229de495bc6ece:templates/trader.yaml:generic-api-key:13 +0a426251fedb8b55111455e35bffd661f4489541:backend/test.py:generic-api-key:13 +daf41a143aa8c483db584ba1e7222e8eafec1d3b:backend/operate.yaml:generic-api-key:13 +daf41a143aa8c483db584ba1e7222e8eafec1d3b:backend/controller.py:generic-api-key:201 +af77e930289cbc87987567bff0efc25936484df2:backend/controller.py:generic-api-key:354b04972639d66053109596d3b73a1d91688964ebb:electron/constants/publishOptions.js:github-fine-grained-pat:3 +b04972639d66053109596d3b73a1d91688964ebb:electron/constants/publishOptions.js:github-fine-grained-pat:3 +af77e930289cbc87987567bff0efc25936484df2:backend/controller.py:generic-api-key:354 diff --git a/README.md b/README.md index fbd11071..8eb3551a 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,34 @@ -# Pearl -Electron + NextJS + Python Backend application to one-click run Agents. +

+Pearl +

+ +Pearl is an application used to run autonomous agents powered by the OLAS Network. ## Technologies Used + - Electron - NodeJS (20.11 LTS) -- AntD -- TypeScript +- AntD (^5) +- NextJS (^14) +- Javascript / TypeScript - Python (3.10) -- Poetry (1.7.1) -- Docker (24) +- Poetry (^1.7.1) +- Docker Engine ## Getting Started ### Installing system dependencies -The following installation scripts assume you have the following on each OS: +The following installation steps assume you have the following on each OS: + - Linux: a debian based operating system such as Ubuntu with `apt` to install packages. - MacOS: [Homebrew](https://brew.sh/) -- Windows: [Chocolatey](https://chocolatey.org/install) -#### NodeJS via NVM +

NodeJS

 -NodeJS is best installed and managed through NVM, which allows you to install and select the version of NodeJS you wish to use. For this project is the current LTS version 20.11. +NodeJS is best installed and managed through NVM. It allows you to install and select specific versions of NodeJS. Pearl has been built using version 20.11, LTS. -##### Linux +
Linux
```bash sudo apt install curl @@ -33,7 +38,7 @@ nvm install --lts nvm use --lts ``` -##### MacOS +
MacOS
```bash brew install nvm @@ -57,115 +62,201 @@ nvm install --lts nvm use --lts ``` -#### Yarn +
+ +

Yarn

 + +Yarn is the package manager used for dependency management of the Electron app and NextJS frontend. ```bash npm install --global yarn ``` +
+ +

Python

 + +
Linux
-#### Python -##### Linux ```bash sudo apt install python3 ``` -##### MacOS -``` + +
MacOS
+ +```bash brew install python ``` -#### PIPX -##### Linux +
+ +

PIPX

 + +
Linux
+ ```bash sudo apt install pipx ``` -##### MacOS + +
MacOS
+ ```bash brew install pipx ``` -#### Poetry +
+ +

Poetry

 + +Poetry is used on the backend to install and manage dependencies, and create a virtual environment for the backend API. + ```bash pipx install poetry ``` -If promoted to run `pipx ensurepath`, run this command. -#### Docker -##### Linux -You can change the `ubuntu.22.04~jammy` version to your OS in the following command: +If promoted to run `pipx ensurepath`, run it. + +
+ +

Docker

 + +
Linux
+ +*Update the `ubuntu.22.04~jammy` version string to your current OS version before running the following command:* + ```bash VERSION_STRING=5:24.0.7-1~ubuntu.22.04~jammy sudo apt-get install docker-ce=$VERSION_STRING docker-ce-cli=$VERSION_STRING containerd.io docker-buildx-plugin docker-compose-plugin sudo usermod -aG docker $USER ``` -If you are unsure of your current release version and codename to update the VERSION_STRING above, you can run: + +If you are unsure about your current OS version/codename, you can find it by running: + ```bash lsb_release -a ``` -##### MacOS -You must install Docker V24 manually, as brew does not allow for versioning with Docker. +
MacOS
-- Docker Desktop version that supports Docker V24: [https://docs.docker.com/desktop/release-notes/#4261](https://docs.docker.com/desktop/release-notes/#4261) -- Guide to install: [https://docs.docker.com/desktop/install/mac-install/](https://docs.docker.com/desktop/install/mac-install/) +You can [install Docker Desktop via the Docker website](https://www.docker.com/products/docker-desktop/). Be sure to select the correct version for your system's CPU architecture. -### Setup ENV file +If you are unsure about your system's CPU architecture, run the following command: -Create a `.env` file in the root directory, or rename `.env.example` to `.env`. +```bash +uname -p +# x86 64 Intel chip +# arm64 Apple chip +``` + +
+ +

Setting up your .env file

+ +Create an `.env` file in the root directory, or rename `.env.example` to `.env`. +Then set the following environment variables. + +

NODE_ENV

 -#### NODE_ENV For development usage, set `NODE_ENV=development`. For production usage, set `NODE_ENV=production`. -#### FORK_URL +
-**Required for forking Gnosis using a Hardhat node during development.** +

FORK_URL

 + +**This variable is required for both development and production.** +**Must be a Gnosis Mainnet RPC URL.** + +- In `development` this RPC url is only used if/when forking mainnet with Hardhat (covered later). This process allows you to test without losing funds. +- In `production` this RPC URL is used as the main RPC for Pearl. You can get a Gnosis RPC from [Nodies](https://www.nodies.app/). -Then, set `FORK_URL=https://....` in your .env file. +Once you have a Gnosis Mainnet RPC URL, set `FORK_URL=YOUR_RPC_URL_HERE` in your .env file. + +Note: this must be an external RPC. If you decide to use Hardhat for testing on a mainnet fork, do _not_ set your Hardhat Node URL here. +
-Be sure to set an external RPC here. +

DEV_RPC

 +This environment variable is only used when `NODE_ENV=development` is set. -### Install project dependencies +In `development` mode, it is used throughout Pearl as the main RPC. -This will install the required dependencies for the backend, frontend, and electron. +If you're using Hardhat, you can set `DEV_RPC=http://localhost:8545`. +Or, you can use another, external RPC URL that wish to test on, ensuring that the chain ID is 100 (Gnosis Mainnet's chain ID). + +
+ +

Installing project dependencies

+ +Run the following command to install all project dependencies. ```bash yarn install-deps ``` -### Run the development app +

Running the application

+ +Provided your system dependencies are installed, environment variables are set, and your RPC is running. -In the root directory, run: +You can start Pearl by running the following command in the root directory: ```bash yarn start ``` -This will run Electron, which launches NextJS and the Backend as child processes. +This will run Electron, which launches the NextJS frontend and the Python backend as child processes. + +

Chain forking (for development)

+ +In the interest of protecting your funds during development, you can run a forked version of Gnosis Mainnet. + +There are two recommended options, choose one: -### Starting Hardhat (for development) +

Tenderly (preferred)

 -In the interest of not losing funds, we can run a Hardhat node that forks Gnosis -- provided the FORK_URL has been set to an external RPC in your .env file. +[Tenderly](https://tenderly.co/) is a service with a plethora of useful blockchain development tools. The tool required here gives you the ability to **fork networks**. -Run the following to start your Hardhat node: +You can also monitor all transactions, and fund your accounts with any token that you please. + +1. Signup to [Tenderly](https://tenderly.co/), and select the plan you desire. **The Free plan should suffice for most users**. +2. Go to *Forks* under the *Development* tab -- in the left sidebar of your dashboard. +3. Click *Create Fork*, select "Gnosis Chain" as the network, and use Chain ID `100`. +4. Copy the RPC url into the appropriate .env variables in your repository. (Recommended to set both `FORK_URL` & `DEV_RPC` to this RPC url during development). +5. Click the *Fund Accounts* button to fund your accounts with XDAI (native token) and [OLAS](https://gnosisscan.io/token/0xce11e14225575945b8e6dc0d4f2dd4c570f79d9f). + +
+ +

Hardhat

 +Note: using Hardhat will result in the loss of chain state once your Hardhat node is turned off. + +Run the following command in the root of your project folder to start your Hardhat node: ```bash npx hardhat node ``` -**Once Hardhat is running, you can use `http://localhost:8545` during the agent spawning process as your RPC.** +**Once Hardhat is running, you will be able to use `http://localhost:8545` as your development RPC.** -### Funding addresses while running a Hardhat fork -There are a number of scripts to fund addresses for testing: +
Funding your addresses with Hardhat
+ +There are scripts to fund addresses during testing/development: - XDAI funding: -``` + +```bash poetry run python scripts/fund.py 0xYOURADDRESS ``` -- OLAS funding: `TBA` -## Further notes / issues +- OLAS funding: + +```bash +poetry run python scripts/transfer_olas.py PATH_TO_KEY_CONTAINING_OLAS ADDRESS_TO_TRANSFER AMOUNT +``` + +
+ +

Notes and Common Issues

-- Only one agent can be run at a time. +- If Pearl is running, it will kill any attempt to run another Pearl instance. This is to ensure there are no port conflicts. +- Enivironment variables are cached in the terminal, if you change them while your terminal is open, you will need to restart the terminal. \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..38e750eb --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,34 @@ +# Security Policy + +This document outlines security procedures and general policies for the `olas-operate-app` project. + +## Supported Versions + +The following table shows which versions of `olas-operate-app` are currently being supported with security updates. + +| Version | Supported | +|-----------------|--------------------| +| `1.0.0` | :white_check_mark: | +| `< 1.0.0` | :x: | + +## Reporting a Vulnerability + +The `olas-operate-app` team and community take all security bugs in `olas-operate-app` seriously. Thank you for improving the security of `olas-operate-app`. We appreciate your efforts and responsible disclosure and will make every effort to acknowledge your contributions. + +Report security bugs by emailing `info@valory.xyz`. + +The lead maintainer will acknowledge your email within 48 hours, and will send a more detailed response within 48 hours indicating the next steps in handling your report. After the initial reply to your report, the security team will endeavour to keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance. + +Report security bugs in third-party modules to the person or team maintaining the module. + +## Disclosure Policy + +When the security team receives a security bug report, they will assign it to a primary handler. This person will coordinate the fix and release process, involving the following steps: + +- Confirm the problem and determine the affected versions. +- Audit code to find any potential similar problems. +- Prepare fixes for all releases still under maintenance. These fixes will be released as fast as possible to PyPI. + +## Comments on this Policy + +If you have suggestions on how this process could be improved please submit a pull request. diff --git a/electron/constants/publishOptions.js b/electron/constants/publishOptions.js index d131bca8..1ecbc1d4 100644 --- a/electron/constants/publishOptions.js +++ b/electron/constants/publishOptions.js @@ -1,15 +1,11 @@ -// update key: READ-ONLY access to Pearl App, delete once public -const updateKey = - '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'; - const publishOptions = { provider: 'github', owner: 'valory-xyz', repo: 'olas-operate-app', releaseType: 'release', token: process.env.GH_TOKEN, - private: true, + private: false, publishAutoUpdate: true, }; -module.exports = { publishOptions, updateKey }; +module.exports = { publishOptions }; diff --git a/electron/update.js b/electron/update.js index efdecc02..d31153f2 100644 --- a/electron/update.js +++ b/electron/update.js @@ -1,21 +1,16 @@ -const { publishOptions, updateKey } = require('./constants/publishOptions'); +const { publishOptions } = require('./constants/publishOptions'); const electronUpdater = require('electron-updater'); const electronLogger = require('electron-log'); const macUpdater = new electronUpdater.MacUpdater({ ...publishOptions, - private: true, - token: updateKey, + private: false, }); macUpdater.logger = electronLogger; macUpdater.setFeedURL({ ...publishOptions, - token: updateKey, - requestHeaders: { - authorization: `Bearer ${updateKey}`, - }, }); macUpdater.autoDownload = true; diff --git a/hardhat.config.js b/hardhat.config.js index a822326c..610348d8 100644 --- a/hardhat.config.js +++ b/hardhat.config.js @@ -8,7 +8,7 @@ const config = { networks: { hardhat: { forking: { - url: "https://gnosis-pokt.nodies.app" + url: process.env.FORK_URL, }, chainId: 100, },