diff --git a/autonomy/cli/deploy.py b/autonomy/cli/deploy.py index b68e5aec75..a61ab81aec 100644 --- a/autonomy/cli/deploy.py +++ b/autonomy/cli/deploy.py @@ -39,6 +39,7 @@ build_and_deploy_from_token, build_deployment, run_deployment, + run_host_deployment, stop_deployment, ) from autonomy.cli.helpers.env import load_env_file @@ -58,6 +59,7 @@ from autonomy.deploy.constants import INFO, LOGGING_LEVELS from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator +from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator def _validate_packages_path(path: Optional[Path] = None) -> Path: @@ -121,12 +123,18 @@ def deploy_group( default=None, help="Number of agents.", ) +@click.option( + "--localhost", + "deployment_type", + flag_value=HostDeploymentGenerator.deployment_type, + help="Use localhost as a backend.", +) @click.option( "--docker", "deployment_type", flag_value=DockerComposeGenerator.deployment_type, default=True, - help="Use docker as a backend.", + help="Use docker as a backend. (default)", ) @click.option( "--kubernetes", @@ -206,6 +214,13 @@ def deploy_group( help="Set agent memory usage limit.", default=DEFAULT_AGENT_MEMORY_LIMIT, ) +@click.option( + "--mkdir", + type=str, + help="Directory names to create in the build directory.", + default=[], + multiple=True, +) @registry_flag() @password_option(confirmation_prompt=True) @image_author_option @@ -217,6 +232,7 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo output_dir: Optional[Path], dev_mode: bool, registry: str, + mkdir: list[str], number_of_agents: Optional[int] = None, password: Optional[str] = None, open_aea_dir: Optional[Path] = None, @@ -269,6 +285,7 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo use_acn=use_acn, use_tm_testnet_setup=use_tm_testnet_setup, image_author=image_author, + mkdir=mkdir, resources={ "agent": { "limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit}, @@ -308,16 +325,42 @@ def build_deployment_command( # pylint: disable=too-many-arguments, too-many-lo default=False, help="Run service in the background.", ) +@click.option( + "--localhost", + "deployment_type", + flag_value="localhost", + help="Use localhost as a backend.", +) +@click.option( + "--docker", + "deployment_type", + flag_value="docker", + help="Use docker as a backend. (default)", + default=True, +) def run( - build_dir: Path, no_recreate: bool, remove_orphans: bool, detach: bool = False + build_dir: Path, + no_recreate: bool, + remove_orphans: bool, + detach: bool, + deployment_type: str, ) -> None: """Run deployment.""" build_dir = Path(build_dir or Path.cwd()).absolute() - if not (build_dir / DockerComposeGenerator.output_name).exists(): + deployment = ( + HostDeploymentGenerator + if deployment_type == "localhost" + else DockerComposeGenerator + ) + if not (build_dir / deployment.output_name).exists(): raise click.ClickException( f"Deployment configuration does not exist @ {build_dir}" ) - run_deployment(build_dir, no_recreate, remove_orphans, detach=detach) + click.echo(f"Running build @ {build_dir}") + if deployment_type == "localhost": + run_host_deployment(build_dir) + else: + run_deployment(build_dir, no_recreate, remove_orphans, detach=detach) @deploy_group.command(name="stop") diff --git a/autonomy/cli/helpers/deployment.py b/autonomy/cli/helpers/deployment.py index 9b8ee42ed5..b6a5c0bddd 100644 --- a/autonomy/cli/helpers/deployment.py +++ b/autonomy/cli/helpers/deployment.py @@ -26,7 +26,7 @@ import sys import time from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple import click from aea.configurations.data_types import PublicId @@ -49,10 +49,14 @@ from autonomy.deploy.build import generate_deployment from autonomy.deploy.constants import ( AGENT_KEYS_DIR, + AGENT_VARS_CONFIG_FILE, BENCHMARKS_DIR, + DEATTACH_WINDOWS_FLAG, INFO, LOG_DIR, PERSISTENT_DATA_DIR, + TENDERMINT_FLASK_APP_PATH, + TENDERMINT_VARS_CONFIG_FILE, TM_STATE_DIR, VENVS_DIR, ) @@ -61,9 +65,11 @@ from autonomy.deploy.image import build_image -def _build_dirs(build_dir: Path) -> None: +def _build_dirs(build_dir: Path, mkdir: Optional[list[str]]) -> None: """Build necessary directories.""" + mkdirs = [(new_dir_name,) for new_dir_name in mkdir] if mkdir else [] + for dir_path in [ (PERSISTENT_DATA_DIR,), (PERSISTENT_DATA_DIR, LOG_DIR), @@ -71,9 +77,9 @@ def _build_dirs(build_dir: Path) -> None: (PERSISTENT_DATA_DIR, BENCHMARKS_DIR), (PERSISTENT_DATA_DIR, VENVS_DIR), (AGENT_KEYS_DIR,), - ]: + ] + mkdirs: path = Path(build_dir, *dir_path) - path.mkdir() + path.mkdir(exist_ok=True, parents=True) # TOFIX: remove this safely try: os.chown(path, 1000, 1000) @@ -124,7 +130,6 @@ def run_deployment( detach: bool = False, ) -> None: """Run deployment.""" - click.echo(f"Running build @ {build_dir}") try: project = _load_compose_project(build_dir=build_dir) commands = docker_compose.TopLevelCommand(project=project) @@ -168,93 +173,27 @@ def run_deployment( stop_deployment(build_dir=build_dir) -def _prepare_agent_env(working_dir: Path) -> None: - """Prepare agent env, add keys, run aea commands.""" - env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8")) - # Patch for trader agent - if "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_STORE_PATH" in env: - data_dir = working_dir / "data" - data_dir.mkdir(exist_ok=True) - env["SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_STORE_PATH"] = str(data_dir) - - # TODO: Dynamic port allocation, backport to service builder - env["CONNECTION_ABCI_CONFIG_HOST"] = "localhost" - env["CONNECTION_ABCI_CONFIG_PORT"] = "26658" - - for var in env: - # Fix tendermint connection params - if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_COM_URL"): - env[var] = "http://localhost:8080" - - if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_URL"): - env[var] = "http://localhost:26657" - - if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_P2P_URL"): - env[var] = "localhost:26656" - - if var.endswith("MODELS_BENCHMARK_TOOL_ARGS_LOG_DIR"): - benchmarks_dir = working_dir / "benchmarks" - benchmarks_dir.mkdir(exist_ok=True, parents=True) - env[var] = str(benchmarks_dir.resolve()) - - (working_dir / "agent.json").write_text( - json.dumps(env, indent=2), - encoding="utf-8", - ) - - -def _run_aea_cmd( - args: List[str], - cwd: Optional[Path] = None, - stdout: int = subprocess.PIPE, - stderr: int = subprocess.PIPE, - **kwargs: Any, -) -> None: - """Run an aea command in a subprocess.""" - result = subprocess.run( # pylint: disable=subprocess-run-check # nosec - args=[sys.executable, "-m", "aea.cli", *args], - cwd=cwd, - stdout=stdout, - stderr=stderr, - **kwargs, - ) - if result.returncode != 0: - std_error = result.stderr.decode() - if "Item with name ethereum already present!" not in std_error: - raise RuntimeError(f"Error running: {args} @ {cwd}\n{std_error}") - - -def _setup_agent(working_dir: Path) -> None: - """Setup agent.""" - _prepare_agent_env(working_dir) - _run_aea_cmd(["add-key", "ethereum"], cwd=working_dir) - _run_aea_cmd(["issue-certificates"], cwd=working_dir) +def _get_deattached_creation_flags() -> int: + """Get Popen creation flag based on the platform.""" + return DEATTACH_WINDOWS_FLAG if platform.system() == "Windows" else 0 def _start_localhost_agent(working_dir: Path) -> None: """Start localhost agent process.""" - env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8")) + env = json.loads((working_dir / AGENT_VARS_CONFIG_FILE).read_text()) subprocess.run( # pylint: disable=subprocess-run-check # nosec args=[sys.executable, "-m", "aea.cli", "run"], cwd=working_dir, env={**os.environ, **env}, - creationflags=( - 0x00000008 if platform.system() == "Windows" else 0 - ), # Detach process from the main process + creationflags=_get_deattached_creation_flags(), # Detach process from the main process ) def _start_localhost_tendermint(working_dir: Path) -> subprocess.Popen: """Start localhost tendermint process.""" check_tendermint_version() - env = json.loads((working_dir / "tendermint.json").read_text(encoding="utf-8")) - flask_app_path = ( - Path(__file__).parent.parent.parent.parent - / "deployments" - / "Dockerfiles" - / "tendermint" - / "app.py" - ) + env = json.loads((working_dir / TENDERMINT_VARS_CONFIG_FILE).read_text()) + flask_app_path = Path(__file__).parents[3] / TENDERMINT_FLASK_APP_PATH process = subprocess.Popen( # pylint: disable=consider-using-with # nosec args=[ "flask", @@ -268,24 +207,20 @@ def _start_localhost_tendermint(working_dir: Path) -> subprocess.Popen: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env={**os.environ, **env, "FLASK_APP": f"{flask_app_path}:create_server"}, - creationflags=( - 0x00000008 if platform.system() == "Windows" else 0 - ), # Detach process from the main process + creationflags=_get_deattached_creation_flags(), # Detach process from the main process ) (working_dir / "tendermint.pid").write_text( data=str(process.pid), - encoding="utf-8", ) return process def run_host_deployment(build_dir: Path) -> None: """Run host deployment.""" - _setup_agent(build_dir) tm_process = _start_localhost_tendermint(build_dir) try: _start_localhost_agent(build_dir) - except Exception: # pylint: disable=broad-except + finally: tm_process.terminate() @@ -320,6 +255,7 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals use_tm_testnet_setup: bool = False, image_author: Optional[str] = None, resources: Optional[Resources] = None, + mkdir: Optional[list[str]] = None, ) -> None: """Build deployment.""" @@ -333,7 +269,7 @@ def build_deployment( # pylint: disable=too-many-arguments, too-many-locals click.echo(f"Building deployment @ {build_dir}") build_dir.mkdir() - _build_dirs(build_dir) + _build_dirs(build_dir, mkdir) report = generate_deployment( service_path=Path.cwd(), diff --git a/autonomy/deploy/base.py b/autonomy/deploy/base.py index 2515077851..56dc262a6f 100644 --- a/autonomy/deploy/base.py +++ b/autonomy/deploy/base.py @@ -65,6 +65,7 @@ KUBERNETES_DEPLOYMENT = "kubernetes" DOCKER_COMPOSE_DEPLOYMENT = "docker-compose" +LOCALHOST_DEPLOYMENT = "localhost" LOOPBACK = "127.0.0.1" LOCALHOST = "localhost" @@ -448,14 +449,14 @@ def _try_update_tendermint_params( """Try update the tendermint parameters""" is_kubernetes_deployment = self.deplopyment_type == KUBERNETES_DEPLOYMENT + is_localhost_deployment = self.deplopyment_type == LOCALHOST_DEPLOYMENT def _update_tendermint_params( param_args: Dict, idx: int, - is_kubernetes_deployment: bool = False, ) -> None: """Update tendermint params""" - if is_kubernetes_deployment: + if is_kubernetes_deployment or is_localhost_deployment: param_args[TENDERMINT_URL_PARAM] = TENDERMINT_NODE_LOCAL param_args[TENDERMINT_COM_URL_PARAM] = TENDERMINT_COM_LOCAL else: @@ -486,7 +487,6 @@ def _update_tendermint_params( _update_tendermint_params( param_args=param_args, idx=0, - is_kubernetes_deployment=is_kubernetes_deployment, ) else: param_args = self._get_config_from_json_path( @@ -495,7 +495,6 @@ def _update_tendermint_params( _update_tendermint_params( param_args=param_args, idx=0, - is_kubernetes_deployment=is_kubernetes_deployment, ) return @@ -512,7 +511,6 @@ def _update_tendermint_params( _update_tendermint_params( param_args=param_args, idx=agent_idx, - is_kubernetes_deployment=is_kubernetes_deployment, ) except KeyError: # pragma: nocover logging.warning( @@ -593,9 +591,9 @@ def _update_abci_connection_config( processed_overrides = deepcopy(overrides) if self.service.number_of_agents == 1: processed_overrides["config"]["host"] = ( - LOOPBACK - if self.deplopyment_type == KUBERNETES_DEPLOYMENT - else self.get_abci_container_name(index=0) + self.get_abci_container_name(index=0) + if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT + else LOOPBACK ) processed_overrides["config"]["port"] = processed_overrides["config"].get( "port", DEFAULT_ABCI_PORT @@ -609,9 +607,9 @@ def _update_abci_connection_config( for idx, override in processed_overrides.items(): override["config"]["host"] = ( - LOOPBACK - if self.deplopyment_type == KUBERNETES_DEPLOYMENT - else self.get_abci_container_name(index=idx) + self.get_abci_container_name(index=idx) + if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT + else LOOPBACK ) override["config"]["port"] = override["config"].get( "port", DEFAULT_ABCI_PORT diff --git a/autonomy/deploy/build.py b/autonomy/deploy/build.py index ea1a1d2d8c..8904411699 100644 --- a/autonomy/deploy/build.py +++ b/autonomy/deploy/build.py @@ -24,11 +24,13 @@ from autonomy.deploy.constants import DEPLOYMENT_REPORT, INFO from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator +from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator DEPLOYMENT_OPTIONS: Dict[str, Type[BaseDeploymentGenerator]] = { "kubernetes": KubernetesGenerator, "docker-compose": DockerComposeGenerator, + "localhost": HostDeploymentGenerator, } @@ -54,9 +56,14 @@ def generate_deployment( # pylint: disable=too-many-arguments, too-many-locals resources: Optional[Resources] = None, ) -> str: """Generate the deployment for the service.""" - if dev_mode and type_of_deployment != DockerComposeGenerator.deployment_type: + if type_of_deployment == HostDeploymentGenerator.deployment_type: + if number_of_agents is not None and number_of_agents > 1: + raise RuntimeError( + "Host deployment currently only supports single agent deployments" + ) + elif dev_mode: raise RuntimeError( - "Development mode can only be used with docker-compose deployments" + "Development mode can only be used with localhost deployments" ) service_builder = ServiceBuilder.from_dir( diff --git a/autonomy/deploy/constants.py b/autonomy/deploy/constants.py index 00c553b647..7c0ba1cfbe 100644 --- a/autonomy/deploy/constants.py +++ b/autonomy/deploy/constants.py @@ -19,6 +19,7 @@ """Constants for generating deployments environment.""" +from pathlib import Path from string import Template @@ -34,6 +35,23 @@ DEPLOYMENT_KEY_DIRECTORY = "agent_keys" DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA = "agent_{agent_n}" KUBERNETES_AGENT_KEY_NAME = DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA + "_private_key.yaml" +TENDERMINT_BIN_UNIX = "tendermint" +TENDERMINT_BIN_WINDOWS = "tendermint.exe" +TENDERMINT_VARS_CONFIG_FILE = "tendermint.json" +AGENT_VARS_CONFIG_FILE = "agent.json" +TENDERMINT_FLASK_APP_PATH = ( + Path("deployments") / "Dockerfiles" / "tendermint" / "app.py" +) +DEATTACH_WINDOWS_FLAG = 0x00000008 + +TM_ENV_TMHOME = "TMHOME" +TM_ENV_TMSTATE = "TMSTATE" +TM_ENV_PROXY_APP = "PROXY_APP" +TM_ENV_P2P_LADDR = "P2P_LADDR" +TM_ENV_RPC_LADDR = "RPC_LADDR" +TM_ENV_PROXY_APP = "PROXY_APP" +TM_ENV_CREATE_EMPTY_BLOCKS = "CREATE_EMPTY_BLOCKS" +TM_ENV_USE_GRPC = "USE_GRPC" DEFAULT_ENCODING = "utf-8" diff --git a/autonomy/deploy/generators/localhost/__init__.py b/autonomy/deploy/generators/localhost/__init__.py index c6b4188f1c..d0103678f9 100644 --- a/autonomy/deploy/generators/localhost/__init__.py +++ b/autonomy/deploy/generators/localhost/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022 Valory AG +# Copyright 2024 Valory AG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/autonomy/deploy/generators/localhost/base.py b/autonomy/deploy/generators/localhost/base.py index c584f5c29a..1a1f7e88a8 100644 --- a/autonomy/deploy/generators/localhost/base.py +++ b/autonomy/deploy/generators/localhost/base.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2021-2024 Valory AG +# Copyright 2024 Valory AG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,12 +19,11 @@ """Localhost Deployment Generator.""" import json -import shutil import subprocess # nosec import typing as t -from pathlib import Path from aea.configurations.constants import ( + DEFAULT_AEA_CONFIG_FILE, DEFAULT_LEDGER, LEDGER, PRIVATE_KEY, @@ -34,14 +33,29 @@ from aea.helpers.yaml_utils import yaml_load_all from autonomy.deploy.base import BaseDeploymentGenerator -from autonomy.deploy.constants import DEFAULT_ENCODING -from autonomy.deploy.generators.localhost.utils import check_tendermint_version +from autonomy.deploy.constants import ( + AGENT_VARS_CONFIG_FILE, + DEFAULT_ENCODING, + TENDERMINT_VARS_CONFIG_FILE, + TM_ENV_CREATE_EMPTY_BLOCKS, + TM_ENV_P2P_LADDR, + TM_ENV_PROXY_APP, + TM_ENV_RPC_LADDR, + TM_ENV_TMHOME, + TM_ENV_TMSTATE, + TM_ENV_USE_GRPC, + TM_STATE_DIR, +) +from autonomy.deploy.generators.localhost.utils import ( + check_tendermint_version, + setup_agent, +) class HostDeploymentGenerator(BaseDeploymentGenerator): """Localhost deployment.""" - output_name: str = "agent.json" + output_name: str = AGENT_VARS_CONFIG_FILE deployment_type: str = "localhost" def generate_config_tendermint(self) -> "HostDeploymentGenerator": @@ -62,17 +76,16 @@ def generate_config_tendermint(self) -> "HostDeploymentGenerator": # TODO: Dynamic port allocation params = { - "TMHOME": tmhome, - "TMSTATE": str(self.build_dir / "tm_state"), - "P2P_LADDR": "tcp://localhost:26656", - "RPC_LADDR": "tcp://localhost:26657", - "PROXY_APP": "tcp://localhost:26658", - "CREATE_EMPTY_BLOCKS": "true", - "USE_GRPC": "false", + TM_ENV_TMHOME: tmhome, + TM_ENV_TMSTATE: str(self.build_dir / TM_STATE_DIR), + TM_ENV_P2P_LADDR: "tcp://localhost:26656", + TM_ENV_RPC_LADDR: "tcp://localhost:26657", + TM_ENV_PROXY_APP: "tcp://localhost:26658", + TM_ENV_CREATE_EMPTY_BLOCKS: "true", + TM_ENV_USE_GRPC: "false", } - (self.build_dir / "tendermint.json").write_text( + (self.build_dir / TENDERMINT_VARS_CONFIG_FILE).write_text( json.dumps(params, indent=2), - encoding="utf-8", ) return self @@ -90,7 +103,6 @@ def generate( ) (self.build_dir / self.output_name).write_text( json.dumps(self.output, indent=2), - encoding="utf-8", ) return self @@ -118,13 +130,8 @@ def write_config(self) -> "BaseDeploymentGenerator": """Write output to build dir""" super().write_config() # copy private keys - with open_file("aea-config.yaml", "r", encoding="utf-8") as fp: + with open_file(DEFAULT_AEA_CONFIG_FILE, "r") as fp: aea_config = yaml_load_all(fp) - for path in aea_config[0].get("private_key_paths", {}).values(): - if Path(path).exists(): - shutil.copy(path, self.build_dir) - # copy config and vendor - shutil.copy("aea-config.yaml", self.build_dir) - shutil.copytree("vendor", self.build_dir / "vendor") + setup_agent(self.build_dir, aea_config[0]) return self diff --git a/autonomy/deploy/generators/localhost/utils.py b/autonomy/deploy/generators/localhost/utils.py index 3868c28c55..a0bddc56cc 100644 --- a/autonomy/deploy/generators/localhost/utils.py +++ b/autonomy/deploy/generators/localhost/utils.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2021-2024 Valory AG +# Copyright 2024 Valory AG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,12 +18,22 @@ # ------------------------------------------------------------------------------ """Localhost Deployment utilities.""" +import json import os import platform import shutil import subprocess # nosec import sys from pathlib import Path +from typing import Any, List, Optional + +from aea.configurations.constants import DEFAULT_AEA_CONFIG_FILE, VENDOR + +from autonomy.deploy.constants import ( + BENCHMARKS_DIR, + TENDERMINT_BIN_UNIX, + TENDERMINT_BIN_WINDOWS, +) LOCAL_TENDERMINT_VERSION = "0.34.19" @@ -31,20 +41,103 @@ def check_tendermint_version() -> Path: """Check tendermint version.""" - tendermint_executable = Path(str(shutil.which("tendermint"))) + tendermint_executable = Path(str(shutil.which(TENDERMINT_BIN_UNIX))) if platform.system() == "Windows": - tendermint_executable = Path(os.path.dirname(sys.executable)) / "tendermint.exe" + tendermint_executable = ( + Path(os.path.dirname(sys.executable)) / TENDERMINT_BIN_WINDOWS + ) if ( # check tendermint version tendermint_executable is None - or subprocess.check_output([tendermint_executable, "version"]) # nosec - .decode("utf-8") - .strip() + or ( + current_version := subprocess.check_output( # nosec + [tendermint_executable, "version"] + ) + .strip() + .decode() + ) != LOCAL_TENDERMINT_VERSION ): raise FileNotFoundError( f"Please install tendermint version {LOCAL_TENDERMINT_VERSION} " f"or build and run via docker by using the --docker flag." + + f"\nYour tendermint version is: {current_version}" + if current_version + else "" ) return tendermint_executable + + +def _run_aea_cmd( + args: List[str], + cwd: Optional[Path] = None, + stdout: int = subprocess.PIPE, + stderr: int = subprocess.PIPE, + ignore_error: str | None = None, + **kwargs: Any, +) -> None: + """Run an aea command in a subprocess.""" + result = subprocess.run( # pylint: disable=subprocess-run-check # nosec + args=[sys.executable, "-m", "aea.cli", *args], + cwd=cwd, + stdout=stdout, + stderr=stderr, + **kwargs, + ) + if result.returncode != 0: + result_error = result.stderr.decode() + if ignore_error and ignore_error not in result_error: + raise RuntimeError(f"Error running: {args} @ {cwd}\n{result_error}") + + +def _prepare_agent_env(working_dir: Path) -> None: + """Prepare agent env, add keys, run aea commands.""" + env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8")) + + # TODO: Dynamic port allocation, backport to service builder + env["CONNECTION_ABCI_CONFIG_HOST"] = "localhost" + env["CONNECTION_ABCI_CONFIG_PORT"] = "26658" + + for var in env: + # Fix tendermint connection params + if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_COM_URL"): + env[var] = "http://localhost:8080" + + if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_URL"): + env[var] = "http://localhost:26657" + + if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_P2P_URL"): + env[var] = "localhost:26656" + + if var.endswith("MODELS_BENCHMARK_TOOL_ARGS_LOG_DIR"): + benchmarks_dir = working_dir / BENCHMARKS_DIR + benchmarks_dir.mkdir(exist_ok=True, parents=True) + env[var] = str(benchmarks_dir.resolve()) + + (working_dir / "agent.json").write_text( + json.dumps(env, indent=2), + encoding="utf-8", + ) + + +def setup_agent(working_dir: Path, agent_config: dict[str, Any]) -> None: + """Setup locally deployed agent.""" + _prepare_agent_env(working_dir) + shutil.copy(DEFAULT_AEA_CONFIG_FILE, working_dir) + + # add dependencies + if (working_dir.parent / VENDOR).exists(): + shutil.copytree(working_dir.parent / VENDOR, working_dir / VENDOR) + + # add private keys + for ledger_name, path in agent_config.get("private_key_paths", {}).items(): + if Path(path).exists(): + shutil.copy(path, working_dir) + _run_aea_cmd( + ["add-key", ledger_name], + cwd=working_dir, + ignore_error="already present", + ) + + _run_aea_cmd(["issue-certificates"], cwd=working_dir) diff --git a/deployments/Dockerfiles/tendermint/app.py b/deployments/Dockerfiles/tendermint/app.py index bd8a5dd473..680835b4b6 100644 --- a/deployments/Dockerfiles/tendermint/app.py +++ b/deployments/Dockerfiles/tendermint/app.py @@ -32,16 +32,30 @@ from flask import Flask, Response, jsonify, request from werkzeug.exceptions import InternalServerError, NotFound -from deployments.Dockerfiles.tendermint.tendermint import ( - DEFAULT_P2P_LISTEN_ADDRESS, - DEFAULT_RPC_LISTEN_ADDRESS, +from autonomy.deploy.constants import ( + TM_ENV_CREATE_EMPTY_BLOCKS, + TM_ENV_P2P_LADDR, + TM_ENV_PROXY_APP, + TM_ENV_RPC_LADDR, + TM_ENV_TMHOME, + TM_ENV_USE_GRPC, ) try: - from .tendermint import TendermintNode, TendermintParams # type: ignore + from .tendermint import ( # type: ignore + DEFAULT_P2P_LISTEN_ADDRESS, + DEFAULT_RPC_LISTEN_ADDRESS, + TendermintNode, + TendermintParams, + ) except ImportError: - from tendermint import TendermintNode, TendermintParams # type: ignore + from tendermint import ( # type: ignore + DEFAULT_P2P_LISTEN_ADDRESS, + DEFAULT_RPC_LISTEN_ADDRESS, + TendermintNode, + TendermintParams, + ) ENCODING = "utf-8" DEFAULT_LOG_FILE = "log.log" @@ -199,12 +213,12 @@ def create_app( # pylint: disable=too-many-statements """Create the Tendermint server app""" write_to_log = os.environ.get("WRITE_TO_LOG", "false").lower() == "true" tendermint_params = TendermintParams( - proxy_app=os.environ["PROXY_APP"], - rpc_laddr=os.environ.get("RPC_LADDR", DEFAULT_RPC_LISTEN_ADDRESS), - p2p_laddr=os.environ.get("P2P_LADDR", DEFAULT_P2P_LISTEN_ADDRESS), - consensus_create_empty_blocks=os.environ["CREATE_EMPTY_BLOCKS"] == "true", - home=os.environ["TMHOME"], - use_grpc=os.environ["USE_GRPC"] == "true", + proxy_app=os.environ[TM_ENV_PROXY_APP], + rpc_laddr=os.environ.get(TM_ENV_RPC_LADDR, DEFAULT_RPC_LISTEN_ADDRESS), + p2p_laddr=os.environ.get(TM_ENV_P2P_LADDR, DEFAULT_P2P_LISTEN_ADDRESS), + consensus_create_empty_blocks=os.environ[TM_ENV_CREATE_EMPTY_BLOCKS] == "true", + home=os.environ[TM_ENV_TMHOME], + use_grpc=os.environ[TM_ENV_USE_GRPC] == "true", ) app = Flask(__name__) diff --git a/docs/api/cli/deploy.md b/docs/api/cli/deploy.md index ece74159b8..9c0c5524f4 100644 --- a/docs/api/cli/deploy.md +++ b/docs/api/cli/deploy.md @@ -46,12 +46,18 @@ Deploy an agent service. default=None, help="Number of agents.", ) +@click.option( + "--localhost", + "deployment_type", + flag_value=HostDeploymentGenerator.deployment_type, + help="Use localhost as a backend.", +) @click.option( "--docker", "deployment_type", flag_value=DockerComposeGenerator.deployment_type, default=True, - help="Use docker as a backend.", + help="Use docker as a backend. (default)", ) @click.option( "--kubernetes", @@ -133,6 +139,13 @@ Deploy an agent service. help="Set agent memory usage limit.", default=DEFAULT_AGENT_MEMORY_LIMIT, ) +@click.option( + "--mkdir", + type=str, + help="Directory names to create in the build directory.", + default=[], + multiple=True, +) @registry_flag() @password_option(confirmation_prompt=True) @image_author_option @@ -144,6 +157,7 @@ def build_deployment_command( output_dir: Optional[Path], dev_mode: bool, registry: str, + mkdir: list[str], number_of_agents: Optional[int] = None, password: Optional[str] = None, open_aea_dir: Optional[Path] = None, @@ -192,10 +206,21 @@ Build deployment setup for n agents. default=False, help="Run service in the background.", ) -def run(build_dir: Path, - no_recreate: bool, - remove_orphans: bool, - detach: bool = False) -> None +@click.option( + "--localhost", + "deployment_type", + flag_value="localhost", + help="Use localhost as a backend.", +) +@click.option( + "--docker", + "deployment_type", + flag_value="docker", + help="Use docker as a backend. (default)", + default=True, +) +def run(build_dir: Path, no_recreate: bool, remove_orphans: bool, detach: bool, + deployment_type: str) -> None ``` Run deployment. diff --git a/docs/api/cli/helpers/deployment.md b/docs/api/cli/helpers/deployment.md index 65e9ab2271..e2a8b7396c 100644 --- a/docs/api/cli/helpers/deployment.md +++ b/docs/api/cli/helpers/deployment.md @@ -59,7 +59,8 @@ def build_deployment(keys_file: Path, use_acn: bool = False, use_tm_testnet_setup: bool = False, image_author: Optional[str] = None, - resources: Optional[Resources] = None) -> None + resources: Optional[Resources] = None, + mkdir: Optional[list[str]] = None) -> None ``` Build deployment. diff --git a/docs/api/deploy/generators/localhost/utils.md b/docs/api/deploy/generators/localhost/utils.md index 9a6e0c2710..b45558406a 100644 --- a/docs/api/deploy/generators/localhost/utils.md +++ b/docs/api/deploy/generators/localhost/utils.md @@ -14,3 +14,13 @@ def check_tendermint_version() -> Path Check tendermint version. + + +#### setup`_`agent + +```python +def setup_agent(working_dir: Path, agent_config: dict[str, Any]) -> None +``` + +Setup locally deployed agent. + diff --git a/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py b/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py index 51a6fa4bc6..689851bcf7 100644 --- a/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py +++ b/tests/test_autonomy/test_cli/test_deploy/test_build/test_deployment.py @@ -27,14 +27,24 @@ from unittest import mock import yaml +from aea.cli.registry.settings import REGISTRY_LOCAL from aea.cli.utils.config import get_default_author_from_cli_config -from aea.configurations.constants import DEFAULT_ENV_DOTFILE, PACKAGES +from aea.cli.utils.constants import CLI_CONFIG_PATH, DEFAULT_CLI_CONFIG +from aea.configurations.constants import ( + CONNECTION, + CONTRACT, + DEFAULT_AEA_CONFIG_FILE, + DEFAULT_ENV_DOTFILE, + PACKAGES, + PROTOCOL, + SKILL, +) from aea_test_autonomy.configurations import ( ETHEREUM_ENCRYPTED_KEYS, ETHEREUM_ENCRYPTION_PASSWORD, ) -from autonomy.constants import DEFAULT_BUILD_FOLDER, DEFAULT_DOCKER_IMAGE_AUTHOR +from autonomy.constants import DEFAULT_BUILD_FOLDER, DEFAULT_DOCKER_IMAGE_AUTHOR, VALORY from autonomy.deploy.base import ( DEFAULT_AGENT_CPU_LIMIT, DEFAULT_AGENT_CPU_REQUEST, @@ -43,12 +53,26 @@ ServiceBuilder, ) from autonomy.deploy.constants import ( + AGENT_VARS_CONFIG_FILE, DEBUG, DEPLOYMENT_AGENT_KEY_DIRECTORY_SCHEMA, DEPLOYMENT_KEY_DIRECTORY, + INFO, KUBERNETES_AGENT_KEY_NAME, + PERSISTENT_DATA_DIR, + TENDERMINT_VARS_CONFIG_FILE, + TM_ENV_CREATE_EMPTY_BLOCKS, + TM_ENV_P2P_LADDR, + TM_ENV_PROXY_APP, + TM_ENV_RPC_LADDR, + TM_ENV_TMHOME, + TM_ENV_TMSTATE, + TM_ENV_USE_GRPC, + TM_STATE_DIR, ) from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator +from autonomy.deploy.generators.localhost.utils import _run_aea_cmd +from autonomy.replay.agent import ETHEREUM_PRIVATE_KEY_FILE from tests.conftest import ROOT_DIR, skip_docker_tests from tests.test_autonomy.base import get_dummy_service_config @@ -131,6 +155,57 @@ def load_and_check_docker_compose_file( return docker_compose + @staticmethod + def check_localhost_build(build_dir: Path) -> None: + """Check localhost build directory.""" + build_tree = list(map(lambda x: x.name, build_dir.iterdir())) + assert all( + [ + child in build_tree + for child in { + ".certs", + DEFAULT_AEA_CONFIG_FILE, + DEPLOYMENT_KEY_DIRECTORY, + ETHEREUM_PRIVATE_KEY_FILE, + PERSISTENT_DATA_DIR, + AGENT_VARS_CONFIG_FILE, + "data", + "node", + TENDERMINT_VARS_CONFIG_FILE, + } + ] + ) + + def load_and_check_localhost_build(self, path: Path) -> None: + """Load localhost build config.""" + with open(path / TENDERMINT_VARS_CONFIG_FILE, "r", encoding="utf-8") as fp: + assert json.load(fp) == { + TM_ENV_TMHOME: ( + self.t / "register_reset" / DEFAULT_BUILD_FOLDER / "node" + ).as_posix(), + TM_ENV_TMSTATE: ( + self.t / "register_reset" / DEFAULT_BUILD_FOLDER / TM_STATE_DIR + ).as_posix(), + TM_ENV_P2P_LADDR: "tcp://localhost:26656", + TM_ENV_RPC_LADDR: "tcp://localhost:26657", + TM_ENV_PROXY_APP: "tcp://localhost:26658", + TM_ENV_CREATE_EMPTY_BLOCKS: "true", + TM_ENV_USE_GRPC: "false", + } + with open(path / AGENT_VARS_CONFIG_FILE, "r", encoding="utf-8") as fp: + assert json.load(fp) == { + "ID": "0", + "AEA_AGENT": "valory/register_reset:0.1.0:bafybeia4pxlphcvco3ttlv3tytklriwvuwxxxr5m2tdry32yc5vogxtm7u", + "LOG_LEVEL": INFO, + "AEA_PASSWORD": "", + "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_ADDRESS": "http://host.docker.internal:8545", + "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_CHAIN_ID": "31337", + "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_POA_CHAIN": "False", + "CONNECTION_LEDGER_CONFIG_LEDGER_APIS_ETHEREUM_DEFAULT_GAS_PRICE_STRATEGY": "eip1559", + "CONNECTION_ABCI_CONFIG_HOST": "localhost", + "CONNECTION_ABCI_CONFIG_PORT": "26658", + } + @staticmethod def check_docker_compose_build( build_dir: Path, @@ -149,6 +224,73 @@ def check_docker_compose_build( ) +class TestLocalhostBuilds(BaseDeployBuildTest): + """Test localhost builds.""" + + def setup(self) -> None: + """Setup test for localhost deployment.""" + super().setup() + shutil.copy( + ROOT_DIR + / PACKAGES + / VALORY + / "agents" + / "register_reset" + / DEFAULT_AEA_CONFIG_FILE, + self.t / "register_reset", + ) + aea_cli_config = DEFAULT_CLI_CONFIG + aea_cli_config["registry_config"]["settings"][REGISTRY_LOCAL][ + "default_packages_path" + ] = (ROOT_DIR / PACKAGES).as_posix() + Path(CLI_CONFIG_PATH).write_text(yaml.dump(aea_cli_config)) + + with open(self.t / "register_reset" / DEFAULT_AEA_CONFIG_FILE, "r") as fp: + agent_config = next(yaml.safe_load_all(fp)) + agent_config["private_key_paths"]["ethereum"] = ETHEREUM_PRIVATE_KEY_FILE + with open(self.t / "register_reset" / DEFAULT_AEA_CONFIG_FILE, "w") as fp: + yaml.dump(agent_config, fp) + + # add all the components + for component_type in (CONNECTION, CONTRACT, SKILL, PROTOCOL): + for component_name in agent_config[component_type + "s"]: + _run_aea_cmd( + [ + "--skip-consistency-check", + "add", + component_type, + component_name, + "--mixed", + ], + cwd=self.t / "register_reset", + ignore_error="already exists", + ) + + # prepare ethereum private key + with open(self.t / "register_reset" / ETHEREUM_PRIVATE_KEY_FILE, "w") as fp: + fp.write( # mock private key + "0x0000000000000000000000000000000000000000000000000000000000000001" + ) + + def test_localhost_build( + self, + ) -> None: + """Test that the build command works.""" + + build_dir = self.t / "register_reset" / DEFAULT_BUILD_FOLDER + with mock.patch("os.chown"), OS_ENV_PATCH: + result = self.run_cli( + (str(self.keys_file), "--localhost", "--mkdir", "data") + ) + + assert result.exit_code == 0, result.output + assert build_dir.exists() + assert (build_dir / "data").exists() + + self.check_localhost_build(build_dir=build_dir) + self.load_and_check_localhost_build(path=build_dir) + + class TestDockerComposeBuilds(BaseDeployBuildTest): """Test docker-compose build.""" diff --git a/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py b/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py index e1b57e7a66..e9722f0d62 100644 --- a/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py +++ b/tests/test_autonomy/test_cli/test_deploy/test_run_deployment.py @@ -20,12 +20,25 @@ """Test `run` command.""" +import json import os +import shutil from unittest import mock -from autonomy.constants import DOCKER_COMPOSE_YAML +from aea.configurations.constants import DEFAULT_AEA_CONFIG_FILE, PACKAGES +from autonomy.constants import DEFAULT_BUILD_FOLDER, DOCKER_COMPOSE_YAML, VALORY +from autonomy.deploy.base import ServiceBuilder +from autonomy.deploy.constants import ( + AGENT_VARS_CONFIG_FILE, + TENDERMINT_VARS_CONFIG_FILE, +) + +from tests.conftest import ROOT_DIR from tests.test_autonomy.test_cli.base import BaseCliTest +from tests.test_autonomy.test_cli.test_deploy.test_build.test_deployment import ( + OS_ENV_PATCH, +) class TestRun(BaseCliTest): @@ -51,6 +64,53 @@ def test_run( assert result.exit_code == 0, result.output assert "Running build @" in result.output + def test_run_local(self) -> None: + """Test that `deploy run` works on localhost.""" + super().setup() + + # setup the service keys and packages + self.keys_file = self.t / "keys.json" + shutil.copytree(ROOT_DIR / PACKAGES, self.t / PACKAGES) + shutil.copy( + ROOT_DIR / "deployments" / "keys" / "hardhat_keys.json", self.keys_file + ) + shutil.copytree( + self.t / PACKAGES / "valory" / "services" / "register_reset", + self.t / "register_reset", + ) + with OS_ENV_PATCH: + self.spec = ServiceBuilder.from_dir( + self.t / "register_reset", + self.keys_file, + ) + os.chdir(self.t / "register_reset") + + # setup aea-config.yaml + shutil.copy( + ROOT_DIR + / PACKAGES + / VALORY + / "agents" + / "register_reset" + / DEFAULT_AEA_CONFIG_FILE, + self.t / "register_reset", + ) + + # setup agent.json and tendermint.json + os.mkdir(build_path := self.t / "register_reset" / DEFAULT_BUILD_FOLDER) + with open(build_path / TENDERMINT_VARS_CONFIG_FILE, "w") as fp: + json.dump({}, fp) + with open(build_path / AGENT_VARS_CONFIG_FILE, "w") as fp: + json.dump({}, fp) + with ( + mock.patch("autonomy.cli.helpers.deployment.subprocess.run"), + mock.patch("autonomy.cli.helpers.deployment.subprocess.Popen"), + mock.patch("autonomy.cli.helpers.deployment.check_tendermint_version"), + ): + result = self.run_cli(("--localhost", "--build-dir", build_path.as_posix())) + assert result.exit_code == 0, result.output + assert "Running build @" in result.output + def test_missing_config_file( self, ) -> None: diff --git a/tests/test_autonomy/test_deploy/test_deployment_generators.py b/tests/test_autonomy/test_deploy/test_deployment_generators.py index 5b2de7fccb..ab5f0ea036 100644 --- a/tests/test_autonomy/test_deploy/test_deployment_generators.py +++ b/tests/test_autonomy/test_deploy/test_deployment_generators.py @@ -33,6 +33,7 @@ from autonomy.deploy.base import BaseDeploymentGenerator, ServiceBuilder from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator +from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator from tests.conftest import ROOT_DIR @@ -52,7 +53,10 @@ def get_dummy_service() -> Service: @skip_docker_tests -@pytest.mark.parametrize("generator_cls", (DockerComposeGenerator, KubernetesGenerator)) +@pytest.mark.parametrize( + "generator_cls", + (DockerComposeGenerator, KubernetesGenerator, HostDeploymentGenerator), +) @pytest.mark.parametrize("image_version", [None, "0.1.0"]) @pytest.mark.parametrize("use_hardhat", [False, True]) @pytest.mark.parametrize("use_acn", [False, True]) @@ -84,5 +88,9 @@ def test_versioning( ) deployment_generator.generate(**generate_kwargs) - expected = f"valory/oar-oracle:{image_version or AGENT.hash}" + oar_image = "oar-" + if generator_cls == HostDeploymentGenerator: + oar_image = "" + image_version = f"latest:{AGENT.hash}" + expected = f"valory/{oar_image}oracle:{image_version or AGENT.hash}" assert expected in deployment_generator.output