Merge pull request #1 from OjusWiZard/feat/non-docker-1

WIP (deploy): Integrate non-docker logic with CLI

autonomy/cli/deploy.py

@@ -39,6 +39,7 @@
     build_and_deploy_from_token,
     build_deployment,
     run_deployment,
+    run_host_deployment,
     stop_deployment,
 )
 from autonomy.cli.helpers.env import load_env_file
@@ -58,6 +59,7 @@
 from autonomy.deploy.constants import INFO, LOGGING_LEVELS
 from autonomy.deploy.generators.docker_compose.base import DockerComposeGenerator
 from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.base import HostDeploymentGenerator
 
 
 def _validate_packages_path(path: Optional[Path] = None) -> Path:
@@ -121,12 +123,18 @@ def deploy_group(
     default=None,
     help="Number of agents.",
 )
+@click.option(
+    "--localhost",
+    "deployment_type",
+    flag_value=HostDeploymentGenerator.deployment_type,
+    help="Use localhost as a backend.",
+)
 @click.option(
     "--docker",
     "deployment_type",
     flag_value=DockerComposeGenerator.deployment_type,
     default=True,
-    help="Use docker as a backend.",
+    help="Use docker as a backend. (default)",
 )
 @click.option(
     "--kubernetes",
@@ -206,6 +214,13 @@ def deploy_group(
     help="Set agent memory usage limit.",
     default=DEFAULT_AGENT_MEMORY_LIMIT,
 )
+@click.option(
+    "--mkdir",
+    type=str,
+    help="Directory names to create in the build directory.",
+    default=[],
+    multiple=True,
+)
 @registry_flag()
 @password_option(confirmation_prompt=True)
 @image_author_option
@@ -217,6 +232,7 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
     output_dir: Optional[Path],
     dev_mode: bool,
     registry: str,
+    mkdir: list[str],
     number_of_agents: Optional[int] = None,
     password: Optional[str] = None,
     open_aea_dir: Optional[Path] = None,
@@ -269,6 +285,7 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
             use_acn=use_acn,
             use_tm_testnet_setup=use_tm_testnet_setup,
             image_author=image_author,
+            mkdir=mkdir,
             resources={
                 "agent": {
                     "limit": {"cpu": agent_cpu_limit, "memory": agent_memory_limit},
@@ -308,16 +325,42 @@ def build_deployment_command(  # pylint: disable=too-many-arguments, too-many-lo
     default=False,
     help="Run service in the background.",
 )
+@click.option(
+    "--localhost",
+    "deployment_type",
+    flag_value="localhost",
+    help="Use localhost as a backend.",
+)
+@click.option(
+    "--docker",
+    "deployment_type",
+    flag_value="docker",
+    help="Use docker as a backend. (default)",
+    default=True,
+)
 def run(
-    build_dir: Path, no_recreate: bool, remove_orphans: bool, detach: bool = False
+    build_dir: Path,
+    no_recreate: bool,
+    remove_orphans: bool,
+    detach: bool,
+    deployment_type: str,
 ) -> None:
     """Run deployment."""
     build_dir = Path(build_dir or Path.cwd()).absolute()
-    if not (build_dir / DockerComposeGenerator.output_name).exists():
+    deployment = (
+        HostDeploymentGenerator
+        if deployment_type == "localhost"
+        else DockerComposeGenerator
+    )
+    if not (build_dir / deployment.output_name).exists():
         raise click.ClickException(
             f"Deployment configuration does not exist @ {build_dir}"
         )
-    run_deployment(build_dir, no_recreate, remove_orphans, detach=detach)
+    click.echo(f"Running build @ {build_dir}")
+    if deployment_type == "localhost":
+        run_host_deployment(build_dir)
+    else:
+        run_deployment(build_dir, no_recreate, remove_orphans, detach=detach)
 
 
 @deploy_group.command(name="stop")

autonomy/cli/helpers/deployment.py

@@ -26,7 +26,7 @@
 import sys
 import time
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
 import click
 from aea.configurations.data_types import PublicId
@@ -49,10 +49,14 @@
 from autonomy.deploy.build import generate_deployment
 from autonomy.deploy.constants import (
     AGENT_KEYS_DIR,
+    AGENT_VARS_CONFIG_FILE,
     BENCHMARKS_DIR,
+    DEATTACH_WINDOWS_FLAG,
     INFO,
     LOG_DIR,
     PERSISTENT_DATA_DIR,
+    TENDERMINT_FLASK_APP_PATH,
+    TENDERMINT_VARS_CONFIG_FILE,
     TM_STATE_DIR,
     VENVS_DIR,
 )
@@ -61,19 +65,21 @@
 from autonomy.deploy.image import build_image
 
 
-def _build_dirs(build_dir: Path) -> None:
+def _build_dirs(build_dir: Path, mkdir: Optional[list[str]]) -> None:
     """Build necessary directories."""
 
+    mkdirs = [(new_dir_name,) for new_dir_name in mkdir] if mkdir else []
+
     for dir_path in [
         (PERSISTENT_DATA_DIR,),
         (PERSISTENT_DATA_DIR, LOG_DIR),
         (PERSISTENT_DATA_DIR, TM_STATE_DIR),
         (PERSISTENT_DATA_DIR, BENCHMARKS_DIR),
         (PERSISTENT_DATA_DIR, VENVS_DIR),
         (AGENT_KEYS_DIR,),
-    ]:
+    ] + mkdirs:
         path = Path(build_dir, *dir_path)
-        path.mkdir()
+        path.mkdir(exist_ok=True, parents=True)
         # TOFIX: remove this safely
         try:
             os.chown(path, 1000, 1000)
@@ -124,7 +130,6 @@ def run_deployment(
     detach: bool = False,
 ) -> None:
     """Run deployment."""
-    click.echo(f"Running build @ {build_dir}")
     try:
         project = _load_compose_project(build_dir=build_dir)
         commands = docker_compose.TopLevelCommand(project=project)
@@ -168,93 +173,27 @@ def run_deployment(
         stop_deployment(build_dir=build_dir)
 
 
-def _prepare_agent_env(working_dir: Path) -> None:
-    """Prepare agent env, add keys, run aea commands."""
-    env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8"))
-    # Patch for trader agent
-    if "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_STORE_PATH" in env:
-        data_dir = working_dir / "data"
-        data_dir.mkdir(exist_ok=True)
-        env["SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_STORE_PATH"] = str(data_dir)
-
-    # TODO: Dynamic port allocation, backport to service builder
-    env["CONNECTION_ABCI_CONFIG_HOST"] = "localhost"
-    env["CONNECTION_ABCI_CONFIG_PORT"] = "26658"
-
-    for var in env:
-        # Fix tendermint connection params
-        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_COM_URL"):
-            env[var] = "http://localhost:8080"
-
-        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_URL"):
-            env[var] = "http://localhost:26657"
-
-        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_P2P_URL"):
-            env[var] = "localhost:26656"
-
-        if var.endswith("MODELS_BENCHMARK_TOOL_ARGS_LOG_DIR"):
-            benchmarks_dir = working_dir / "benchmarks"
-            benchmarks_dir.mkdir(exist_ok=True, parents=True)
-            env[var] = str(benchmarks_dir.resolve())
-
-    (working_dir / "agent.json").write_text(
-        json.dumps(env, indent=2),
-        encoding="utf-8",
-    )
-
-
-def _run_aea_cmd(
-    args: List[str],
-    cwd: Optional[Path] = None,
-    stdout: int = subprocess.PIPE,
-    stderr: int = subprocess.PIPE,
-    **kwargs: Any,
-) -> None:
-    """Run an aea command in a subprocess."""
-    result = subprocess.run(  # pylint: disable=subprocess-run-check # nosec
-        args=[sys.executable, "-m", "aea.cli", *args],
-        cwd=cwd,
-        stdout=stdout,
-        stderr=stderr,
-        **kwargs,
-    )
-    if result.returncode != 0:
-        std_error = result.stderr.decode()
-        if "Item with name ethereum already present!" not in std_error:
-            raise RuntimeError(f"Error running: {args} @ {cwd}\n{std_error}")
-
-
-def _setup_agent(working_dir: Path) -> None:
-    """Setup agent."""
-    _prepare_agent_env(working_dir)
-    _run_aea_cmd(["add-key", "ethereum"], cwd=working_dir)
-    _run_aea_cmd(["issue-certificates"], cwd=working_dir)
+def _get_deattached_creation_flags() -> int:
+    """Get Popen creation flag based on the platform."""
+    return DEATTACH_WINDOWS_FLAG if platform.system() == "Windows" else 0
 
 
 def _start_localhost_agent(working_dir: Path) -> None:
     """Start localhost agent process."""
-    env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8"))
+    env = json.loads((working_dir / AGENT_VARS_CONFIG_FILE).read_text())
     subprocess.run(  # pylint: disable=subprocess-run-check # nosec
         args=[sys.executable, "-m", "aea.cli", "run"],
         cwd=working_dir,
         env={**os.environ, **env},
-        creationflags=(
-            0x00000008 if platform.system() == "Windows" else 0
-        ),  # Detach process from the main process
+        creationflags=_get_deattached_creation_flags(),  # Detach process from the main process
     )
 
 
 def _start_localhost_tendermint(working_dir: Path) -> subprocess.Popen:
     """Start localhost tendermint process."""
     check_tendermint_version()
-    env = json.loads((working_dir / "tendermint.json").read_text(encoding="utf-8"))
-    flask_app_path = (
-        Path(__file__).parent.parent.parent.parent
-        / "deployments"
-        / "Dockerfiles"
-        / "tendermint"
-        / "app.py"
-    )
+    env = json.loads((working_dir / TENDERMINT_VARS_CONFIG_FILE).read_text())
+    flask_app_path = Path(__file__).parents[3] / TENDERMINT_FLASK_APP_PATH
     process = subprocess.Popen(  # pylint: disable=consider-using-with # nosec
         args=[
             "flask",
@@ -268,24 +207,20 @@ def _start_localhost_tendermint(working_dir: Path) -> subprocess.Popen:
         stdout=subprocess.DEVNULL,
         stderr=subprocess.DEVNULL,
         env={**os.environ, **env, "FLASK_APP": f"{flask_app_path}:create_server"},
-        creationflags=(
-            0x00000008 if platform.system() == "Windows" else 0
-        ),  # Detach process from the main process
+        creationflags=_get_deattached_creation_flags(),  # Detach process from the main process
     )
     (working_dir / "tendermint.pid").write_text(
         data=str(process.pid),
-        encoding="utf-8",
     )
     return process
 
 
 def run_host_deployment(build_dir: Path) -> None:
     """Run host deployment."""
-    _setup_agent(build_dir)
     tm_process = _start_localhost_tendermint(build_dir)
     try:
         _start_localhost_agent(build_dir)
-    except Exception:  # pylint: disable=broad-except
+    finally:
         tm_process.terminate()
 
 
@@ -320,6 +255,7 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
     use_tm_testnet_setup: bool = False,
     image_author: Optional[str] = None,
     resources: Optional[Resources] = None,
+    mkdir: Optional[list[str]] = None,
 ) -> None:
     """Build deployment."""
 
@@ -333,7 +269,7 @@ def build_deployment(  # pylint: disable=too-many-arguments, too-many-locals
 
     click.echo(f"Building deployment @ {build_dir}")
     build_dir.mkdir()
-    _build_dirs(build_dir)
+    _build_dirs(build_dir, mkdir)
 
     report = generate_deployment(
         service_path=Path.cwd(),

autonomy/deploy/base.py

@@ -65,6 +65,7 @@
 
 KUBERNETES_DEPLOYMENT = "kubernetes"
 DOCKER_COMPOSE_DEPLOYMENT = "docker-compose"
+LOCALHOST_DEPLOYMENT = "localhost"
 
 LOOPBACK = "127.0.0.1"
 LOCALHOST = "localhost"
@@ -448,14 +449,14 @@ def _try_update_tendermint_params(
         """Try update the tendermint parameters"""
 
         is_kubernetes_deployment = self.deplopyment_type == KUBERNETES_DEPLOYMENT
+        is_localhost_deployment = self.deplopyment_type == LOCALHOST_DEPLOYMENT
 
         def _update_tendermint_params(
             param_args: Dict,
             idx: int,
-            is_kubernetes_deployment: bool = False,
         ) -> None:
             """Update tendermint params"""
-            if is_kubernetes_deployment:
+            if is_kubernetes_deployment or is_localhost_deployment:
                 param_args[TENDERMINT_URL_PARAM] = TENDERMINT_NODE_LOCAL
                 param_args[TENDERMINT_COM_URL_PARAM] = TENDERMINT_COM_LOCAL
             else:
@@ -486,7 +487,6 @@ def _update_tendermint_params(
                         _update_tendermint_params(
                             param_args=param_args,
                             idx=0,
-                            is_kubernetes_deployment=is_kubernetes_deployment,
                         )
                 else:
                     param_args = self._get_config_from_json_path(
@@ -495,7 +495,6 @@ def _update_tendermint_params(
                     _update_tendermint_params(
                         param_args=param_args,
                         idx=0,
-                        is_kubernetes_deployment=is_kubernetes_deployment,
                     )
                 return
 
@@ -512,7 +511,6 @@ def _update_tendermint_params(
                 _update_tendermint_params(
                     param_args=param_args,
                     idx=agent_idx,
-                    is_kubernetes_deployment=is_kubernetes_deployment,
                 )
         except KeyError:  # pragma: nocover
             logging.warning(
@@ -593,9 +591,9 @@ def _update_abci_connection_config(
         processed_overrides = deepcopy(overrides)
         if self.service.number_of_agents == 1:
             processed_overrides["config"]["host"] = (
-                LOOPBACK
-                if self.deplopyment_type == KUBERNETES_DEPLOYMENT
-                else self.get_abci_container_name(index=0)
+                self.get_abci_container_name(index=0)
+                if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT
+                else LOOPBACK
             )
             processed_overrides["config"]["port"] = processed_overrides["config"].get(
                 "port", DEFAULT_ABCI_PORT
@@ -609,9 +607,9 @@ def _update_abci_connection_config(
 
         for idx, override in processed_overrides.items():
             override["config"]["host"] = (
-                LOOPBACK
-                if self.deplopyment_type == KUBERNETES_DEPLOYMENT
-                else self.get_abci_container_name(index=idx)
+                self.get_abci_container_name(index=idx)
+                if self.deplopyment_type == DOCKER_COMPOSE_DEPLOYMENT
+                else LOOPBACK
             )
             override["config"]["port"] = override["config"].get(
                 "port", DEFAULT_ABCI_PORT