WIP (deploy): Port localhost deployment code

Signed-off-by: OjusWiZard <[email protected]>

autonomy/cli/helpers/deployment.py

@@ -18,11 +18,15 @@
 # ------------------------------------------------------------------------------
 
 """Deployment helpers."""
+import json
 import os
+import platform
 import shutil
+import subprocess  # nosec
+import sys
 import time
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 import click
 from aea.configurations.data_types import PublicId
@@ -53,6 +57,7 @@
     VENVS_DIR,
 )
 from autonomy.deploy.generators.kubernetes.base import KubernetesGenerator
+from autonomy.deploy.generators.localhost.utils import check_tendermint_version
 from autonomy.deploy.image import build_image
 
 
@@ -163,6 +168,127 @@ def run_deployment(
         stop_deployment(build_dir=build_dir)
 
 
+def _prepare_agent_env(working_dir: Path) -> None:
+    """Prepare agent env, add keys, run aea commands."""
+    env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8"))
+    # Patch for trader agent
+    if "SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_STORE_PATH" in env:
+        data_dir = working_dir / "data"
+        data_dir.mkdir(exist_ok=True)
+        env["SKILL_TRADER_ABCI_MODELS_PARAMS_ARGS_STORE_PATH"] = str(data_dir)
+
+    # TODO: Dynamic port allocation, backport to service builder
+    env["CONNECTION_ABCI_CONFIG_HOST"] = "localhost"
+    env["CONNECTION_ABCI_CONFIG_PORT"] = "26658"
+
+    for var in env:
+        # Fix tendermint connection params
+        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_COM_URL"):
+            env[var] = "http://localhost:8080"
+
+        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_URL"):
+            env[var] = "http://localhost:26657"
+
+        if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_P2P_URL"):
+            env[var] = "localhost:26656"
+
+        if var.endswith("MODELS_BENCHMARK_TOOL_ARGS_LOG_DIR"):
+            benchmarks_dir = working_dir / "benchmarks"
+            benchmarks_dir.mkdir(exist_ok=True, parents=True)
+            env[var] = str(benchmarks_dir.resolve())
+
+    (working_dir / "agent.json").write_text(
+        json.dumps(env, indent=2),
+        encoding="utf-8",
+    )
+
+
+def _run_aea_cmd(
+    args: list[str],
+    cwd: Optional[Path] = None,
+    stdout: int = subprocess.PIPE,
+    stderr: int = subprocess.PIPE,
+    **kwargs: Any,
+) -> None:
+    """Run an aea command in a subprocess."""
+    result = subprocess.run(  # pylint: disable=subprocess-run-check # nosec
+        args=[sys.executable, "-m", "aea.cli", *args],
+        cwd=cwd,
+        stdout=stdout,
+        stderr=stderr,
+        **kwargs,
+    )
+    if result.returncode != 0:
+        std_error = result.stderr.decode()
+        if "Item with name ethereum already present!" not in std_error:
+            raise RuntimeError(f"Error running: {args} @ {cwd}\n{std_error}")
+
+
+def _setup_agent(working_dir: Path) -> None:
+    """Setup agent."""
+    _prepare_agent_env(working_dir)
+    _run_aea_cmd(["add-key", "ethereum"], cwd=working_dir)
+    _run_aea_cmd(["issue-certificates"], cwd=working_dir)
+
+
+def _start_localhost_agent(working_dir: Path) -> None:
+    """Start localhost agent process."""
+    env = json.loads((working_dir / "agent.json").read_text(encoding="utf-8"))
+    subprocess.run(  # pylint: disable=subprocess-run-check # nosec
+        args=[sys.executable, "-m", "aea.cli", "run"],
+        cwd=working_dir,
+        env={**os.environ, **env},
+        creationflags=(
+            0x00000008 if platform.system() == "Windows" else 0
+        ),  # Detach process from the main process
+    )
+
+
+def _start_localhost_tendermint(working_dir: Path) -> subprocess.Popen:
+    """Start localhost tendermint process."""
+    check_tendermint_version()
+    env = json.loads((working_dir / "tendermint.json").read_text(encoding="utf-8"))
+    flask_app_path = (
+        Path(__file__).parent.parent.parent.parent
+        / "deployments"
+        / "Dockerfiles"
+        / "tendermint"
+        / "app.py"
+    )
+    process = subprocess.Popen(  # pylint: disable=consider-using-with # nosec
+        args=[
+            "flask",
+            "run",
+            "--host",
+            "localhost",
+            "--port",
+            "8080",
+        ],
+        cwd=working_dir,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        env={**os.environ, **env, "FLASK_APP": f"{flask_app_path}:create_server"},
+        creationflags=(
+            0x00000008 if platform.system() == "Windows" else 0
+        ),  # Detach process from the main process
+    )
+    (working_dir / "tendermint.pid").write_text(
+        data=str(process.pid),
+        encoding="utf-8",
+    )
+    return process
+
+
+def run_host_deployment(build_dir: Path) -> None:
+    """Run host deployment."""
+    _setup_agent(build_dir)
+    tm_process = _start_localhost_tendermint(build_dir)
+    try:
+        _start_localhost_agent(build_dir)
+    except Exception:  # pylint: disable=broad-except
+        tm_process.terminate()
+
+
 def stop_deployment(build_dir: Path) -> None:
     """Stop running deployment."""
     try:

autonomy/deploy/generators/localhost/__init__.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+#   Copyright 2022 Valory AG
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Localhost Deployment Generator."""

autonomy/deploy/generators/localhost/base.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+#   Copyright 2021-2024 Valory AG
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Localhost Deployment Generator."""
+import json
+import shutil
+import subprocess  # nosec
+import typing as t
+from pathlib import Path
+
+from aea.configurations.constants import (
+    DEFAULT_LEDGER,
+    LEDGER,
+    PRIVATE_KEY,
+    PRIVATE_KEY_PATH_SCHEMA,
+)
+from aea.helpers.io import open_file
+from aea.helpers.yaml_utils import yaml_load_all
+
+from autonomy.deploy.base import BaseDeploymentGenerator
+from autonomy.deploy.constants import DEFAULT_ENCODING
+from autonomy.deploy.generators.localhost.utils import check_tendermint_version
+
+
+class HostDeploymentGenerator(BaseDeploymentGenerator):
+    """Localhost deployment."""
+
+    output_name: str = "agent.json"
+    deployment_type: str = "localhost"
+
+    def generate_config_tendermint(self) -> "HostDeploymentGenerator":
+        """Generate tendermint configuration."""
+        tmhome = str(self.build_dir / "node")
+        tendermint_executable = check_tendermint_version()
+        # if executable found, setup its configs
+        subprocess.run(  # pylint: disable=subprocess-run-check # nosec
+            args=[
+                tendermint_executable,
+                "--home",
+                tmhome,
+                "init",
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        # TODO: Dynamic port allocation
+        params = {
+            "TMHOME": tmhome,
+            "TMSTATE": str(self.build_dir / "tm_state"),
+            "P2P_LADDR": "tcp://localhost:26656",
+            "RPC_LADDR": "tcp://localhost:26657",
+            "PROXY_APP": "tcp://localhost:26658",
+            "CREATE_EMPTY_BLOCKS": "true",
+            "USE_GRPC": "false",
+        }
+        (self.build_dir / "tendermint.json").write_text(
+            json.dumps(params, indent=2),
+            encoding="utf-8",
+        )
+
+        return self
+
+    def generate(
+        self,
+        image_version: t.Optional[str] = None,
+        use_hardhat: bool = False,
+        use_acn: bool = False,
+    ) -> "HostDeploymentGenerator":
+        """Generate agent and tendermint configurations"""
+        agent = self.service_builder.generate_agent(agent_n=0)
+        self.output = json.dumps(
+            {key: f"{value}" for key, value in agent.items()}, indent=2
+        )
+        (self.build_dir / self.output_name).write_text(
+            json.dumps(self.output, indent=2),
+            encoding="utf-8",
+        )
+
+        return self
+
+    def _populate_keys(self) -> None:
+        """Populate the keys directory"""
+        kp, *_ = t.cast(t.List[t.Dict[str, str]], self.service_builder.keys)
+        key = kp[PRIVATE_KEY]
+        ledger = kp.get(LEDGER, DEFAULT_LEDGER)
+        keys_file = self.build_dir / PRIVATE_KEY_PATH_SCHEMA.format(ledger)
+        keys_file.write_text(key, encoding=DEFAULT_ENCODING)
+
+    def _populate_keys_multiledger(self) -> None:
+        """Populate the keys directory with multiple set of keys"""
+
+    def populate_private_keys(self) -> "HostDeploymentGenerator":
+        """Populate the private keys to the build directory for host mapping."""
+        if self.service_builder.multiledger:
+            self._populate_keys_multiledger()
+        else:
+            self._populate_keys()
+        return self
+
+    def write_config(self) -> "BaseDeploymentGenerator":
+        """Write output to build dir"""
+        super().write_config()
+        # copy private keys
+        with open_file("aea-config.yaml", "r", encoding="utf-8") as fp:
+            aea_config = yaml_load_all(fp)
+        for path in aea_config[0].get("private_key_paths", {}).values():
+            if Path(path).exists():
+                shutil.copy(path, self.build_dir)
+
+        # copy config and vendor
+        shutil.copy("aea-config.yaml", self.build_dir)
+        shutil.copytree("vendor", self.build_dir / "vendor")
+        return self

autonomy/deploy/generators/localhost/utils.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+#   Copyright 2021-2024 Valory AG
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+"""Localhost Deployment utilities."""
+import os
+import platform
+import shutil
+import subprocess  # nosec
+import sys
+from pathlib import Path
+
+
+LOCAL_TENDERMINT_VERSION = "0.34.19"
+
+
+def check_tendermint_version() -> Path:
+    """Check tendermint version."""
+    tendermint_executable = Path(str(shutil.which("tendermint")))
+    if platform.system() == "Windows":
+        tendermint_executable = Path(os.path.dirname(sys.executable)) / "tendermint.exe"
+
+    if (  # check tendermint version
+        tendermint_executable is None
+        or subprocess.check_output([tendermint_executable, "version"])  # nosec
+        .decode("utf-8")
+        .strip()
+        != LOCAL_TENDERMINT_VERSION
+    ):
+        raise FileNotFoundError(
+            f"Please install tendermint version {LOCAL_TENDERMINT_VERSION} "
+            f"or build and run via docker by using the --docker flag."
+        )
+
+    return tendermint_executable

deployments/Dockerfiles/tendermint/app.py

@@ -154,7 +154,7 @@ def __init__(self, logger: logging.Logger, dump_dir: Optional[Path] = None) -> N
 
         self.resets = 0
         self.logger = logger
-        self.dump_dir = dump_dir or Path("/tm_state")
+        self.dump_dir = dump_dir or Path(os.environ["TMSTATE"] or "/tm_state")
 
         if self.dump_dir.is_dir():
             shutil.rmtree(str(self.dump_dir), onerror=self.readonly_handler)
@@ -195,6 +195,8 @@ def create_app(  # pylint: disable=too-many-statements
     write_to_log = os.environ.get("WRITE_TO_LOG", "false").lower() == "true"
     tendermint_params = TendermintParams(
         proxy_app=os.environ["PROXY_APP"],
+        rpc_laddr=os.environ["RPC_LADDR"],
+        p2p_laddr=os.environ["P2P_LADDR"],
         consensus_create_empty_blocks=os.environ["CREATE_EMPTY_BLOCKS"] == "true",
         home=os.environ["TMHOME"],
         use_grpc=os.environ["USE_GRPC"] == "true",