TM graceful stop

operate/constants.py

@@ -37,3 +37,5 @@
 ON_CHAIN_INTERACT_SLEEP = 3.0
 
 HEALTH_CHECK_URL = "http://127.0.0.1:8716/healthcheck"  # possible DNS issues on windows so use IP address
+
+TM_CONTROL_URL = "http://localhost:8080"

operate/services/deployment_runner.py

@@ -28,12 +28,16 @@
 import typing as t
 from abc import ABC, ABCMeta, abstractmethod
 from pathlib import Path
+from traceback import print_exc
 from typing import Any
 from venv import main as venv_cli
 
 import psutil
 from aea.__version__ import __version__ as aea_version
 from autonomy.__version__ import __version__ as autonomy_version
+from requests import get
+
+from operate import constants
 
 
 class AbstractDeploymentRunner(ABC):
@@ -86,6 +90,9 @@ def kill_process(pid: int) -> None:
 class BaseDeploymentRunner(AbstractDeploymentRunner, metaclass=ABCMeta):
     """Base deployment with aea support."""
 
+    TM_CONTROL_URL = constants.TM_CONTROL_URL
+    SLEEP_BEFORE_TM_KILL = 2  # seconds
+
     def _run_aea(self, *args: str, cwd: Path) -> Any:
         """Run aea command."""
         return self._run_cmd(args=[self._aea_bin, *args], cwd=cwd)
@@ -123,7 +130,7 @@ def _prepare_agent_env(self) -> Any:
         for var in env:
             # Fix tendermint connection params
             if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_COM_URL"):
-                env[var] = "http://localhost:8080"
+                env[var] = self.TM_CONTROL_URL
 
             if var.endswith("MODELS_PARAMS_ARGS_TENDERMINT_URL"):
                 env[var] = "http://localhost:26657"
@@ -189,8 +196,17 @@ def _stop_agent(self) -> None:
             return
         kill_process(int(pid.read_text(encoding="utf-8")))
 
+    def _get_tm_exit_url(self) -> str:
+        return f"{self.TM_CONTROL_URL}/exit"
+
     def _stop_tendermint(self) -> None:
         """Start tendermint process."""
+        try:
+            get(self._get_tm_exit_url())
+            time.sleep(self.SLEEP_BEFORE_TM_KILL)
+        except Exception:  # pylint: disable=broad-except
+            print_exc()
+
         pid = self._work_directory / "tendermint.pid"
         if not pid.exists():
             return

operate/services/utils/tendermint.py

@@ -18,8 +18,10 @@
 # ------------------------------------------------------------------------------
 
 """Tendermint manager."""
+import contextlib
 import json
 import logging
+import multiprocessing
 import os
 import platform
 import re
@@ -32,6 +34,7 @@
 from logging import Logger
 from pathlib import Path
 from threading import Event, Thread
+from time import sleep
 from typing import Any, Callable, Dict, List, Optional, Tuple, cast
 
 import requests
@@ -310,8 +313,8 @@ def _stop_monitoring_thread(self) -> None:
 
     def stop(self) -> None:
         """Stop a Tendermint node process."""
-        self._stop_tm_process()
         self._stop_monitoring_thread()
+        self._stop_tm_process()
 
     @staticmethod
     def _write_to_console(line: str) -> None:
@@ -503,6 +506,9 @@ def create_app(  # pylint: disable=too-many-statements
     )
 
     app = Flask(__name__)  # pylint: disable=redefined-outer-name
+    app._is_on_exit = (  # pylint: disable=protected-access
+        False  # ugly but better than global ver
+    )
     period_dumper = PeriodDumper(
         logger=app.logger,
         dump_dir=Path(os.environ["TMSTATE"]),
@@ -570,6 +576,8 @@ def update_params() -> Dict:
     @app.route("/gentle_reset")
     def gentle_reset() -> Tuple[Any, int]:
         """Reset the tendermint node gently."""
+        if app._is_on_exit:  # pylint: disable=protected-access
+            raise RuntimeError("server exit now")
         try:
             tendermint_node.stop()
             tendermint_node.start()
@@ -597,6 +605,8 @@ def app_hash() -> Tuple[Any, int]:
     @app.route("/hard_reset")
     def hard_reset() -> Tuple[Any, int]:
         """Reset the node forcefully, and prune the blocks"""
+        if app._is_on_exit:  # pylint: disable=protected-access
+            raise RuntimeError("server exit now")
         try:
             tendermint_node.stop()
             if IS_DEV_MODE:
@@ -639,7 +649,46 @@ def create_server() -> Any:
     return flask_app
 
 
-if __name__ == "__main__":
-    # Start the Flask server programmatically
+def run_app_in_subprocess(q: multiprocessing.Queue) -> None:
+    """Run flask app in a subprocess to kill it when needed."""
+    print("app in subprocess")
+    app, tendermint_node = create_app()
+
+    @app.route("/exit")
+    def handle_server_exit() -> Response:
+        """Handle server exit."""
+        app._is_on_exit = True  # pylint: disable=protected-access
+        tendermint_node.stop()
+
+        q.put(True)
+        return {"node": "stopped"}
+
+    app.run(host="localhost", port=8080)
+
+
+def run_stoppable_main() -> None:
+    """Main to spawn flask in a subprocess."""
+    print("run stoppable main!")
+    q: multiprocessing.Queue = multiprocessing.Queue()
+    p = multiprocessing.Process(target=run_app_in_subprocess, args=(q,))
+    p.start()
+    # wait for stop marker
+    q.get(block=True)
+    sleep(1)
+    p.terminate()
+
+
+def main() -> None:
+    """Main entrance."""
     app = create_server()
     app.run(host="localhost", port=8080)
+
+
+if __name__ == "__main__":
+    # Start the Flask server programmatically
+
+    with contextlib.suppress(Exception):
+        # support for pyinstaller multiprocessing
+        multiprocessing.freeze_support()
+
+    run_stoppable_main()

package.json

@@ -63,5 +63,5 @@
     "download-binaries": "sh download_binaries.sh",
     "build:pearl": "sh build_pearl.sh"
   },
-  "version": "0.1.0-rc191"
+  "version": "0.1.0-rc194"
 }

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "olas-operate-middleware"
-version = "0.1.0-rc191"
+version = "0.1.0-rc194"
 description = ""
 authors = ["David Vilela <[email protected]>", "Viraj Patel <[email protected]>"]
 readme = "README.md"