From be6b40c3f188c39db86009170fafb8dc8cf6b7bb Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 8 Sep 2021 06:56:41 +0300 Subject: [PATCH 1/4] bulk install python dependencies --- aea/cli/install.py | 8 ++--- aea/helpers/install_dependency.py | 22 +++++++++++++ tests/test_cli/test_install.py | 8 ++--- tests/test_helpers/test_install_dependency.py | 32 ++++++++++++++++++- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/aea/cli/install.py b/aea/cli/install.py index 376e2a2a16..11acab0ea6 100644 --- a/aea/cli/install.py +++ b/aea/cli/install.py @@ -18,7 +18,6 @@ # ------------------------------------------------------------------------------ """Implementation of the 'aea install' subcommand.""" -import pprint from typing import Optional, cast import click @@ -29,7 +28,7 @@ from aea.configurations.data_types import Dependencies from aea.configurations.pypi import is_satisfiable, is_simple_dep, to_set_specifier from aea.exceptions import AEAException -from aea.helpers.install_dependency import call_pip, install_dependency +from aea.helpers.install_dependency import call_pip, install_dependencies @click.command() @@ -79,10 +78,7 @@ def do_install(ctx: Context, requirement: Optional[str] = None) -> None: ] ) ) - - for name, d in dependencies.items(): - click.echo(f"Installing {pprint.pformat(name)}...") - install_dependency(name, d, logger) + install_dependencies(list(dependencies.values()), logger=logger) except AEAException as e: raise click.ClickException(str(e)) diff --git a/aea/helpers/install_dependency.py b/aea/helpers/install_dependency.py index d0385d0717..e0f15a38b2 100644 --- a/aea/helpers/install_dependency.py +++ b/aea/helpers/install_dependency.py @@ -19,6 +19,7 @@ """Helper to install python dependencies.""" import subprocess # nosec import sys +from itertools import chain from logging import Logger from subprocess import PIPE # nosec from typing import List @@ -51,6 +52,27 @@ def install_dependency( ) +def install_dependencies( + dependencies: List[Dependency], logger: Logger, install_timeout: float = 300, +) -> None: + """ + Install python dependencies to the current python environment. + + :param dependencies: dict of dependency name and specification + :param logger: the logger + :param install_timeout: timeout to wait pip to install + """ + try: + pip_args = list(chain(*[d.get_pip_install_args() for d in dependencies])) + pip_args = [("--extra-index" if i == "-i" else i) for i in pip_args] + logger.debug("Calling 'pip install {}'".format(" ".join(pip_args))) + call_pip(["install", *pip_args], timeout=install_timeout, retry=True) + except Exception as e: + raise AEAException( + f"An error occurred while installing with pip install {' '.join(pip_args)}: {e}" + ) + + def call_pip(pip_args: List[str], timeout: float = 300, retry: bool = False) -> None: """ Run pip install command. diff --git a/tests/test_cli/test_install.py b/tests/test_cli/test_install.py index ff4435a7ce..4aadf9f466 100644 --- a/tests/test_cli/test_install.py +++ b/tests/test_cli/test_install.py @@ -70,6 +70,8 @@ def test_exit_code_equal_to_zero(self): class TestInstallFailsWhenDependencyDoesNotExist(AEATestCaseEmpty): """Test that the command 'aea install' fails when a dependency is not found.""" + capture_log = True + @classmethod def setup_class(cls): """Set the test up.""" @@ -94,10 +96,6 @@ def setup_class(cls): "version": "==0.1.0", "index": "https://test.pypi.org/simple", }, - "this_is_a_test_dependency_on_git": { - "git": "https://github.com/an_user/a_repo.git", - "ref": "master", - }, } ) @@ -108,7 +106,7 @@ def test_error(self): """Assert an error occurs.""" with pytest.raises( ClickException, - match="An error occurred while installing this_is_a_test_dependency.*", + match="An error occurred while installing.*this_is_a_test_dependency.*", ): self.run_cli_command("install", cwd=self._get_cwd()) diff --git a/tests/test_helpers/test_install_dependency.py b/tests/test_helpers/test_install_dependency.py index 22bf39f6d2..17c3839577 100644 --- a/tests/test_helpers/test_install_dependency.py +++ b/tests/test_helpers/test_install_dependency.py @@ -24,7 +24,7 @@ from aea.configurations.base import Dependency from aea.exceptions import AEAException -from aea.helpers.install_dependency import install_dependency +from aea.helpers.install_dependency import install_dependencies, install_dependency class InstallDependencyTestCase(TestCase): @@ -51,3 +51,33 @@ def test__install_dependency_fails_real_pip_call(self): install_dependency( "testnotexists", Dependency("testnotexists", "==10.0.0"), mock.Mock() ) + + +class InstallDependenciesTestCase(TestCase): + """Test case for _install_dependencies method.""" + + def test_fails(self, *mocks): + """Test for install_dependency method fails.""" + result = mock.Mock() + result.returncode = 1 + with mock.patch("subprocess.run", return_value=result): + with self.assertRaises(AEAException): + install_dependencies([Dependency("test", "==10.0.0")], mock.Mock()) + + def test_ok(self, *mocks): + """Test for install_dependency method ok.""" + result = mock.Mock() + result.returncode = 0 + with mock.patch("subprocess.run", return_value=result): + install_dependencies([Dependency("test", "==10.0.0")], mock.Mock()) + + def test_fails_real_pip_call(self): + """Test for install_dependency method fails.""" + with pytest.raises(AEAException, match=r"No matching distribution found"): + install_dependencies([Dependency("testnotexists", "==10.0.0")], mock.Mock()) + + """Test for install_dependency method fails.""" + with pytest.raises(AEAException, match=r"No matching distribution found"): + install_dependency( + "testnotexists", Dependency("testnotexists", "==10.0.0"), mock.Mock() + ) From 5bca00b0f7c62323ca009562d6b63532d031d516 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Fri, 10 Sep 2021 05:35:29 +0300 Subject: [PATCH 2/4] direct usage of pip install command --- aea/helpers/install_dependency.py | 30 +++++++++++++++++-- setup.py | 3 +- tests/test_helpers/test_install_dependency.py | 14 +++++---- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/aea/helpers/install_dependency.py b/aea/helpers/install_dependency.py index e0f15a38b2..8d58b4fd44 100644 --- a/aea/helpers/install_dependency.py +++ b/aea/helpers/install_dependency.py @@ -17,12 +17,16 @@ # # ------------------------------------------------------------------------------ """Helper to install python dependencies.""" +import logging import subprocess # nosec import sys +from io import StringIO from itertools import chain from logging import Logger from subprocess import PIPE # nosec -from typing import List +from typing import List, Tuple + +from pip._internal.commands.install import InstallCommand from aea.configurations.base import Dependency from aea.exceptions import AEAException, enforce @@ -62,17 +66,39 @@ def install_dependencies( :param logger: the logger :param install_timeout: timeout to wait pip to install """ + del install_timeout try: pip_args = list(chain(*[d.get_pip_install_args() for d in dependencies])) pip_args = [("--extra-index" if i == "-i" else i) for i in pip_args] logger.debug("Calling 'pip install {}'".format(" ".join(pip_args))) - call_pip(["install", *pip_args], timeout=install_timeout, retry=True) + exit_code, err_logs = pip_install(*pip_args) + if exit_code != 0: + raise Exception(err_logs) except Exception as e: raise AEAException( f"An error occurred while installing with pip install {' '.join(pip_args)}: {e}" ) +def pip_install(*args: str) -> Tuple[int, str]: + """Use pip install command directly.""" + buf = StringIO() + buf_handler = logging.StreamHandler(buf) + logger = logging.getLogger("pip") + logger.addHandler(buf_handler) + logger.setLevel(logging.ERROR) + exit_code = -100 + try: + cmd = InstallCommand("install", "install") + exit_code = cmd.main(list(args)) + finally: + error_logs = buf.getvalue() + logger.removeHandler(buf_handler) + del buf_handler + del buf + return exit_code, error_logs + + def call_pip(pip_args: List[str], timeout: float = 300, retry: bool = False) -> None: """ Run pip install command. diff --git a/setup.py b/setup.py index 45b39f4de2..9a5b1f44d6 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def get_all_extras() -> Dict: "pyyaml>=4.2b1,<6.0", "requests>=2.22.0,<3.0.0", "python-dotenv>=0.14.0,<0.18.0", - "ecdsa>=0.15,<0.17.0" + "ecdsa>=0.15,<0.17.0", + "pip", ] if os.name == "nt" or os.getenv("WIN_BUILD_WHEEL", None) == "1": diff --git a/tests/test_helpers/test_install_dependency.py b/tests/test_helpers/test_install_dependency.py index 17c3839577..8941ddc314 100644 --- a/tests/test_helpers/test_install_dependency.py +++ b/tests/test_helpers/test_install_dependency.py @@ -58,17 +58,19 @@ class InstallDependenciesTestCase(TestCase): def test_fails(self, *mocks): """Test for install_dependency method fails.""" - result = mock.Mock() - result.returncode = 1 - with mock.patch("subprocess.run", return_value=result): + result = 1 + with mock.patch( + "pip._internal.commands.install.InstallCommand.main", return_value=result + ): with self.assertRaises(AEAException): install_dependencies([Dependency("test", "==10.0.0")], mock.Mock()) def test_ok(self, *mocks): """Test for install_dependency method ok.""" - result = mock.Mock() - result.returncode = 0 - with mock.patch("subprocess.run", return_value=result): + result = 0 + with mock.patch( + "pip._internal.commands.install.InstallCommand.main", return_value=result + ): install_dependencies([Dependency("test", "==10.0.0")], mock.Mock()) def test_fails_real_pip_call(self): From ef451e639fa038d6a4663c3e0a539a3b8d4cfe3d Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Tue, 14 Sep 2021 03:04:28 +0300 Subject: [PATCH 3/4] Revert "direct usage of pip install command" This reverts commit 5bca00b0f7c62323ca009562d6b63532d031d516. --- aea/helpers/install_dependency.py | 30 ++----------------- setup.py | 3 +- tests/test_helpers/test_install_dependency.py | 14 ++++----- 3 files changed, 9 insertions(+), 38 deletions(-) diff --git a/aea/helpers/install_dependency.py b/aea/helpers/install_dependency.py index 8d58b4fd44..e0f15a38b2 100644 --- a/aea/helpers/install_dependency.py +++ b/aea/helpers/install_dependency.py @@ -17,16 +17,12 @@ # # ------------------------------------------------------------------------------ """Helper to install python dependencies.""" -import logging import subprocess # nosec import sys -from io import StringIO from itertools import chain from logging import Logger from subprocess import PIPE # nosec -from typing import List, Tuple - -from pip._internal.commands.install import InstallCommand +from typing import List from aea.configurations.base import Dependency from aea.exceptions import AEAException, enforce @@ -66,39 +62,17 @@ def install_dependencies( :param logger: the logger :param install_timeout: timeout to wait pip to install """ - del install_timeout try: pip_args = list(chain(*[d.get_pip_install_args() for d in dependencies])) pip_args = [("--extra-index" if i == "-i" else i) for i in pip_args] logger.debug("Calling 'pip install {}'".format(" ".join(pip_args))) - exit_code, err_logs = pip_install(*pip_args) - if exit_code != 0: - raise Exception(err_logs) + call_pip(["install", *pip_args], timeout=install_timeout, retry=True) except Exception as e: raise AEAException( f"An error occurred while installing with pip install {' '.join(pip_args)}: {e}" ) -def pip_install(*args: str) -> Tuple[int, str]: - """Use pip install command directly.""" - buf = StringIO() - buf_handler = logging.StreamHandler(buf) - logger = logging.getLogger("pip") - logger.addHandler(buf_handler) - logger.setLevel(logging.ERROR) - exit_code = -100 - try: - cmd = InstallCommand("install", "install") - exit_code = cmd.main(list(args)) - finally: - error_logs = buf.getvalue() - logger.removeHandler(buf_handler) - del buf_handler - del buf - return exit_code, error_logs - - def call_pip(pip_args: List[str], timeout: float = 300, retry: bool = False) -> None: """ Run pip install command. diff --git a/setup.py b/setup.py index 9a5b1f44d6..45b39f4de2 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,7 @@ def get_all_extras() -> Dict: "pyyaml>=4.2b1,<6.0", "requests>=2.22.0,<3.0.0", "python-dotenv>=0.14.0,<0.18.0", - "ecdsa>=0.15,<0.17.0", - "pip", + "ecdsa>=0.15,<0.17.0" ] if os.name == "nt" or os.getenv("WIN_BUILD_WHEEL", None) == "1": diff --git a/tests/test_helpers/test_install_dependency.py b/tests/test_helpers/test_install_dependency.py index 8941ddc314..17c3839577 100644 --- a/tests/test_helpers/test_install_dependency.py +++ b/tests/test_helpers/test_install_dependency.py @@ -58,19 +58,17 @@ class InstallDependenciesTestCase(TestCase): def test_fails(self, *mocks): """Test for install_dependency method fails.""" - result = 1 - with mock.patch( - "pip._internal.commands.install.InstallCommand.main", return_value=result - ): + result = mock.Mock() + result.returncode = 1 + with mock.patch("subprocess.run", return_value=result): with self.assertRaises(AEAException): install_dependencies([Dependency("test", "==10.0.0")], mock.Mock()) def test_ok(self, *mocks): """Test for install_dependency method ok.""" - result = 0 - with mock.patch( - "pip._internal.commands.install.InstallCommand.main", return_value=result - ): + result = mock.Mock() + result.returncode = 0 + with mock.patch("subprocess.run", return_value=result): install_dependencies([Dependency("test", "==10.0.0")], mock.Mock()) def test_fails_real_pip_call(self): From 18e5cdac3a76fa077eb161b519621ff4eba0bbcc Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Tue, 14 Sep 2021 03:04:57 +0300 Subject: [PATCH 4/4] Revert "Revert "direct usage of pip install command"" This reverts commit ef451e639fa038d6a4663c3e0a539a3b8d4cfe3d. --- aea/helpers/install_dependency.py | 30 +++++++++++++++++-- setup.py | 3 +- tests/test_helpers/test_install_dependency.py | 14 +++++---- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/aea/helpers/install_dependency.py b/aea/helpers/install_dependency.py index e0f15a38b2..8d58b4fd44 100644 --- a/aea/helpers/install_dependency.py +++ b/aea/helpers/install_dependency.py @@ -17,12 +17,16 @@ # # ------------------------------------------------------------------------------ """Helper to install python dependencies.""" +import logging import subprocess # nosec import sys +from io import StringIO from itertools import chain from logging import Logger from subprocess import PIPE # nosec -from typing import List +from typing import List, Tuple + +from pip._internal.commands.install import InstallCommand from aea.configurations.base import Dependency from aea.exceptions import AEAException, enforce @@ -62,17 +66,39 @@ def install_dependencies( :param logger: the logger :param install_timeout: timeout to wait pip to install """ + del install_timeout try: pip_args = list(chain(*[d.get_pip_install_args() for d in dependencies])) pip_args = [("--extra-index" if i == "-i" else i) for i in pip_args] logger.debug("Calling 'pip install {}'".format(" ".join(pip_args))) - call_pip(["install", *pip_args], timeout=install_timeout, retry=True) + exit_code, err_logs = pip_install(*pip_args) + if exit_code != 0: + raise Exception(err_logs) except Exception as e: raise AEAException( f"An error occurred while installing with pip install {' '.join(pip_args)}: {e}" ) +def pip_install(*args: str) -> Tuple[int, str]: + """Use pip install command directly.""" + buf = StringIO() + buf_handler = logging.StreamHandler(buf) + logger = logging.getLogger("pip") + logger.addHandler(buf_handler) + logger.setLevel(logging.ERROR) + exit_code = -100 + try: + cmd = InstallCommand("install", "install") + exit_code = cmd.main(list(args)) + finally: + error_logs = buf.getvalue() + logger.removeHandler(buf_handler) + del buf_handler + del buf + return exit_code, error_logs + + def call_pip(pip_args: List[str], timeout: float = 300, retry: bool = False) -> None: """ Run pip install command. diff --git a/setup.py b/setup.py index 45b39f4de2..9a5b1f44d6 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def get_all_extras() -> Dict: "pyyaml>=4.2b1,<6.0", "requests>=2.22.0,<3.0.0", "python-dotenv>=0.14.0,<0.18.0", - "ecdsa>=0.15,<0.17.0" + "ecdsa>=0.15,<0.17.0", + "pip", ] if os.name == "nt" or os.getenv("WIN_BUILD_WHEEL", None) == "1": diff --git a/tests/test_helpers/test_install_dependency.py b/tests/test_helpers/test_install_dependency.py index 17c3839577..8941ddc314 100644 --- a/tests/test_helpers/test_install_dependency.py +++ b/tests/test_helpers/test_install_dependency.py @@ -58,17 +58,19 @@ class InstallDependenciesTestCase(TestCase): def test_fails(self, *mocks): """Test for install_dependency method fails.""" - result = mock.Mock() - result.returncode = 1 - with mock.patch("subprocess.run", return_value=result): + result = 1 + with mock.patch( + "pip._internal.commands.install.InstallCommand.main", return_value=result + ): with self.assertRaises(AEAException): install_dependencies([Dependency("test", "==10.0.0")], mock.Mock()) def test_ok(self, *mocks): """Test for install_dependency method ok.""" - result = mock.Mock() - result.returncode = 0 - with mock.patch("subprocess.run", return_value=result): + result = 0 + with mock.patch( + "pip._internal.commands.install.InstallCommand.main", return_value=result + ): install_dependencies([Dependency("test", "==10.0.0")], mock.Mock()) def test_fails_real_pip_call(self):