diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d7c4cde --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,39 @@ +name: ci-tracksuite + +# Controls when the workflow will run +on: + push: + branches: [ "master", "develop" ] + pull_request: + branches: [ "master", "develop" ] + +jobs: + qa: + name: qa + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + - run: pip install black flake8 isort + - run: isort --check . + - run: black --check . + - run: flake8 . + + setup: + name: setup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install . + + test: + name: pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: git config --global user.email "dummy.user@ecmwf.int" + - run: git config --global user.name "Dummy User" + - run: python -m pip install .[test] + - run: python -m pytest . -v diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml new file mode 100644 index 0000000..4697c0d --- /dev/null +++ b/.github/workflows/on-push.yml @@ -0,0 +1,56 @@ +name: on-push + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -l {0} + +jobs: + setup: + name: setup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install . + + test: + name: pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: git config --global user.email "dummy.user@ecmwf.int" + - run: git config --global user.name "Dummy User" + - run: python -m pip install .[test] + - run: python -m pytest . -v + + distribution: + runs-on: ubuntu-latest + needs: [setup, test] + + steps: + - uses: actions/checkout@v3 + - name: Build distributions + run: | + $CONDA/bin/python -m pip install build + $CONDA/bin/python -m build + - name: Publish a Python distribution to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index 9afbe58..70963fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,41 @@ [build-system] -# NOTE: `pip install build` to build with `python -m build` -requires = [ - "setuptools >= 40.9.0", - "wheel" -] +requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" -[tool.pytest.ini_options] -addopts = "" -doctest_optionflags = "" -testpaths = "tests" -markers = [] +[project] +name = "tracksuite" +version = "0.3.1" +description = "ecflow suite tracking and deploying toolkit" +authors = [ + { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, + { name = "Corentin Carton de Wiart", email = "corentin.carton@ecmwf.int" }, +] +license = { text = "Apache License Version 2.0" } +requires-python = ">=3.8" +readme = "README.md" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: Unix", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering", +] + +dependencies = [ + "gitpython >= 3.1.25" +] + +[project.optional-dependencies] +test = ["pytest", "mocker", "pytest-mock"] + +[project.urls] +"Source code" = "https://github.com/ecmwf/tracksuite" + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["tests"] + +[project.scripts] + tracksuite-init = "tracksuite.init:main" + tracksuite-deploy = "tracksuite.deploy:main" diff --git a/requirements.txt b/requirements.txt index bc396e8..6d3fef0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ gitpython >= 3.1.25 -paramiko diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d177466..0000000 --- a/setup.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[metadata] -name = tracksuite -version = attr: tracksuite.version.__version__ -author = European Centre for Medium-Range Weather Forecasts (ECMWF) -author_email = software.support@ecmwf.int -license = Apache 2.0 -license_files = LICENSE -description = Suite deployment tools -long_description = file: README.md -long_description_content_type=text/markdown - -[options] -packages = find: -include_package_data = True -install_requires = - gitpython >= 3.1.25 - paramiko - -[options.packages.find] -include = tracksuite* - -[options.entry_points] -console_scripts = - tracksuite-init = tracksuite.init:main - tracksuite-deploy = tracksuite.deploy:main diff --git a/setup.py b/setup.py deleted file mode 100644 index 6068493..0000000 --- a/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() diff --git a/tests/test_deploy.py b/tests/test_deploy.py new file mode 100644 index 0000000..c8d33ef --- /dev/null +++ b/tests/test_deploy.py @@ -0,0 +1,160 @@ +import os +import tempfile + +import git +import pytest + +from tracksuite.deploy import GitDeployment +from tracksuite.init import setup_remote + + +@pytest.fixture +def git_deployment(): + + temp_dir = tempfile.TemporaryDirectory().name + staging_dir = os.path.join(temp_dir, "staging") + target_repo = os.path.join(temp_dir, "target") + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=target_repo, + ) + + deployer = GitDeployment( + host="localhost", + user=current_user, + staging_dir=staging_dir, + target_repo=target_repo, + ) + + return deployer + + +@pytest.fixture +def git_deployment_with_backup(): + + temp_dir = tempfile.TemporaryDirectory().name + staging_dir = os.path.join(temp_dir, "staging") + target_repo = os.path.join(temp_dir, "target") + backup_path = os.path.join(temp_dir, "backup.git") + git.Repo.init(backup_path, bare=True) + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=target_repo, + remote=backup_path, + ) + + deployer = GitDeployment( + host="localhost", + user=current_user, + staging_dir=staging_dir, + target_repo=target_repo, + backup_repo=backup_path, + ) + + return deployer + + +def test_git_deployment_constructor(git_deployment): + + assert git_deployment.host == "localhost" + assert git_deployment.user == os.getenv("USER") + print(git_deployment.target_dir) + assert os.path.exists(git_deployment.target_dir) + + +def test_git_deployment_constructor_with_backup(git_deployment_with_backup): + + assert git_deployment_with_backup.host == "localhost" + assert git_deployment_with_backup.user == os.getenv("USER") + print(git_deployment_with_backup.target_dir) + assert os.path.exists(git_deployment_with_backup.target_dir) + assert os.path.exists(git_deployment_with_backup.backup_repo) + + +def test_deploy_default(git_deployment): + + deployer = git_deployment + staging_dir = deployer.staging_dir + + os.mkdir(staging_dir) + with open(os.path.join(staging_dir, "dummy.txt"), "w") as f: + f.write("dummy content") + + deployer.pull_remotes() + deployer.diff_staging() + deployer.deploy() + + with open(os.path.join(deployer.target_dir, "dummy.txt"), "r") as f: + assert f.read() == "dummy content" + + repo = git.Repo(deployer.target_dir) + commit_history = repo.iter_commits() + all_commits = [commit for commit in commit_history] + assert f"deployed by {deployer.user}" in all_commits[0].message + + +def test_deploy_message(git_deployment): + + deployer = git_deployment + staging_dir = deployer.staging_dir + + os.mkdir(staging_dir) + with open(os.path.join(staging_dir, "dummy.txt"), "w") as f: + f.write("dummy content") + + deployer.pull_remotes() + deployer.diff_staging() + deployer.deploy("This is my change") + + with open(os.path.join(deployer.target_dir, "dummy.txt"), "r") as f: + assert f.read() == "dummy content" + + repo = git.Repo(deployer.target_dir) + commit_history = repo.iter_commits() + all_commits = [commit for commit in commit_history] + assert "This is my change" in all_commits[0].message + + +def test_deploy_with_backup(git_deployment_with_backup): + + deployer = git_deployment_with_backup + staging_dir = deployer.staging_dir + + os.mkdir(staging_dir) + with open(os.path.join(staging_dir, "dummy.txt"), "w") as f: + f.write("dummy content") + + deployer.pull_remotes() + deployer.diff_staging() + deployer.deploy("This is my change") + + with open(os.path.join(deployer.target_dir, "dummy.txt"), "r") as f: + assert f.read() == "dummy content" + + repo = git.Repo(deployer.target_repo) + commit_history = repo.iter_commits() + all_commits = [commit for commit in commit_history] + assert "This is my change" in all_commits[0].message + + +def test_deploy_files(git_deployment): + + deployer = git_deployment + staging_dir = deployer.staging_dir + + os.mkdir(staging_dir) + for file in ["file1.txt", "file2.txt", "file3.txt"]: + with open(os.path.join(deployer.staging_dir, file), "w") as f: + f.write("dummy content") + + deployer.pull_remotes() + deployer.diff_staging() + deployer.deploy(files=["file1.txt", "file2.txt"]) + + assert os.path.exists(os.path.join(deployer.target_dir, "file1.txt")) + assert os.path.exists(os.path.join(deployer.target_dir, "file2.txt")) + assert not os.path.exists(os.path.join(deployer.target_dir, "file3.txt")) diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 0000000..5d130d3 --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,186 @@ +import os +import tempfile +from unittest.mock import patch + +import git +import pytest + +from tracksuite.init import LocalHostClient, SSHClient, setup_remote + + +@pytest.fixture +def ssh_client(): + return SSHClient("test_host", "test_user") + + +def side_effect_for_run_cmd(ssh_command): + return ssh_command + + +@pytest.fixture +def mock_run_cmd_returns_input(mocker): + with patch("tracksuite.init.run_cmd") as mock: + mock.side_effect = side_effect_for_run_cmd + yield mock + + +def test_ssh_client_exec(ssh_client, mock_run_cmd_returns_input): + + # Execute the method under test + command_to_execute = ["echo Hello World"] + result = ssh_client.exec(command_to_execute) + + # Assert that the mock was called with the expected command + expected_ssh_command = f'ssh test_user@test_host "{command_to_execute[0]}; "' + mock_run_cmd_returns_input.assert_called_with(expected_ssh_command) + + # Assert that the result is what our side_effect function returns + assert ( + result == expected_ssh_command + ), "The mock did not return the expected dynamic value" + + +class DummyReturnCode: + def __init__(self, returncode): + self.returncode = returncode + + +@pytest.fixture +def mock_run_cmd_returns_true(mocker): + with patch("tracksuite.init.run_cmd") as mock: + mock.return_value = DummyReturnCode(0) + yield mock + + +def test_ssh_client_is_path(ssh_client, mock_run_cmd_returns_true): + + ssh_client = SSHClient("test_host", "test_user") + + # Execute the method under test + result = ssh_client.is_path("/tmp") + assert result is True + + +def test_localhost_client_different_user(): + with pytest.raises(Exception): + LocalHostClient("localhost", "invalid_user") + + +def test_localhost_client_same_user(): + current_user = os.getenv("USER") + LocalHostClient("localhost", current_user) + + +def test_localhost_client_exec(): + current_user = os.getenv("USER") + localhost = LocalHostClient("localhost", current_user) + command_to_execute = ["echo Hello World"] + result = localhost.exec(command_to_execute) + assert result.returncode == 0 + + +def test_localhost_client_is_path(): + current_user = os.getenv("USER") + localhost = LocalHostClient("localhost", current_user) + + with tempfile.TemporaryDirectory() as temp_dir: + test_dir = os.path.join(temp_dir, "test_dir") + localhost.exec(f"mkdir {test_dir}") + localhost.is_path(test_dir) + + +def test_setup_remote(): + + with tempfile.TemporaryDirectory() as temp_dir: + + remote_path = os.path.join(temp_dir, "remote") + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=remote_path, + ) + assert os.path.exists(remote_path) + assert os.path.exists(os.path.join(remote_path, ".git")) + assert os.path.exists(os.path.join(remote_path, "dummy.txt")) + + repo = git.Repo(remote_path) + commit_history = repo.iter_commits() + for commit in commit_history: + print(commit.message) + assert "first commit" in commit.message + + +def test_setup_remote_with_backup(): + + with tempfile.TemporaryDirectory() as temp_dir: + + repo_path = os.path.join(temp_dir, "my_repo.git") + repo = git.Repo.init(repo_path, bare=True) + + remote_path = os.path.join(temp_dir, "remote") + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=remote_path, + remote=repo_path, + ) + assert os.path.exists(remote_path) + assert os.path.exists(os.path.join(remote_path, ".git")) + assert os.path.exists(os.path.join(remote_path, "dummy.txt")) + + commit_history = repo.iter_commits() + for commit in commit_history: + assert "first commit" in commit.message + print(commit.message) + + +def test_setup_remote_with_backup_fail(): + + with tempfile.TemporaryDirectory() as temp_dir: + + repo_path = os.path.join(temp_dir, "my_repo.git") + repo = git.Repo.init(repo_path, bare=True) + + # Add a dummy commit + repo.index.commit("Dummy commit 1") + repo.index.commit("Dummy commit 2") + commit_history = repo.iter_commits() + for commit in commit_history: + print(commit.message) + + remote_path = os.path.join(temp_dir, "remote") + current_user = os.getenv("USER") + with pytest.raises(Exception): + setup_remote( + host="localhost", + user=current_user, + target_dir=remote_path, + remote=repo_path, + ) + + +def test_setup_remote_with_backup_force(): + + with tempfile.TemporaryDirectory() as temp_dir: + + repo_path = os.path.join(temp_dir, "my_repo.git") + repo = git.Repo.init(repo_path, bare=True) + + # Add a dummy commit + repo.index.commit("Dummy commit 1") + repo.index.commit("Dummy commit 2") + commit_history = repo.iter_commits() + for commit in commit_history: + print(commit.message) + + remote_path = os.path.join(temp_dir, "remote") + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=remote_path, + remote=repo_path, + force=True, + ) diff --git a/tracksuite/__init__.py b/tracksuite/__init__.py index c0501c6..9f913c3 100644 --- a/tracksuite/__init__.py +++ b/tracksuite/__init__.py @@ -2,4 +2,3 @@ from .deploy import GitDeployment from .init import setup_remote -from .version import __version__ diff --git a/tracksuite/init.py b/tracksuite/init.py index 1c1cd51..06b548f 100644 --- a/tracksuite/init.py +++ b/tracksuite/init.py @@ -53,10 +53,11 @@ def __init__(self, host, user, ssh_options=None): def is_path(self, path): # Build the ssh command cmd = [f"[ -d {path} ] && exit 0 || exit 1"] + ret = self.exec(cmd) try: ret = self.exec(cmd) return ret.returncode == 0 - except: + except Exception: return False def exec(self, commands, dir=None): @@ -68,6 +69,7 @@ def exec(self, commands, dir=None): ssh_command += f"cd {dir}; " for cmd in commands: ssh_command += f"{cmd}; " + ssh_command = ssh_command + '"' value = run_cmd(ssh_command) return value @@ -79,6 +81,12 @@ class LocalHostClient(Client): def __init__(self, host, user): assert host == "localhost" + if user != os.getenv("USER"): + raise ValueError( + "Localhost user cannot be different than executing user. " + + "To deploy with a different user, use a different host." + ) + super().__init__(host, user) def is_path(self, path): @@ -124,7 +132,8 @@ def setup_remote(host, user, target_dir, remote=None, force=False): ret = ssh.exec(f"mkdir -p {target_dir}; exit 0") if not ssh.is_path(target_dir): raise Exception( - f"Target directory {target_dir} not properly created on {host} with user {user}\n\n" + ret.stdout + f"Target directory {target_dir} not properly created on {host} with user {user}\n\n" + + ret.stdout ) target_git = os.path.join(target_dir, ".git") @@ -150,7 +159,9 @@ def setup_remote(host, user, target_dir, remote=None, force=False): # making sure we can clone the repository if not ssh.is_path(target_git): - print(f'Target directory {target_dir} not properaly created on {host} with user {user}') + print( + f"Target directory {target_dir} not properaly created on {host} with user {user}" + ) raise Exception(ret.stdout) with tempfile.TemporaryDirectory() as tmp_repo: diff --git a/tracksuite/utils.py b/tracksuite/utils.py index b789887..bbe460e 100644 --- a/tracksuite/utils.py +++ b/tracksuite/utils.py @@ -4,9 +4,9 @@ class CmdError(Exception): def __init__(self, cause, process_ret): super().__init__( - f'ERROR: Command failed with {cause} ({process_ret.returncode})' - f'\nCalled: {process_ret.args}' - f'\nOutput: {process_ret.output}' + f"ERROR: Command failed with {cause} ({process_ret.returncode})" + f"\nCalled: {process_ret.args}" + f"\nOutput: {process_ret.output}" ) @@ -22,17 +22,23 @@ def run_cmd(cmd, timeout=300, **kwargs): """ try: ret = subprocess.run( - cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - timeout=timeout, encoding='utf8', - **kwargs + cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + encoding="utf8", + **kwargs, ) except subprocess.TimeoutExpired as exc: exc.returncode = -1 - raise CmdError('timeout', exc) + raise CmdError("timeout", exc) except subprocess.CalledProcessError as exc: - raise CmdError('error', exc) + exc.output = str(exc) + raise CmdError("error", exc) except Exception as exc: exc.returncode = 99 exc.output = str(exc) - raise CmdError('foreign error', exc) + raise CmdError("foreign error", exc) return ret diff --git a/tracksuite/version.py b/tracksuite/version.py deleted file mode 100644 index 493f741..0000000 --- a/tracksuite/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.3.0"