From ad3969aadbb9532cfb433967e7dae3d5bdfbc35b Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:03:22 +0000 Subject: [PATCH 01/29] added tests for init module --- tests/test_init.py | 183 ++++++++++++++++++++++++++++++++++++++++++++ tracksuite/init.py | 17 +++- tracksuite/utils.py | 23 +++--- 3 files changed, 211 insertions(+), 12 deletions(-) create mode 100644 tests/test_init.py diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 0000000..48819ed --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,183 @@ +import os +import tempfile +from unittest.mock import patch + +import git +import pytest + +from tracksuite.init import LocalHostClient, SSHClient, setup_remote + + +@pytest.fixture +def ssh_client(): + return SSHClient("test_host", "test_user") + + +def side_effect_for_run_cmd(ssh_command): + return ssh_command + + +@pytest.fixture +def mock_run_cmd_returns_input(mocker): + with patch("tracksuite.init.run_cmd") as mock: + mock.side_effect = side_effect_for_run_cmd + yield mock + + +def test_ssh_client_exec(ssh_client, mock_run_cmd_returns_input): + + # Execute the method under test + command_to_execute = ["echo Hello World"] + result = ssh_client.exec(command_to_execute) + + # Assert that the mock was called with the expected command + expected_ssh_command = f'ssh test_user@test_host "{command_to_execute[0]}; "' + mock_run_cmd_returns_input.assert_called_with(expected_ssh_command) + + # Assert that the result is what our side_effect function returns + assert ( + result == expected_ssh_command + ), "The mock did not return the expected dynamic value" + + +class DummyReturnCode: + def __init__(self, returncode): + self.returncode = returncode + + +@pytest.fixture +def mock_run_cmd_returns_true(mocker): + with patch("tracksuite.init.run_cmd") as mock: + mock.return_value = DummyReturnCode(0) + yield mock + + +def test_ssh_client_is_path(ssh_client, mock_run_cmd_returns_true): + + ssh_client = SSHClient("test_host", "test_user") + + # Execute the method under test + result = ssh_client.is_path("/tmp") + assert result is True + + +def test_localhost_client_different_user(): + with pytest.raises(Exception): + LocalHostClient("localhost", "invalid_user") + + +def test_localhost_client_same_user(): + current_user = os.getenv("USER") + LocalHostClient("localhost", current_user) + + +def test_localhost_client_exec(): + current_user = os.getenv("USER") + localhost = LocalHostClient("localhost", current_user) + command_to_execute = ["echo Hello World"] + result = localhost.exec(command_to_execute) + assert result.returncode == 0 + + +def test_localhost_client_is_path(): + current_user = os.getenv("USER") + localhost = LocalHostClient("localhost", current_user) + + with tempfile.TemporaryDirectory() as temp_dir: + test_dir = os.path.join(temp_dir, "test_dir") + localhost.exec(f"mkdir {test_dir}") + localhost.is_path(test_dir) + + +def test_setup_remote(): + + with tempfile.TemporaryDirectory() as temp_dir: + + remote_path = os.path.join(temp_dir, "remote") + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=remote_path, + ) + assert os.path.exists(remote_path) + assert os.path.exists(os.path.join(remote_path, ".git")) + + repo = git.Repo(remote_path) + commit_history = repo.iter_commits() + for commit in commit_history: + assert "first commit" in commit.message + + +def test_setup_remote_with_backup(): + + with tempfile.TemporaryDirectory() as temp_dir: + + repo_path = os.path.join(temp_dir, "my_repo.git") + repo = git.Repo.init(repo_path, bare=True) + + remote_path = os.path.join(temp_dir, "remote") + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=remote_path, + remote=repo_path, + ) + assert os.path.exists(remote_path) + assert os.path.exists(os.path.join(remote_path, ".git")) + + commit_history = repo.iter_commits() + for commit in commit_history: + assert "first commit" in commit.message + print(commit.message) + + +def test_setup_remote_with_backup_fail(): + + with tempfile.TemporaryDirectory() as temp_dir: + + repo_path = os.path.join(temp_dir, "my_repo.git") + repo = git.Repo.init(repo_path, bare=True) + + # Add a dummy commit + repo.index.commit("Dummy commit 1") + repo.index.commit("Dummy commit 2") + commit_history = repo.iter_commits() + for commit in commit_history: + print(commit.message) + + remote_path = os.path.join(temp_dir, "remote") + current_user = os.getenv("USER") + with pytest.raises(Exception): + setup_remote( + host="localhost", + user=current_user, + target_dir=remote_path, + remote=repo_path, + ) + + +def test_setup_remote_with_backup_force(): + + with tempfile.TemporaryDirectory() as temp_dir: + + repo_path = os.path.join(temp_dir, "my_repo.git") + repo = git.Repo.init(repo_path, bare=True) + + # Add a dummy commit + repo.index.commit("Dummy commit 1") + repo.index.commit("Dummy commit 2") + commit_history = repo.iter_commits() + for commit in commit_history: + print(commit.message) + + remote_path = os.path.join(temp_dir, "remote") + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=remote_path, + remote=repo_path, + force=True, + ) diff --git a/tracksuite/init.py b/tracksuite/init.py index 1c1cd51..1303ab3 100644 --- a/tracksuite/init.py +++ b/tracksuite/init.py @@ -53,10 +53,11 @@ def __init__(self, host, user, ssh_options=None): def is_path(self, path): # Build the ssh command cmd = [f"[ -d {path} ] && exit 0 || exit 1"] + ret = self.exec(cmd) try: ret = self.exec(cmd) return ret.returncode == 0 - except: + except Exception: return False def exec(self, commands, dir=None): @@ -68,6 +69,7 @@ def exec(self, commands, dir=None): ssh_command += f"cd {dir}; " for cmd in commands: ssh_command += f"{cmd}; " + ssh_command = ssh_command + '"' value = run_cmd(ssh_command) return value @@ -79,6 +81,12 @@ class LocalHostClient(Client): def __init__(self, host, user): assert host == "localhost" + if user != os.getenv("USER"): + raise ValueError( + "Localhost user cannot be different than executing user. "+ + "To deploy with a different user, use a different host." + ) + super().__init__(host, user) def is_path(self, path): @@ -124,7 +132,8 @@ def setup_remote(host, user, target_dir, remote=None, force=False): ret = ssh.exec(f"mkdir -p {target_dir}; exit 0") if not ssh.is_path(target_dir): raise Exception( - f"Target directory {target_dir} not properly created on {host} with user {user}\n\n" + ret.stdout + f"Target directory {target_dir} not properly created on {host} with user {user}\n\n" + + ret.stdout ) target_git = os.path.join(target_dir, ".git") @@ -150,7 +159,9 @@ def setup_remote(host, user, target_dir, remote=None, force=False): # making sure we can clone the repository if not ssh.is_path(target_git): - print(f'Target directory {target_dir} not properaly created on {host} with user {user}') + print( + f"Target directory {target_dir} not properaly created on {host} with user {user}" + ) raise Exception(ret.stdout) with tempfile.TemporaryDirectory() as tmp_repo: diff --git a/tracksuite/utils.py b/tracksuite/utils.py index b789887..8f5da16 100644 --- a/tracksuite/utils.py +++ b/tracksuite/utils.py @@ -4,9 +4,9 @@ class CmdError(Exception): def __init__(self, cause, process_ret): super().__init__( - f'ERROR: Command failed with {cause} ({process_ret.returncode})' - f'\nCalled: {process_ret.args}' - f'\nOutput: {process_ret.output}' + f"ERROR: Command failed with {cause} ({process_ret.returncode})" + f"\nCalled: {process_ret.args}" + f"\nOutput: {process_ret.output}" ) @@ -22,17 +22,22 @@ def run_cmd(cmd, timeout=300, **kwargs): """ try: ret = subprocess.run( - cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - timeout=timeout, encoding='utf8', - **kwargs + cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + encoding="utf8", + **kwargs, ) except subprocess.TimeoutExpired as exc: exc.returncode = -1 - raise CmdError('timeout', exc) + raise CmdError("timeout", exc) except subprocess.CalledProcessError as exc: - raise CmdError('error', exc) + raise CmdError("error", exc) except Exception as exc: exc.returncode = 99 exc.output = str(exc) - raise CmdError('foreign error', exc) + raise CmdError("foreign error", exc) return ret From 78f6066c6fa3a832710ec3f36cc5b96d2e617e4e Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:10:01 +0000 Subject: [PATCH 02/29] add github actions --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++ .github/workflows/on-push.yml | 57 +++++++++++++++++++++++++++++++++++ requirements.txt | 1 - 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/on-push.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..280d6e3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: ci-tracksuite + +# Controls when the workflow will run +on: + push: + branches: [ "master", "develop" ] + pull_request: + branches: [ "master", "develop" ] + +jobs: + qa: + name: qa + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + - run: pip install black flake8 isort + - run: isort --check . + - run: black --check . + - run: flake8 . + + setup: + name: setup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install -r requirements.txt + - run: python -m pip install . + + test: + name: pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install -r requirements.txt + - run: python -m pip install pytest + - run: python -m pip install . + - run: python -m pytest . diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml new file mode 100644 index 0000000..2e14c17 --- /dev/null +++ b/.github/workflows/on-push.yml @@ -0,0 +1,57 @@ +name: on-push + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -l {0} + +jobs: + setup: + name: setup + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install -r requirements.txt + - run: python -m pip install . + + test: + name: pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install -r requirements.txt + - run: python -m pip install pytest + - run: python -m pip install . + - run: python -m pytest . + + distribution: + runs-on: ubuntu-latest + needs: [setup, test] + + steps: + - uses: actions/checkout@v3 + - name: Build distributions + run: | + $CONDA/bin/python -m pip install build + $CONDA/bin/python -m build + - name: Publish a Python distribution to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/requirements.txt b/requirements.txt index bc396e8..6d3fef0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ gitpython >= 3.1.25 -paramiko From 5fbcb4989af870db19f2ff5bb9b2b36a55f43366 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:12:32 +0000 Subject: [PATCH 03/29] minor formatting updatge --- setup.cfg | 1 - tracksuite/init.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index d177466..13a9a84 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,6 @@ packages = find: include_package_data = True install_requires = gitpython >= 3.1.25 - paramiko [options.packages.find] include = tracksuite* diff --git a/tracksuite/init.py b/tracksuite/init.py index 1303ab3..06b548f 100644 --- a/tracksuite/init.py +++ b/tracksuite/init.py @@ -83,8 +83,8 @@ def __init__(self, host, user): assert host == "localhost" if user != os.getenv("USER"): raise ValueError( - "Localhost user cannot be different than executing user. "+ - "To deploy with a different user, use a different host." + "Localhost user cannot be different than executing user. " + + "To deploy with a different user, use a different host." ) super().__init__(host, user) From 0ca1af1ee82190bbad22a6c02fa765f6a50e4ad2 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:18:30 +0000 Subject: [PATCH 04/29] fix tests --- .github/workflows/ci.yml | 5 +---- .github/workflows/on-push.yml | 5 +---- setup.cfg | 6 ++++++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 280d6e3..33ed3b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,6 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: python -m pip install -r requirements.txt - run: python -m pip install . test: @@ -34,7 +33,5 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: python -m pip install -r requirements.txt - - run: python -m pip install pytest - - run: python -m pip install . + - run: python -m pip install .[test] - run: python -m pytest . diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 2e14c17..fd7def0 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -25,7 +25,6 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: python -m pip install -r requirements.txt - run: python -m pip install . test: @@ -34,9 +33,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: python -m pip install -r requirements.txt - - run: python -m pip install pytest - - run: python -m pip install . + - run: python -m pip install .[test] - run: python -m pytest . distribution: diff --git a/setup.cfg b/setup.cfg index 13a9a84..34e7324 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,6 +15,12 @@ include_package_data = True install_requires = gitpython >= 3.1.25 +[options.extras_require] +test = + pytest + mocker + pytest-mock + [options.packages.find] include = tracksuite* From 0eb1417e2ad41a7dafa412afdf236484687f4193 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:23:31 +0000 Subject: [PATCH 05/29] debug ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33ed3b0..2daa915 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,4 +34,4 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: python -m pip install .[test] - - run: python -m pytest . + - run: python -m pytest . -v From 64b140dcf024cf7a9bb6888da14453c8d432fa45 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:31:41 +0000 Subject: [PATCH 06/29] debug ci --- tests/test_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_init.py b/tests/test_init.py index 48819ed..3e2b4f5 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -104,7 +104,7 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, ".git")) repo = git.Repo(remote_path) - commit_history = repo.iter_commits() + commit_history = repo.iter_commits(branch='main') for commit in commit_history: assert "first commit" in commit.message From 168893bf896d0c00fa9a40add798f1680d6dec2b Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:34:56 +0000 Subject: [PATCH 07/29] debug ci --- tests/test_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_init.py b/tests/test_init.py index 3e2b4f5..bf267d0 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -104,7 +104,7 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, ".git")) repo = git.Repo(remote_path) - commit_history = repo.iter_commits(branch='main') + commit_history = repo.iter_commits("main") for commit in commit_history: assert "first commit" in commit.message From d08b1cb724bd29cd33e9136f2e85e714f49bf7b6 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:40:25 +0000 Subject: [PATCH 08/29] debug ci --- tests/test_init.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index bf267d0..c6a66b7 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -104,9 +104,8 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, ".git")) repo = git.Repo(remote_path) - commit_history = repo.iter_commits("main") - for commit in commit_history: - assert "first commit" in commit.message + branch = repo.heads.main + assert "first commit" in branch.commit.message def test_setup_remote_with_backup(): From 238009a1d81e02112744592684512c826e9c5ace Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:42:38 +0000 Subject: [PATCH 09/29] debug ci --- tests/test_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_init.py b/tests/test_init.py index c6a66b7..d228e34 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -104,7 +104,7 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, ".git")) repo = git.Repo(remote_path) - branch = repo.heads.main + branch = repo.heads[0] assert "first commit" in branch.commit.message From 47d2656ab17b96dffdbfc75d44b5acda87d16ef8 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:46:28 +0000 Subject: [PATCH 10/29] debug ci --- tests/test_init.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_init.py b/tests/test_init.py index d228e34..d1cf6ad 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -104,6 +104,7 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, ".git")) repo = git.Repo(remote_path) + print(repo.heads) branch = repo.heads[0] assert "first commit" in branch.commit.message From 9f6cbfd43798ae4d6823d96f367e6b15d886ad98 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:48:15 +0000 Subject: [PATCH 11/29] debug ci --- tests/test_init.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_init.py b/tests/test_init.py index d1cf6ad..8d3e0e0 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -102,7 +102,8 @@ def test_setup_remote(): ) assert os.path.exists(remote_path) assert os.path.exists(os.path.join(remote_path, ".git")) - + + print(os.listdir(os.path.join(remote_path, ".git"))) repo = git.Repo(remote_path) print(repo.heads) branch = repo.heads[0] From 81c058c9dae32cc6a52eeb28cba36ae3f567cc87 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:56:52 +0000 Subject: [PATCH 12/29] debug ci --- tests/test_init.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_init.py b/tests/test_init.py index 8d3e0e0..f2b3ade 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1,5 +1,6 @@ import os import tempfile +import time from unittest.mock import patch import git @@ -102,7 +103,9 @@ def test_setup_remote(): ) assert os.path.exists(remote_path) assert os.path.exists(os.path.join(remote_path, ".git")) - + + time.sleep(2) + print(os.listdir(os.path.join(remote_path, ".git"))) repo = git.Repo(remote_path) print(repo.heads) From eef46e35b08aec5fdb9d741cb18a1785c52ae858 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 21:58:10 +0000 Subject: [PATCH 13/29] debug ci --- tests/test_init.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index f2b3ade..8e24a57 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -104,13 +104,18 @@ def test_setup_remote(): assert os.path.exists(remote_path) assert os.path.exists(os.path.join(remote_path, ".git")) - time.sleep(2) - - print(os.listdir(os.path.join(remote_path, ".git"))) repo = git.Repo(remote_path) - print(repo.heads) - branch = repo.heads[0] - assert "first commit" in branch.commit.message + commit_history = repo.iter_commits() + for commit in commit_history: + assert "first commit" in commit.message + print(commit.message) + # time.sleep(2) + + # print(os.listdir(os.path.join(remote_path, ".git"))) + # repo = git.Repo(remote_path) + # print(repo.heads) + # branch = repo.heads[0] + # assert "first commit" in branch.commit.message def test_setup_remote_with_backup(): From 65a29af419196f4504f49fe6bff48500dd2d2e7e Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Wed, 14 Feb 2024 22:09:00 +0000 Subject: [PATCH 14/29] debug ci --- tests/test_init.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index 8e24a57..3c94c59 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1,6 +1,5 @@ import os import tempfile -import time from unittest.mock import patch import git @@ -103,19 +102,13 @@ def test_setup_remote(): ) assert os.path.exists(remote_path) assert os.path.exists(os.path.join(remote_path, ".git")) + assert os.path.exists(os.path.join(remote_path, "dummy.txt")) repo = git.Repo(remote_path) commit_history = repo.iter_commits() for commit in commit_history: - assert "first commit" in commit.message print(commit.message) - # time.sleep(2) - - # print(os.listdir(os.path.join(remote_path, ".git"))) - # repo = git.Repo(remote_path) - # print(repo.heads) - # branch = repo.heads[0] - # assert "first commit" in branch.commit.message + assert "first commit" in commit.message def test_setup_remote_with_backup(): From 271e92b38aded621fba9c1a5b06d8c57d2c8e01b Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 08:35:02 +0000 Subject: [PATCH 15/29] debug ci --- tests/test_init.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_init.py b/tests/test_init.py index 3c94c59..7a541ab 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -93,7 +93,8 @@ def test_setup_remote(): with tempfile.TemporaryDirectory() as temp_dir: - remote_path = os.path.join(temp_dir, "remote") + # remote_path = os.path.join(temp_dir, "remote") + remote_path = "remote" current_user = os.getenv("USER") setup_remote( host="localhost", From bfedb9de4ae68cf2848262888c2cd22afc0ba9af Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 09:05:58 +0000 Subject: [PATCH 16/29] debug ci --- tests/test_init.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index 7a541ab..7a0e75e 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -93,8 +93,7 @@ def test_setup_remote(): with tempfile.TemporaryDirectory() as temp_dir: - # remote_path = os.path.join(temp_dir, "remote") - remote_path = "remote" + remote_path = os.path.join(temp_dir, "remote") current_user = os.getenv("USER") setup_remote( host="localhost", @@ -105,7 +104,9 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, ".git")) assert os.path.exists(os.path.join(remote_path, "dummy.txt")) - repo = git.Repo(remote_path) + test_path = os.path.join(temp_dir, "test") + repo = git.Repo.clone_from(remote_path, test_path) + # repo = git.Repo(remote_path) commit_history = repo.iter_commits() for commit in commit_history: print(commit.message) @@ -129,6 +130,7 @@ def test_setup_remote_with_backup(): ) assert os.path.exists(remote_path) assert os.path.exists(os.path.join(remote_path, ".git")) + assert os.path.exists(os.path.join(remote_path, "dummy.txt")) commit_history = repo.iter_commits() for commit in commit_history: From f76533183bc2df9a7694ecfc4e4c7f82c4c76091 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 09:13:15 +0000 Subject: [PATCH 17/29] debug ci --- tests/test_init.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index 7a0e75e..249aaa9 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -104,14 +104,25 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, ".git")) assert os.path.exists(os.path.join(remote_path, "dummy.txt")) - test_path = os.path.join(temp_dir, "test") - repo = git.Repo.clone_from(remote_path, test_path) - # repo = git.Repo(remote_path) + repo = git.Repo(remote_path) commit_history = repo.iter_commits() + print(repo) + print(repo.head) + print(repo.heads) + print(repo.branches) + print(repo.index) + print(repo.git_dir) + print(repo.re_envvars) + print(repo.references) + print(repo.refs) + print(repo.unsafe_git_clone_options) + print(repo.untracked_files) + print(repo.working_dir) + print(repo.working_tree_dir) for commit in commit_history: print(commit.message) assert "first commit" in commit.message - + assert False def test_setup_remote_with_backup(): From 70a32f0ef706d5933fe36231a034a020069db556 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 09:15:24 +0000 Subject: [PATCH 18/29] debug ci --- tests/test_init.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_init.py b/tests/test_init.py index 249aaa9..933df8e 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -105,7 +105,6 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, "dummy.txt")) repo = git.Repo(remote_path) - commit_history = repo.iter_commits() print(repo) print(repo.head) print(repo.heads) @@ -119,11 +118,13 @@ def test_setup_remote(): print(repo.untracked_files) print(repo.working_dir) print(repo.working_tree_dir) + commit_history = repo.iter_commits() for commit in commit_history: print(commit.message) assert "first commit" in commit.message assert False + def test_setup_remote_with_backup(): with tempfile.TemporaryDirectory() as temp_dir: From dc71a56e72f61dc270578a59350c526f938b3a47 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 09:25:34 +0000 Subject: [PATCH 19/29] debug ci --- .github/workflows/ci.yml | 10 ++++++++++ tests/test_init.py | 3 +++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2daa915..a5702a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,3 +35,13 @@ jobs: - uses: actions/setup-python@v2 - run: python -m pip install .[test] - run: python -m pytest . -v + + debug: + name: pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: python -m pip install .[test] + - run: mkdir git_test; cd git_test; git init; touch test.txt; git add .; git commit -am 'dummy commit'; git log; git branch -v + - run: python -c 'import git; repo = git.Repo("git_test"); print(repo.git.log()); print(repo.heads)' \ No newline at end of file diff --git a/tests/test_init.py b/tests/test_init.py index 933df8e..a270874 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -118,6 +118,9 @@ def test_setup_remote(): print(repo.untracked_files) print(repo.working_dir) print(repo.working_tree_dir) + print(repo.git.log()) + print(repo.git.branch()) + print(repo.git.status()) commit_history = repo.iter_commits() for commit in commit_history: print(commit.message) From 50efc04265fe0fa09963a64437a73129d06e706e Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 09:28:01 +0000 Subject: [PATCH 20/29] debug ci --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5702a6..eecbf02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,15 +33,19 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 + - run: git config --global user.email "dummy.user@ecmwf.int" + - run: git config --global user.name "Dummy User" - run: python -m pip install .[test] - run: python -m pytest . -v debug: - name: pytest + name: debug runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 + - run: git config --global user.email "dummy.user@ecmwf.int" + - run: git config --global user.name "Dummy User" - run: python -m pip install .[test] - run: mkdir git_test; cd git_test; git init; touch test.txt; git add .; git commit -am 'dummy commit'; git log; git branch -v - run: python -c 'import git; repo = git.Repo("git_test"); print(repo.git.log()); print(repo.heads)' \ No newline at end of file From 32de26bb704a11741d68db6ce31194e00c146203 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 09:32:33 +0000 Subject: [PATCH 21/29] debug ci --- .github/workflows/ci.yml | 2 +- tests/test_init.py | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eecbf02..311936f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: git config --global user.email "dummy.user@ecmwf.int" + - run: git config --global user.email "dummy.user@dummy.com" - run: git config --global user.name "Dummy User" - run: python -m pip install .[test] - run: mkdir git_test; cd git_test; git init; touch test.txt; git add .; git commit -am 'dummy commit'; git log; git branch -v diff --git a/tests/test_init.py b/tests/test_init.py index a270874..5d130d3 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -105,27 +105,10 @@ def test_setup_remote(): assert os.path.exists(os.path.join(remote_path, "dummy.txt")) repo = git.Repo(remote_path) - print(repo) - print(repo.head) - print(repo.heads) - print(repo.branches) - print(repo.index) - print(repo.git_dir) - print(repo.re_envvars) - print(repo.references) - print(repo.refs) - print(repo.unsafe_git_clone_options) - print(repo.untracked_files) - print(repo.working_dir) - print(repo.working_tree_dir) - print(repo.git.log()) - print(repo.git.branch()) - print(repo.git.status()) commit_history = repo.iter_commits() for commit in commit_history: print(commit.message) assert "first commit" in commit.message - assert False def test_setup_remote_with_backup(): From aeef704ffd9c37891033036f5afc89dd98d87ae8 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 09:41:10 +0000 Subject: [PATCH 22/29] debug ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 311936f..67f3d3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,5 +47,5 @@ jobs: - run: git config --global user.email "dummy.user@dummy.com" - run: git config --global user.name "Dummy User" - run: python -m pip install .[test] - - run: mkdir git_test; cd git_test; git init; touch test.txt; git add .; git commit -am 'dummy commit'; git log; git branch -v + - run: ssh localhost; mkdir git_test; cd git_test; git init; touch test.txt; git add .; git commit -am 'dummy commit'; git log; git branch -v - run: python -c 'import git; repo = git.Repo("git_test"); print(repo.git.log()); print(repo.heads)' \ No newline at end of file From f43b57619b394c86f3e0fb5180e457277052e825 Mon Sep 17 00:00:00 2001 From: Corentin Carton De Wiart Date: Thu, 15 Feb 2024 17:16:39 +0000 Subject: [PATCH 23/29] minor change in run_cmd function --- tracksuite/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tracksuite/utils.py b/tracksuite/utils.py index 8f5da16..bbe460e 100644 --- a/tracksuite/utils.py +++ b/tracksuite/utils.py @@ -35,6 +35,7 @@ def run_cmd(cmd, timeout=300, **kwargs): exc.returncode = -1 raise CmdError("timeout", exc) except subprocess.CalledProcessError as exc: + exc.output = str(exc) raise CmdError("error", exc) except Exception as exc: exc.returncode = 99 From 40d1250a83aa11291a785f94786f7c614abb4220 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 17:18:17 +0000 Subject: [PATCH 24/29] remove debug in github actions --- .github/workflows/ci.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67f3d3c..d7c4cde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,15 +37,3 @@ jobs: - run: git config --global user.name "Dummy User" - run: python -m pip install .[test] - run: python -m pytest . -v - - debug: - name: debug - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: git config --global user.email "dummy.user@dummy.com" - - run: git config --global user.name "Dummy User" - - run: python -m pip install .[test] - - run: ssh localhost; mkdir git_test; cd git_test; git init; touch test.txt; git add .; git commit -am 'dummy commit'; git log; git branch -v - - run: python -c 'import git; repo = git.Repo("git_test"); print(repo.git.log()); print(repo.heads)' \ No newline at end of file From 4254fcc2c3dc06d57d5a114f51f3181e665a1e8c Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 18:03:20 +0000 Subject: [PATCH 25/29] add test_deploy --- tests/test_deploy.py | 160 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/test_deploy.py diff --git a/tests/test_deploy.py b/tests/test_deploy.py new file mode 100644 index 0000000..c8d33ef --- /dev/null +++ b/tests/test_deploy.py @@ -0,0 +1,160 @@ +import os +import tempfile + +import git +import pytest + +from tracksuite.deploy import GitDeployment +from tracksuite.init import setup_remote + + +@pytest.fixture +def git_deployment(): + + temp_dir = tempfile.TemporaryDirectory().name + staging_dir = os.path.join(temp_dir, "staging") + target_repo = os.path.join(temp_dir, "target") + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=target_repo, + ) + + deployer = GitDeployment( + host="localhost", + user=current_user, + staging_dir=staging_dir, + target_repo=target_repo, + ) + + return deployer + + +@pytest.fixture +def git_deployment_with_backup(): + + temp_dir = tempfile.TemporaryDirectory().name + staging_dir = os.path.join(temp_dir, "staging") + target_repo = os.path.join(temp_dir, "target") + backup_path = os.path.join(temp_dir, "backup.git") + git.Repo.init(backup_path, bare=True) + current_user = os.getenv("USER") + setup_remote( + host="localhost", + user=current_user, + target_dir=target_repo, + remote=backup_path, + ) + + deployer = GitDeployment( + host="localhost", + user=current_user, + staging_dir=staging_dir, + target_repo=target_repo, + backup_repo=backup_path, + ) + + return deployer + + +def test_git_deployment_constructor(git_deployment): + + assert git_deployment.host == "localhost" + assert git_deployment.user == os.getenv("USER") + print(git_deployment.target_dir) + assert os.path.exists(git_deployment.target_dir) + + +def test_git_deployment_constructor_with_backup(git_deployment_with_backup): + + assert git_deployment_with_backup.host == "localhost" + assert git_deployment_with_backup.user == os.getenv("USER") + print(git_deployment_with_backup.target_dir) + assert os.path.exists(git_deployment_with_backup.target_dir) + assert os.path.exists(git_deployment_with_backup.backup_repo) + + +def test_deploy_default(git_deployment): + + deployer = git_deployment + staging_dir = deployer.staging_dir + + os.mkdir(staging_dir) + with open(os.path.join(staging_dir, "dummy.txt"), "w") as f: + f.write("dummy content") + + deployer.pull_remotes() + deployer.diff_staging() + deployer.deploy() + + with open(os.path.join(deployer.target_dir, "dummy.txt"), "r") as f: + assert f.read() == "dummy content" + + repo = git.Repo(deployer.target_dir) + commit_history = repo.iter_commits() + all_commits = [commit for commit in commit_history] + assert f"deployed by {deployer.user}" in all_commits[0].message + + +def test_deploy_message(git_deployment): + + deployer = git_deployment + staging_dir = deployer.staging_dir + + os.mkdir(staging_dir) + with open(os.path.join(staging_dir, "dummy.txt"), "w") as f: + f.write("dummy content") + + deployer.pull_remotes() + deployer.diff_staging() + deployer.deploy("This is my change") + + with open(os.path.join(deployer.target_dir, "dummy.txt"), "r") as f: + assert f.read() == "dummy content" + + repo = git.Repo(deployer.target_dir) + commit_history = repo.iter_commits() + all_commits = [commit for commit in commit_history] + assert "This is my change" in all_commits[0].message + + +def test_deploy_with_backup(git_deployment_with_backup): + + deployer = git_deployment_with_backup + staging_dir = deployer.staging_dir + + os.mkdir(staging_dir) + with open(os.path.join(staging_dir, "dummy.txt"), "w") as f: + f.write("dummy content") + + deployer.pull_remotes() + deployer.diff_staging() + deployer.deploy("This is my change") + + with open(os.path.join(deployer.target_dir, "dummy.txt"), "r") as f: + assert f.read() == "dummy content" + + repo = git.Repo(deployer.target_repo) + commit_history = repo.iter_commits() + all_commits = [commit for commit in commit_history] + assert "This is my change" in all_commits[0].message + + +def test_deploy_files(git_deployment): + + deployer = git_deployment + staging_dir = deployer.staging_dir + + os.mkdir(staging_dir) + for file in ["file1.txt", "file2.txt", "file3.txt"]: + with open(os.path.join(deployer.staging_dir, file), "w") as f: + f.write("dummy content") + + deployer.pull_remotes() + deployer.diff_staging() + deployer.deploy(files=["file1.txt", "file2.txt"]) + + assert os.path.exists(os.path.join(deployer.target_dir, "file1.txt")) + assert os.path.exists(os.path.join(deployer.target_dir, "file2.txt")) + assert not os.path.exists(os.path.join(deployer.target_dir, "file3.txt")) From 05dcdbffbd1228fe12de137840f753c668c7114b Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Thu, 15 Feb 2024 18:10:04 +0000 Subject: [PATCH 26/29] debug on push github actions --- .github/workflows/on-push.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index fd7def0..4697c0d 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -33,8 +33,10 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 + - run: git config --global user.email "dummy.user@ecmwf.int" + - run: git config --global user.name "Dummy User" - run: python -m pip install .[test] - - run: python -m pytest . + - run: python -m pytest . -v distribution: runs-on: ubuntu-latest From e62db1d0ebe8d8f1e24c621b73f1909d54462659 Mon Sep 17 00:00:00 2001 From: corentincarton Date: Tue, 5 Mar 2024 09:38:58 +0000 Subject: [PATCH 27/29] Update version.py --- tracksuite/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracksuite/version.py b/tracksuite/version.py index 493f741..260c070 100644 --- a/tracksuite/version.py +++ b/tracksuite/version.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.3.1" From 8e9b6689e7c868ff63cd68ed3633b7732e06e3d2 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Tue, 5 Mar 2024 10:16:36 +0000 Subject: [PATCH 28/29] update packaging --- pyproject.toml | 48 +++++++++++++++++++++++++++++++++--------- setup.cfg | 30 -------------------------- setup.py | 3 --- tracksuite/__init__.py | 1 - tracksuite/version.py | 1 - 5 files changed, 38 insertions(+), 45 deletions(-) delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 tracksuite/version.py diff --git a/pyproject.toml b/pyproject.toml index 9afbe58..c17eba3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,41 @@ [build-system] -# NOTE: `pip install build` to build with `python -m build` -requires = [ - "setuptools >= 40.9.0", - "wheel" -] +requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" -[tool.pytest.ini_options] -addopts = "" -doctest_optionflags = "" -testpaths = "tests" -markers = [] +[project] +name = "tracksuite" +version = "0.3.1" +description = "ecflow suite tracking and deploying toolkit" +authors = [ + { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, + { name = "Corentin Carton de Wiart", email = "corentin.carton@ecmwf.int" }, +] +license = { text = "Apache License Version 2.0" } +requires-python = ">=3.8" +readme = "README.md" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: Unix", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering", +] + +dependencies = [ + "gitpython >= 3.1.25" +] + +[project.optional-dependencies] +tests = ["pytest", "mocker", "pytest-mock"] + +[project.urls] +"Source code" = "https://github.com/ecmwf/tracksuite" + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["tests"] + +[project.scripts] + tracksuite-init = "tracksuite.init:main" + tracksuite-deploy = "tracksuite.deploy:main" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 34e7324..0000000 --- a/setup.cfg +++ /dev/null @@ -1,30 +0,0 @@ -[metadata] -name = tracksuite -version = attr: tracksuite.version.__version__ -author = European Centre for Medium-Range Weather Forecasts (ECMWF) -author_email = software.support@ecmwf.int -license = Apache 2.0 -license_files = LICENSE -description = Suite deployment tools -long_description = file: README.md -long_description_content_type=text/markdown - -[options] -packages = find: -include_package_data = True -install_requires = - gitpython >= 3.1.25 - -[options.extras_require] -test = - pytest - mocker - pytest-mock - -[options.packages.find] -include = tracksuite* - -[options.entry_points] -console_scripts = - tracksuite-init = tracksuite.init:main - tracksuite-deploy = tracksuite.deploy:main diff --git a/setup.py b/setup.py deleted file mode 100644 index 6068493..0000000 --- a/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() diff --git a/tracksuite/__init__.py b/tracksuite/__init__.py index c0501c6..9f913c3 100644 --- a/tracksuite/__init__.py +++ b/tracksuite/__init__.py @@ -2,4 +2,3 @@ from .deploy import GitDeployment from .init import setup_remote -from .version import __version__ diff --git a/tracksuite/version.py b/tracksuite/version.py deleted file mode 100644 index 260c070..0000000 --- a/tracksuite/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.3.1" From cfe8922003167c575369fbc6a5de110a79755e51 Mon Sep 17 00:00:00 2001 From: Corentin Carton de Wiart Date: Tue, 5 Mar 2024 10:19:34 +0000 Subject: [PATCH 29/29] debug tests --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c17eba3..70963fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ ] [project.optional-dependencies] -tests = ["pytest", "mocker", "pytest-mock"] +test = ["pytest", "mocker", "pytest-mock"] [project.urls] "Source code" = "https://github.com/ecmwf/tracksuite"