diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..bb396be --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,32 @@ +name: publish +on: + push: + tags: + - 'v*.*.*' + +jobs: + pypi-publish: + name: upload release to PyPI + runs-on: ubuntu-latest + # Specifying a GitHub environment is optional, but strongly encouraged + environment: release + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10.4' + cache: pip + cache-dependency-path: '**/pyproject.toml' + - name: Install dependencies + run: | + pip install setuptools wheel build twine setuptools_scm + - name: Build + run: | + python -m build + # retrieve your distributions here + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4185379 --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +.DS_Store +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e13e337 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Agah Karakuzu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9f0d8c0 --- /dev/null +++ b/README.md @@ -0,0 +1,165 @@ + +# MyST Libre + +![PyPI - Version](https://img.shields.io/pypi/v/myst-libre?style=flat&logo=python&logoColor=white&logoSize=8&labelColor=rgb(255%2C0%2C0)&color=white) + +## JupyterHub in Docker for MyST + +A small library to manage reproducible execution environments using Docker and JupyterHub +to build MyST articles in containers. + +## Table of Contents + +- [Myst Libre](#myst-libre) + - [Table of Contents](#table-of-contents) + - [Installation](#installation) + - [Usage](#usage) + - [Authentication](#authentication) + - [Docker Registry Client](#docker-registry-client) + - [Build Source Manager](#build-source-manager) + - [JupyterHub Local Spawner](#jupyterhub-local-spawner) + - [MyST Markdown Client](#myst-markdown-client) + - [Module and Class Descriptions](#module-and-class-descriptions) + - [Contributing](#contributing) + - [License](#license) + +## Installation + +1. **Clone the repository:** + ```sh + git clone https://github.com/yourusername/myst_libre.git + cd myst_libre + ``` + +2. **Create a virtual environment:** + ```sh + python -m venv venv + source venv/bin/activate # On Windows use `venv\Scripts\activate` + ``` + +3. **Install the required packages:** + ```sh + pip install -r requirements.txt + ``` + +4. **Set up environment variables:** + Create a `.env` file in the project root and add the following: + ```env + DOCKER_PRIVATE_REGISTRY_USERNAME=your_username + DOCKER_PRIVATE_REGISTRY_PASSWORD=your_password + ``` + +## External requirements + +- Node.js (For MyST) +- Docker + +## Quick Start + +```python +from myst_libre.rees import REES +from myst_libre.tools import JupyterHubLocalSpawner + +resources = REES(dict(registry_url="https://binder-registry.conp.cloud", + gh_user_repo_name = "agahkarakuzu/mriscope", + gh_repo_commit_hash = "6d3f64da214441bbb55b2005234fd4fd745fb372", + binder_image_tag = "489ae0eb0d08fe30e45bc31201524a6570b9b7dd")) + +hub = JupyterHubLocalSpawner(resources, + host_data_parent_dir = "~/neurolibre/mriscope/data", + host_build_source_parent_dir = '~/Desktop/tmp', + container_data_mount_dir = '/home/jovyan/data', + container_build_source_mount_dir = '/home/jovyan') + +hub.spawn_jupyter_hub() +``` +## Usage + +### Authentication + +The `Authenticator` class handles loading authentication credentials from environment variables. + +```python +from myst_libre.tools.authenticator import Authenticator + +auth = Authenticator() +print(auth._auth) +``` + + +### Docker Registry Client + +The DockerRegistryClient class provides methods to interact with a Docker registry. + +```python +from myst_libre.tools.docker_registry_client import DockerRegistryClient + +client = DockerRegistryClient(registry_url='https://my-registry.example.com', gh_user_repo_name='user/repo') +token = client.get_token() +print(token) +``` + +### Build Source Manager + +The BuildSourceManager class manages source code repositories. + +```python +from myst_libre.tools.build_source_manager import BuildSourceManager + +manager = BuildSourceManager(gh_user_repo_name='user/repo', gh_repo_commit_hash='commit_hash') +manager.git_clone_repo('/path/to/clone') +project_name = manager.get_project_name() +print(project_name) +``` + +## Module and Class Descriptions + +### AbstractClass +**Description**: Provides basic logging functionality and colored printing capabilities. + +### Authenticator +**Description**: Handles authentication by loading credentials from environment variables. +**Inherited from**: AbstractClass +**Inputs**: Environment variables `DOCKER_PRIVATE_REGISTRY_USERNAME` and `DOCKER_PRIVATE_REGISTRY_PASSWORD` + +### RestClient +**Description**: Provides a client for making REST API calls. +**Inherited from**: Authenticator + +### DockerRegistryClient +**Description**: Manages interactions with a Docker registry. +**Inherited from**: Authenticator +**Inputs**: +- `registry_url`: URL of the Docker registry +- `gh_user_repo_name`: GitHub user/repository name +- `auth`: Authentication credentials + +### BuildSourceManager +**Description**: Manages source code repositories. +**Inherited from**: AbstractClass +**Inputs**: +- `gh_user_repo_name`: GitHub user/repository name +- `gh_repo_commit_hash`: Commit hash of the repository + +### JupyterHubLocalSpawner +**Description**: Manages JupyterHub instances locally. +**Inherited from**: AbstractClass +**Inputs**: +- `rees`: Instance of the REES class +- `registry_url`: URL of the Docker registry +- `gh_user_repo_name`: GitHub user/repository name +- `auth`: Authentication credentials +- `binder_image_tag`: Docker image tag +- `build_src_commit_hash`: Commit hash of the repository +- `container_data_mount_dir`: Directory to mount data in the container +- `container_build_source_mount_dir`: Directory to mount build source in the container +- `host_data_parent_dir`: Host directory for data +- `host_build_source_parent_dir`: Host directory for build source + +### MystMD +**Description**: Manages MyST markdown operations such as building and converting files. +**Inherited from**: AbstractClass +**Inputs**: +- `build_dir`: Directory where the build will take place +- `env_vars`: Environment variables needed for the build process +- `executable`: Name of the MyST executable (default is 'myst') diff --git a/example.py b/example.py new file mode 100644 index 0000000..da5744e --- /dev/null +++ b/example.py @@ -0,0 +1,20 @@ +from myst_libre.tools import JupyterHubLocalSpawner, MystMD +from myst_libre.rees import REES +from myst_libre.builders import MystBuilder + +resources = REES(dict( + registry_url="https://binder-registry.conp.cloud", + gh_user_repo_name = "agahkarakuzu/mriscope", + gh_repo_commit_hash = "ae64d9ed17e6ce66ecf94d585d7b68a19a435d70", + binder_image_tag = "489ae0eb0d08fe30e45bc31201524a6570b9b7dd")) + + +hub = JupyterHubLocalSpawner(resources, + host_data_parent_dir = "/Users/agah/Desktop/neurolibre/mriscope/data", + host_build_source_parent_dir = '/Users/agah/Desktop/tmp', + container_data_mount_dir = '/home/jovyan/data', + container_build_source_mount_dir = '/home/jovyan') + +hub.spawn_jupyter_hub() + +MystBuilder(hub).build() \ No newline at end of file diff --git a/myst_libre/__init__.py b/myst_libre/__init__.py new file mode 100644 index 0000000..45853bf --- /dev/null +++ b/myst_libre/__init__.py @@ -0,0 +1,3 @@ +from setuptools_scm import get_version + +__version__ = get_version() \ No newline at end of file diff --git a/myst_libre/abstract_class.py b/myst_libre/abstract_class.py new file mode 100644 index 0000000..65c879e --- /dev/null +++ b/myst_libre/abstract_class.py @@ -0,0 +1,47 @@ +""" +abstract_class.py + +This module contains the AbstractClass which provides basic logging functionality +and colored printing capabilities. +""" + +import logging +from termcolor import colored + +class AbstractClass: + """ + AbstractClass + + A base class that provides logging functionality and methods for printing colored messages. + """ + def __init__(self): + """ + Initialize the AbstractClass with default logging settings. + """ + self.logging_level = logging.INFO + self.logger = logging.getLogger(self.__class__.__name__) + self.logger.setLevel(self.logging_level) + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) + if not self.logger.handlers: # Avoid adding multiple handlers if already set + self.logger.addHandler(handler) + + def set_log_level(self, level): + """ + Set the logging level. + + Args: + level (str): Logging level. + """ + self.logging_level = level + self.logger = logging.basicConfig(level=self.logging_level, format='%(asctime)s - %(levelname)s - %(message)s') + + def cprint(self, message, color): + """ + Print a message in a specified color using termcolor. + + Args: + message (str): The message to print. + color (str): The color to use for printing the message. + """ + print(colored(message, color)) \ No newline at end of file diff --git a/myst_libre/builders/__init__.py b/myst_libre/builders/__init__.py new file mode 100644 index 0000000..1933fd1 --- /dev/null +++ b/myst_libre/builders/__init__.py @@ -0,0 +1 @@ +from .myst_builder import MystBuilder \ No newline at end of file diff --git a/myst_libre/builders/myst_builder.py b/myst_libre/builders/myst_builder.py new file mode 100644 index 0000000..87bfb6c --- /dev/null +++ b/myst_libre/builders/myst_builder.py @@ -0,0 +1,21 @@ +from myst_libre.tools import JupyterHubLocalSpawner, MystMD +from myst_libre.abstract_class import AbstractClass + +class MystBuilder(AbstractClass): + def __init__(self, hub): + if not isinstance(hub, JupyterHubLocalSpawner): + raise TypeError(f"Expected 'hub' to be an instance of JupyterHubLocalSpawner, got {type(hub).__name__} instead") + super().__init__() + #aa = MystMD('/Users/agah/Desktop', {"anan":5}) + self.env_vars = {} + self.build_dir = "" + self.hub = hub + self.env_vars = {"JUPYTER_BASE_URL":f"{self.hub.jh_url}", + "JUPYTER_TOKEN":f"{self.hub.jh_token}", + "port":f"{self.hub.port}" + } + self.myst_client = MystMD(hub.rees.build_dir, self.env_vars) + + def build(self): + self.cprint(f'Starting MyST build {self.hub.jh_url}','yellow') + self.myst_client.build() \ No newline at end of file diff --git a/myst_libre/rees/__init__.py b/myst_libre/rees/__init__.py new file mode 100644 index 0000000..781c322 --- /dev/null +++ b/myst_libre/rees/__init__.py @@ -0,0 +1 @@ +from .rees import REES \ No newline at end of file diff --git a/myst_libre/rees/rees.py b/myst_libre/rees/rees.py new file mode 100644 index 0000000..7125b51 --- /dev/null +++ b/myst_libre/rees/rees.py @@ -0,0 +1,60 @@ +from myst_libre.abstract_class import AbstractClass +from myst_libre.tools.docker_registry_client import DockerRegistryClient +from myst_libre.tools.build_source_manager import BuildSourceManager +import docker +import subprocess + +class REES(DockerRegistryClient,BuildSourceManager): + def __init__(self, rees_dict): + # These are needed in the scope of the base classes + self.registry_url = rees_dict['registry_url'] + self.gh_user_repo_name = rees_dict['gh_user_repo_name'] + self.gh_repo_commit_hash = rees_dict['gh_repo_commit_hash'] + self.binder_image_tag = rees_dict['binder_image_tag'] + # Initialize as base to rees + BuildSourceManager.__init__(self) + DockerRegistryClient.__init__(self) + + self.cprint(f"␤[Preflight checks]","light_grey") + self.check_docker_installed() + + self.pull_image_name = "" + self.use_public_registry = False + self.repo_commit_info = {} + self.binder_commit_info = {} + self.docker_client = docker.from_env() + + def check_docker_installed(self): + """ + Check if Docker is installed and available in the system PATH. + + Raises: + EnvironmentError: If Docker is not installed or not found in PATH. + """ + try: + result = subprocess.run(['docker', '--version'], capture_output=True, text=True, check=True) + self.cprint(f"✓ Docker is installed: {result.stdout.strip()}",'green') + except subprocess.CalledProcessError as e: + raise EnvironmentError("Docker is not installed or not found in PATH. Please install Docker to proceed.") from e + + def login_to_registry(self): + """ + Login to a private docker registry. + """ + self.docker_client.login(username=self._auth['username'], password=self._auth['password'], registry=self.registry_url) + + def pull_image(self): + """ + Pull the Docker image from the registry. + """ + if bool(self._auth) or not self.use_public_registry: + self.login_to_registry() + self.logger.info(f"Logging into {self.registry_url_bare}") + + try: + self.pull_image_name = f'{self.registry_url_bare}/{self.found_image_name}' + self.logger.info(f'Pulling image {self.pull_image_name}:{self.binder_image_tag} from {self.registry_url}.') + self.docker_image = self.docker_client.images.pull(self.pull_image_name, tag=self.binder_image_tag) + except: + self.logger.info(f'Pulling image {self.found_image_name}:{self.binder_image_tag} from {self.registry_url}.') + self.docker_image = self.docker_client.images.pull(self.found_image_name, tag=self.binder_image_tag) \ No newline at end of file diff --git a/myst_libre/test/__init__.py b/myst_libre/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myst_libre/test/test_myst_libre.py b/myst_libre/test/test_myst_libre.py new file mode 100644 index 0000000..a5b4fe3 --- /dev/null +++ b/myst_libre/test/test_myst_libre.py @@ -0,0 +1,107 @@ +import unittest +from unittest.mock import patch, MagicMock +from tools import DockerRegistryClient, BuildSourceManager, JupyterHubLocalSpawner, request_set_decorator + +class TestDockerRegistryClient(unittest.TestCase): + + def setUp(self): + self.registry_url = 'http://example.com' + self.gh_user_repo_name = 'user/repo' + self.auth = {'username': 'user', 'password': 'pass'} + self.client = DockerRegistryClient(self.registry_url, self.gh_user_repo_name, self.auth) + + @patch('myst_libre.tools.RestClient.get') + def test_get_token_success(self, mock_get): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_get.return_value = mock_response + + result = self.client.get_token() + self.assertTrue(result) + + @patch('myst_libre.tools.RestClient.get') + def test_get_token_failure(self, mock_get): + mock_response = MagicMock() + mock_response.status_code = 401 + mock_get.return_value = mock_response + + result = self.client.get_token() + self.assertFalse(result) + + @patch('myst_libre.tools.RestClient.get') + @patch('myst_libre.tools.DockerRegistryClient.get_image_list') + def test_search_img_by_repo_name(self, mock_get_image_list, mock_get): + self.client.docker_images = ['example.com/binder-user-2drepo'] + mock_get_image_list.return_value = True + mock_response = MagicMock() + mock_response.json.return_value = {'tags': ['latest']} + mock_response.status_code = 200 + mock_get.return_value = mock_response + + result = self.client.search_img_by_repo_name() + self.assertTrue(result) + +class TestBuildSourceManager(unittest.TestCase): + + def setUp(self): + self.gh_user_repo_name = 'user/repo' + self.gh_repo_commit_hash = 'commit_hash' + self.manager = BuildSourceManager(self.gh_user_repo_name, self.gh_repo_commit_hash) + + @patch('os.makedirs') + @patch('os.path.exists', return_value=False) + def test_create_build_dir_host(self, mock_exists, mock_makedirs): + result = self.manager.create_build_dir_host() + mock_makedirs.assert_called_once_with(self.manager.build_dir) + self.assertTrue(result) + + @patch('os.path.exists', return_value=True) + def test_create_build_dir_host_exists(self, mock_exists): + result = self.manager.create_build_dir_host() + self.assertFalse(result) + + @patch('myst_libre.Repo.clone_from') + @patch('myst_libre.BuildSourceManager.create_build_dir_host', return_value=True) + def test_git_clone_repo(self, mock_create_build_dir_host, mock_clone_from): + result = self.manager.git_clone_repo() + mock_clone_from.assert_called_once_with(f'https://github.com/{self.gh_user_repo_name}', self.manager.build_dir) + self.assertTrue(result) + +class TestJupyterHubLocalSpawner(unittest.TestCase): + + def setUp(self): + self.registry_url = 'http://example.com' + self.gh_user_repo_name = 'user/repo' + self.auth = {'username': 'user', 'password': 'pass'} + self.gh_repo_commit_hash = 'commit_hash' + self.spawner = JupyterHubLocalSpawner(self.registry_url, self.gh_user_repo_name, self.auth, self.gh_repo_commit_hash) + + @patch('myst_libre.docker.from_env') + def test_login_to_registry(self, mock_docker): + mock_docker_client = MagicMock() + mock_docker.return_value = mock_docker_client + + self.spawner.login_to_registry() + mock_docker_client.login.assert_called_once_with(username=self.auth['username'], password=self.auth['password'], registry=self.registry_url) + + @patch('myst_libre.tools.DockerRegistryClient.search_img_by_repo_name', return_value=True) + @patch('myst_libre.tools.BuildSourceManager.git_clone_repo') + @patch('myst_libre.tools.BuildSourceManager.git_checkout_commit') + @patch('myst_libre.tools.JupyterHubLocalSpawner.pull_image') + @patch('myst_libre.tools.JupyterHubLocalSpawner.find_open_port', return_value=8888) + @patch('myst_libre.tools.JupyterHubLocalSpawner._is_port_in_use', return_value=False) + @patch('myst_libre.rees.docker.from_env') + def test_spawn_jupyter_hub(self, mock_docker, mock_is_port_in_use, mock_find_open_port, mock_pull_image, mock_git_checkout_commit, mock_git_clone_repo, mock_search_img_by_repo_name): + mock_docker_client = MagicMock() + mock_docker.return_value = mock_docker_client + + self.spawner.spawn_jupyter_hub() + + self.assertTrue(mock_search_img_by_repo_name.called) + self.assertTrue(mock_git_clone_repo.called) + self.assertTrue(mock_git_checkout_commit.called) + self.assertTrue(mock_pull_image.called) + self.assertTrue(mock_docker_client.containers.run.called) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/myst_libre/tools/__init__.py b/myst_libre/tools/__init__.py new file mode 100644 index 0000000..6cf2149 --- /dev/null +++ b/myst_libre/tools/__init__.py @@ -0,0 +1,8 @@ +from .decorators import request_set_decorator +from ..abstract_class import AbstractClass +from .rest_client import RestClient +from .docker_registry_client import DockerRegistryClient +from .build_source_manager import BuildSourceManager +from .jupyter_hub_local_spawner import JupyterHubLocalSpawner +from .myst_client import MystMD +from .authenticator import Authenticator \ No newline at end of file diff --git a/myst_libre/tools/authenticator.py b/myst_libre/tools/authenticator.py new file mode 100644 index 0000000..18d3875 --- /dev/null +++ b/myst_libre/tools/authenticator.py @@ -0,0 +1,24 @@ +import os +from dotenv import load_dotenv +from myst_libre.abstract_class import AbstractClass + +class Authenticator(AbstractClass): + def __init__(self): + super().__init__() + self._auth = {} + self._load_auth_from_env() + + def _load_auth_from_env(self): + load_dotenv() + username = os.getenv('DOCKER_PRIVATE_REGISTRY_USERNAME') + password = os.getenv('DOCKER_PRIVATE_REGISTRY_PASSWORD') + + if not username or not password: + self._auth['username'] = None + self._auth['password'] = None + else: + self._auth['username'] = username + self._auth['password'] = password + + del os.environ['DOCKER_PRIVATE_REGISTRY_USERNAME'] + del os.environ['DOCKER_PRIVATE_REGISTRY_PASSWORD'] \ No newline at end of file diff --git a/myst_libre/tools/build_source_manager.py b/myst_libre/tools/build_source_manager.py new file mode 100644 index 0000000..8091ab4 --- /dev/null +++ b/myst_libre/tools/build_source_manager.py @@ -0,0 +1,114 @@ +""" +build_source_manager.py + +This module contains the BuildSourceManager class for handling source code repositories. +""" + +import os +import json +os.environ["GIT_PYTHON_REFRESH"] = "quiet" +from git import Repo +from datetime import datetime +from myst_libre.abstract_class import AbstractClass + +class BuildSourceManager(AbstractClass): + """ + Manager for handling source code repositories. + + Args: + gh_user_repo_name (str): GitHub user/repository name. + gh_repo_commit_hash (str): Commit hash of the repository. + """ + def __init__(self): + super().__init__() + self.build_dir = "" + self.branch = 'main' + self.provider = 'https://github.com' + self.username = self.gh_user_repo_name.split('/')[0] + self.repo_name = self.gh_user_repo_name.split('/')[1] + now = datetime.now() + self.created_at = now.strftime("%Y-%m-%dT%H:%M:%S") + + def create_build_dir_host(self): + """ + Create build directory on the host machine. + + Returns: + bool: True if directory created, else False. + """ + if not os.path.exists(self.build_dir): + os.makedirs(self.build_dir) + return True + return False + + def git_clone_repo(self,clone_parent_directory): + """ + Clone the GitHub repository. + + Returns: + bool: True if cloned successfully, else False. + """ + self.host_build_source_parent_dir = clone_parent_directory + self.build_dir = os.path.join(self.host_build_source_parent_dir, self.username, self.repo_name, self.gh_repo_commit_hash) + if self.create_build_dir_host(): + self.logger.info(f'Cloning into {self.build_dir}') + self.repo_object = Repo.clone_from(f'{self.provider}/{self.gh_user_repo_name}', self.build_dir) + else: + self.logger.warning(f'Source {self.build_dir} already exists.') + self.repo_object = Repo(self.build_dir) + + self.set_commit_info() + self.validate_commits() + + def git_checkout_commit(self): + """ + Checkout the specified commit in the repository. + + Returns: + bool: True if checked out successfully. + """ + self.logger.info(f'Checking out {self.gh_repo_commit_hash}') + self.repo_object.git.checkout(self.gh_repo_commit_hash) + return True + + def get_project_name(self): + """ + Get the project name from the data requirement file. + + Returns: + str: Project name. + """ + data_config_dir = os.path.join(self.build_dir, 'binder', 'data_requirement.json') + with open(data_config_dir, 'r') as file: + data = json.load(file) + return data['projectName'] + + def set_commit_info(self): + self.binder_commit_info['datetime'] = self.repo_object.commit(self.binder_image_tag).committed_datetime + self.binder_commit_info['message'] = self.repo_object.commit(self.binder_image_tag).message + self.repo_commit_info['datetime'] = self.repo_object.commit(self.gh_repo_commit_hash).committed_datetime + self.repo_commit_info['message'] = self.repo_object.commit(self.gh_repo_commit_hash).message + self.validate_commits() + + def validate_commits(self): + if self.repo_commit_info['datetime'] < self.binder_commit_info['datetime']: + raise ValueError("The repo commit datetime cannot be older than the binder commit datetime.") + + def create_latest_symlink(self): + """ + Create a symlink to the latest build directory. + """ + self.latest_dir = os.path.join(self.host_build_source_parent_dir, self.username, self.repo_name, 'latest') + self.logger.info(f'Creating symlink {self.gh_repo_commit_hash} --> latest') + if not os.path.exists(self.latest_dir): + os.makedirs(self.latest_dir) + else: + for item in os.listdir(self.build_dir): + os.unlink(item) + for item in os.listdir(self.build_dir): + source_path = os.path.join(self.build_dir, item) + target_path = os.path.join(self.latest_dir, item) + if os.path.isdir(source_path): + os.symlink(source_path, target_path, target_is_directory=True) + else: + os.symlink(source_path, target_path) diff --git a/myst_libre/tools/decorators.py b/myst_libre/tools/decorators.py new file mode 100644 index 0000000..edbb55b --- /dev/null +++ b/myst_libre/tools/decorators.py @@ -0,0 +1,18 @@ +import logging + +def request_set_decorator(success_status_code=200, set_attribute=None, json_key=None): + def decorator(func): + def wrapper(self, *args, **kwargs): + response = func(self, *args, **kwargs) + if response.status_code != success_status_code: + logging.error(f"HTTP Error: {response.status_code} - {response.text}") + return [] + json_data = response.json() + if json_key: + data_list = json_data.get(json_key, []) + if set_attribute: + setattr(self, set_attribute, data_list) + return data_list + return json_data + return wrapper + return decorator \ No newline at end of file diff --git a/myst_libre/tools/docker_registry_client.py b/myst_libre/tools/docker_registry_client.py new file mode 100644 index 0000000..956f3c1 --- /dev/null +++ b/myst_libre/tools/docker_registry_client.py @@ -0,0 +1,89 @@ +""" +docker_registry_client.py + +This module contains the DockerRegistryClient class for interacting with a Docker registry. +""" + +import re +from .rest_client import RestClient +from .authenticator import Authenticator +from .decorators import request_set_decorator +from dotenv import load_dotenv +import os + +load_dotenv() + +class DockerRegistryClient(Authenticator): + """ + DockerRegistryClient + + Client for interacting with a Docker registry. + + Args: + registry_url (str): URL of the Docker registry. + gh_user_repo_name (str): GitHub user/repository name. + auth (dict): Authentication credentials. + """ + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + super().__init__() + self.registry_url_bare = self.registry_url.replace("http://", "").replace("https://", "") + self.found_image_name = None + self.found_image_tags = None + self.docker_images = [] + self.rest_client = RestClient() + + def get_token(self): + """ + Authenticate and get a token from the Docker registry. + + Returns: + bool: True if authenticated successfully, else False. + """ + auth_url = f"{self.registry_url}/v2/" + response = self.rest_client.get(auth_url) + if response.status_code == 200: + return True + else: + self.logger.error(f"Failed to authenticate: {response.status_code} {response.text}") + return False + + def search_img_by_repo_name(self): + """ + Search for a Docker image by repository name. + + Returns: + bool: True if image found, else False. + """ + self.get_image_list() + user_repo_formatted = self.gh_user_repo_name.replace('-', '-2d').replace('_', '-5f').replace('/', '-2d') + pattern = f'{self.registry_url_bare}/binder-{user_repo_formatted}.*' + for image in self.docker_images: + if re.match(pattern, image): + self.found_image_name = image + self.list_tags() + return True + return False + + @request_set_decorator(success_status_code=200, set_attribute="docker_images", json_key="repositories") + def get_image_list(self): + """ + Get the list of images from the Docker registry. + + Returns: + Response: HTTP response object. + """ + repo_url = f"{self.registry_url}/v2/_catalog" + return self.rest_client.get(repo_url) + + @request_set_decorator(success_status_code=200, set_attribute="found_image_tags", json_key="tags") + def list_tags(self): + """ + List tags for the found Docker image. + + Returns: + Response: HTTP response object. + """ + tags_url = f"{self.registry_url}/v2/{self.found_image_name}/tags/list" + return self.rest_client.get(tags_url) diff --git a/myst_libre/tools/jupyter_hub_local_spawner.py b/myst_libre/tools/jupyter_hub_local_spawner.py new file mode 100644 index 0000000..70f5168 --- /dev/null +++ b/myst_libre/tools/jupyter_hub_local_spawner.py @@ -0,0 +1,147 @@ +""" +jupyter_hub_local_spawner.py + +This module contains the JupyterHubLocalSpawner class for managing JupyterHub instances locally. +""" + +import os +import logging +import socket +from hashlib import blake2b +from myst_libre.abstract_class import AbstractClass +from myst_libre.rees import REES + +class JupyterHubLocalSpawner(AbstractClass): + """ + Spawner for managing JupyterHub instances locally. + + Args: + registry_url (str): URL of the Docker registry (https://my-registry.example.com). + gh_user_repo_name (str): GitHub user/repository name. + auth (dict): Authentication credentials {username:"***","password":"***"}. + binder_image_tag (str): Docker image tag of the container in which the article will be built. + build_src_commit_hash (str): Commit hash of the repository from which the article will be built. + """ + def __init__(self,rees,**kwargs): + if not isinstance(rees, REES): + raise TypeError(f"Expected 'rees' to be an instance of REES, got {type(rees).__name__} instead") + super().__init__() + self.rees = rees + required_inputs = ['container_data_mount_dir','container_build_source_mount_dir', + 'host_data_parent_dir','host_build_source_parent_dir'] + + for inp in required_inputs: + if inp not in kwargs.keys(): + raise(f'{inp} is not provided for JupyterHubLocalSpawner.') + else: + setattr(self, inp, kwargs[inp]) + + self.container = None + self.port = None + self.jh_token = None + + def find_open_port(self): + """ + Find an open port to use. + + Returns: + int: Available port number. + + Raises: + Exception: If no open ports are available. + """ + for port in range(8888, 10000): + if not self._is_port_in_use(port): + return port + raise Exception("No open ports available") + + def _is_port_in_use(self, port): + """ + Check if a port is in use. + + Args: + port (int): Port number to check. + + Returns: + bool: True if port is in use, else False. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(('127.0.0.1', port)) == 0 + + def spawn_jupyter_hub(self): + """ + Spawn a JupyterHub instance. + """ + self.port = self.find_open_port() + h = blake2b(digest_size=20) + h.update(os.urandom(20)) + self.jh_token = h.hexdigest() + + if not self.rees.search_img_by_repo_name(): + raise Exception(f"[ERROR] A docker image has not been found for {self.rees.gh_user_repo_name} at {self.binder_image_tag}.") + if self.rees.binder_image_tag not in self.rees.found_image_tags: + raise Exception(f"[ERROR] A docker image exists for {self.rees.gh_user_repo_name}, yet the tag {self.binder_image_tag} is missing.") + + # self.rees.found_image_name is assigned if above not fails + + self.rees.git_clone_repo(self.host_build_source_parent_dir) + self.rees.git_checkout_commit() + + mnt_vol = {f'{os.path.join(self.host_data_parent_dir, self.rees.get_project_name())}': {'bind': os.path.join(self.container_data_mount_dir,self.rees.get_project_name()), 'mode': 'ro'}, + self.rees.build_dir: {'bind': f'{self.container_build_source_mount_dir}', 'mode': 'rw'}} + + self.rees.pull_image() + self.jh_url = f"http://localhost:{self.port}" + try: + self.container = self.rees.docker_client.containers.run( + self.rees.docker_image, + ports={f'{self.port}/tcp': self.port}, + environment={"JUPYTER_TOKEN": f'{self.jh_token}',"port": f'{self.port}',"JUPYTER_BASE_URL": f'{self.jh_url}'}, + entrypoint=f'jupyter server --allow-root --ip 0.0.0.0 --log-level=DEBUG --IdentityProvider.token="{self.jh_token}" --ServerApp.port="{self.port}"', + volumes=mnt_vol, + detach=True) + logging.info(f'Jupyter hub is {self.container.status}') + self.cprint(f'␤[Status]', 'light_grey') + self.cprint(f' ├─────── ⏺ running', 'green') + self.cprint(f' └─────── Container {self.container.short_id} {self.container.name}', 'green') + self.cprint(f' ℹ Run the following commands in the terminal if you are debugging locally:', 'yellow') + self.cprint(f' port=\"{self.port}\"', 'cyan') + self.cprint(f' export JUPYTER_BASE_URL=\"{self.jh_url}\"', 'cyan') + self.cprint(f' export JUPYTER_TOKEN=\"{self.jh_token}\"', 'cyan') + self.cprint(f'␤[Resources]', 'light_grey') + self.cprint(f' ├── MyST repository', 'magenta') + self.cprint(f' │ ├───────── ✸ {self.rees.gh_user_repo_name}','light_blue') + self.cprint(f' │ ├───────── ⎌ {self.rees.gh_repo_commit_hash}','light_blue') + self.cprint(f" │ └───────── ⏲ {self.rees.repo_commit_info['datetime']}: {self.rees.repo_commit_info['message']}".replace('\n', ''),'light_blue') + self.cprint(f' └── Docker container', 'magenta') + self.cprint(f' ├───────── ✸ {self.rees.pull_image_name}','light_blue') + self.cprint(f' ├───────── ⎌ {self.rees.binder_image_tag}','light_blue') + self.cprint(f" ├───────── ⏲ {self.rees.binder_commit_info['datetime']}: {self.rees.binder_commit_info['message']}".replace('\n', ''),'light_blue') + self.cprint(f' └───────── ℹ This image was built from REES-compliant {self.rees.gh_user_repo_name} repository at the commit above','yellow') + except Exception as e: + logging.error(f'Could not spawn a JH: \n {e}') + + def delete_stopped_containers(self): + """ + Delete all stopped Docker containers. + """ + stopped_containers = self.rees.docker_client.containers.list(all=True, filters={"status": "exited"}) + for container in stopped_containers: + logging.info(f"Deleting stopped container: {container.id}") + container.remove() + + def delete_image(self): + """ + Delete the pulled Docker image. + """ + if self.docker_image: + logging.info(f"Deleting image: {self.docker_image.id}") + self.rees.docker_client.images.remove(image=self.docker_image.id) + + def stop_container(self): + """ + Stop and remove the running container. + """ + if self.container: + self.container.stop() + self.container.remove() \ No newline at end of file diff --git a/myst_libre/tools/myst_client.py b/myst_libre/tools/myst_client.py new file mode 100644 index 0000000..45251a1 --- /dev/null +++ b/myst_libre/tools/myst_client.py @@ -0,0 +1,133 @@ +""" +myst_client.py + +This module contains the MystMD class for managing MyST markdown operations such as building and converting files. +""" + +import subprocess +import os +from myst_libre.abstract_class import AbstractClass +import sys + +class MystMD(AbstractClass): + """ + MystMD + + A class to manage MyST markdown operations such as building and converting files. + + Args: + build_dir (str): Directory where the build will take place. + env_vars (dict): Environment variables needed for the build process. + executable (str): Name of the MyST executable. Default is 'myst'. + """ + def __init__(self, build_dir, env_vars, executable='myst'): + """ + Initialize the MystMD class with build directory, environment variables, and executable name. + """ + super().__init__() + self.executable = executable + self.build_dir = build_dir + self.env_vars = env_vars + self.cprint(f"␤[Preflight checks]","light_grey") + self.check_node_installed() + self.check_mystmd_installed() + + def check_node_installed(self): + """ + Check if Node.js is installed and available in the system PATH. + + Raises: + EnvironmentError: If Node.js is not installed or not found in PATH. + """ + try: + result = subprocess.run(['node', '--version'], capture_output=True, text=True, check=True) + self.cprint(f"✓ Node.js is installed: {result.stdout.strip()}","green") + except subprocess.CalledProcessError as e: + raise EnvironmentError("Node.js is not installed or not found in PATH. Please install Node.js to proceed.") from e + + def check_mystmd_installed(self): + """ + Check if MyST markdown tool is installed and available in the system PATH. + + Raises: + EnvironmentError: If MyST markdown tool is not installed or not found in PATH. + """ + try: + result = subprocess.run([self.executable, '--version'], capture_output=True, text=True, check=True) + self.cprint(f"✓ mystmd is installed: {result.stdout.strip()}","green") + except subprocess.CalledProcessError as e: + raise EnvironmentError(f"{self.executable} is not installed or not found in PATH. Please install mystmd to proceed.") from e + + def run_command(self, *args, env_vars={}): + """ + Run a command using the MyST executable. + + Args: + *args: Arguments for the MyST executable command. + env_vars (dict): Environment variables to set for the command. + + Returns: + str: Command output or None if failed. + """ + command = [self.executable] + list(args) + try: + # Combine the current environment with the provided env_vars + env = os.environ.copy() + env.update(env_vars) + + process = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + output = [] + error_output = [] + + # Stream stdout and stderr in real-time + while True: + stdout_line = process.stdout.readline() + stderr_line = process.stderr.readline() + if not stdout_line and not stderr_line and process.poll() is not None: + break + if stdout_line: + print(stdout_line, end='') + output.append(stdout_line) + if stderr_line: + print(stderr_line, end='', file=sys.stderr) + error_output.append(stderr_line) + + process.wait() + + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, command, output='\n'.join(output), stderr='\n'.join(error_output)) + + return ''.join(output) + except subprocess.CalledProcessError as e: + print(f"Error running command: {e}") + print(f"Command output: {e.output}") + print(f"Error output: {e.stderr}") + return None + except Exception as e: + print(f"Unexpected error: {e}") + return None + + def build(self): + """ + Build the MyST markdown project. + + Returns: + str: Command output or None if failed. + """ + os.chdir(self.build_dir) + env = os.environ.copy() + self.env_vars.update(env) + return self.run_command('build', '--execute', '--html',env_vars=self.env_vars) + + def convert(self, input_file, output_file): + """ + Convert a MyST markdown file to another format. + + Args: + input_file (str): Path to the input MyST markdown file. + output_file (str): Path to the output file. + + Returns: + str: Command output or None if failed. + """ + return self.run_command('convert', input_file, '-o', output_file,env_vars=[]) \ No newline at end of file diff --git a/myst_libre/tools/rest_client.py b/myst_libre/tools/rest_client.py new file mode 100644 index 0000000..aa9c6ee --- /dev/null +++ b/myst_libre/tools/rest_client.py @@ -0,0 +1,51 @@ +""" +rest_client.py + +This module contains the RestClient class for making REST API calls. +""" + +import requests +from requests.auth import HTTPBasicAuth +from .authenticator import Authenticator + +class RestClient(Authenticator): + """ + RestClient + + A client for making REST API calls. + + Args: + auth (dict): Authentication credentials. + """ + def __init__(self): + super().__init__() + self.session = requests.Session() + self.session.auth = HTTPBasicAuth(self._auth['username'], self._auth['password']) + + def get(self, url): + """ + Perform a GET request. + + Args: + url (str): URL for the GET request. + + Returns: + Response: HTTP response object. + """ + response = self.session.get(url) + return response + + def post(self, url, data=None, json=None): + """ + Perform a POST request. + + Args: + url (str): URL for the POST request. + data (dict, optional): Data to send in the request body. + json (dict, optional): JSON data to send in the request body. + + Returns: + Response: HTTP response object. + """ + response = self.session.post(url, data=data, json=json) + return response \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..24a534b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "myst_libre" +dynamic = ["version"] +description = "A Python library for managing source code repositories, interacting with Docker registries, handling MyST markdown operations, and spawning JupyterHub instances locally." +authors = [ + { name="agahkarakuzu", email="agahkarakuzu@gmail.com" } +] +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.7" +keywords = ["myst", "docker", "jupyterhub", "markdown", "repository"] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ + "requests", + "docker", + "python-dotenv", + "PyGithub", + "termcolor", + "mystmd" +] + +[project.urls] +Homepage = "https://github.com/agahkarakuzu/myst_libre" + +[tool.setuptools_scm] +version_scheme = "post-release" +local_scheme = "dirty-tag" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6144008 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +dotenv +PyGithub +docker +hashlib +termcolor +mystmd +requests \ No newline at end of file