Merge pull request #697 from AlmaLinux/albs-deploy-fix-2

albs-deploy-fix 2

.github/workflows/migrations-check.yml

@@ -28,14 +28,16 @@ jobs:
         run: |
           git rebase --abort
           git merge --no-ff --no-log origin/master
-      - name: Create dirs
+      - name: Create dirs and files
         run: |
           mkdir -p /home/runner/work/albs-web-server/alts
           mkdir -p /home/runner/work/albs-web-server/albs-frontend
           mkdir -p /home/runner/work/albs-web-server/albs-node
           mkdir -p /home/runner/work/albs-web-server/albs-sign-node
           mkdir -p /home/runner/work/albs-web-server/alma-tests-cacher
           touch /home/runner/work/albs-web-server/alma-tests-cacher/vars.env
+          mkdir -p /home/runner/work/albs-web-server/albs-sign-file
+          touch /home/runner/work/albs-web-server/albs-sign-file/.env
       - name: Add env file
         run: |
           touch ./vars.env

.github/workflows/pytest.yml

@@ -15,14 +15,16 @@ jobs:
         name: Check out repository
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      - name: Create dirs
+      - name: Create dirs and files
         run: |
           mkdir -p /home/runner/work/albs-web-server/alts
           mkdir -p /home/runner/work/albs-web-server/albs-frontend
           mkdir -p /home/runner/work/albs-web-server/albs-node
+          mkdir -p /home/runner/work/albs-web-server/albs-sign-file
           mkdir -p /home/runner/work/albs-web-server/albs-sign-node
           mkdir -p /home/runner/work/albs-web-server/alma-tests-cacher
           touch /home/runner/work/albs-web-server/alma-tests-cacher/vars.env
+          touch /home/runner/work/albs-web-server/albs-sign-file/.env
       - name: Add env file
         run: |
           touch ./vars.env

alws/config.py

@@ -53,8 +53,12 @@ class Settings(BaseSettings):
     rabbitmq_default_host: str = 'rabbitmq'
     rabbitmq_default_vhost: str = 'test_system'
 
-    sign_server_url: Optional[str] = 'http://web_server:8000/api/v1/'
-    sign_server_token: Optional[str] = None
+    albs_api_url: Optional[str] = 'http://web_server:8000/api/v1/'
+    albs_jwt_token: Optional[str] = None
+
+    sign_server_api_url: Optional[str] = 'http://sign_file:8000/'
+    sign_server_username: Optional[str] = None
+    sign_server_password: Optional[str] = None
 
     documentation_path: str = 'alws/documentation/'
 

docker-compose.yml

@@ -74,9 +74,11 @@ services:
     environment:
       CELERY_CONFIG_PATH: "/scheduler_config.yaml"
       SCHEDULER_CONFIG_PATH: "/scheduler_config.yaml"
-    command: "bash -c 'source env/bin/activate && pip3 install --upgrade pip &&
-        pip3 install -r /tmp/requirements/scheduler.txt --no-cache-dir &&
-        uvicorn --host 0.0.0.0 alts.scheduler.app:app --limit-concurrency 100 --backlog 1000'"
+    command: "bash -c 'source env/bin/activate && \
+                       pip3 install --upgrade pip && \
+                       pip3 install -r /tmp/requirements/scheduler.txt --no-cache-dir && \
+                       /wait_for_it.sh web_server:8000 && \
+                       uvicorn --host 0.0.0.0 alts.scheduler.app:app --limit-concurrency 100 --backlog 1000'"
     restart: on-failure
     volumes:
       - ../alts/configs/alts_config.yaml:/scheduler_config.yaml
@@ -524,6 +526,27 @@ services:
         max-size: "100Mb"
         max-file: "3"
 
+  sign_file:
+    image: sign-file:latest
+    build:
+      dockerfile: Dockerfile
+      context: ../albs-sign-file
+    command: >
+      bash -c "/app/env/bin/python /app/db_manage.py dev_init
+      && /app/env/bin/python /app/start.py"
+    restart: on-failure
+    volumes:
+      - "~/.gnupg:/home/alt/.gnupg"
+      - ../albs-sign-file/sign:/app/sign
+      - ../albs-sign-file/start.py:/app/start.py
+      - ../albs-sign-file/db_manage.py:/app/db_manage.py
+    ports:
+      - "8083:8000"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100Mb"
+        max-file: "3"
 
 volumes:
   frontend_node_modules:

nginx_configs/albs.conf

@@ -10,6 +10,10 @@ upstream pulp {
     server pulp:80;
 }
 
+upstream signfile {
+    server sign_file:8000;
+}
+
 map $http_upgrade $connection_upgrade {
     default upgrade;
     '' close;
@@ -64,4 +68,9 @@ server {
         }
         proxy_pass http://pulp;
     }
+
+    location /sign-file/ {
+        proxy_set_header Host $http_host;
+        proxy_pass http://signfile/;
+    }
 }

scripts/packages_exporter.py

@@ -210,8 +210,8 @@ def __init__(
         self.logger = logging.getLogger(LOGGER_NAME)
         self.pulp_client = pulp_client
         self.createrepo_c = local["createrepo_c"]
-        self.headers = {
-            "Authorization": f"Bearer {settings.sign_server_token}",
+        self.web_server_headers = {
+            "Authorization": f"Bearer {settings.albs_jwt_token}",
         }
         self.pulp_system_user = pulp_user
         self.common_group = pulp_group
@@ -271,18 +271,36 @@ async def make_request(
         method: str,
         endpoint: str,
         params: dict = None,
-        data: dict = None,
+        body: dict = None,
+        user_headers: dict = None,
+        data: typing.Optional[list] = None,
+        send_to: typing.Literal['web_server', 'sign_server'] = 'web_server',
     ):
-        full_url = urllib.parse.urljoin(settings.sign_server_url, endpoint)
+        if send_to == 'web_server':
+            headers = {**self.web_server_headers}
+            full_url = urllib.parse.urljoin(settings.albs_api_url, endpoint)
+        elif send_to == 'sign_server':
+            headers = {}
+            full_url = urllib.parse.urljoin(
+                settings.sign_server_api_url, endpoint
+            )
+        else:
+            raise ValueError(
+                "send_to parameter must be either web_server or sign_server"
+            )
+
+        if user_headers:
+            headers.update(user_headers)
+
         async with aiohttp.ClientSession(
-            headers=self.headers, raise_for_status=True
+            headers=headers, raise_for_status=True
         ) as session:
             async with session.request(
-                method, full_url, json=data, params=params
+                method, full_url, json=body, params=params, data=data
             ) as response:
-                json_data = await response.read()
-                json_data = json.loads(json_data)
-                return json_data
+                if response.headers['Content-Type'] == 'application/json':
+                    return await response.json()
+                return await response.text()
 
     async def create_filesystem_exporters(
         self,
@@ -348,9 +366,24 @@ async def get_exporter_data(
 
         return list(dict(results).values())
 
-    async def sign_repomd_xml(self, data):
-        endpoint = "sign-tasks/sync_sign_task/"
-        return await self.make_request("POST", endpoint, data=data)
+    async def sign_repomd_xml(
+        self, path_to_file: str, key_id: str, token: str
+    ):
+        endpoint = "sign"
+        result = {"asc_content": None, "error": None}
+        try:
+            response = await self.make_request(
+                "POST",
+                endpoint,
+                params={"keyid": key_id},
+                data={"file": Path(path_to_file).read_bytes()},
+                user_headers={"Authorization": f"Bearer {token}"},
+                send_to="sign_server",
+            )
+            result["asc_content"] = response
+        except Exception as err:
+            result['error'] = err
+        return result
 
     async def get_sign_keys(self):
         endpoint = "sign-keys/"
@@ -469,7 +502,7 @@ async def export_repositories(self, repo_ids: list) -> typing.List[str]:
         exported_paths = [i for i in results if i]
         return exported_paths
 
-    async def repomd_signer(self, repodata_path, key_id):
+    async def repomd_signer(self, repodata_path, key_id, token):
         string_repodata_path = str(repodata_path)
         if key_id is None:
             self.logger.info(
@@ -478,13 +511,9 @@ async def repomd_signer(self, repodata_path, key_id):
             )
             return
 
-        with open(os.path.join(repodata_path, "repomd.xml"), "rt") as f:
-            file_content = f.read()
-        sign_data = {
-            "content": file_content,
-            "pgp_keyid": key_id,
-        }
-        result = await self.sign_repomd_xml(sign_data)
+        file_path = os.path.join(repodata_path, "repomd.xml")
+        result = await self.sign_repomd_xml(file_path, key_id, token)
+        self.logger.info('PGP key id: %s', key_id)
         result_data = result.get("asc_content")
         if result_data is None:
             self.logger.error(
@@ -741,6 +770,18 @@ def regenerate_repo_metadata(self, repo_path):
 
         self.logger.info(stdout)
 
+    async def get_sign_server_token(self) -> str:
+        body = {
+            'email': settings.sign_server_username,
+            'password': settings.sign_server_password,
+        }
+        endpoint = 'token'
+        method = 'POST'
+        response = await self.make_request(
+            method=method, endpoint=endpoint, body=body, send_to='sign_server'
+        )
+        return response['token']
+
 
 async def sign_repodata(
     exporter: Exporter,
@@ -752,6 +793,7 @@ async def sign_repodata(
     repodata_paths = []
 
     tasks = []
+    token = await exporter.get_sign_server_token()
 
     for repo_path in exported_paths:
         path = Path(repo_path)
@@ -775,8 +817,8 @@ async def sign_repodata(
                         None,
                     )
                     break
-
-        tasks.append(exporter.repomd_signer(repodata, key_id))
+        exporter.logger.info('Key ID: %s', str(key_id))
+        tasks.append(exporter.repomd_signer(repodata, key_id, token))
 
     await asyncio.gather(*tasks)
 
@@ -993,7 +1035,11 @@ def main():
                 exporter.logger.debug("JSON dump is done")
                 exporter.logger.debug("Generating OVAL data")
                 oval = sync(
-                    exporter.get_oval_xml(platform, only_released=True)
+                    # aiohttp is not able to send booleans in params.
+                    # For this reason, we're passing only_released as a string,
+                    # which in turn will be converted into boolean on backend
+                    # side by fastapi/pydantic.
+                    exporter.get_oval_xml(platform, only_released="true")
                 )
                 with open(os.path.join(platform_path, "oval.xml"), "w") as fd:
                     fd.write(oval)

tests/test_scripts/test_packages_exporter.py

@@ -0,0 +1,24 @@
+import os
+import tempfile
+
+import pytest
+from syncer import sync
+
+from scripts.packages_exporter import Exporter
+
+IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") is not None
+
+
+@pytest.mark.skip(
+    reason="See https://github.com/AlmaLinux/build-system/issues/204"
+)
+def test_repomd_signer():
+    exporter = Exporter(
+        pulp_client=None, repodata_cache_dir='~/.cache/pulp_exporter'
+    )
+    db_keys = sync(exporter.get_sign_keys())
+    key_id = db_keys[0]['keyid']
+    with tempfile.NamedTemporaryFile() as fp:
+        fp.write(b'Hello world!')
+        res = sync(exporter.sign_repomd_xml(fp.name, key_id))
+        assert res['error'] is None