Merge branch 'main' into feature/add-observed_at-to-task-list-object-…

…hyperlinks

.github/workflows/build-rdo-package.yml

@@ -8,6 +8,7 @@ on:
   push:
     tags:
       - v*
+  workflow_dispatch:
 
 env:
   PKGDIR: /home/runner/work/nl-kat-coordination
@@ -86,7 +87,7 @@ jobs:
         run: python3.8 -m venv /var/www/html/.venv
 
       - name: Rocky Install requirements
-        run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
+        run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
 
       - name: Rocky Create rocky_venv tarball
         run: tar -zcvf ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
@@ -111,13 +112,18 @@ jobs:
         shell: bash --login {0}
         working-directory: ./rocky
 
-      - name: Rocky Collectstatic
-        run: SECRET_KEY="whatever" /var/www/html/.venv/bin/python3.8 manage.py collectstatic
-        working-directory: ./rocky
-
       - name: Rocky Compilemessages
-        run: SECRET_KEY="whatever" /var/www/html/.venv/bin/python3.8 manage.py compilemessages
+        run: /var/www/html/.venv/bin/python3.8 manage.py collectstatic && /var/www/html/.venv/bin/python3.8 manage.py compress && /var/www/html/.venv/bin/python3.8 manage.py compilemessages
         working-directory: ./rocky
+        env:
+          BYTES_API: http://bytes:8000
+          BYTES_PASSWORD: password
+          BYTES_USERNAME: username
+          KATALOGUS_API: http://katalogus:8000
+          KEIKO_API: http://keiko:8000
+          OCTOPOES_API: http://octopoes_api:80
+          SCHEDULER_API: http://scheduler:8000
+          SECRET_KEY: whatever
 
       - name: Rocky Create rocky release
         run: tar -cvzf ${{ env.PKGDIR }}/rocky_${{ env.RELEASE_VERSION }}.tar.gz --exclude node_modules --exclude rocky_venv* --exclude=.git* --exclude .parcel-cache --exclude Dockerfile .

.pre-commit-config.yaml

@@ -12,6 +12,11 @@ repos:
   - id: check-json
   - id: check-toml
   - id: debug-statements
+    exclude: |
+      (?x)(
+      ^boefjes/tools |
+      ^octopoes/tools
+      )
   - id: end-of-file-fixer
     exclude: |
       (?x)(
@@ -102,10 +107,18 @@ repos:
   rev: v1.32.1
   hooks:
   - id: djlint-reformat-django
-    files: '^rocky/.*/templates/.*$'
+    files: |
+      (?x)(
+      ^rocky/.*/templates/.*$ |
+      ^rocky/reports/report_types/.*/.*\.html
+      )
 
   - id: djlint-django
-    files: '^rocky/.*/templates/.*$'
+    files: |
+      (?x)(
+      ^rocky/.*/templates/.*$ |
+      ^rocky/reports/report_types/.*/.*\.html
+      )
 
 - repo: https://github.com/thibaudcolas/pre-commit-stylelint
   rev: v15.10.1

CONTRIBUTING.rst

@@ -0,0 +1,20 @@
+============
+Contributing
+============
+
+Thank you, dear developer, who is considering to help out with OpenKAT! Feel welcome. If you want to get in touch, please do so!
+
+Documentation
+=============
+
+We keep `our documentation here <https://docs.openkat.nl>`_, generated from our github repo.
+
+Guidelines
+==========
+
+`Our contribution guidelines <https://docs.openkat.nl/guidelines/contributions.html>`_ might help you find your way.
+
+Contact
+=======
+
+`Get in touch <https://github.com/minvws/nl-kat-coordination/blob/main/README.rst#contact>`_ with our dev team or community managers here.

boefjes/boefjes/api.py

@@ -14,14 +14,14 @@
 from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus
 from boefjes.config import settings
 from boefjes.job_handler import (
-    _collect_default_mime_types,
     _find_ooi_in_past,
     get_environment_settings,
     get_octopoes_api_connector,
     serialize_ooi,
 )
 from boefjes.job_models import BoefjeMeta
 from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
+from boefjes.plugins.models import _default_meta_mime_types
 from octopoes.models import Reference
 
 app = FastAPI(title="Boefje API")
@@ -128,7 +128,7 @@ async def boefje_output(
     bytes_client.save_boefje_meta(boefje_meta)
 
     if boefje_output.files:
-        mime_types = _collect_default_mime_types(task.p_item.data)
+        mime_types = _default_meta_mime_types(task.p_item.data)
         for file in boefje_output.files:
             raw = base64.b64decode(file.content)
             # when supported, also save file.name to Bytes

boefjes/boefjes/clients/bytes_client.py

@@ -2,13 +2,14 @@
 import typing
 from functools import wraps
 from typing import Any, Callable, Dict, Set, Union
+from uuid import UUID
 
 import requests
 from requests.adapters import HTTPAdapter
 from requests.models import HTTPError
 
 from boefjes.clients.scheduler_client import LogRetry
-from boefjes.job_models import BoefjeMeta, NormalizerMeta
+from boefjes.job_models import BoefjeMeta, NormalizerMeta, RawDataMeta
 
 BYTES_API_CLIENT_VERSION = "0.3"
 logger = logging.getLogger(__name__)
@@ -103,10 +104,7 @@ def save_normalizer_meta(self, normalizer_meta: NormalizerMeta) -> None:
         self._verify_response(response)
 
     @retry_with_login
-    def save_raw(self, boefje_meta_id: str, raw: bytes, mime_types: Set[str] = None) -> None:
-        if not mime_types:
-            mime_types = set()
-
+    def save_raw(self, boefje_meta_id: str, raw: bytes, mime_types: Set[str] = frozenset()) -> UUID:
         headers = {"content-type": "application/octet-stream"}
         headers.update(self.headers)
 
@@ -119,9 +117,18 @@ def save_raw(self, boefje_meta_id: str, raw: bytes, mime_types: Set[str] = None)
 
         self._verify_response(response)
 
+        return UUID(response.json()["id"])
+
     @retry_with_login
     def get_raw(self, raw_data_id: str) -> bytes:
         response = self._session.get(f"/bytes/raw/{raw_data_id}", headers=self.headers)
         self._verify_response(response)
 
         return response.content
+
+    @retry_with_login
+    def get_raw_meta(self, raw_data_id: str) -> RawDataMeta:
+        response = self._session.get(f"/bytes/raw/{raw_data_id}/meta", headers=self.headers)
+        self._verify_response(response)
+
+        return RawDataMeta.parse_obj(response.json())

boefjes/boefjes/docker_boefjes_runner.py

@@ -31,9 +31,9 @@ def run(self) -> None:
             raise RuntimeError("Boefje does not have OCI image")
 
         # local import to prevent circular dependency
-        from boefjes import job_handler
+        import boefjes.plugins.models
 
-        stderr_mime_types = job_handler._collect_default_mime_types(self.boefje_meta)
+        stderr_mime_types = boefjes.plugins.models._default_meta_mime_types(self.boefje_meta)
 
         task_id = str(self.boefje_meta.id)
         self.scheduler_client.patch_task(task_id, TaskStatus.RUNNING)

boefjes/boefjes/job_handler.py

@@ -3,7 +3,7 @@
 import traceback
 from datetime import datetime, timedelta, timezone
 from enum import Enum
-from typing import Any, Dict, List, Set
+from typing import Any, Dict, List
 
 import requests
 from pydantic.tools import parse_obj_as
@@ -18,6 +18,7 @@
     NormalizerPlainOOI,
 )
 from boefjes.katalogus.local_repository import LocalPluginRepository
+from boefjes.plugins.models import _default_meta_mime_types
 from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner
 from octopoes.api.models import Declaration, Observation
 from octopoes.connector.octopoes import OctopoesAPIConnector
@@ -99,28 +100,6 @@ def get_environment_settings(boefje_meta: BoefjeMeta, environment_keys: List[str
         logger.exception("Error getting environment settings")
         raise
 
-    return {}
-
-
-def _collect_default_mime_types(boefje_meta: BoefjeMeta) -> Set[str]:
-    boefje_id = boefje_meta.boefje.id
-
-    mime_types = {
-        boefje_id,
-        f"boefje/{boefje_id}",
-        f"boefje/{boefje_id}-{boefje_meta.parameterized_arguments_hash}",
-    }
-
-    if boefje_meta.boefje.version is not None:
-        mime_types = mime_types.union(
-            {
-                f"boefje/{boefje_id}-{boefje_meta.boefje.version}",
-                f"boefje/{boefje_id}-{boefje_meta.parameterized_arguments_hash}-{boefje_meta.boefje.version}",
-            }
-        )
-
-    return mime_types
-
 
 class BoefjeHandler(Handler):
     def __init__(self, job_runner, local_repository: LocalPluginRepository):
@@ -155,7 +134,7 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
         boefje_meta.runnable_hash = boefje_resource.runnable_hash
         boefje_meta.environment = get_environment_settings(boefje_meta, env_keys) if env_keys else {}
 
-        mime_types = _collect_default_mime_types(boefje_meta)
+        mime_types = _default_meta_mime_types(boefje_meta)
 
         logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 
@@ -172,14 +151,19 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
             raise
         finally:
             boefje_meta.ended_at = datetime.now(timezone.utc)
-            logger.info("Saving to Bytes for boefje boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
+            logger.info("Saving to Bytes for boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 
             bytes_api_client.login()
             bytes_api_client.save_boefje_meta(boefje_meta)
 
             if boefje_results:
                 for boefje_added_mime_types, output in boefje_results:
-                    bytes_api_client.save_raw(boefje_meta.id, output, mime_types.union(boefje_added_mime_types))
+                    raw_file_id = bytes_api_client.save_raw(
+                        boefje_meta.id, output, mime_types.union(boefje_added_mime_types)
+                    )
+                    logger.debug(
+                        "Saved raw file %s for boefje %s[%s]", raw_file_id, boefje_meta.boefje.id, boefje_meta.id
+                    )
 
             logger.info("Done with boefje for %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 

boefjes/boefjes/katalogus/local_repository.py

@@ -1,11 +1,10 @@
-import hashlib
 import json
 import logging
 import pkgutil
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
-from boefjes.katalogus.models import RESERVED_LOCAL_ID, Boefje, Normalizer, PluginType
+from boefjes.katalogus.models import PluginType
 from boefjes.plugins.models import (
     BOEFJE_DEFINITION_FILE,
     BOEFJES_DIR,
@@ -27,23 +26,25 @@ def __init__(self, path: Path):
         self._cached_normalizers = None
 
     def get_all(self) -> List[PluginType]:
-        all_plugins = [self._boefje_to_plugin(boefje) for boefje in self.resolve_boefjes().values()]
-        normalizers = [self._normalizer_to_plugin(normalizer) for normalizer in self.resolve_normalizers().values()]
+        all_plugins = [boefje_resource.boefje for boefje_resource in self.resolve_boefjes().values()]
+        normalizers = [normalizer_resource.normalizer for normalizer_resource in self.resolve_normalizers().values()]
 
         all_plugins += normalizers
 
         return all_plugins
 
-    def by_id(self, plugin_id: str) -> Optional[PluginType]:
+    def by_id(self, plugin_id: str) -> PluginType:
         boefjes = self.resolve_boefjes()
 
         if plugin_id in boefjes:
-            return self._boefje_to_plugin(boefjes[plugin_id])
+            return boefjes[plugin_id].boefje
 
         normalizers = self.resolve_normalizers()
 
         if plugin_id in normalizers:
-            return self._normalizer_to_plugin(normalizers[plugin_id])
+            return normalizers[plugin_id].normalizer
+
+        raise Exception(f"Can't find plugin {plugin_id}")
 
     def schema(self, id_: str) -> Optional[Dict]:
         boefjes = self.resolve_boefjes()
@@ -149,42 +150,6 @@ def create_relative_import_statement_from_cwd(package_dir: Path) -> str:
 
         return f"{relative_path[1:].replace('/', '.')}."  # Turns into "boefjes.plugins."
 
-    @staticmethod
-    def _boefje_to_plugin(boefje: BoefjeResource) -> Boefje:
-        def_file = boefje.path / "boefje.json"
-        def_obj = json.loads(def_file.read_text())
-        def_obj["repository_id"] = RESERVED_LOCAL_ID
-        def_obj["runnable_hash"] = get_runnable_hash(boefje.path)
-
-        return Boefje.parse_obj(def_obj)
-
-    @staticmethod
-    def _normalizer_to_plugin(normalizer: NormalizerResource) -> Normalizer:
-        def_file = normalizer.path / "normalizer.json"
-        def_obj = json.loads(def_file.read_text())
-        def_obj["repository_id"] = RESERVED_LOCAL_ID
-
-        normalizer: Normalizer = Normalizer.parse_obj(def_obj)
-        normalizer.consumes.append(f"normalizer/{normalizer.id}")
-
-        return normalizer
-
 
 def get_local_repository():
     return LocalPluginRepository(BOEFJES_DIR)
-
-
-def get_runnable_hash(path: Path) -> str:
-    """Returns sha256(file1 + file2 + ...) of all files in the given path."""
-
-    folder_hash = hashlib.sha256()
-
-    for file in sorted(path.glob("**/*")):
-        # Note that the hash does not include *.pyc files
-        # Thus there may be a desync between the source code and the cached, compiled bytecode
-        if file.is_file() and file.suffix != ".pyc":
-            with file.open("rb") as f:
-                while chunk := f.read(32768):
-                    folder_hash.update(chunk)
-
-    return folder_hash.hexdigest()

boefjes/boefjes/katalogus/models.py

@@ -39,6 +39,7 @@ class Boefje(Plugin):
     scan_level: int = 1
     consumes: Set[str] = Field(default_factory=set)
     produces: List[str] = Field(default_factory=list)
+    mime_types: Set[str] = Field(default_factory=set)
     options: Optional[List[str]]
     runnable_hash: Optional[str]
     oci_image: Optional[str]

boefjes/boefjes/katalogus/tests/test_plugins.py

@@ -107,6 +107,20 @@ def test_get_plugin(self):
         res = self.client.get("/v1/organisations/test-org/repositories/test-repo/plugins/test-boefje-1")
         self.assertEqual(200, res.status_code)
 
+        # Simpler endpoint works as well, but due to the mock the default mime_types are not dynamically added
+        res = self.client.get("/v1/organisations/test-org/plugins/test-boefje-1")
+        self.assertEqual(200, res.status_code)
+        assert "mime_types" in res.json()
+        assert not res.json()["mime_types"]
+
+        # For boefjes that are pulled from the local repository, we actually get the default mime_types
+        assert set(self.client.get("/v1/organisations/test-org/plugins/kat_test").json()["mime_types"]) == set(
+            [
+                "kat_test",
+                "boefje/kat_test",
+            ]
+        )
+
     def test_non_existing_plugin(self):
         res = self.client.get("/v1/organisations/test-org/repositories/test-repo/plugins/future-plugin")
         self.assertEqual(404, res.status_code)

boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json

@@ -8,7 +8,6 @@
     "produces": [
         "ADRFindingType"
     ],
-    "environment_keys": [],
     "scan_level": 0,
     "enabled": true
 }

boefjes/boefjes/plugins/kat_adr_validator/main.py

@@ -28,9 +28,4 @@ def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[bytes, str]]]:
 
     output = run_adr_validator(url)
 
-    return [
-        (
-            set(),
-            output,
-        ),
-    ]
+    return [(set(), output)]

boefjes/boefjes/plugins/kat_crt_sh/boefje.json

@@ -9,6 +9,5 @@
         "Hostname",
         "X509Certificate"
     ],
-    "environment_keys": [],
     "scan_level": 1
 }