Merge branch 'chore/mula/add-tests-external-services' into chore/mula…

…/add-tests-external-services-octopoes

* chore/mula/add-tests-external-services: (27 commits)
  Skip unittest
  Fix observed_at parsing (#1959)
  Add code review suggestions
  Update mula/.ci/docker-compose.yml
  Translations update from Hosted Weblate (#1941)
  Pre-commit
  Remove octopoes integration
  Fix error adding settings when schema doesn't have required attribute (#1952)
  Fix installing rocky packages in RDO workflow (#1946)
  Add reporting templates to djlint pre-commit hook (#1943)
  Fix reports and normalizer detail page (#1942)
  Fix report tables overflowing whole page (#1940)
  Add 1.13 release notes (#1935)
  Update granian and use new of configuration via env vars (#1931)
  Create general feature flag context processor (#1925)
  Revert accidental pillow downgrade (#1929)
  Bumped `urllib3` package (#1927)
  New feature: Generate DNS and TLS report from a list of OOIs (basic Report page) (#1908)
  Translations update from Hosted Weblate (#1915)
  Fix import (#1917)
  ...

.github/workflows/build-rdo-package.yml

@@ -8,6 +8,7 @@ on:
   push:
     tags:
       - v*
+  workflow_dispatch:
 
 env:
   PKGDIR: /home/runner/work/nl-kat-coordination
@@ -86,7 +87,7 @@ jobs:
         run: python3.8 -m venv /var/www/html/.venv
 
       - name: Rocky Install requirements
-        run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
+        run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
 
       - name: Rocky Create rocky_venv tarball
         run: tar -zcvf ${{ env.PKGDIR }}/rocky_venv_${{ env.RELEASE_VERSION }}.tar.gz -C /var/www/html/ .venv
@@ -111,13 +112,18 @@ jobs:
         shell: bash --login {0}
         working-directory: ./rocky
 
-      - name: Rocky Collectstatic
-        run: SECRET_KEY="whatever" /var/www/html/.venv/bin/python3.8 manage.py collectstatic
-        working-directory: ./rocky
-
       - name: Rocky Compilemessages
-        run: SECRET_KEY="whatever" /var/www/html/.venv/bin/python3.8 manage.py compilemessages
+        run: /var/www/html/.venv/bin/python3.8 manage.py collectstatic && /var/www/html/.venv/bin/python3.8 manage.py compress && /var/www/html/.venv/bin/python3.8 manage.py compilemessages
         working-directory: ./rocky
+        env:
+          BYTES_API: http://bytes:8000
+          BYTES_PASSWORD: password
+          BYTES_USERNAME: username
+          KATALOGUS_API: http://katalogus:8000
+          KEIKO_API: http://keiko:8000
+          OCTOPOES_API: http://octopoes_api:80
+          SCHEDULER_API: http://scheduler:8000
+          SECRET_KEY: whatever
 
       - name: Rocky Create rocky release
         run: tar -cvzf ${{ env.PKGDIR }}/rocky_${{ env.RELEASE_VERSION }}.tar.gz --exclude node_modules --exclude rocky_venv* --exclude=.git* --exclude .parcel-cache --exclude Dockerfile .

.pre-commit-config.yaml

@@ -12,6 +12,11 @@ repos:
   - id: check-json
   - id: check-toml
   - id: debug-statements
+    exclude: |
+      (?x)(
+      ^boefjes/tools |
+      ^octopoes/tools
+      )
   - id: end-of-file-fixer
     exclude: |
       (?x)(
@@ -102,10 +107,18 @@ repos:
   rev: v1.32.1
   hooks:
   - id: djlint-reformat-django
-    files: '^rocky/.*/templates/.*$'
+    files: |
+      (?x)(
+      ^rocky/.*/templates/.*$ |
+      ^rocky/reports/report_types/.*/.*\.html
+      )
 
   - id: djlint-django
-    files: '^rocky/.*/templates/.*$'
+    files: |
+      (?x)(
+      ^rocky/.*/templates/.*$ |
+      ^rocky/reports/report_types/.*/.*\.html
+      )
 
 - repo: https://github.com/thibaudcolas/pre-commit-stylelint
   rev: v15.10.1

CONTRIBUTING.rst

@@ -0,0 +1,20 @@
+============
+Contributing
+============
+
+Thank you, dear developer, who is considering to help out with OpenKAT! Feel welcome. If you want to get in touch, please do so!
+
+Documentation
+=============
+
+We keep `our documentation here <https://docs.openkat.nl>`_, generated from our github repo.
+
+Guidelines
+==========
+
+`Our contribution guidelines <https://docs.openkat.nl/guidelines/contributions.html>`_ might help you find your way.
+
+Contact
+=======
+
+`Get in touch <https://github.com/minvws/nl-kat-coordination/blob/main/README.rst#contact>`_ with our dev team or community managers here.

boefjes/boefjes/clients/bytes_client.py

@@ -2,13 +2,14 @@
 import typing
 from functools import wraps
 from typing import Any, Callable, Dict, Set, Union
+from uuid import UUID
 
 import requests
 from requests.adapters import HTTPAdapter
 from requests.models import HTTPError
 
 from boefjes.clients.scheduler_client import LogRetry
-from boefjes.job_models import BoefjeMeta, NormalizerMeta
+from boefjes.job_models import BoefjeMeta, NormalizerMeta, RawDataMeta
 
 BYTES_API_CLIENT_VERSION = "0.3"
 logger = logging.getLogger(__name__)
@@ -103,10 +104,7 @@ def save_normalizer_meta(self, normalizer_meta: NormalizerMeta) -> None:
         self._verify_response(response)
 
     @retry_with_login
-    def save_raw(self, boefje_meta_id: str, raw: bytes, mime_types: Set[str] = None) -> None:
-        if not mime_types:
-            mime_types = set()
-
+    def save_raw(self, boefje_meta_id: str, raw: bytes, mime_types: Set[str] = frozenset()) -> UUID:
         headers = {"content-type": "application/octet-stream"}
         headers.update(self.headers)
 
@@ -119,9 +117,18 @@ def save_raw(self, boefje_meta_id: str, raw: bytes, mime_types: Set[str] = None)
 
         self._verify_response(response)
 
+        return UUID(response.json()["id"])
+
     @retry_with_login
     def get_raw(self, raw_data_id: str) -> bytes:
         response = self._session.get(f"/bytes/raw/{raw_data_id}", headers=self.headers)
         self._verify_response(response)
 
         return response.content
+
+    @retry_with_login
+    def get_raw_meta(self, raw_data_id: str) -> RawDataMeta:
+        response = self._session.get(f"/bytes/raw/{raw_data_id}/meta", headers=self.headers)
+        self._verify_response(response)
+
+        return RawDataMeta.parse_obj(response.json())

boefjes/boefjes/job_handler.py

@@ -172,14 +172,19 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
             raise
         finally:
             boefje_meta.ended_at = datetime.now(timezone.utc)
-            logger.info("Saving to Bytes for boefje boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
+            logger.info("Saving to Bytes for boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 
             bytes_api_client.login()
             bytes_api_client.save_boefje_meta(boefje_meta)
 
             if boefje_results:
                 for boefje_added_mime_types, output in boefje_results:
-                    bytes_api_client.save_raw(boefje_meta.id, output, mime_types.union(boefje_added_mime_types))
+                    raw_file_id = bytes_api_client.save_raw(
+                        boefje_meta.id, output, mime_types.union(boefje_added_mime_types)
+                    )
+                    logger.debug(
+                        "Saved raw file %s for boefje %s[%s]", raw_file_id, boefje_meta.boefje.id, boefje_meta.id
+                    )
 
             logger.info("Done with boefje for %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 

boefjes/boefjes/katalogus/local_repository.py

@@ -34,7 +34,7 @@ def get_all(self) -> List[PluginType]:
 
         return all_plugins
 
-    def by_id(self, plugin_id: str) -> Optional[PluginType]:
+    def by_id(self, plugin_id: str) -> PluginType:
         boefjes = self.resolve_boefjes()
 
         if plugin_id in boefjes:
@@ -45,6 +45,8 @@ def by_id(self, plugin_id: str) -> Optional[PluginType]:
         if plugin_id in normalizers:
             return self._normalizer_to_plugin(normalizers[plugin_id])
 
+        raise Exception(f"Can't find plugin {plugin_id}")
+
     def schema(self, id_: str) -> Optional[Dict]:
         boefjes = self.resolve_boefjes()
 

boefjes/boefjes/plugins/kat_cve_2023_34039/main.py

@@ -17,7 +17,7 @@
 import os
 from typing import List, Tuple, Union
 
-from boefjes.boefjes.job_models import BoefjeMeta
+from boefjes.job_models import BoefjeMeta
 
 
 def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[str, bytes]]]:

boefjes/boefjes/plugins/kat_cve_2023_35078/main.py

@@ -18,6 +18,6 @@ def run(boefje_meta: BoefjeMeta) -> List[Tuple[set, Union[str, bytes]]]:
     response = requests.get(full_url, verify=False, allow_redirects=False)
 
     if response.status_code == 200:
-        return [({response.headers.get("content-type")}, response.content)]
+        return [(set(), response.content)]
     else:
         return [(set(), "Ivanti Endpoint Manager Mobile (EPMM), formerly MobileIron Core not found")]

boefjes/requirements-dev.txt

@@ -1248,9 +1248,9 @@ tomli==2.0.1 ; python_version >= "3.8" and python_version < "3.11" \
 typing-extensions==4.7.1 ; python_version >= "3.8" and python_version < "4.0" \
     --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \
     --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2
-urllib3==1.26.17 ; python_version >= "3.8" and python_version < "4.0" \
-    --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \
-    --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b
+urllib3==1.26.18 ; python_version >= "3.8" and python_version < "4.0" \
+    --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \
+    --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0
 uvicorn==0.20.0 ; python_version >= "3.8" and python_version < "4.0" \
     --hash=sha256:a4e12017b940247f836bc90b72e725d7dfd0c8ed1c51eb365f5ba30d9f5127d8 \
     --hash=sha256:c3ed1598a5668208723f2bb49336f4509424ad198d6ab2615b7783db58d919fd

boefjes/requirements.txt

@@ -1145,9 +1145,9 @@ tldextract==3.4.4 ; python_version >= "3.8" and python_version < "4.0" \
 typing-extensions==4.7.1 ; python_version >= "3.8" and python_version < "4.0" \
     --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \
     --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2
-urllib3==1.26.17 ; python_version >= "3.8" and python_version < "4.0" \
-    --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \
-    --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b
+urllib3==1.26.18 ; python_version >= "3.8" and python_version < "4.0" \
+    --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \
+    --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0
 uvicorn==0.20.0 ; python_version >= "3.8" and python_version < "4.0" \
     --hash=sha256:a4e12017b940247f836bc90b72e725d7dfd0c8ed1c51eb365f5ba30d9f5127d8 \
     --hash=sha256:c3ed1598a5668208723f2bb49336f4509424ad198d6ab2615b7783db58d919fd

boefjes/tools/run_boefje.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# ruff: noqa: E402
+
+import logging
+import pdb
+import sys
+import uuid
+from pathlib import Path
+
+import click
+
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+
+from boefjes.job_handler import BoefjeHandler
+from boefjes.job_models import Boefje, BoefjeMeta
+from boefjes.katalogus.local_repository import get_local_repository
+from boefjes.local import LocalBoefjeJobRunner
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+
+@click.command()
+@click.option("--pdb", "start_pdb", is_flag=True, help="Start pdb on exceptions")
+@click.argument("organization_code")
+@click.argument("boefje_id")
+@click.argument("input_ooi")
+def run_boefje(start_pdb, organization_code, boefje_id, input_ooi):
+    """Run boefje"""
+
+    meta = BoefjeMeta(id=uuid.uuid4(), boefje=Boefje(id=boefje_id), organization=organization_code, input_ooi=input_ooi)
+
+    local_repository = get_local_repository()
+
+    handler = BoefjeHandler(LocalBoefjeJobRunner(local_repository), local_repository)
+    try:
+        handler.handle(meta)
+    except Exception:
+        if start_pdb:
+            pdb.post_mortem()
+
+        raise
+
+
+if __name__ == "__main__":
+    run_boefje()

boefjes/tools/run_normalizer.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# ruff: noqa: E402
+
+import logging
+import pdb
+import sys
+import uuid
+from pathlib import Path
+
+import click
+
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+
+from boefjes.job_handler import NormalizerHandler, bytes_api_client
+from boefjes.job_models import Normalizer, NormalizerMeta
+from boefjes.katalogus.local_repository import get_local_repository
+from boefjes.local import LocalNormalizerJobRunner
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+
+@click.command()
+@click.option("--pdb", "start_pdb", is_flag=True, help="Start pdb on exceptions")
+@click.argument("normalizer_id")
+@click.argument("raw_id")
+def run_normalizer(start_pdb, normalizer_id, raw_id):
+    """Run normalizer"""
+
+    bytes_api_client.login()
+    raw = bytes_api_client.get_raw_meta(raw_id)
+
+    meta = NormalizerMeta(id=uuid.uuid4(), raw_data=raw, normalizer=Normalizer(id=normalizer_id))
+
+    local_repository = get_local_repository()
+
+    handler = NormalizerHandler(LocalNormalizerJobRunner(local_repository))
+    try:
+        handler.handle(meta)
+    except Exception:
+        if start_pdb:
+            pdb.post_mortem()
+
+        raise
+
+
+if __name__ == "__main__":
+    run_normalizer()

boefjes/tools/show_raw.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# ruff: noqa: E402, T201
+
+import json
+import sys
+from pathlib import Path
+
+import click
+from pygments import highlight
+from pygments.formatters import TerminalFormatter
+from pygments.lexers import JsonLexer
+
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+
+from boefjes.job_handler import bytes_api_client
+
+
+@click.command()
+@click.option("--json", "print_json", is_flag=True, help="Pretty print raw as json")
+@click.argument("raw_id")
+def show_raw(print_json, raw_id):
+    """Show raw file"""
+
+    bytes_api_client.login()
+    raw = bytes_api_client.get_raw(raw_id)
+
+    raw_str = raw.decode("utf-8")
+
+    if print_json:
+        json_object = json.loads(raw_str)
+        formatted_json_str = json.dumps(json_object, indent=4, sort_keys=True)
+        print(highlight(formatted_json_str, JsonLexer(), TerminalFormatter()))
+    else:
+        print(raw_str)
+
+
+if __name__ == "__main__":
+    show_raw()

bytes/bytes/api/root.py

@@ -33,10 +33,13 @@ class ServiceHealth(BaseModel):
 
 
 def validation_exception_handler(_: Request, exc: Union[RequestValidationError, ValidationError]) -> JSONResponse:
-    logger.info(exc.json())
-    logger.debug(exc.__traceback__)
-
-    return JSONResponse({"errors": exc.errors()}, status_code=HTTP_422_UNPROCESSABLE_ENTITY)
+    logger.critical(exc)
+    return JSONResponse(
+        {
+            "value": str(exc),
+        },
+        HTTP_422_UNPROCESSABLE_ENTITY,
+    )
 
 
 @router.get("/", include_in_schema=False)