Merge branch 'main' into feature/octopoes-research-doc

.github/workflows/build-rdo-package.yml

@@ -15,7 +15,7 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Generic bits
     steps:
@@ -26,7 +26,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.10'
           cache: 'pip'
 
       - name: Prep Install requirements
@@ -48,12 +48,12 @@ jobs:
 
       - name: Octopoes Build whl package
         run: |
-          python3.8 -m pip install build
-          python3.8 -m build
+          python3.10 -m pip install build
+          python3.10 -m build
         working-directory: ./octopoes
 
       - name: Octopoes Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Octopoes Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
@@ -84,7 +84,7 @@ jobs:
         working-directory: ./rocky
 
       - name: Rocky Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Rocky Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
@@ -113,7 +113,7 @@ jobs:
         working-directory: ./rocky
 
       - name: Rocky Compilemessages
-        run: /var/www/html/.venv/bin/python3.8 manage.py collectstatic && /var/www/html/.venv/bin/python3.8 manage.py compress && /var/www/html/.venv/bin/python3.8 manage.py compilemessages
+        run: /var/www/html/.venv/bin/python3.10 manage.py collectstatic && /var/www/html/.venv/bin/python3.10 manage.py compress && /var/www/html/.venv/bin/python3.10 manage.py compilemessages
         working-directory: ./rocky
         env:
           BYTES_API: http://bytes:8000
@@ -138,7 +138,7 @@ jobs:
         working-directory: ./bytes
 
       - name: Bytes Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Bytes Install requirements
         run: cd /var/www/html; source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
@@ -159,14 +159,14 @@ jobs:
         working-directory: ./mula
 
       - name: Mula Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Create scheduler release archive
         run: tar -cvzf ${{ env.PKGDIR }}/scheduler_${{ env.RELEASE_VERSION }}.tar.gz --exclude=./.git* --exclude=Makefile --exclude=Dockerfile --exclude=base.yml --exclude=requirements* --exclude=tests .
         working-directory: ./mula
 
       - name: Create virtual env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; pip install --requirement requirements.txt
@@ -184,7 +184,7 @@ jobs:
         working-directory: ./boefjes
 
       - name: Boefjes Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; find . -name requirements.txt | xargs -L 1 pip install -r; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
@@ -206,7 +206,7 @@ jobs:
         working-directory: ./keiko
 
       - name: Keiko Create env
-        run: python3.8 -m venv /var/www/html/.venv
+        run: python3.10 -m venv /var/www/html/.venv
 
       - name: Keiko Install requirements
         run: source .venv/bin/activate; pip install --upgrade pip; find . -name requirements.txt | xargs -L 1 pip install -r

.github/workflows/sigrid-publish.yml

@@ -0,0 +1,20 @@
+name: Publish to Sigrid
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  sigridci:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Sigrid CI
+        uses: Software-Improvement-Group/sigridci@main
+        with:
+          customer: vws
+          system: kat
+          publishonly: true
+        env:
+          SIGRID_CI_TOKEN: "${{ secrets.SIGRID_CI_TOKEN }}"

.github/workflows/sigrid-pullrequest.yml

@@ -0,0 +1,22 @@
+name: Sigrid pull request feedback
+on: [pull_request]
+
+jobs:
+  sigridci:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Sigrid CI
+        uses: Software-Improvement-Group/sigridci@main
+        with:
+          customer: vws
+          system: kat
+        env:
+          SIGRID_CI_TOKEN: "${{ secrets.SIGRID_CI_TOKEN }}"
+#      - name: "Sigrid pull request feedback"
+#        uses: mshick/add-pr-comment@v2
+#        if: always()
+#        with:
+#          message-id: sigrid
+#          message-path: sigrid-ci-output/feedback.md

.pre-commit-config.yaml

@@ -64,7 +64,7 @@ repos:
   rev: v1.2.0
   hooks:
   - id: mypy
-    additional_dependencies: ['types-PyYAML', 'types-requests', 'types-cachetools', 'pydantic', 'pynacl']
+    additional_dependencies: ['types-PyYAML', 'types-requests', 'types-cachetools', 'types-retry', 'pydantic', 'pynacl']
     exclude: |
       (?x)(
       ^boefjes/ |

boefjes/boefjes/api.py

@@ -21,7 +21,7 @@
 )
 from boefjes.job_models import BoefjeMeta
 from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
-from boefjes.plugins.models import _default_meta_mime_types
+from boefjes.plugins.models import _default_mime_types
 from octopoes.models import Reference
 
 app = FastAPI(title="Boefje API")
@@ -128,7 +128,7 @@ async def boefje_output(
     bytes_client.save_boefje_meta(boefje_meta)
 
     if boefje_output.files:
-        mime_types = _default_meta_mime_types(task.p_item.data)
+        mime_types = _default_mime_types(task.p_item.data.boefje)
         for file in boefje_output.files:
             raw = base64.b64decode(file.content)
             # when supported, also save file.name to Bytes

boefjes/boefjes/docker_boefjes_runner.py

@@ -33,7 +33,7 @@ def run(self) -> None:
         # local import to prevent circular dependency
         import boefjes.plugins.models
 
-        stderr_mime_types = boefjes.plugins.models._default_meta_mime_types(self.boefje_meta)
+        stderr_mime_types = boefjes.plugins.models._default_mime_types(self.boefje_meta.boefje)
 
         task_id = str(self.boefje_meta.id)
         self.scheduler_client.patch_task(task_id, TaskStatus.RUNNING)

boefjes/boefjes/job_handler.py

@@ -18,7 +18,7 @@
     NormalizerPlainOOI,
 )
 from boefjes.katalogus.local_repository import LocalPluginRepository
-from boefjes.plugins.models import _default_meta_mime_types
+from boefjes.plugins.models import _default_mime_types
 from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner
 from octopoes.api.models import Declaration, Observation
 from octopoes.connector.octopoes import OctopoesAPIConnector
@@ -27,6 +27,7 @@
 from octopoes.models.types import OOIType
 
 logger = logging.getLogger(__name__)
+
 bytes_api_client = BytesAPIClient(
     settings.bytes_api,
     username=settings.bytes_username,
@@ -134,7 +135,7 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
         boefje_meta.runnable_hash = boefje_resource.runnable_hash
         boefje_meta.environment = get_environment_settings(boefje_meta, env_keys) if env_keys else {}
 
-        mime_types = _default_meta_mime_types(boefje_meta)
+        mime_types = _default_mime_types(boefje_meta.boefje)
 
         logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 
@@ -153,7 +154,6 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
             boefje_meta.ended_at = datetime.now(timezone.utc)
             logger.info("Saving to Bytes for boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
 
-            bytes_api_client.login()
             bytes_api_client.save_boefje_meta(boefje_meta)
 
             if boefje_results:
@@ -175,7 +175,6 @@ def __init__(self, job_runner):
     def handle(self, normalizer_meta: NormalizerMeta) -> None:
         logger.info("Handling normalizer %s[%s]", normalizer_meta.normalizer.id, normalizer_meta.id)
 
-        bytes_api_client.login()
         raw = bytes_api_client.get_raw(normalizer_meta.raw_data.id)
 
         normalizer_meta.started_at = datetime.now(timezone.utc)

boefjes/boefjes/job_models.py

@@ -1,4 +1,3 @@
-import hashlib
 from datetime import datetime, timedelta
 from typing import Dict, List, Literal, Optional, Union
 from uuid import UUID
@@ -48,12 +47,6 @@ class BoefjeMeta(Job):
     runnable_hash: Optional[str]
     environment: Optional[Dict[str, str]]
 
-    @property
-    def parameterized_arguments_hash(self) -> str:
-        encoded_arguments = ",".join(f"{k}={v}" for k, v in self.arguments.items())
-
-        return hashlib.sha256(encoded_arguments.encode("utf-8")).hexdigest()
-
 
 class RawDataMeta(BaseModel):
     id: UUID

boefjes/boefjes/katalogus/models.py

@@ -38,8 +38,7 @@ class Boefje(Plugin):
     type: Literal["boefje"] = "boefje"
     scan_level: int = 1
     consumes: Set[str] = Field(default_factory=set)
-    produces: List[str] = Field(default_factory=list)
-    mime_types: Set[str] = Field(default_factory=set)
+    produces: Set[str] = Field(default_factory=set)
     options: Optional[List[str]]
     runnable_hash: Optional[str]
     oci_image: Optional[str]

boefjes/boefjes/katalogus/tests/boefjes_test_dir/kat_test/boefje.json

@@ -5,10 +5,6 @@
     "consumes": [
         "DNSZone"
     ],
-    "produces": [
-        "X509Certificate",
-        "Hostname"
-    ],
     "environment_keys": [],
     "scan_level": 1
 }

boefjes/boefjes/katalogus/tests/boefjes_test_dir/kat_test/kat_test_2/boefje.json

@@ -6,8 +6,7 @@
         "DNSZone"
     ],
     "produces": [
-        "X509Certificate",
-        "Hostname"
+        "text/html"
     ],
     "environment_keys": [],
     "scan_level": 1

boefjes/boefjes/katalogus/tests/test_plugin_service.py

@@ -125,7 +125,7 @@ def test_get_plugins(self):
         self.assertEqual("kat_test", kat_test.id)
         self.assertEqual("Kat test name", kat_test.name)
         self.assertEqual({"DNSZone"}, kat_test.consumes)
-        self.assertSetEqual({"Hostname", "X509Certificate"}, set(kat_test.produces))
+        self.assertSetEqual({"boefje/kat_test"}, set(kat_test.produces))
 
         kat_test_norm = list(filter(lambda x: x.id == "kat_test_normalize", plugins)).pop()
         self.assertIn("kat_test_normalize", kat_test_norm.id)

boefjes/boefjes/katalogus/tests/test_plugins.py

@@ -110,15 +110,12 @@ def test_get_plugin(self):
         # Simpler endpoint works as well, but due to the mock the default mime_types are not dynamically added
         res = self.client.get("/v1/organisations/test-org/plugins/test-boefje-1")
         self.assertEqual(200, res.status_code)
-        assert "mime_types" in res.json()
-        assert not res.json()["mime_types"]
+        assert "produces" in res.json()
+        assert res.json()["produces"] == ["text/html"]
 
         # For boefjes that are pulled from the local repository, we actually get the default mime_types
-        assert set(self.client.get("/v1/organisations/test-org/plugins/kat_test").json()["mime_types"]) == set(
-            [
-                "kat_test",
-                "boefje/kat_test",
-            ]
+        assert set(self.client.get("/v1/organisations/test-org/plugins/kat_test").json()["produces"]) == set(
+            ["boefje/kat_test"]
         )
 
     def test_non_existing_plugin(self):

boefjes/boefjes/plugins/kat_adr_finding_types/boefje.json

@@ -5,9 +5,6 @@
     "consumes": [
         "ADRFindingType"
     ],
-    "produces": [
-        "ADRFindingType"
-    ],
     "scan_level": 0,
     "enabled": true
 }

boefjes/boefjes/plugins/kat_adr_finding_types/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_adr_finding_types_normalize",
     "consumes": [
-        "adr-finding-types"
+        "boefje/adr-finding-types"
     ],
     "produces": [
         "ADRFindingType"

boefjes/boefjes/plugins/kat_adr_validator/boefje.json

@@ -5,9 +5,5 @@
     "consumes": [
         "RESTAPI"
     ],
-    "produces": [
-        "ADRFindingType",
-        "Finding"
-    ],
     "scan_level": 1
 }

boefjes/boefjes/plugins/kat_adr_validator/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "adr-validator",
     "consumes": [
-        "adr-validator"
+        "boefje/adr-validator"
     ],
     "produces": [
         "APIDesignRule",

boefjes/boefjes/plugins/kat_binaryedge/boefje.json

@@ -6,16 +6,6 @@
         "IPAddressV4",
         "IPAddressV6"
     ],
-    "produces": [
-        "KATFindingType",
-        "SoftwareInstance",
-        "Service",
-        "IPPort",
-        "Finding",
-        "Software",
-        "IPService",
-        "CVEFindingType"
-    ],
     "environment_keys": ["BINARYEDGE_API"],
     "scan_level": 2
 }

boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_containers",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_databases",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_http_web",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_message_queues",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_protocols",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",