diff --git a/boefjes/boefjes/katalogus/api/plugins.py b/boefjes/boefjes/katalogus/api/plugins.py index 0f15d3a3d10..7b639a88ba0 100644 --- a/boefjes/boefjes/katalogus/api/plugins.py +++ b/boefjes/boefjes/katalogus/api/plugins.py @@ -1,8 +1,9 @@ +import datetime from functools import partial from fastapi import APIRouter, Body, Depends, HTTPException, status from fastapi.responses import FileResponse, JSONResponse, Response -from httpx import HTTPStatusError +from pydantic import BaseModel, Field from boefjes.katalogus.api.organisations import check_organisation_exists from boefjes.katalogus.dependencies.plugins import ( @@ -12,6 +13,8 @@ get_plugins_filter_parameters, ) from boefjes.katalogus.models import FilterParameters, PaginationParameters, PluginType +from boefjes.katalogus.storage.interfaces import PluginStorage +from boefjes.sql.plugin_storage import get_plugin_storage router = APIRouter( prefix="/organisations/{organisation_id}", @@ -82,34 +85,99 @@ def get_plugin( return p.by_plugin_id(plugin_id, organisation_id) except KeyError: raise HTTPException(status.HTTP_404_NOT_FOUND, "Plugin not found") - except HTTPStatusError as ex: - raise HTTPException(ex.response.status_code) -@router.patch("/plugins/{plugin_id}") +@router.post("/plugins", status_code=status.HTTP_201_CREATED) +def add_plugin(plugin: PluginType, plugin_service: PluginService = Depends(get_plugin_service)): + with plugin_service as service: + if plugin.type == "boefje": + return service.create_boefje(plugin) + + if plugin.type == "normalizer": + return service.create_normalizer(plugin) + + raise HTTPException(status.HTTP_400_BAD_REQUEST, "Creation of Bits is not supported") + + +@router.patch("/plugins/{plugin_id}", status_code=status.HTTP_204_NO_CONTENT) def update_plugin_state( plugin_id: str, organisation_id: str, enabled: bool = Body(False, embed=True), plugin_service: PluginService = Depends(get_plugin_service), ): - try: - with plugin_service as p: - p.update_by_id(plugin_id, organisation_id, enabled) - except HTTPStatusError as ex: - raise HTTPException(ex.response.status_code) + with plugin_service as p: + p.set_enabled_by_id(plugin_id, organisation_id, enabled) + + +class BoefjeIn(BaseModel): + """ + For patching, we need all fields to be optional, hence we overwrite the definition here. + Also see https://fastapi.tiangolo.com/tutorial/body-updates/ as a reference. + """ + + name: str | None = None + version: str | None = None + created: datetime.datetime | None = None + description: str | None = None + environment_keys: list[str] = Field(default_factory=list) + scan_level: int = 1 + consumes: set[str] = Field(default_factory=set) + produces: set[str] = Field(default_factory=set) + oci_image: str | None = None + oci_arguments: list[str] = Field(default_factory=list) + + +@router.patch("/boefjes/{boefje_id}", status_code=status.HTTP_204_NO_CONTENT) +def update_boefje( + boefje_id: str, + boefje: BoefjeIn, + storage: PluginStorage = Depends(get_plugin_storage), +): + with storage as p: + p.update_boefje(boefje_id, boefje.model_dump(exclude_unset=True)) + + +@router.delete("/boefjes/{boefje_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_boefje(boefje_id: str, plugin_storage: PluginStorage = Depends(get_plugin_storage)): + with plugin_storage as p: + p.delete_boefje_by_id(boefje_id) + + +class NormalizerIn(BaseModel): + """ + For patching, we need all fields to be optional, hence we overwrite the definition here. + Also see https://fastapi.tiangolo.com/tutorial/body-updates/ as a reference. + """ + + name: str | None = None + version: str | None = None + created: datetime.datetime | None = None + description: str | None = None + environment_keys: list[str] = Field(default_factory=list) + consumes: list[str] = Field(default_factory=list) # mime types (and/ or boefjes) + produces: list[str] = Field(default_factory=list) # oois + + +@router.patch("/normalizers/{normalizer_id}", status_code=status.HTTP_204_NO_CONTENT) +def update_normalizer( + normalizer_id: str, + normalizer: NormalizerIn, + storage: PluginStorage = Depends(get_plugin_storage), +): + with storage as p: + p.update_normalizer(normalizer_id, normalizer.model_dump(exclude_unset=True)) + + +@router.delete("/normalizers/{normalizer_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_normalizer(normalizer_id: str, plugin_storage: PluginStorage = Depends(get_plugin_storage)): + with plugin_storage as p: + p.delete_normalizer_by_id(normalizer_id) @router.get("/plugins/{plugin_id}/schema.json", include_in_schema=False) -def get_plugin_schema( - plugin_id: str, - plugin_service: PluginService = Depends(get_plugin_service), -) -> JSONResponse: - try: - with plugin_service as p: - return JSONResponse(p.schema(plugin_id)) - except HTTPStatusError as ex: - raise HTTPException(ex.response.status_code) +def get_plugin_schema(plugin_id: str, plugin_service: PluginService = Depends(get_plugin_service)) -> JSONResponse: + return JSONResponse(plugin_service.schema(plugin_id)) @router.get("/plugins/{plugin_id}/cover.jpg", include_in_schema=False) @@ -117,11 +185,7 @@ def get_plugin_cover( plugin_id: str, plugin_service: PluginService = Depends(get_plugin_service), ) -> FileResponse: - try: - with plugin_service as p: - return FileResponse(p.cover(plugin_id)) - except HTTPStatusError as ex: - raise HTTPException(ex.response.status_code) + return FileResponse(plugin_service.cover(plugin_id)) @router.get("/plugins/{plugin_id}/description.md", include_in_schema=False) @@ -130,18 +194,10 @@ def get_plugin_description( organisation_id: str, plugin_service: PluginService = Depends(get_plugin_service), ) -> Response: - try: - with plugin_service as p: - return Response(p.description(plugin_id, organisation_id)) - except HTTPStatusError as ex: - raise HTTPException(ex.response.status_code) + return Response(plugin_service.description(plugin_id, organisation_id)) -@router.post("/settings/clone/{to_organisation_id}") -def clone_organisation_settings( - organisation_id: str, - to_organisation_id: str, - storage: PluginService = Depends(get_plugin_service), -): +@router.post("/settings/clone/{to_org}") +def clone_settings(organisation_id: str, to_org: str, storage: PluginService = Depends(get_plugin_service)): with storage as store: - store.clone_settings_to_organisation(organisation_id, to_organisation_id) + store.clone_settings_to_organisation(organisation_id, to_org) diff --git a/boefjes/boefjes/katalogus/api/root.py b/boefjes/boefjes/katalogus/api/root.py index 1e0062dda58..0b3a7ac2bae 100644 --- a/boefjes/boefjes/katalogus/api/root.py +++ b/boefjes/boefjes/katalogus/api/root.py @@ -17,7 +17,7 @@ from boefjes.config import settings from boefjes.katalogus.api import organisations, plugins from boefjes.katalogus.api import settings as settings_router -from boefjes.katalogus.storage.interfaces import StorageError +from boefjes.katalogus.storage.interfaces import NotFound, StorageError from boefjes.katalogus.version import __version__ with settings.log_cfg.open() as f: @@ -51,16 +51,22 @@ app.include_router(router, prefix="/v1") -@app.exception_handler(StorageError) -def entity_not_found_handler(request: Request, exc: StorageError): - logger.exception("some error", exc_info=exc) - +@app.exception_handler(NotFound) +def entity_not_found_handler(request: Request, exc: NotFound): return JSONResponse( status_code=status.HTTP_404_NOT_FOUND, content={"message": exc.message}, ) +@app.exception_handler(StorageError) +def storage_error_handler(request: Request, exc: StorageError): + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"message": exc.message}, + ) + + class ServiceHealth(BaseModel): service: str healthy: bool = False diff --git a/boefjes/boefjes/katalogus/dependencies/plugins.py b/boefjes/boefjes/katalogus/dependencies/plugins.py index c1364cb0ea9..d254afa8ca7 100644 --- a/boefjes/boefjes/katalogus/dependencies/plugins.py +++ b/boefjes/boefjes/katalogus/dependencies/plugins.py @@ -10,15 +10,18 @@ from sqlalchemy.orm import Session from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository -from boefjes.katalogus.models import FilterParameters, PaginationParameters, PluginType +from boefjes.katalogus.models import Boefje, FilterParameters, Normalizer, PaginationParameters, PluginType from boefjes.katalogus.storage.interfaces import ( + ExistingPluginId, NotFound, PluginEnabledStorage, + PluginStorage, SettingsNotConformingToSchema, SettingsStorage, ) from boefjes.sql.db import session_managed_iterator from boefjes.sql.plugin_enabled_storage import create_plugin_enabled_storage +from boefjes.sql.plugin_storage import create_plugin_storage from boefjes.sql.setting_storage import create_setting_storage logger = logging.getLogger(__name__) @@ -27,24 +30,40 @@ class PluginService: def __init__( self, + plugin_storage: PluginStorage, plugin_enabled_store: PluginEnabledStorage, settings_storage: SettingsStorage, local_repo: LocalPluginRepository, ): + self.plugin_storage = plugin_storage self.plugin_enabled_store = plugin_enabled_store self.settings_storage = settings_storage self.local_repo = local_repo def __enter__(self): self.plugin_enabled_store.__enter__() + self.plugin_storage.__enter__() + self.settings_storage.__enter__() return self def __exit__(self, exc_type, exc_val, exc_tb): self.plugin_enabled_store.__exit__(exc_type, exc_val, exc_tb) + self.plugin_storage.__exit__(exc_type, exc_val, exc_tb) + self.settings_storage.__exit__(exc_type, exc_val, exc_tb) def get_all(self, organisation_id: str) -> list[PluginType]: - return [self._set_plugin_enabled(plugin, organisation_id) for plugin in self.local_repo.get_all()] + all_plugins = self.get_all_without_enabled() + + return [self._set_plugin_enabled(plugin, organisation_id) for plugin in all_plugins.values()] + + def get_all_without_enabled(self): + all_plugins = {plugin.id: plugin for plugin in self.local_repo.get_all()} + + for plugin in self.plugin_storage.get_all(): + all_plugins[plugin.id] = plugin + + return all_plugins def by_plugin_id(self, plugin_id: str, organisation_id: str) -> PluginType: all_plugins = self.get_all(organisation_id) @@ -72,22 +91,36 @@ def get_all_settings(self, organisation_id: str, plugin_id: str): return self.settings_storage.get_all(organisation_id, plugin_id) def clone_settings_to_organisation(self, from_organisation: str, to_organisation: str): - # One requirement is that we also do not keep previously enabled boefjes enabled of they are not copied. + # One requirement is that only boefjes enabled in the from_organisation end up being enabled for the target. for plugin_id in self.plugin_enabled_store.get_all_enabled(to_organisation): - self.update_by_id(plugin_id, to_organisation, enabled=False) + self.set_enabled_by_id(plugin_id, to_organisation, enabled=False) for plugin in self.get_all(from_organisation): if all_settings := self.get_all_settings(from_organisation, plugin.id): self.upsert_settings(all_settings, to_organisation, plugin.id) for plugin_id in self.plugin_enabled_store.get_all_enabled(from_organisation): - self.update_by_id(plugin_id, to_organisation, enabled=True) + self.set_enabled_by_id(plugin_id, to_organisation, enabled=True) def upsert_settings(self, values: dict, organisation_id: str, plugin_id: str): self._assert_settings_match_schema(values, organisation_id, plugin_id) return self.settings_storage.upsert(values, organisation_id, plugin_id) + def create_boefje(self, boefje: Boefje) -> None: + try: + self.local_repo.by_id(boefje.id) + raise ExistingPluginId(boefje.id) + except KeyError: + self.plugin_storage.create_boefje(boefje) + + def create_normalizer(self, normalizer: Normalizer) -> None: + try: + self.local_repo.by_id(normalizer.id) + raise ExistingPluginId(normalizer.id) + except KeyError: + self.plugin_storage.create_normalizer(normalizer) + def delete_settings(self, organisation_id: str, plugin_id: str): self.settings_storage.delete(organisation_id, plugin_id) @@ -96,7 +129,7 @@ def delete_settings(self, organisation_id: str, plugin_id: str): except SettingsNotConformingToSchema: logger.warning("Making sure %s is disabled for %s because settings are deleted", plugin_id, organisation_id) - self.update_by_id(plugin_id, organisation_id, False) + self.set_enabled_by_id(plugin_id, organisation_id, False) def schema(self, plugin_id: str) -> dict | None: return self.local_repo.schema(plugin_id) @@ -119,7 +152,7 @@ def description(self, plugin_id: str, organisation_id: str) -> str: logger.error("Plugin not found: %s", plugin_id) return "" - def update_by_id(self, plugin_id: str, organisation_id: str, enabled: bool): + def set_enabled_by_id(self, plugin_id: str, organisation_id: str, enabled: bool): if enabled: all_settings = self.settings_storage.get_all(organisation_id, plugin_id) self._assert_settings_match_schema(all_settings, organisation_id, plugin_id) @@ -149,6 +182,7 @@ def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> Plugi def get_plugin_service(organisation_id: str) -> Iterator[PluginService]: def closure(session: Session): return PluginService( + create_plugin_storage(session), create_plugin_enabled_storage(session), create_setting_storage(session), get_local_repository(), diff --git a/boefjes/boefjes/katalogus/local_repository.py b/boefjes/boefjes/katalogus/local_repository.py index d6f2a5e0242..3e9071123f8 100644 --- a/boefjes/boefjes/katalogus/local_repository.py +++ b/boefjes/boefjes/katalogus/local_repository.py @@ -43,7 +43,7 @@ def by_id(self, plugin_id: str) -> PluginType: if plugin_id in normalizers: return normalizers[plugin_id].normalizer - raise Exception(f"Can't find plugin {plugin_id}") + raise KeyError(f"Can't find plugin {plugin_id}") def schema(self, id_: str) -> dict | None: boefjes = self.resolve_boefjes() diff --git a/boefjes/boefjes/katalogus/models.py b/boefjes/boefjes/katalogus/models.py index b9fe473a735..69d8cc2c637 100644 --- a/boefjes/boefjes/katalogus/models.py +++ b/boefjes/boefjes/katalogus/models.py @@ -14,12 +14,11 @@ class Plugin(BaseModel): id: str name: str | None = None version: str | None = None - authors: list[str] | None = None created: datetime.datetime | None = None description: str | None = None environment_keys: list[str] = Field(default_factory=list) - related: list[str] | None = None enabled: bool = False + static: bool = True # We need to differentiate between local and remote plugins to know which ones can be deleted def __str__(self): return f"{self.id}:{self.version}" @@ -30,7 +29,6 @@ class Boefje(Plugin): scan_level: int = 1 consumes: set[str] = Field(default_factory=set) produces: set[str] = Field(default_factory=set) - options: list[str] | None = None runnable_hash: str | None = None oci_image: str | None = None oci_arguments: list[str] = Field(default_factory=list) diff --git a/boefjes/boefjes/katalogus/storage/interfaces.py b/boefjes/boefjes/katalogus/storage/interfaces.py index 61c30d5fd8a..86bbfaba62b 100644 --- a/boefjes/boefjes/katalogus/storage/interfaces.py +++ b/boefjes/boefjes/katalogus/storage/interfaces.py @@ -1,6 +1,6 @@ from abc import ABC -from boefjes.katalogus.models import Organisation +from boefjes.katalogus.models import Boefje, Normalizer, Organisation, PluginType class StorageError(Exception): @@ -28,10 +28,20 @@ def __init__(self, organisation_id: str): class PluginNotFound(NotFound): + def __init__(self, plugin_id: str): + super().__init__(f"Plugin with id '{plugin_id}' not found") + + +class PluginStateNotFound(NotFound): def __init__(self, plugin_id: str, organisation_id: str): super().__init__(f"State for plugin with id '{plugin_id}' not found for organisation '{organisation_id}'") +class ExistingPluginId(StorageError): + def __init__(self, plugin_id: str): + super().__init__(f"Plugin id '{plugin_id}' is already used") + + class SettingsNotFound(NotFound): def __init__(self, organisation_id: str, plugin_id: str): super().__init__(f"Setting not found for organisation '{organisation_id}' and plugin '{plugin_id}'") @@ -57,6 +67,41 @@ def delete_by_id(self, organisation_id: str) -> None: raise NotImplementedError +class PluginStorage(ABC): + def __enter__(self): + return self + + def __exit__(self, exc_type: type[Exception], exc_value: str, exc_traceback: str) -> None: # noqa: F841 + pass + + def get_all(self) -> list[PluginType]: + raise NotImplementedError + + def boefje_by_id(self, boefje_id: str) -> Boefje: + raise NotImplementedError + + def normalizer_by_id(self, normalizer_id: str) -> Normalizer: + raise NotImplementedError + + def create_boefje(self, boefje: Boefje) -> None: + raise NotImplementedError + + def create_normalizer(self, normalizer: Normalizer) -> None: + raise NotImplementedError + + def update_boefje(self, boefje_id: str, data: dict) -> None: + raise NotImplementedError + + def update_normalizer(self, normalizer_id: str, data: dict) -> None: + raise NotImplementedError + + def delete_boefje_by_id(self, boefje_id: str) -> None: + raise NotImplementedError + + def delete_normalizer_by_id(self, normalizer_id: str) -> None: + raise NotImplementedError + + class SettingsStorage(ABC): def __enter__(self): return self diff --git a/boefjes/boefjes/katalogus/storage/memory.py b/boefjes/boefjes/katalogus/storage/memory.py index bd72f116b8b..d89e972ca32 100644 --- a/boefjes/boefjes/katalogus/storage/memory.py +++ b/boefjes/boefjes/katalogus/storage/memory.py @@ -1,5 +1,10 @@ -from boefjes.katalogus.models import Organisation -from boefjes.katalogus.storage.interfaces import OrganisationStorage, PluginEnabledStorage, SettingsStorage +from boefjes.katalogus.models import Boefje, Normalizer, Organisation, PluginType +from boefjes.katalogus.storage.interfaces import ( + OrganisationStorage, + PluginEnabledStorage, + PluginStorage, + SettingsStorage, +) # key = organisation id; value = organisation organisations: dict[str, Organisation] = {} @@ -25,6 +30,33 @@ def delete_by_id(self, organisation_id: str) -> None: del self._data[organisation_id] +class PluginStorageMemory(PluginStorage): + def __init__(self): + self._boefjes = {} + self._normalizers = {} + + def get_all(self) -> list[PluginType]: + return list(self._boefjes.values()) + list(self._normalizers.values()) + + def boefje_by_id(self, boefje_id: str) -> Boefje: + return self._boefjes[boefje_id] + + def normalizer_by_id(self, normalizer_id: str) -> Normalizer: + return self._normalizers[normalizer_id] + + def create_boefje(self, boefje: Boefje) -> None: + self._boefjes[boefje.id] = boefje + + def create_normalizer(self, normalizer: Normalizer) -> None: + self._normalizers[normalizer.id] = normalizer + + def delete_boefje_by_id(self, boefje_id: str) -> None: + del self._boefjes[boefje_id] + + def delete_normalizer_by_id(self, normalizer_id: str) -> None: + del self._normalizers[normalizer_id] + + class SettingsStorageMemory(SettingsStorage): def __init__(self): self._data = {} diff --git a/boefjes/boefjes/katalogus/tests/integration/test_api.py b/boefjes/boefjes/katalogus/tests/integration/test_api.py index 1f7252c45af..e54a91514c6 100644 --- a/boefjes/boefjes/katalogus/tests/integration/test_api.py +++ b/boefjes/boefjes/katalogus/tests/integration/test_api.py @@ -8,7 +8,7 @@ from boefjes.config import settings from boefjes.katalogus.api.root import app from boefjes.katalogus.dependencies.encryption import IdentityMiddleware -from boefjes.katalogus.models import Organisation +from boefjes.katalogus.models import Boefje, Normalizer, Organisation from boefjes.sql.db import SQL_BASE, get_engine from boefjes.sql.organisation_storage import SQLOrganisationStorage from boefjes.sql.plugin_enabled_storage import SQLPluginEnabledStorage @@ -34,12 +34,12 @@ def tearDown(self) -> None: session = sessionmaker(bind=get_engine())() for table in SQL_BASE.metadata.tables: - session.execute(f"DELETE FROM {table} CASCADE") # noqa: S608 + session.execute(f"TRUNCATE {table} CASCADE") # noqa: S608 session.commit() session.close() - def test_plugin_api(self): + def test_get_local_plugin(self): response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/dns-records") self.assertEqual(response.status_code, 200) @@ -47,6 +47,103 @@ def test_plugin_api(self): self.assertEqual("dns-records", data["id"]) + def test_filter_plugins(self): + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/") + self.assertEqual(len(response.json()), 93) + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins?plugin_type=boefje") + self.assertEqual(len(response.json()), 41) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins?limit=10") + self.assertEqual(len(response.json()), 10) + + def test_cannot_add_plugin_reserved_id(self): + boefje = Boefje(id="dns-records", name="My test boefje", static=False) + response = self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=boefje.json()) + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json(), {"message": "Plugin id 'dns-records' is already used"}) + + normalizer = Normalizer(id="kat_nmap_normalize", name="My test normalizer", static=False) + response = self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=normalizer.json()) + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json(), {"message": "Plugin id 'kat_nmap_normalize' is already used"}) + + def test_add_boefje(self): + boefje = Boefje(id="test_plugin", name="My test boefje", static=False) + response = self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=boefje.json()) + self.assertEqual(response.status_code, 201) + + response = self.client.post(f"/v1/organisations/{self.org.id}/plugins", json={"a": "b"}) + self.assertEqual(response.status_code, 422) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/?plugin_type=boefje") + self.assertEqual(len(response.json()), 42) + + boefje_dict = boefje.dict() + boefje_dict["consumes"] = list(boefje_dict["consumes"]) + boefje_dict["produces"] = list(boefje_dict["produces"]) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/test_plugin") + self.assertEqual(response.json(), boefje_dict) + + def test_delete_boefje(self): + boefje = Boefje(id="test_plugin", name="My test boefje", static=False) + response = self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=boefje.json()) + self.assertEqual(response.status_code, 201) + + response = self.client.delete(f"/v1/organisations/{self.org.id}/boefjes/test_plugin") + self.assertEqual(response.status_code, 204) + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/test_plugin") + self.assertEqual(response.status_code, 404) + + def test_add_normalizer(self): + normalizer = Normalizer(id="test_normalizer", name="My test normalizer", static=False) + response = self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=normalizer.json()) + self.assertEqual(response.status_code, 201) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/?plugin_type=normalizer") + self.assertEqual(len(response.json()), 53) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/test_normalizer") + self.assertEqual(response.json(), normalizer.dict()) + + def test_delete_normalizer(self): + normalizer = Normalizer(id="test_normalizer", name="My test normalizer", static=False) + response = self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=normalizer.json()) + self.assertEqual(response.status_code, 201) + + response = self.client.delete(f"/v1/organisations/{self.org.id}/normalizers/test_normalizer") + self.assertEqual(response.status_code, 204) + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/test_normalizer") + self.assertEqual(response.status_code, 404) + + def test_update_plugins(self): + normalizer = Normalizer(id="norm_id", name="My test normalizer", static=False) + boefje = Boefje(id="test_plugin", name="My test boefje", description="123", static=False) + + self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=boefje.json()) + self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/{boefje.id}", json={"description": "4"}) + self.client.patch(f"/v1/organisations/{self.org.id}/plugins/{boefje.id}", json={"enabled": True}) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/{boefje.id}") + self.assertEqual(response.json()["description"], "4") + self.assertTrue(response.json()["enabled"]) + + r = self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/dns-records", json={"id": "4", "version": "s"}) + self.assertEqual(r.status_code, 404) + r = self.client.patch(f"/v1/organisations/{self.org.id}/boefjes/dns-records", json={"name": "Overwrite name"}) + self.assertEqual(r.status_code, 404) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/dns-records") + self.assertEqual(response.json()["name"], "DnsRecords") + self.assertIsNone(response.json()["version"]) + self.assertEqual(response.json()["id"], "dns-records") + + self.client.post(f"/v1/organisations/{self.org.id}/plugins", content=normalizer.json()) + self.client.patch(f"/v1/organisations/{self.org.id}/normalizers/{normalizer.id}", json={"version": "v1.2"}) + + response = self.client.get(f"/v1/organisations/{self.org.id}/plugins/{normalizer.id}") + self.assertEqual(response.json()["version"], "v1.2") + def test_basic_settings_api(self): plug = "dns-records" diff --git a/boefjes/boefjes/katalogus/tests/integration/test_sql_repositories.py b/boefjes/boefjes/katalogus/tests/integration/test_sql_repositories.py index 36fee9ba1cd..319a4ed2f1d 100644 --- a/boefjes/boefjes/katalogus/tests/integration/test_sql_repositories.py +++ b/boefjes/boefjes/katalogus/tests/integration/test_sql_repositories.py @@ -1,3 +1,4 @@ +import datetime import os from unittest import TestCase, skipIf @@ -5,11 +6,18 @@ from sqlalchemy.orm import sessionmaker from boefjes.config import settings -from boefjes.katalogus.models import Boefje, Organisation -from boefjes.katalogus.storage.interfaces import OrganisationNotFound, PluginNotFound, SettingsNotFound, StorageError +from boefjes.katalogus.models import Boefje, Normalizer, Organisation +from boefjes.katalogus.storage.interfaces import ( + OrganisationNotFound, + PluginNotFound, + PluginStateNotFound, + SettingsNotFound, + StorageError, +) from boefjes.sql.db import SQL_BASE, get_engine from boefjes.sql.organisation_storage import SQLOrganisationStorage from boefjes.sql.plugin_enabled_storage import SQLPluginEnabledStorage +from boefjes.sql.plugin_storage import SQLPluginStorage from boefjes.sql.setting_storage import SQLSettingsStorage, create_encrypter @@ -22,6 +30,7 @@ def setUp(self) -> None: self.organisation_storage = SQLOrganisationStorage(session, settings) self.settings_storage = SQLSettingsStorage(session, create_encrypter()) self.plugin_state_storage = SQLPluginEnabledStorage(session, settings) + self.plugin_storage = SQLPluginStorage(session, settings) def tearDown(self) -> None: session = sessionmaker(bind=get_engine())() @@ -51,15 +60,6 @@ def test_organisation_storage(self): with self.assertRaises(OrganisationNotFound): storage.get_by_id(organisation_id) - def test_organisations(self): - org = Organisation(id="org1", name="Test") - - with self.organisation_storage as storage: - storage.create(org) - - returned_org = storage.get_by_id(org.id) - self.assertEqual(org, returned_org) - def test_settings_storage(self): organisation_id = "test" plugin_id = 64 * "a" @@ -152,11 +152,119 @@ def test_plugin_enabled_storage(self): returned_state = plugin_state_storage.get_by_id(plugin.id, org.id) self.assertFalse(returned_state) - with self.assertRaises(PluginNotFound): + with self.assertRaises(PluginStateNotFound): plugin_state_storage.get_by_id("wrong", org.id) - with self.assertRaises(PluginNotFound): + with self.assertRaises(PluginStateNotFound): plugin_state_storage.get_by_id("wrong", org.id) - with self.assertRaises(PluginNotFound): + with self.assertRaises(PluginStateNotFound): plugin_state_storage.get_by_id(plugin.id, "wrong") + + def test_bare_boefje_storage(self): + boefje = Boefje(id="test_boefje", name="Test", static=False) + + with self.plugin_storage as storage: + storage.create_boefje(boefje) + + returned_boefje = storage.boefje_by_id(boefje.id) + self.assertEqual(boefje, returned_boefje) + + storage.update_boefje(boefje.id, {"description": "4"}) + self.assertEqual(storage.boefje_by_id(boefje.id).description, "4") + boefje.description = "4" + + all_plugins = storage.get_all() + self.assertEqual(all_plugins, [boefje]) + + with self.plugin_storage as storage: + storage.delete_boefje_by_id(boefje.id) + + with self.assertRaises(PluginNotFound): + storage.boefje_by_id(boefje.id) + + def test_rich_boefje_storage(self): + boefje = Boefje( + id="test_boefje", + name="Test", + version="v1.09", + created=datetime.datetime(2010, 10, 10, 10, 10, 10, tzinfo=datetime.UTC), + description="My Boefje", + environment_keys=["api_key", "TOKEN"], + scan_level=4, + consumes=["Internet"], + produces=[ + "image/png", + "application/zip+json", + "application/har+json", + "application/json", + "application/localstorage+json", + ], + oci_image="ghcr.io/test/image:123", + oci_arguments=["host", "-n", "123123123123123123123"], + static=False, + ) + + with self.plugin_storage as storage: + storage.create_boefje(boefje) + + returned_boefje = storage.boefje_by_id(boefje.id) + self.assertEqual(boefje, returned_boefje) + + def test_bare_normalizer_storage(self): + normalizer = Normalizer(id="test_boefje", name="Test", static=False) + + with self.plugin_storage as storage: + storage.create_normalizer(normalizer) + + returned_normalizer = storage.normalizer_by_id(normalizer.id) + self.assertEqual(normalizer, returned_normalizer) + + storage.update_normalizer(normalizer.id, {"version": "v4"}) + self.assertEqual(storage.normalizer_by_id(normalizer.id).version, "v4") + normalizer.version = "v4" + + all_plugins = storage.get_all() + self.assertEqual(all_plugins, [normalizer]) + + with self.plugin_storage as storage: + storage.delete_normalizer_by_id(normalizer.id) + + with self.assertRaises(PluginNotFound): + storage.normalizer_by_id(normalizer.id) + + def test_rich_normalizer_storage(self): + normalizer = Normalizer( + id="test_normalizer", + name="Test", + version="v1.19", + created=datetime.datetime(2010, 10, 10, 10, 10, 10, tzinfo=datetime.UTC), + description="My Normalizer", + environment_keys=["api_key", "TOKEN"], + scan_level=4, + consumes=["Internet"], + produces=[ + "image/png", + "application/zip+json", + "application/har+json", + "application/json", + "application/localstorage+json", + ], + static=False, + ) + + with self.plugin_storage as storage: + storage.create_normalizer(normalizer) + + returned_normalizer = storage.normalizer_by_id(normalizer.id) + self.assertEqual(normalizer, returned_normalizer) + + def test_plugin_storage(self): + boefje = Boefje(id="test_boefje", name="Test", static=False) + normalizer = Normalizer(id="test_boefje", name="Test", static=False) + + with self.plugin_storage as storage: + storage.create_boefje(boefje) + storage.create_normalizer(normalizer) + + self.assertEqual(storage.get_all(), [boefje, normalizer]) diff --git a/boefjes/boefjes/katalogus/tests/test_organisations.py b/boefjes/boefjes/katalogus/tests/test_organisation_api.py similarity index 100% rename from boefjes/boefjes/katalogus/tests/test_organisations.py rename to boefjes/boefjes/katalogus/tests/test_organisation_api.py diff --git a/boefjes/boefjes/katalogus/tests/test_plugin_service.py b/boefjes/boefjes/katalogus/tests/test_plugin_service.py index 48f31024f22..62a6a8c55bb 100644 --- a/boefjes/boefjes/katalogus/tests/test_plugin_service.py +++ b/boefjes/boefjes/katalogus/tests/test_plugin_service.py @@ -4,7 +4,7 @@ from boefjes.katalogus.dependencies.plugins import PluginService from boefjes.katalogus.local_repository import LocalPluginRepository from boefjes.katalogus.storage.interfaces import SettingsNotConformingToSchema -from boefjes.katalogus.storage.memory import PluginStatesStorageMemory, SettingsStorageMemory +from boefjes.katalogus.storage.memory import PluginStatesStorageMemory, PluginStorageMemory, SettingsStorageMemory def mock_plugin_service(organisation_id: str) -> PluginService: @@ -14,6 +14,7 @@ def mock_plugin_service(organisation_id: str) -> PluginService: test_boefjes_dir = BASE_DIR / "katalogus" / "tests" / "boefjes_test_dir" return PluginService( + PluginStorageMemory(), PluginStatesStorageMemory(organisation_id), storage, LocalPluginRepository(test_boefjes_dir), @@ -48,7 +49,7 @@ def test_get_plugin_by_id(self): self.assertTrue(plugin.enabled) def test_update_by_id(self): - self.service.update_by_id("kat_test_normalize", self.organisation, False) + self.service.set_enabled_by_id("kat_test_normalize", self.organisation, False) plugin = self.service.by_plugin_id("kat_test_normalize", self.organisation) self.assertFalse(plugin.enabled) @@ -56,7 +57,7 @@ def test_update_by_id_bad_schema(self): plugin_id = "kat_test" with self.assertRaises(SettingsNotConformingToSchema) as ctx: - self.service.update_by_id(plugin_id, self.organisation, True) + self.service.set_enabled_by_id(plugin_id, self.organisation, True) msg = ( "Settings for organisation test and plugin kat_test are not conform the plugin schema: 'api_key' is a " @@ -65,12 +66,12 @@ def test_update_by_id_bad_schema(self): self.assertEqual(ctx.exception.message, msg) self.service.settings_storage.upsert({"api_key": 128 * "a"}, self.organisation, plugin_id) - self.service.update_by_id(plugin_id, self.organisation, True) + self.service.set_enabled_by_id(plugin_id, self.organisation, True) value = 129 * "a" self.service.settings_storage.upsert({"api_key": 129 * "a"}, self.organisation, plugin_id) with self.assertRaises(SettingsNotConformingToSchema) as ctx: - self.service.update_by_id(plugin_id, self.organisation, True) + self.service.set_enabled_by_id(plugin_id, self.organisation, True) msg = ( f"Settings for organisation test and plugin kat_test are not conform the plugin schema: " @@ -97,7 +98,7 @@ def test_removing_mandatory_setting_disables_plugin(self): plugin_id = "kat_test" self.service.settings_storage.upsert({"api_key": 128 * "a"}, self.organisation, plugin_id) - self.service.update_by_id(plugin_id, self.organisation, True) + self.service.set_enabled_by_id(plugin_id, self.organisation, True) plugin = self.service.by_plugin_id(plugin_id, self.organisation) self.assertTrue(plugin.enabled) @@ -113,17 +114,17 @@ def test_adding_integer_settings_within_given_constraints(self): self.service.settings_storage.upsert({"api_key": "24"}, self.organisation, plugin_id) with self.assertRaises(SettingsNotConformingToSchema) as ctx: - self.service.update_by_id(plugin_id, self.organisation, True) + self.service.set_enabled_by_id(plugin_id, self.organisation, True) self.assertIn("'24' is not of type 'integer'", ctx.exception.message) self.service.settings_storage.upsert({"api_key": 24}, self.organisation, plugin_id) - self.service.update_by_id(plugin_id, self.organisation, True) + self.service.set_enabled_by_id(plugin_id, self.organisation, True) plugin = self.service.by_plugin_id(plugin_id, self.organisation) self.assertTrue(plugin.enabled) - self.service.update_by_id(plugin_id, self.organisation, False) + self.service.set_enabled_by_id(plugin_id, self.organisation, False) def test_clone_one_setting(self): new_org_id = "org2" @@ -131,8 +132,8 @@ def test_clone_one_setting(self): self.service.settings_storage.upsert({"api_key": "24"}, self.organisation, plugin_id) assert self.service.get_all_settings(self.organisation, plugin_id) == {"api_key": "24"} - self.service.update_by_id(plugin_id, self.organisation, True) - self.service.update_by_id("kat_test_normalize", new_org_id, True) + self.service.set_enabled_by_id(plugin_id, self.organisation, True) + self.service.set_enabled_by_id("kat_test_normalize", new_org_id, True) assert "api_key" not in self.service.get_all_settings(new_org_id, plugin_id) diff --git a/boefjes/boefjes/katalogus/tests/test_plugins.py b/boefjes/boefjes/katalogus/tests/test_plugins_api.py similarity index 98% rename from boefjes/boefjes/katalogus/tests/test_plugins.py rename to boefjes/boefjes/katalogus/tests/test_plugins_api.py index 2471c29cc7b..77a8a2b97e8 100644 --- a/boefjes/boefjes/katalogus/tests/test_plugins.py +++ b/boefjes/boefjes/katalogus/tests/test_plugins_api.py @@ -107,7 +107,7 @@ def test_patching_enabled_state(self): "/v1/organisations/test-org/plugins/test-boefje-1", json={"enabled": False}, ) - self.assertEqual(200, res.status_code) + self.assertEqual(204, res.status_code) res = self.client.get("/v1/organisations/test-org/plugins") self.assertEqual(200, res.status_code) @@ -128,7 +128,7 @@ def test_patching_enabled_state_non_existing_org(self): json={"enabled": False}, ) - self.assertEqual(200, res.status_code) + self.assertEqual(204, res.status_code) res = self.client.get("/v1/organisations/non-existing-org/plugins") self.assertEqual(200, res.status_code) diff --git a/boefjes/boefjes/migrations/versions/6f99834a4a5a_introduce_boefje_and_normalizer_models.py b/boefjes/boefjes/migrations/versions/6f99834a4a5a_introduce_boefje_and_normalizer_models.py new file mode 100644 index 00000000000..b843700c12b --- /dev/null +++ b/boefjes/boefjes/migrations/versions/6f99834a4a5a_introduce_boefje_and_normalizer_models.py @@ -0,0 +1,64 @@ +"""Introduce Boefje and Normalizer models + +Revision ID: 6f99834a4a5a +Revises: 7c88b9cd96aa +Create Date: 2024-05-28 13:00:12.338182 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "6f99834a4a5a" +down_revision = "7c88b9cd96aa" +branch_labels = None +depends_on = None + + +scan_level_enum = sa.Enum("0", "1", "2", "3", "4", name="scan_level") + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + op.create_table( + "boefje", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("plugin_id", sa.String(length=64), nullable=False), + sa.Column("created", sa.DateTime(timezone=True), nullable=True), + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("scan_level", scan_level_enum, nullable=False), + sa.Column("consumes", sa.ARRAY(sa.String(length=128)), nullable=False), + sa.Column("produces", sa.ARRAY(sa.String(length=128)), nullable=False), + sa.Column("environment_keys", sa.ARRAY(sa.String(length=128)), nullable=False), + sa.Column("oci_image", sa.String(length=256), nullable=True), + sa.Column("oci_arguments", sa.ARRAY(sa.String(length=128)), nullable=False), + sa.Column("version", sa.String(length=16), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("plugin_id"), + ) + op.create_table( + "normalizer", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("plugin_id", sa.String(length=64), nullable=False), + sa.Column("created", sa.DateTime(timezone=True), nullable=True), + sa.Column("name", sa.String(length=64), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("consumes", sa.ARRAY(sa.String(length=128)), nullable=False), + sa.Column("produces", sa.ARRAY(sa.String(length=128)), nullable=False), + sa.Column("environment_keys", sa.ARRAY(sa.String(length=128)), nullable=False), + sa.Column("version", sa.String(length=16), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("plugin_id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("normalizer") + op.drop_table("boefje") + scan_level_enum.drop(op.get_bind(), checkfirst=False) + # ### end Alembic commands ### diff --git a/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json b/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json index 9312a54673b..52c21e9a03e 100644 --- a/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json +++ b/boefjes/boefjes/plugins/kat_adr_validator/normalizer.json @@ -1,5 +1,5 @@ { - "id": "adr-validator", + "id": "adr-validator-normalize", "consumes": [ "boefje/adr-validator" ], diff --git a/boefjes/boefjes/sql/db_models.py b/boefjes/boefjes/sql/db_models.py index 939c2c10629..1a870e84028 100644 --- a/boefjes/boefjes/sql/db_models.py +++ b/boefjes/boefjes/sql/db_models.py @@ -1,9 +1,19 @@ -from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, UniqueConstraint +from enum import Enum + +from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, UniqueConstraint, types from sqlalchemy.orm import relationship from boefjes.sql.db import SQL_BASE +class ScanLevel(Enum): + L0 = 0 + L1 = 1 + L2 = 2 + L3 = 3 + L4 = 4 + + class OrganisationInDB(SQL_BASE): __tablename__ = "organisation" @@ -46,3 +56,44 @@ class PluginStateInDB(SQL_BASE): organisation_pk = Column(Integer, ForeignKey("organisation.pk", ondelete="CASCADE"), nullable=False) organisation = relationship("OrganisationInDB") + + +class BoefjeInDB(SQL_BASE): + __tablename__ = "boefje" + + id = Column(types.Integer, primary_key=True, autoincrement=True) + plugin_id = Column(types.String(length=64), nullable=False, unique=True) + created = Column(types.DateTime(timezone=True), nullable=True) + + # Metadata + name = Column(String(length=64), nullable=False) + description = Column(types.Text, nullable=True) + scan_level = Column(types.Enum(*[str(x.value) for x in ScanLevel], name="scan_level"), nullable=False, default="4") + + # Job specifications + consumes = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) + produces = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) + environment_keys = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) + + # Image specifications + oci_image = Column(types.String(length=256), nullable=True) + oci_arguments = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) + version = Column(types.String(length=16), nullable=True) + + +class NormalizerInDB(SQL_BASE): + __tablename__ = "normalizer" + + id = Column(types.Integer, primary_key=True, autoincrement=True) + plugin_id = Column(types.String(length=64), nullable=False, unique=True) + created = Column(types.DateTime(timezone=True), nullable=True) + + # Metadata + name = Column(String(length=64), nullable=False) + description = Column(types.Text, nullable=True) + + # Job specifications + consumes = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) + produces = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) + environment_keys = Column(types.ARRAY(types.String(length=128)), default=lambda: [], nullable=False) + version = Column(types.String(length=16), nullable=True) diff --git a/boefjes/boefjes/sql/plugin_enabled_storage.py b/boefjes/boefjes/sql/plugin_enabled_storage.py index f6d874fa02f..6576a70c3e8 100644 --- a/boefjes/boefjes/sql/plugin_enabled_storage.py +++ b/boefjes/boefjes/sql/plugin_enabled_storage.py @@ -3,7 +3,7 @@ from sqlalchemy.orm import Session, sessionmaker from boefjes.config import Settings, settings -from boefjes.katalogus.storage.interfaces import OrganisationNotFound, PluginEnabledStorage, PluginNotFound +from boefjes.katalogus.storage.interfaces import OrganisationNotFound, PluginEnabledStorage, PluginStateNotFound from boefjes.sql.db import ObjectNotFoundException, get_engine from boefjes.sql.db_models import OrganisationInDB, PluginStateInDB from boefjes.sql.session import SessionMixin @@ -47,7 +47,7 @@ def update_or_create_by_id(self, plugin_id: str, enabled: bool, organisation_id: try: instance = self._db_instance_by_id(plugin_id, organisation_id) instance.enabled = enabled - except PluginNotFound: + except PluginStateNotFound: logger.info("Plugin state not found, creating new instance") self.create(plugin_id, enabled, organisation_id) @@ -62,7 +62,7 @@ def _db_instance_by_id(self, plugin_id: str, organisation_id: str) -> PluginStat ) if instance is None: - raise PluginNotFound(plugin_id, organisation_id) from ObjectNotFoundException( + raise PluginStateNotFound(plugin_id, organisation_id) from ObjectNotFoundException( PluginStateInDB, plugin_id=plugin_id, organisation_id=organisation_id, diff --git a/boefjes/boefjes/sql/plugin_storage.py b/boefjes/boefjes/sql/plugin_storage.py new file mode 100644 index 00000000000..4b76f4c0dc3 --- /dev/null +++ b/boefjes/boefjes/sql/plugin_storage.py @@ -0,0 +1,159 @@ +import logging +from collections.abc import Iterator + +from sqlalchemy.orm import Session + +from boefjes.config import Settings, settings +from boefjes.katalogus.models import Boefje, Normalizer, PluginType +from boefjes.katalogus.storage.interfaces import PluginNotFound, PluginStorage +from boefjes.sql.db import ObjectNotFoundException, session_managed_iterator +from boefjes.sql.db_models import BoefjeInDB, NormalizerInDB +from boefjes.sql.session import SessionMixin + +logger = logging.getLogger(__name__) + + +class SQLPluginStorage(SessionMixin, PluginStorage): + def __init__(self, session: Session, app_settings: Settings): + self.app_settings = app_settings + + super().__init__(session) + + def get_all(self) -> list[PluginType]: + boefjes = [self.to_boefje(boefje) for boefje in self.session.query(BoefjeInDB).all()] + normalizers = [self.to_normalizer(normalizer) for normalizer in self.session.query(NormalizerInDB).all()] + return boefjes + normalizers + + def boefje_by_id(self, boefje_id: str) -> Boefje: + instance = self._db_boefje_instance_by_id(boefje_id) + + return self.to_boefje(instance) + + def normalizer_by_id(self, normalizer_id: str) -> Normalizer: + instance = self._db_normalizer_instance_by_id(normalizer_id) + + return self.to_normalizer(instance) + + def create_boefje(self, boefje: Boefje) -> None: + logger.info("Saving plugin: %s", boefje.json()) + + boefje_in_db = self.to_boefje_in_db(boefje) + self.session.add(boefje_in_db) + + def update_boefje(self, boefje_id: str, data: dict) -> None: + instance = self._db_boefje_instance_by_id(boefje_id) + + for key, value in data.items(): + setattr(instance, key, value) + + self.session.add(instance) + + def create_normalizer(self, normalizer: Normalizer) -> None: + logger.info("Saving plugin: %s", normalizer.json()) + + normalizer_in_db = self.to_normalizer_in_db(normalizer) + self.session.add(normalizer_in_db) + + def update_normalizer(self, normalizer_id: str, data: dict) -> None: + instance = self._db_normalizer_instance_by_id(normalizer_id) + + for key, value in data.items(): + setattr(instance, key, value) + + self.session.add(instance) + + def delete_boefje_by_id(self, boefje_id: str) -> None: + instance = self._db_boefje_instance_by_id(boefje_id) + + self.session.delete(instance) + + def delete_normalizer_by_id(self, normalizer_id: str) -> None: + instance = self._db_normalizer_instance_by_id(normalizer_id) + + self.session.delete(instance) + + def _db_boefje_instance_by_id(self, boefje_id: str) -> BoefjeInDB: + instance = self.session.query(BoefjeInDB).filter(BoefjeInDB.plugin_id == boefje_id).first() + + if instance is None: + raise PluginNotFound(boefje_id) from ObjectNotFoundException(BoefjeInDB, id=boefje_id) + + return instance + + def _db_normalizer_instance_by_id(self, normalizer_id: str) -> NormalizerInDB: + instance = self.session.query(NormalizerInDB).filter(NormalizerInDB.plugin_id == normalizer_id).first() + + if instance is None: + raise PluginNotFound(normalizer_id) from ObjectNotFoundException(NormalizerInDB, id=normalizer_id) + + return instance + + @staticmethod + def to_boefje_in_db(boefje: Boefje) -> BoefjeInDB: + return BoefjeInDB( + plugin_id=boefje.id, + created=boefje.created, + name=boefje.name, + description=boefje.description, + scan_level=str(boefje.scan_level), + consumes=boefje.consumes, + produces=boefje.produces, + environment_keys=boefje.environment_keys, + oci_image=boefje.oci_image, + oci_arguments=boefje.oci_arguments, + version=boefje.version, + ) + + @staticmethod + def to_normalizer_in_db(normalizer: Normalizer) -> NormalizerInDB: + return NormalizerInDB( + plugin_id=normalizer.id, + created=normalizer.created, + name=normalizer.name, + description=normalizer.description, + consumes=normalizer.consumes, + produces=normalizer.produces, + environment_keys=normalizer.environment_keys, + version=normalizer.version, + ) + + @staticmethod + def to_boefje(boefje_in_db: BoefjeInDB) -> Boefje: + return Boefje( + id=boefje_in_db.plugin_id, + name=boefje_in_db.name, + plugin_id=boefje_in_db.id, + created=boefje_in_db.created, + description=boefje_in_db.description, + scan_level=int(boefje_in_db.scan_level), + consumes=boefje_in_db.consumes, + produces=boefje_in_db.produces, + environment_keys=boefje_in_db.environment_keys, + oci_image=boefje_in_db.oci_image, + oci_arguments=boefje_in_db.oci_arguments, + version=boefje_in_db.version, + static=False, + ) + + @staticmethod + def to_normalizer(normalizer_in_db: NormalizerInDB) -> Normalizer: + return Normalizer( + id=normalizer_in_db.plugin_id, + name=normalizer_in_db.name, + plugin_id=normalizer_in_db.id, + created=normalizer_in_db.created, + description=normalizer_in_db.description, + consumes=normalizer_in_db.consumes, + produces=normalizer_in_db.produces, + environment_keys=normalizer_in_db.environment_keys, + version=normalizer_in_db.version, + static=False, + ) + + +def create_plugin_storage(session) -> SQLPluginStorage: + return SQLPluginStorage(session, settings) + + +def get_plugin_storage() -> Iterator[PluginStorage]: + yield from session_managed_iterator(create_plugin_storage) diff --git a/boefjes/tests/examples/adr-validator-normalize.json b/boefjes/tests/examples/adr-validator-normalize.json index 7b459296e47..04ae8585aaf 100644 --- a/boefjes/tests/examples/adr-validator-normalize.json +++ b/boefjes/tests/examples/adr-validator-normalize.json @@ -35,6 +35,6 @@ ] }, "normalizer": { - "id": "adr-validator" + "id": "adr-validator-normalize" } } diff --git a/rocky/tests/stubs/katalogus_boefjes.json b/rocky/tests/stubs/katalogus_boefjes.json index bb79a029eb4..40c79a6cb1b 100644 --- a/rocky/tests/stubs/katalogus_boefjes.json +++ b/rocky/tests/stubs/katalogus_boefjes.json @@ -3,13 +3,11 @@ "id": "binaryedge", "name": "BinaryEdge", "version": null, - "authors": null, "created": null, "description": "Use BinaryEdge to find open ports with vulnerabilities that are found on that port", "environment_keys": [ "BINARYEDGE_API" ], - "related": null, "enabled": true, "type": "boefje", "scan_level": 2, @@ -33,11 +31,9 @@ "id": "ssl-certificates", "name": "SSLCertificates", "version": null, - "authors": null, "created": null, "description": "Scan SSL certificates of websites", "environment_keys": [], - "related": null, "enabled": false, "type": "boefje", "scan_level": 1, @@ -53,11 +49,9 @@ "id": "ssl-version", "name": "SSLScan", "version": null, - "authors": null, "created": null, "description": "Scan SSL/TLS versions of websites", "environment_keys": [], - "related": null, "enabled": false, "type": "boefje", "scan_level": 2, @@ -74,13 +68,11 @@ "id": "wp-scan", "name": "WPScantest", "version": null, - "authors": null, "created": null, "description": "Scan wordpress sites", "environment_keys": [ "WP_SCAN_API" ], - "related": null, "enabled": false, "type": "boefje", "scan_level": 2, @@ -97,11 +89,9 @@ "id": "kat_binaryedge_containers", "name": "test_binary_edge_normalizer", "version": null, - "authors": null, "created": null, "description": null, "environment_keys": [], - "related": null, "enabled": true, "type": "normalizer", "consumes": [ diff --git a/rocky/tests/stubs/katalogus_normalizers.json b/rocky/tests/stubs/katalogus_normalizers.json index cdd1b68139b..182b5bbb549 100644 --- a/rocky/tests/stubs/katalogus_normalizers.json +++ b/rocky/tests/stubs/katalogus_normalizers.json @@ -3,11 +3,9 @@ "id": "kat_adr_finding_types_normalize", "name": "Adr Finding Types Normalize", "version": null, - "authors": null, "created": null, "description": null, "environment_keys": [], - "related": null, "enabled": true, "type": "normalizer", "consumes": [ @@ -22,11 +20,9 @@ "id": "adr-validator", "name": null, "version": null, - "authors": null, "created": null, "description": null, "environment_keys": [], - "related": null, "enabled": true, "type": "normalizer", "consumes": [ @@ -44,11 +40,9 @@ "id": "kat_answer_parser", "name": null, "version": null, - "authors": null, "created": null, "description": null, "environment_keys": [], - "related": null, "enabled": true, "type": "normalizer", "consumes": [ @@ -63,11 +57,9 @@ "id": "kat_binaryedge_containers", "name": null, "version": null, - "authors": null, "created": null, "description": null, "environment_keys": [], - "related": null, "enabled": true, "type": "normalizer", "consumes": [ @@ -89,11 +81,9 @@ "id": "kat_binaryedge_databases", "name": null, "version": null, - "authors": null, "created": null, "description": null, "environment_keys": [], - "related": null, "enabled": true, "type": "normalizer", "consumes": [