Add Boefje and Normalizer models in the KATalogus

boefjes/boefjes/katalogus/api/plugins.py

@@ -1,8 +1,9 @@
+import datetime
 from functools import partial
 
 from fastapi import APIRouter, Body, Depends, HTTPException, status
 from fastapi.responses import FileResponse, JSONResponse, Response
-from httpx import HTTPStatusError
+from pydantic import BaseModel, Field
 
 from boefjes.katalogus.api.organisations import check_organisation_exists
 from boefjes.katalogus.dependencies.plugins import (
@@ -12,6 +13,8 @@
     get_plugins_filter_parameters,
 )
 from boefjes.katalogus.models import FilterParameters, PaginationParameters, PluginType
+from boefjes.katalogus.storage.interfaces import PluginStorage
+from boefjes.sql.plugin_storage import get_plugin_storage
 
 router = APIRouter(
     prefix="/organisations/{organisation_id}",
@@ -82,46 +85,107 @@ def get_plugin(
             return p.by_plugin_id(plugin_id, organisation_id)
     except KeyError:
         raise HTTPException(status.HTTP_404_NOT_FOUND, "Plugin not found")
-    except HTTPStatusError as ex:
-        raise HTTPException(ex.response.status_code)
 
 
-@router.patch("/plugins/{plugin_id}")
+@router.post("/plugins", status_code=status.HTTP_201_CREATED)
+def add_plugin(plugin: PluginType, plugin_service: PluginService = Depends(get_plugin_service)):
+    with plugin_service as service:
+        if plugin.type == "boefje":
+            return service.create_boefje(plugin)
+
+        if plugin.type == "normalizer":
+            return service.create_normalizer(plugin)
+
+    raise HTTPException(status.HTTP_400_BAD_REQUEST, "Creation of Bits is not supported")
+
+
+@router.patch("/plugins/{plugin_id}", status_code=status.HTTP_204_NO_CONTENT)
 def update_plugin_state(
     plugin_id: str,
     organisation_id: str,
     enabled: bool = Body(False, embed=True),
     plugin_service: PluginService = Depends(get_plugin_service),
 ):
-    try:
-        with plugin_service as p:
-            p.update_by_id(plugin_id, organisation_id, enabled)
-    except HTTPStatusError as ex:
-        raise HTTPException(ex.response.status_code)
+    with plugin_service as p:
+        p.set_enabled_by_id(plugin_id, organisation_id, enabled)
+
+
+class BoefjeIn(BaseModel):
+    """
+    For patching, we need all fields to be optional, hence we overwrite the definition here.
+    Also see https://fastapi.tiangolo.com/tutorial/body-updates/ as a reference.
+    """
+
+    name: str | None = None
+    version: str | None = None
+    created: datetime.datetime | None = None
+    description: str | None = None
+    environment_keys: list[str] = Field(default_factory=list)
+    scan_level: int = 1
+    consumes: set[str] = Field(default_factory=set)
+    produces: set[str] = Field(default_factory=set)
+    oci_image: str | None = None
+    oci_arguments: list[str] = Field(default_factory=list)
+
+
+@router.patch("/boefjes/{boefje_id}", status_code=status.HTTP_204_NO_CONTENT)
+def update_boefje(
+    boefje_id: str,
+    boefje: BoefjeIn,
+    storage: PluginStorage = Depends(get_plugin_storage),
+):
+    with storage as p:
+        p.update_boefje(boefje_id, boefje.model_dump(exclude_unset=True))
+
+
+@router.delete("/boefjes/{boefje_id}", status_code=status.HTTP_204_NO_CONTENT)
+def delete_boefje(boefje_id: str, plugin_storage: PluginStorage = Depends(get_plugin_storage)):
+    with plugin_storage as p:
+        p.delete_boefje_by_id(boefje_id)
+
+
+class NormalizerIn(BaseModel):
+    """
+    For patching, we need all fields to be optional, hence we overwrite the definition here.
+    Also see https://fastapi.tiangolo.com/tutorial/body-updates/ as a reference.
+    """
+
+    name: str | None = None
+    version: str | None = None
+    created: datetime.datetime | None = None
+    description: str | None = None
+    environment_keys: list[str] = Field(default_factory=list)
+    consumes: list[str] = Field(default_factory=list)  # mime types (and/ or boefjes)
+    produces: list[str] = Field(default_factory=list)  # oois
+
+
+@router.patch("/normalizers/{normalizer_id}", status_code=status.HTTP_204_NO_CONTENT)
+def update_normalizer(
+    normalizer_id: str,
+    normalizer: NormalizerIn,
+    storage: PluginStorage = Depends(get_plugin_storage),
+):
+    with storage as p:
+        p.update_normalizer(normalizer_id, normalizer.model_dump(exclude_unset=True))
+
+
+@router.delete("/normalizers/{normalizer_id}", status_code=status.HTTP_204_NO_CONTENT)
+def delete_normalizer(normalizer_id: str, plugin_storage: PluginStorage = Depends(get_plugin_storage)):
+    with plugin_storage as p:
+        p.delete_normalizer_by_id(normalizer_id)
 
 
 @router.get("/plugins/{plugin_id}/schema.json", include_in_schema=False)
-def get_plugin_schema(
-    plugin_id: str,
-    plugin_service: PluginService = Depends(get_plugin_service),
-) -> JSONResponse:
-    try:
-        with plugin_service as p:
-            return JSONResponse(p.schema(plugin_id))
-    except HTTPStatusError as ex:
-        raise HTTPException(ex.response.status_code)
+def get_plugin_schema(plugin_id: str, plugin_service: PluginService = Depends(get_plugin_service)) -> JSONResponse:
+    return JSONResponse(plugin_service.schema(plugin_id))
 
 
 @router.get("/plugins/{plugin_id}/cover.jpg", include_in_schema=False)
 def get_plugin_cover(
     plugin_id: str,
     plugin_service: PluginService = Depends(get_plugin_service),
 ) -> FileResponse:
-    try:
-        with plugin_service as p:
-            return FileResponse(p.cover(plugin_id))
-    except HTTPStatusError as ex:
-        raise HTTPException(ex.response.status_code)
+    return FileResponse(plugin_service.cover(plugin_id))
 
 
 @router.get("/plugins/{plugin_id}/description.md", include_in_schema=False)
@@ -130,18 +194,10 @@ def get_plugin_description(
     organisation_id: str,
     plugin_service: PluginService = Depends(get_plugin_service),
 ) -> Response:
-    try:
-        with plugin_service as p:
-            return Response(p.description(plugin_id, organisation_id))
-    except HTTPStatusError as ex:
-        raise HTTPException(ex.response.status_code)
+    return Response(plugin_service.description(plugin_id, organisation_id))
 
 
-@router.post("/settings/clone/{to_organisation_id}")
-def clone_organisation_settings(
-    organisation_id: str,
-    to_organisation_id: str,
-    storage: PluginService = Depends(get_plugin_service),
-):
+@router.post("/settings/clone/{to_org}")
+def clone_settings(organisation_id: str, to_org: str, storage: PluginService = Depends(get_plugin_service)):
     with storage as store:
-        store.clone_settings_to_organisation(organisation_id, to_organisation_id)
+        store.clone_settings_to_organisation(organisation_id, to_org)

boefjes/boefjes/katalogus/api/root.py

@@ -17,7 +17,7 @@
 from boefjes.config import settings
 from boefjes.katalogus.api import organisations, plugins
 from boefjes.katalogus.api import settings as settings_router
-from boefjes.katalogus.storage.interfaces import StorageError
+from boefjes.katalogus.storage.interfaces import NotFound, StorageError
 from boefjes.katalogus.version import __version__
 
 with settings.log_cfg.open() as f:
@@ -51,16 +51,22 @@
 app.include_router(router, prefix="/v1")
 
 
-@app.exception_handler(StorageError)
-def entity_not_found_handler(request: Request, exc: StorageError):
-    logger.exception("some error", exc_info=exc)
-
+@app.exception_handler(NotFound)
+def entity_not_found_handler(request: Request, exc: NotFound):
     return JSONResponse(
         status_code=status.HTTP_404_NOT_FOUND,
         content={"message": exc.message},
     )
 
 
+@app.exception_handler(StorageError)
+def storage_error_handler(request: Request, exc: StorageError):
+    return JSONResponse(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        content={"message": exc.message},
+    )
+
+
 class ServiceHealth(BaseModel):
     service: str
     healthy: bool = False

boefjes/boefjes/katalogus/dependencies/plugins.py

@@ -10,15 +10,18 @@
 from sqlalchemy.orm import Session
 
 from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
-from boefjes.katalogus.models import FilterParameters, PaginationParameters, PluginType
+from boefjes.katalogus.models import Boefje, FilterParameters, Normalizer, PaginationParameters, PluginType
 from boefjes.katalogus.storage.interfaces import (
+    ExistingPluginId,
     NotFound,
     PluginEnabledStorage,
+    PluginStorage,
     SettingsNotConformingToSchema,
     SettingsStorage,
 )
 from boefjes.sql.db import session_managed_iterator
 from boefjes.sql.plugin_enabled_storage import create_plugin_enabled_storage
+from boefjes.sql.plugin_storage import create_plugin_storage
 from boefjes.sql.setting_storage import create_setting_storage
 
 logger = logging.getLogger(__name__)
@@ -27,24 +30,40 @@
 class PluginService:
     def __init__(
         self,
+        plugin_storage: PluginStorage,
         plugin_enabled_store: PluginEnabledStorage,
         settings_storage: SettingsStorage,
         local_repo: LocalPluginRepository,
     ):
+        self.plugin_storage = plugin_storage
         self.plugin_enabled_store = plugin_enabled_store
         self.settings_storage = settings_storage
         self.local_repo = local_repo
 
     def __enter__(self):
         self.plugin_enabled_store.__enter__()
+        self.plugin_storage.__enter__()
+        self.settings_storage.__enter__()
 
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.plugin_enabled_store.__exit__(exc_type, exc_val, exc_tb)
+        self.plugin_storage.__exit__(exc_type, exc_val, exc_tb)
+        self.settings_storage.__exit__(exc_type, exc_val, exc_tb)
 
     def get_all(self, organisation_id: str) -> list[PluginType]:
-        return [self._set_plugin_enabled(plugin, organisation_id) for plugin in self.local_repo.get_all()]
+        all_plugins = self.get_all_without_enabled()
+
+        return [self._set_plugin_enabled(plugin, organisation_id) for plugin in all_plugins.values()]
+
+    def get_all_without_enabled(self):
+        all_plugins = {plugin.id: plugin for plugin in self.local_repo.get_all()}
+
+        for plugin in self.plugin_storage.get_all():
+            all_plugins[plugin.id] = plugin
+
+        return all_plugins
 
     def by_plugin_id(self, plugin_id: str, organisation_id: str) -> PluginType:
         all_plugins = self.get_all(organisation_id)
@@ -72,22 +91,36 @@ def get_all_settings(self, organisation_id: str, plugin_id: str):
         return self.settings_storage.get_all(organisation_id, plugin_id)
 
     def clone_settings_to_organisation(self, from_organisation: str, to_organisation: str):
-        # One requirement is that we also do not keep previously enabled boefjes enabled of they are not copied.
+        # One requirement is that only boefjes enabled in the from_organisation end up being enabled for the target.
         for plugin_id in self.plugin_enabled_store.get_all_enabled(to_organisation):
-            self.update_by_id(plugin_id, to_organisation, enabled=False)
+            self.set_enabled_by_id(plugin_id, to_organisation, enabled=False)
 
         for plugin in self.get_all(from_organisation):
             if all_settings := self.get_all_settings(from_organisation, plugin.id):
                 self.upsert_settings(all_settings, to_organisation, plugin.id)
 
         for plugin_id in self.plugin_enabled_store.get_all_enabled(from_organisation):
-            self.update_by_id(plugin_id, to_organisation, enabled=True)
+            self.set_enabled_by_id(plugin_id, to_organisation, enabled=True)
 
     def upsert_settings(self, values: dict, organisation_id: str, plugin_id: str):
         self._assert_settings_match_schema(values, organisation_id, plugin_id)
 
         return self.settings_storage.upsert(values, organisation_id, plugin_id)
 
+    def create_boefje(self, boefje: Boefje) -> None:
+        try:
+            self.local_repo.by_id(boefje.id)
+            raise ExistingPluginId(boefje.id)
+        except KeyError:
+            self.plugin_storage.create_boefje(boefje)
+
+    def create_normalizer(self, normalizer: Normalizer) -> None:
+        try:
+            self.local_repo.by_id(normalizer.id)
+            raise ExistingPluginId(normalizer.id)
+        except KeyError:
+            self.plugin_storage.create_normalizer(normalizer)
+
     def delete_settings(self, organisation_id: str, plugin_id: str):
         self.settings_storage.delete(organisation_id, plugin_id)
 
@@ -96,7 +129,7 @@ def delete_settings(self, organisation_id: str, plugin_id: str):
         except SettingsNotConformingToSchema:
             logger.warning("Making sure %s is disabled for %s because settings are deleted", plugin_id, organisation_id)
 
-            self.update_by_id(plugin_id, organisation_id, False)
+            self.set_enabled_by_id(plugin_id, organisation_id, False)
 
     def schema(self, plugin_id: str) -> dict | None:
         return self.local_repo.schema(plugin_id)
@@ -119,7 +152,7 @@ def description(self, plugin_id: str, organisation_id: str) -> str:
             logger.error("Plugin not found: %s", plugin_id)
             return ""
 
-    def update_by_id(self, plugin_id: str, organisation_id: str, enabled: bool):
+    def set_enabled_by_id(self, plugin_id: str, organisation_id: str, enabled: bool):
         if enabled:
             all_settings = self.settings_storage.get_all(organisation_id, plugin_id)
             self._assert_settings_match_schema(all_settings, organisation_id, plugin_id)
@@ -149,6 +182,7 @@ def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> Plugi
 def get_plugin_service(organisation_id: str) -> Iterator[PluginService]:
     def closure(session: Session):
         return PluginService(
+            create_plugin_storage(session),
             create_plugin_enabled_storage(session),
             create_setting_storage(session),
             get_local_repository(),

boefjes/boefjes/katalogus/local_repository.py

@@ -43,7 +43,7 @@ def by_id(self, plugin_id: str) -> PluginType:
         if plugin_id in normalizers:
             return normalizers[plugin_id].normalizer
 
-        raise Exception(f"Can't find plugin {plugin_id}")
+        raise KeyError(f"Can't find plugin {plugin_id}")
 
     def schema(self, id_: str) -> dict | None:
         boefjes = self.resolve_boefjes()

boefjes/boefjes/katalogus/models.py

@@ -14,12 +14,11 @@ class Plugin(BaseModel):
     id: str
     name: str | None = None
     version: str | None = None
-    authors: list[str] | None = None
     created: datetime.datetime | None = None
     description: str | None = None
     environment_keys: list[str] = Field(default_factory=list)
-    related: list[str] | None = None
     enabled: bool = False
+    static: bool = True  # We need to differentiate between local and remote plugins to know which ones can be deleted
 
     def __str__(self):
         return f"{self.id}:{self.version}"
@@ -30,7 +29,6 @@ class Boefje(Plugin):
     scan_level: int = 1
     consumes: set[str] = Field(default_factory=set)
     produces: set[str] = Field(default_factory=set)
-    options: list[str] | None = None
     runnable_hash: str | None = None
     oci_image: str | None = None
     oci_arguments: list[str] = Field(default_factory=list)