Merge branch 'poc/mula/combined-schedulers' into feat/boefje/combined…

…-schedulers-integration

.pre-commit-config.yaml

@@ -27,6 +27,10 @@ repos:
       - id: fix-byte-order-marker
       - id: pretty-format-json
         args: ["--autofix", "--no-ensure-ascii", "--no-sort-keys"]
+        exclude: |
+          (?x)(
+          ^boefjes/boefjes/plugins/kat_wappalyzer/technologies.json |
+          )
 
   - repo: https://github.com/abravalheri/validate-pyproject
     rev: v0.23
@@ -122,6 +126,7 @@ repos:
           requirements-.*.txt$ |
           retirejs.json$ |
           ^boefjes/boefjes/plugins/kat_fierce/lists |
+          ^boefjes/boefjes/plugins/kat_wappalyzer/data/.*.json |
           ^boefjes/tests/examples/inputs/cve-result-without-cvss.json |
           ^boefjes/tests/examples |
           ^keiko/glossaries |

boefjes/Dockerfile

@@ -20,11 +20,9 @@ RUN --mount=type=cache,target=/root/.cache \
     pip install --upgrade pip \
     && if [ "$ENVIRONMENT" = "dev" ]; \
     then \
-    grep -v git+https:// requirements-dev.txt | pip install -r /dev/stdin && \
-    grep git+https:// requirements-dev.txt | pip install -r /dev/stdin ; \
+    pip install -r requirements-dev.txt; \
     else \
-    grep -v git+https:// requirements.txt | pip install -r /dev/stdin && \
-    grep git+https:// requirements.txt | pip install -r /dev/stdin ; \
+    pip install -r requirements.txt; \
     fi
 
 FROM dev

boefjes/boefjes/clients/scheduler_client.py

@@ -83,7 +83,10 @@ def pop_item(self, scheduler_id: str) -> Task | None:
         self._verify_response(response)
 
         page = TypeAdapter(PaginatedTasksResponse | None).validate_json(response.content)
-        return TypeAdapter(Task | None).validate_json(page.results[0]) if page else None
+        if page.count == 0:
+            return None
+
+        return page.results[0]
 
     def pop_items(self, scheduler_id: str, filters: dict[str, Any]) -> PaginatedTasksResponse | None:
         response = self._session.post(f"/schedulers/{scheduler_id}/pop", json=filters)

boefjes/boefjes/plugins/kat_dns/main.py

@@ -85,6 +85,8 @@ def get_parent_zone_soa(resolver: dns.resolver.Resolver, name: Name) -> Answer:
 def get_email_security_records(resolver: dns.resolver.Resolver, hostname: str, record_subdomain: str) -> str:
     try:
         answer = resolver.resolve(f"{record_subdomain}.{hostname}", "TXT", raise_on_no_answer=False)
+        if answer.rrset is None:
+            return "NXDOMAIN"
         return answer.response.to_text()
     except dns.resolver.NoNameservers as error:
         # no servers responded happily, we'll check the response from the first

boefjes/boefjes/plugins/kat_kat_finding_types/kat_finding_types.json

@@ -419,6 +419,20 @@
     "impact": "E-mail from this domain can potentially be spoofed if DMARC is not (properly) implemented in combination with DKIM and SPF.",
     "recommendation": "Fix the syntax of the SPF record."
   },
+  "KAT-EXPENSIVE-SPF": {
+    "description": "This SPF record contains an expensive SPF construction.",
+    "source": "https://www.rfc-editor.org/rfc/rfc7208#section-4.6.4",
+    "risk": "low",
+    "impact": "Various recipient mailservers might not perform all requested lookups and bounce email because of missed allowed addresses, or bounce mail entirely due to too many dns lookups.",
+    "recommendation": "Consolidate the SPF record, remove unneeded lookups and mechanisms."
+  },
+  "KAT-DEPRECATED-SPF-MECHANISM": {
+    "description": "This SPF record contains a deprecated SPF mechanism.",
+    "source": "https://www.rfc-editor.org/rfc/rfc7208#section-5.5",
+    "risk": "low",
+    "impact": "Deprecated mechanism is used. It should not be used.",
+    "recommendation": "Fix the SPF record, remove deprecated mechanisms."
+  },
   "SUB-DOMAIN-TAKEOVER": {
     "description": "Subdomain takeover is when an attacker takes control of an unused or improperly configured subdomain, potentially accessing sensitive information or conducting phishing attacks.",
     "source": "https://developer.mozilla.org/en-US/docs/Web/Security/Subdomain_takeovers",