Merge branch 'main' into fix/upgrade-pydantic-v2

boefjes/boefjes/plugins/kat_binaryedge/containers/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_containers",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/databases/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_databases",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/http_web/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_http_web",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_message_queues",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/protocols/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_protocols",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_remote_desktop",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_service_identification",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_binaryedge/services/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_binaryedge_services",
     "consumes": [
-        "binaryedge"
+        "boefje/binaryedge"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_rdns/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_rdns_normalize",
     "consumes": [
-        "rdns"
+        "boefje/rdns"
     ],
     "produces": [
         "DNSPTRRecord"

boefjes/boefjes/plugins/kat_retirejs_finding_types/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_retirejs_finding_types_normalize",
     "consumes": [
-        "retirejs-finding-types"
+        "boefje/retirejs-finding-types"
     ],
     "produces": [
         "RetireJSFindingType"

boefjes/boefjes/plugins/kat_security_txt_downloader/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_sec_txt_downloader_normalize",
     "consumes": [
-        "security_txt_downloader"
+        "boefje/security_txt_downloader"
     ],
     "produces": [
         "SecurityTXT",

boefjes/boefjes/plugins/kat_shodan/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_shodan_normalize",
     "consumes": [
-        "shodan"
+        "boefje/shodan"
     ],
     "produces": [
         "Finding",

boefjes/boefjes/plugins/kat_snyk/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_snyk_normalize",
     "consumes": [
-        "snyk"
+        "boefje/snyk"
     ],
     "produces": [
         "Finding",

boefjes/boefjes/plugins/kat_snyk_finding_types/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_snyk_finding_types_normalize",
     "consumes": [
-        "snyk-finding-types"
+        "boefje/snyk-finding-types"
     ],
     "produces": [
         "SNYKFindingType"

boefjes/boefjes/plugins/kat_ssl_certificates/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_ssl_certificates_normalize",
     "consumes": [
-        "ssl-certificates"
+        "boefje/ssl-certificates"
     ],
     "produces": [
         "X509Certificate"

boefjes/boefjes/plugins/kat_ssl_scan/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_ssl_scan_normalize",
     "consumes": [
-        "ssl-version"
+        "boefje/ssl-version"
     ],
     "produces": [
         "KATFindingType",

boefjes/boefjes/plugins/kat_testssl_sh_ciphers/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_ssl_test_ciphers_normalize",
     "consumes": [
-        "testssl-sh-ciphers"
+        "boefje/testssl-sh-ciphers"
     ],
     "produces": [
         "TLSCipher"

boefjes/boefjes/plugins/kat_website_software/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_website_software_normalize",
     "consumes": [
-        "website-software"
+        "boefje/website-software"
     ],
     "produces": [
         "Software",

boefjes/boefjes/plugins/kat_wpscan/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "kat_wpscan_normalize",
     "consumes": [
-        "wp-scan"
+        "boefje/wp-scan"
     ],
     "produces": [
         "Finding",

boefjes/boefjes/plugins/pdio_subfinder/normalizer.json

@@ -1,7 +1,7 @@
 {
     "id": "pdio-subfinder-normalizer",
     "consumes": [
-        "pdio-subfinder"
+        "boefje/pdio-subfinder"
     ],
     "produces": [
         "Hostname"

bytes/tests/integration/test_meta_repository.py

@@ -110,7 +110,7 @@ def test_save_raw(meta_repository: SQLMetaDataRepository) -> None:
     )
     first_updated_raw = meta_repository.get_raw(query_filter).pop()
 
-    assert first_updated_raw.signing_provider_url == "https://test"
+    assert first_updated_raw.signing_provider_url in ["https://test", "https://freetsa.org/tsr"]  # Depends on CI env
     assert "hash_retrieval_link" in first_updated_raw.json()
     assert "secure_hash" in first_updated_raw.json()
     assert "signing_provider" in first_updated_raw.json()

octopoes/octopoes/repositories/ooi_repository.py

@@ -599,7 +599,7 @@ def count_findings_by_severity(self, valid_time: datetime) -> Counter:
         """
 
         for finding_type_name, finding_type_object, finding_count in self.session.client.query(
-            str(query), valid_time=valid_time
+            query, valid_time=valid_time
         ):
             if not finding_type_object:
                 logger.warning(
@@ -712,4 +712,4 @@ def list_findings(
         )
 
     def query(self, query: Query, valid_time: datetime) -> List[OOI]:
-        return [self.deserialize(row[0]) for row in self.session.client.query(str(query), valid_time=valid_time)]
+        return [self.deserialize(row[0]) for row in self.session.client.query(query, valid_time=valid_time)]

octopoes/octopoes/xtdb/client.py

@@ -10,6 +10,7 @@
 from requests import HTTPError, Response
 
 from octopoes.xtdb.exceptions import NodeNotFound, NoMultinode, XTDBException
+from octopoes.xtdb.query import Query
 
 logger = logging.getLogger(__name__)
 
@@ -93,13 +94,13 @@ def get_entity(self, entity_id: str, valid_time: Optional[datetime] = None) -> d
         self._verify_response(res)
         return res.json()
 
-    def query(self, query: str, valid_time: Optional[datetime] = None) -> List[List[Any]]:
+    def query(self, query: Union[str, Query], valid_time: Optional[datetime] = None) -> List[List[Any]]:
         if valid_time is None:
             valid_time = datetime.now(timezone.utc)
         res = self._session.post(
             f"{self.client_url()}/query",
             params={"valid-time": valid_time.isoformat()},
-            data=query,
+            data=str(query),
             headers={"Content-Type": "application/edn"},
         )
         self._verify_response(res)

octopoes/octopoes/xtdb/query.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from typing import List, Optional, Set, Type, Union
+from typing import Dict, List, Optional, Set, Type, Union
 from uuid import UUID, uuid4
 
 from octopoes.models import OOI
@@ -93,7 +93,8 @@ def from_path(cls, path: Path) -> "Query":
 
         ooi_type = path.segments[-1].target_type
         query = cls(ooi_type)
-        alias_map = {}
+        target_ref = None
+        alias_map: Dict[str, Ref] = {}
 
         for segment in path.segments:
             source_ref = alias_map.get(segment.source_type.get_object_type(), segment.source_type)
@@ -113,15 +114,18 @@ def from_path(cls, path: Path) -> "Query":
             else:
                 query = query.where(target_ref, **{segment.property_name: source_ref})
 
+        if target_ref:  # Make sure we use the last reference in the path as a target
+            query.result_type = target_ref
+
         return query
 
-    def count(self, ooi_type: Ref) -> "Query":
-        self._find_clauses.append(f"(count {self._get_object_alias(ooi_type)})")
+    def pull(self, ooi_type: Ref) -> "Query":
+        self._find_clauses.append(f"(pull {self._get_object_alias(ooi_type)} [*])")
 
         return self
 
-    def group_by(self, ooi_type: Ref) -> "Query":
-        self._find_clauses.append(f"(pull {self._get_object_alias(ooi_type)} [*])")
+    def count(self, ooi_type: Ref) -> "Query":
+        self._find_clauses.append(f"(count {self._get_object_alias(ooi_type)})")
 
         return self
 

octopoes/tests/conftest.py

@@ -1,5 +1,6 @@
+import uuid
 from datetime import datetime, timezone
-from ipaddress import IPv4Address
+from ipaddress import IPv4Address, ip_address
 from typing import Dict, Iterator, List, Optional, Set
 from unittest.mock import Mock
 
@@ -8,13 +9,15 @@
 from requests.adapters import HTTPAdapter, Retry
 
 from octopoes.api.api import app
+from octopoes.api.models import Declaration, Observation
 from octopoes.api.router import settings
 from octopoes.config.settings import Settings, XTDBType
 from octopoes.connector.octopoes import OctopoesAPIConnector
 from octopoes.core.app import get_xtdb_client
 from octopoes.core.service import OctopoesService
 from octopoes.events.manager import EventManager
 from octopoes.models import OOI, DeclaredScanProfile, EmptyScanProfile, Reference, ScanProfileBase
+from octopoes.models.ooi.network import IPAddressV6
 from octopoes.models.path import Direction, Path
 from octopoes.models.types import (
     DNSZone,
@@ -256,3 +259,54 @@ def mock_xtdb_session():
 @pytest.fixture
 def origin_repository(mock_xtdb_session):
     yield XTDBOriginRepository(Mock(spec=EventManager), mock_xtdb_session, XTDBType.XTDB_MULTINODE)
+
+
+def seed_system(octopoes_api_connector: OctopoesAPIConnector, valid_time):
+    network = Network(name="test")
+    octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time))
+
+    hostnames = [
+        Hostname(network=network.reference, name="example.com"),
+        Hostname(network=network.reference, name="a.example.com"),
+        Hostname(network=network.reference, name="b.example.com"),
+        Hostname(network=network.reference, name="c.example.com"),
+        Hostname(network=network.reference, name="d.example.com"),
+        Hostname(network=network.reference, name="e.example.com"),
+        Hostname(network=network.reference, name="f.example.com"),
+    ]
+
+    addresses = [
+        IPAddressV4(network=network.reference, address=ip_address("192.0.2.3")),
+        IPAddressV6(network=network.reference, address=ip_address("3e4d:64a2:cb49:bd48:a1ba:def3:d15d:9230")),
+    ]
+    ports = [
+        IPPort(address=addresses[0].reference, protocol="tcp", port=25),
+        IPPort(address=addresses[0].reference, protocol="tcp", port=443),
+        IPPort(address=addresses[0].reference, protocol="tcp", port=22),
+        IPPort(address=addresses[1].reference, protocol="tcp", port=80),
+    ]
+    services = [Service(name="smtp"), Service(name="https"), Service(name="http"), Service(name="ssh")]
+    ip_services = [
+        IPService(ip_port=ports[0].reference, service=services[0].reference),
+        IPService(ip_port=ports[1].reference, service=services[1].reference),
+        IPService(ip_port=ports[2].reference, service=services[3].reference),
+        IPService(ip_port=ports[3].reference, service=services[2].reference),
+    ]
+
+    resolved_hostnames = [
+        ResolvedHostname(hostname=hostnames[0].reference, address=addresses[0].reference),  # ipv4
+        ResolvedHostname(hostname=hostnames[0].reference, address=addresses[1].reference),  # ipv6
+        ResolvedHostname(hostname=hostnames[1].reference, address=addresses[0].reference),
+        ResolvedHostname(hostname=hostnames[2].reference, address=addresses[0].reference),
+        ResolvedHostname(hostname=hostnames[3].reference, address=addresses[0].reference),
+        ResolvedHostname(hostname=hostnames[4].reference, address=addresses[0].reference),
+        ResolvedHostname(hostname=hostnames[5].reference, address=addresses[0].reference),
+        ResolvedHostname(hostname=hostnames[3].reference, address=addresses[1].reference),
+        ResolvedHostname(hostname=hostnames[4].reference, address=addresses[1].reference),
+        ResolvedHostname(hostname=hostnames[6].reference, address=addresses[1].reference),
+    ]
+
+    oois = hostnames + addresses + ports + services + ip_services + resolved_hostnames
+    octopoes_api_connector.save_observation(
+        Observation(method="", source=network.reference, task_id=uuid.uuid4(), valid_time=valid_time, result=oois)
+    )