From 89cf4f04db9ee5dda435553f0661a1d5af5e65a2 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 8 Jan 2025 11:11:11 +0100 Subject: [PATCH 01/12] DEMO --- octopoes/bits/https_availability/bit.py | 15 ---- .../https_availability/__init__.py | 0 .../https_availability/https_availability.py | 5 +- octopoes/nibbles/https_availability/nibble.py | 50 +++++++++++++ .../octopoes/repositories/ooi_repository.py | 1 + .../test_https_availability_nibble.py | 71 +++++++++++++++++++ 6 files changed, 124 insertions(+), 18 deletions(-) delete mode 100644 octopoes/bits/https_availability/bit.py rename octopoes/{bits => nibbles}/https_availability/__init__.py (100%) rename octopoes/{bits => nibbles}/https_availability/https_availability.py (78%) create mode 100644 octopoes/nibbles/https_availability/nibble.py create mode 100644 octopoes/tests/integration/test_https_availability_nibble.py diff --git a/octopoes/bits/https_availability/bit.py b/octopoes/bits/https_availability/bit.py deleted file mode 100644 index 762dde5e3dc..00000000000 --- a/octopoes/bits/https_availability/bit.py +++ /dev/null @@ -1,15 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.network import IPAddress, IPPort -from octopoes.models.ooi.web import Website - -BIT = BitDefinition( - id="https-availability", - consumes=IPAddress, - parameters=[ - BitParameterDefinition(ooi_type=IPPort, relation_path="address"), - BitParameterDefinition( - ooi_type=Website, relation_path="ip_service.ip_port.address" - ), # we place the findings on the http websites - ], - module="bits.https_availability.https_availability", -) diff --git a/octopoes/bits/https_availability/__init__.py b/octopoes/nibbles/https_availability/__init__.py similarity index 100% rename from octopoes/bits/https_availability/__init__.py rename to octopoes/nibbles/https_availability/__init__.py diff --git a/octopoes/bits/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py similarity index 78% rename from octopoes/bits/https_availability/https_availability.py rename to octopoes/nibbles/https_availability/https_availability.py index 2abec18b340..b3386c684ff 100644 --- a/octopoes/bits/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -1,13 +1,12 @@ from collections.abc import Iterator -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.network import IPAddress, IPPort +from octopoes.models.ooi.network import IPPort from octopoes.models.ooi.web import Website -def run(input_ooi: IPAddress, additional_oois: list[IPPort | Website], config: dict[str, Any]) -> Iterator[OOI]: +def nibble() -> Iterator[OOI]: websites = [website for website in additional_oois if isinstance(website, Website)] open_ports = [port.port for port in additional_oois if isinstance(port, IPPort)] diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py new file mode 100644 index 00000000000..df5c6f40867 --- /dev/null +++ b/octopoes/nibbles/https_availability/nibble.py @@ -0,0 +1,50 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.network import IPAddress, IPPort +from octopoes.models.ooi.web import Website + + +def query(targets: list[Reference | None]) -> str: + def pull(statements: list[str]) -> str: + return f""" + {{ + :query {{ + :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?ipport443 [*]) (pull ?website [*])] + :where [ + {" ".join(statements)} + ] + }} + }} + """ + + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "1000": + return pull( + [ + f""" + [?ipaddress :object_type "IPAddressV4"] + [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] + [?ipport80 :IPPort/address ?ipaddress] + [?ipport80 :IPPort/port 80] + (or + (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) + (and [(identity 0) ?ipport443]) + ) + [?ip_service :IPService/ip_port ?ipport80] + [?website :Website/ip_service ?ip_service] + """ + ] + ) + return "potato" + + +NIBBLE = NibbleDefinition( + id="https-availability", + signature=[ + NibbleParameter(object_type=IPAddress, parser="[*][?object_type == 'IPAddressV4'][]"), + NibbleParameter(object_type=IPPort, parser='[*][?"IPPort/port" == "80"][]'), + NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), + NibbleParameter(object_type=int, parser='[length([*][?"IPPort/port" == "443"])]'), + ], + query=query, +) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index c171c42c19a..a67707765c9 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -927,6 +927,7 @@ def nibble_query( arguments = [None for _ in nibble.signature] query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) data = self.session.client.query(query, valid_time) + breakpoint() objects = [ {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} for element in nibble.signature diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py new file mode 100644 index 00000000000..168210d197c --- /dev/null +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -0,0 +1,71 @@ +import os +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.https_availability.nibble import NIBBLE as https_availability +from nibbles.runner import NibblesRunner + +from octopoes.core.service import OctopoesService +from octopoes.models import Reference +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol +from octopoes.models.ooi.service import IPService, Service +from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + +STATIC_IP = ".".join((4 * "1 ").split()) + + +def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {https_availability.id: https_availability} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + web_url = HostnameHTTPURL( + network=network.reference, netloc=hostname.reference, port=443, path="/", scheme=WebScheme.HTTP + ) + xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) + + service = Service(name="https") + xtdb_octopoes_service.ooi_repository.save(service, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=80, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port443, valid_time) + + ip_service = IPService(ip_port=port.reference, service=service.reference) + xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) + + website = Website(ip_service=ip_service.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(website, valid_time) + + resource = HTTPResource(website=website.reference, web_url=web_url.reference) + xtdb_octopoes_service.ooi_repository.save(resource, valid_time) + + header = HTTPHeader( + resource=resource.reference, key="strict-transport-security", value="max-age=21536000; includeSubDomains" + ) + xtdb_octopoes_service.ooi_repository.save(header, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + args: list[Reference | None] = [ip_address.reference, None, None, None] + print(xtdb_octopoes_service.ooi_repository.nibble_query(ip_address, https_availability, valid_time, args)) From abb845d4c996cbb44fa331821bd1401d35d86e00 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 8 Jan 2025 13:27:10 +0100 Subject: [PATCH 02/12] DEMO2 --- octopoes/.ci/docker-compose.yml | 2 +- .../https_availability/https_availability.py | 20 ++++++++----------- octopoes/nibbles/https_availability/nibble.py | 16 +++++++-------- .../octopoes/repositories/ooi_repository.py | 1 - .../test_https_availability_nibble.py | 17 ++++++++++------ 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/octopoes/.ci/docker-compose.yml b/octopoes/.ci/docker-compose.yml index d4e33b42e13..ae46e5cd8ae 100644 --- a/octopoes/.ci/docker-compose.yml +++ b/octopoes/.ci/docker-compose.yml @@ -17,7 +17,7 @@ services: args: ENVIRONMENT: dev context: . - command: pytest tests/integration --timeout=300 + command: pytest -s tests/integration/test_https_availability_nibble.py --timeout=300 depends_on: - xtdb - ci_octopoes diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py index b3386c684ff..8f917e52cd9 100644 --- a/octopoes/nibbles/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -2,19 +2,15 @@ from octopoes.models import OOI from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.network import IPPort +from octopoes.models.ooi.network import IPAddressV4, IPPort from octopoes.models.ooi.web import Website -def nibble() -> Iterator[OOI]: - websites = [website for website in additional_oois if isinstance(website, Website)] - - open_ports = [port.port for port in additional_oois if isinstance(port, IPPort)] - if 80 in open_ports and 443 not in open_ports: +def nibble(ipv4: IPAddressV4, port80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: + if port443s < 2: ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") - for website in websites: - yield Finding( - ooi=website.reference, - finding_type=ft.reference, - description="HTTP port is open, but HTTPS port is not open", - ) + yield Finding( + ooi=website.reference, + finding_type=ft.reference, + description="HTTP port is open, but HTTPS port is not open", + ) diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index df5c6f40867..8edd3dd8a1a 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -1,6 +1,6 @@ from nibbles.definitions import NibbleDefinition, NibbleParameter from octopoes.models import Reference -from octopoes.models.ooi.network import IPAddress, IPPort +from octopoes.models.ooi.network import IPAddressV4, IPPort from octopoes.models.ooi.web import Website @@ -9,7 +9,7 @@ def pull(statements: list[str]) -> str: return f""" {{ :query {{ - :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?ipport443 [*]) (pull ?website [*])] + :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?website [*]) (count ?ipport443)] :where [ {" ".join(statements)} ] @@ -26,12 +26,12 @@ def pull(statements: list[str]) -> str: [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] [?ipport80 :IPPort/address ?ipaddress] [?ipport80 :IPPort/port 80] + [?ip_service :IPService/ip_port ?ipport80] + [?website :Website/ip_service ?ip_service] (or (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) (and [(identity 0) ?ipport443]) ) - [?ip_service :IPService/ip_port ?ipport80] - [?website :Website/ip_service ?ip_service] """ ] ) @@ -39,12 +39,12 @@ def pull(statements: list[str]) -> str: NIBBLE = NibbleDefinition( - id="https-availability", + id="https_availability", signature=[ - NibbleParameter(object_type=IPAddress, parser="[*][?object_type == 'IPAddressV4'][]"), - NibbleParameter(object_type=IPPort, parser='[*][?"IPPort/port" == "80"][]'), + NibbleParameter(object_type=IPAddressV4, parser="[*][?object_type == 'IPAddressV4'][]"), + NibbleParameter(object_type=IPPort, parser="[*][?object_type == 'IPPort'][]"), NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), - NibbleParameter(object_type=int, parser='[length([*][?"IPPort/port" == "443"])]'), + NibbleParameter(object_type=int, parser='[*][-1][]'), ], query=query, ) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index a67707765c9..c171c42c19a 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -927,7 +927,6 @@ def nibble_query( arguments = [None for _ in nibble.signature] query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) data = self.session.client.query(query, valid_time) - breakpoint() objects = [ {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} for element in nibble.signature diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py index 168210d197c..09e26ae07e0 100644 --- a/octopoes/tests/integration/test_https_availability_nibble.py +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -7,8 +7,8 @@ from nibbles.runner import NibblesRunner from octopoes.core.service import OctopoesService -from octopoes.models import Reference from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol from octopoes.models.ooi.service import IPService, Service from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website @@ -48,9 +48,6 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_ port = IPPort(port=80, address=ip_address.reference, protocol=Protocol.TCP) xtdb_octopoes_service.ooi_repository.save(port, valid_time) - port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) - xtdb_octopoes_service.ooi_repository.save(port443, valid_time) - ip_service = IPService(ip_port=port.reference, service=service.reference) xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) @@ -67,5 +64,13 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_ event_manager.complete_process_events(xtdb_octopoes_service) - args: list[Reference | None] = [ip_address.reference, None, None, None] - print(xtdb_octopoes_service.ooi_repository.nibble_query(ip_address, https_availability, valid_time, args)) + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 1 + + port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port443, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 0 From 6c51943591f84e45ee334175411d6a9169ac0fca Mon Sep 17 00:00:00 2001 From: Benny Date: Thu, 9 Jan 2025 12:10:23 +0100 Subject: [PATCH 03/12] Make precommit happy --- .../https_availability/https_availability.py | 4 +- octopoes/nibbles/https_availability/nibble.py | 37 ++++++++++++------- octopoes/nibbles/runner.py | 3 +- .../test_https_availability_nibble.py | 9 +---- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py index 8f917e52cd9..f88781f7cc2 100644 --- a/octopoes/nibbles/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -2,11 +2,11 @@ from octopoes.models import OOI from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.network import IPAddressV4, IPPort +from octopoes.models.ooi.network import IPAddress, IPPort from octopoes.models.ooi.web import Website -def nibble(ipv4: IPAddressV4, port80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: +def nibble(ipv4: IPAddress, port80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: if port443s < 2: ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") yield Finding( diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index 8edd3dd8a1a..b1720698bab 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -1,6 +1,6 @@ from nibbles.definitions import NibbleDefinition, NibbleParameter from octopoes.models import Reference -from octopoes.models.ooi.network import IPAddressV4, IPPort +from octopoes.models.ooi.network import IPAddress, IPPort from octopoes.models.ooi.web import Website @@ -22,29 +22,38 @@ def pull(statements: list[str]) -> str: return pull( [ f""" - [?ipaddress :object_type "IPAddressV4"] - [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] - [?ipport80 :IPPort/address ?ipaddress] - [?ipport80 :IPPort/port 80] - [?ip_service :IPService/ip_port ?ipport80] - [?website :Website/ip_service ?ip_service] - (or - (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) - (and [(identity 0) ?ipport443]) - ) + (or + (and [?ipaddress :object_type "IPAddressV4"] + [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] + ) + (and [?ipaddress :object_type "IPAddressV6"] + [?ipaddress :IPAddressV6/primary_key "{str(targets[0])}"] + ) + ) + [?ipport80 :IPPort/address ?ipaddress] + [?ipport80 :IPPort/port 80] + [?ip_service :IPService/ip_port ?ipport80] + [?website :Website/ip_service ?ip_service] + (or + (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) + [(identity nil) ?ipport443] + ) """ ] ) - return "potato" + elif sgn == "0100" or sgn == "0010" or sgn == "1110": + return "TODO" + else: + return "TODO" NIBBLE = NibbleDefinition( id="https_availability", signature=[ - NibbleParameter(object_type=IPAddressV4, parser="[*][?object_type == 'IPAddressV4'][]"), + NibbleParameter(object_type=IPAddress, parser="[*][?object_type == 'IPAddressV4'][]"), NibbleParameter(object_type=IPPort, parser="[*][?object_type == 'IPPort'][]"), NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), - NibbleParameter(object_type=int, parser='[*][-1][]'), + NibbleParameter(object_type=int, parser="[*][-1][]"), ], query=query, ) diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py index 1a8828a7c6b..925b4e05a13 100644 --- a/octopoes/nibbles/runner.py +++ b/octopoes/nibbles/runner.py @@ -179,7 +179,8 @@ def _run(self, ooi: OOI, valid_time: datetime) -> dict[str, dict[tuple[Any, ...] nibblet_nibbles = {self.nibbles[nibblet.method] for nibblet in nibblets if nibblet.method in self.nibbles} for nibble in filter( - lambda x: any(isinstance(ooi, param.object_type) for param in x.signature) and x not in nibblet_nibbles, + lambda nibb: any(isinstance(ooi, sgn.object_type) for sgn in nibb.signature) + and nibb not in nibblet_nibbles, self.nibbles.values(), ): if nibble.enabled: diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py index 09e26ae07e0..951064a0e8c 100644 --- a/octopoes/tests/integration/test_https_availability_nibble.py +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -4,7 +4,6 @@ import pytest from nibbles.https_availability.nibble import NIBBLE as https_availability -from nibbles.runner import NibblesRunner from octopoes.core.service import OctopoesService from octopoes.models.ooi.dns.zone import Hostname @@ -20,13 +19,7 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): - nibbler = NibblesRunner( - xtdb_octopoes_service.ooi_repository, - xtdb_octopoes_service.origin_repository, - xtdb_octopoes_service.nibbler.nibble_repository, - ) - xtdb_octopoes_service.nibbler.disable() - nibbler.nibbles = {https_availability.id: https_availability} + xtdb_octopoes_service.nibbler.nibbles = {https_availability.id: https_availability} network = Network(name="internet") xtdb_octopoes_service.ooi_repository.save(network, valid_time) From ca29c3e9602f859e6c267d8c982fcc256ff711cb Mon Sep 17 00:00:00 2001 From: Benny Date: Mon, 13 Jan 2025 15:26:57 +0100 Subject: [PATCH 04/12] Finalize --- octopoes/.ci/docker-compose.yml | 2 +- octopoes/nibbles/definitions.py | 4 ++ .../https_availability/https_availability.py | 6 ++- octopoes/nibbles/https_availability/nibble.py | 51 +++++++++---------- octopoes/nibbles/runner.py | 16 +++--- .../test_https_availability_nibble.py | 1 - 6 files changed, 43 insertions(+), 37 deletions(-) diff --git a/octopoes/.ci/docker-compose.yml b/octopoes/.ci/docker-compose.yml index ae46e5cd8ae..d4e33b42e13 100644 --- a/octopoes/.ci/docker-compose.yml +++ b/octopoes/.ci/docker-compose.yml @@ -17,7 +17,7 @@ services: args: ENVIRONMENT: dev context: . - command: pytest -s tests/integration/test_https_availability_nibble.py --timeout=300 + command: pytest tests/integration --timeout=300 depends_on: - xtdb - ci_octopoes diff --git a/octopoes/nibbles/definitions.py b/octopoes/nibbles/definitions.py index 51d048ed325..78775c36057 100644 --- a/octopoes/nibbles/definitions.py +++ b/octopoes/nibbles/definitions.py @@ -53,6 +53,10 @@ def __hash__(self): def _ini(self) -> dict[str, Any]: return {"id": self.id, "enabled": self.enabled, "checksum": self._checksum} + @property + def triggers(self) -> set[type[OOI]]: + return {sgn.object_type for sgn in self.signature if issubclass(sgn.object_type, OOI)} + def get_nibble_definitions() -> dict[str, NibbleDefinition]: nibble_definitions = {} diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py index f88781f7cc2..99909cf9cd8 100644 --- a/octopoes/nibbles/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -6,9 +6,13 @@ from octopoes.models.ooi.web import Website -def nibble(ipv4: IPAddress, port80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: +def nibble(ipaddress: IPAddress, ipport80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: + _ = ipaddress + _ = ipport80 + # The Null in the XTDB query is counted for one, hence any port443 object starts at > 1 if port443s < 2: ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") + yield ft yield Finding( ooi=website.reference, finding_type=ft.reference, diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index b1720698bab..53aa18e62dc 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -17,40 +17,39 @@ def pull(statements: list[str]) -> str: }} """ + base_query = [ + """ + [?website :Website/ip_service ?ip_service] + [?ipservice :IPService/ip_port ?ipport80] + [?ipport80 :IPPort/port 80] + [?ipport80 :IPPort/address ?ipaddress] + (or + (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) + [(identity nil) ?ipport443] + ) + """ + ] + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) if sgn == "1000": - return pull( - [ - f""" - (or - (and [?ipaddress :object_type "IPAddressV4"] - [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] - ) - (and [?ipaddress :object_type "IPAddressV6"] - [?ipaddress :IPAddressV6/primary_key "{str(targets[0])}"] - ) - ) - [?ipport80 :IPPort/address ?ipaddress] - [?ipport80 :IPPort/port 80] - [?ip_service :IPService/ip_port ?ipport80] - [?website :Website/ip_service ?ip_service] - (or - (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) - [(identity nil) ?ipport443] - ) - """ - ] - ) - elif sgn == "0100" or sgn == "0010" or sgn == "1110": - return "TODO" + return pull([f'[?ipaddress :IPAddress/primary_key "{str(targets[0])}"]'] + base_query) + elif sgn == "0100": + if int(str(targets[1]).split("|")[-1]) == 80: + return pull([f'[?ipport80 :IPPort/primary_key "{str(targets[1])}"]'] + base_query) + else: + return pull(base_query) + elif sgn == "0010": + return pull([f'[?website :Website/primary_key "{str(targets[2])}"]'] + base_query) else: - return "TODO" + return pull(base_query) NIBBLE = NibbleDefinition( id="https_availability", signature=[ - NibbleParameter(object_type=IPAddress, parser="[*][?object_type == 'IPAddressV4'][]"), + NibbleParameter( + object_type=IPAddress, parser="[*][?object_type == 'IPAddressV6' || object_type == 'IPAddressV4'][]" + ), NibbleParameter(object_type=IPPort, parser="[*][?object_type == 'IPPort'][]"), NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), NibbleParameter(object_type=int, parser="[*][-1][]"), diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py index 925b4e05a13..96882691161 100644 --- a/octopoes/nibbles/runner.py +++ b/octopoes/nibbles/runner.py @@ -179,16 +179,16 @@ def _run(self, ooi: OOI, valid_time: datetime) -> dict[str, dict[tuple[Any, ...] nibblet_nibbles = {self.nibbles[nibblet.method] for nibblet in nibblets if nibblet.method in self.nibbles} for nibble in filter( - lambda nibb: any(isinstance(ooi, sgn.object_type) for sgn in nibb.signature) - and nibb not in nibblet_nibbles, + lambda nibbly: nibbly.enabled + and nibbly not in nibblet_nibbles + and any(isinstance(ooi, t) for t in nibbly.triggers), self.nibbles.values(), ): - if nibble.enabled: - if len(nibble.signature) > 1: - self._write(valid_time) - args = self.ooi_repository.nibble_query(ooi, nibble, valid_time) - results = {tuple(arg): set(flatten([nibble(arg)])) for arg in args} - return_value |= {nibble.id: results} + if len(nibble.signature) > 1: + self._write(valid_time) + args = self.ooi_repository.nibble_query(ooi, nibble, valid_time) + results = {tuple(arg): set(flatten([nibble(arg)])) for arg in args} + return_value |= {nibble.id: results} self.cache = merge_results(self.cache, {ooi: return_value}) return return_value diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py index 951064a0e8c..de2f01c8938 100644 --- a/octopoes/tests/integration/test_https_availability_nibble.py +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -62,7 +62,6 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_ port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) xtdb_octopoes_service.ooi_repository.save(port443, valid_time) - event_manager.complete_process_events(xtdb_octopoes_service) assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 From 2373d0c068d8306693ac4c519e15cc95c36cb90a Mon Sep 17 00:00:00 2001 From: Benny Date: Mon, 13 Jan 2025 15:53:34 +0100 Subject: [PATCH 05/12] Close the corners --- .../nibbles/disallowed_csp_hostnames/nibble.py | 2 +- octopoes/nibbles/https_availability/nibble.py | 14 +++++++++++--- octopoes/tests/test_bits.py | 4 ++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/octopoes/nibbles/disallowed_csp_hostnames/nibble.py b/octopoes/nibbles/disallowed_csp_hostnames/nibble.py index fcf230be3e3..e4430a1455f 100644 --- a/octopoes/nibbles/disallowed_csp_hostnames/nibble.py +++ b/octopoes/nibbles/disallowed_csp_hostnames/nibble.py @@ -100,7 +100,7 @@ def query(targets: list[Reference | None]) -> str: NIBBLE = NibbleDefinition( - id="disallowed-csp-hostnames", + id="disallowed_csp_hostnames", signature=[ NibbleParameter(object_type=HTTPHeaderHostname, parser="[*][?object_type == 'HTTPHeaderHostname'][]"), NibbleParameter(object_type=Config, parser="[*][?object_type == 'Config'][]", optional=True), diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index 53aa18e62dc..91f4617f293 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -30,16 +30,24 @@ def pull(statements: list[str]) -> str: """ ] + ref_queries = [ + f'[?ipaddress :IPAddress/primary_key "{str(targets[0])}"]', + f'[?ipport80 :IPPort/primary_key "{str(targets[1])}"]', + f'[?website :Website/primary_key "{str(targets[2])}"]', + ] + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) if sgn == "1000": - return pull([f'[?ipaddress :IPAddress/primary_key "{str(targets[0])}"]'] + base_query) + return pull(ref_queries[0:1] + base_query) elif sgn == "0100": if int(str(targets[1]).split("|")[-1]) == 80: - return pull([f'[?ipport80 :IPPort/primary_key "{str(targets[1])}"]'] + base_query) + return pull(ref_queries[1:2] + base_query) else: return pull(base_query) elif sgn == "0010": - return pull([f'[?website :Website/primary_key "{str(targets[2])}"]'] + base_query) + return pull(ref_queries[2:3] + base_query) + elif sgn == "1110": + return pull(ref_queries + base_query) else: return pull(base_query) diff --git a/octopoes/tests/test_bits.py b/octopoes/tests/test_bits.py index e2e4f154755..d7af1df3e5e 100644 --- a/octopoes/tests/test_bits.py +++ b/octopoes/tests/test_bits.py @@ -1,4 +1,4 @@ -from bits.https_availability.https_availability import run as run_https_availability +from nibbles.https_availability.https_availability import nibble as run_https_availability from nibbles.oois_in_headers.oois_in_headers import nibble as run_oois_in_headers from octopoes.models.ooi.config import Config @@ -42,7 +42,7 @@ def test_url_extracted_by_oois_in_headers_relative_path(http_resource_https): def test_finding_generated_when_443_not_open_and_80_is_open(): port_80 = IPPort(address="fake", protocol="tcp", port=80) website = Website(ip_service="fake", hostname="fake") - results = list(run_https_availability(None, [port_80, website], {})) + results = list(run_https_availability(None, port_80, website, 1)) finding = results[0] assert isinstance(finding, Finding) assert finding.description == "HTTP port is open, but HTTPS port is not open" From 368bf699fe262467ae7deb5bcec540d054d33a57 Mon Sep 17 00:00:00 2001 From: Benny Date: Mon, 13 Jan 2025 15:59:35 +0100 Subject: [PATCH 06/12] Fix unit test --- octopoes/tests/test_bits.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octopoes/tests/test_bits.py b/octopoes/tests/test_bits.py index d7af1df3e5e..a998565eb8f 100644 --- a/octopoes/tests/test_bits.py +++ b/octopoes/tests/test_bits.py @@ -2,7 +2,7 @@ from nibbles.oois_in_headers.oois_in_headers import nibble as run_oois_in_headers from octopoes.models.ooi.config import Config -from octopoes.models.ooi.findings import Finding +from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.network import IPPort from octopoes.models.ooi.web import URL, HTTPHeader, HTTPHeaderURL, Website @@ -43,6 +43,7 @@ def test_finding_generated_when_443_not_open_and_80_is_open(): port_80 = IPPort(address="fake", protocol="tcp", port=80) website = Website(ip_service="fake", hostname="fake") results = list(run_https_availability(None, port_80, website, 1)) - finding = results[0] - assert isinstance(finding, Finding) + finding = [result for result in results if isinstance(result, Finding)][0] assert finding.description == "HTTP port is open, but HTTPS port is not open" + katfindingtype = [result for result in results if not isinstance(result, Finding)][0] + assert isinstance(katfindingtype, KATFindingType) From b8488168e751797afa6eba01bc644a8c2a0bd580 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 09:12:02 +0100 Subject: [PATCH 07/12] Init --- octopoes/.ci/docker-compose.yml | 2 +- octopoes/bits/internetnl/bit.py | 24 ---- octopoes/bits/internetnl/internetnl.py | 50 ------- octopoes/nibbles/definitions.py | 13 +- .../{bits => nibbles}/internetnl/__init__.py | 0 octopoes/nibbles/internetnl/internetnl.py | 18 +++ octopoes/nibbles/internetnl/nibble.py | 130 ++++++++++++++++++ .../octopoes/repositories/ooi_repository.py | 3 +- .../integration/test_internetnl_nibble.py | 69 ++++++++++ octopoes/tools/xtdb-cli.py | 4 +- octopoes/tools/xtdb_client.py | 3 +- 11 files changed, 235 insertions(+), 81 deletions(-) delete mode 100644 octopoes/bits/internetnl/bit.py delete mode 100644 octopoes/bits/internetnl/internetnl.py rename octopoes/{bits => nibbles}/internetnl/__init__.py (100%) create mode 100644 octopoes/nibbles/internetnl/internetnl.py create mode 100644 octopoes/nibbles/internetnl/nibble.py create mode 100644 octopoes/tests/integration/test_internetnl_nibble.py diff --git a/octopoes/.ci/docker-compose.yml b/octopoes/.ci/docker-compose.yml index d4e33b42e13..d03dc811b4d 100644 --- a/octopoes/.ci/docker-compose.yml +++ b/octopoes/.ci/docker-compose.yml @@ -17,7 +17,7 @@ services: args: ENVIRONMENT: dev context: . - command: pytest tests/integration --timeout=300 + command: pytest -s tests/integration/test_internetnl_nibble.py --timeout=300 depends_on: - xtdb - ci_octopoes diff --git a/octopoes/bits/internetnl/bit.py b/octopoes/bits/internetnl/bit.py deleted file mode 100644 index e4b07dc359f..00000000000 --- a/octopoes/bits/internetnl/bit.py +++ /dev/null @@ -1,24 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.dns.zone import Hostname -from octopoes.models.ooi.findings import Finding -from octopoes.models.ooi.web import Website - -BIT = BitDefinition( - id="internet-nl", - consumes=Hostname, - parameters=[ - BitParameterDefinition(ooi_type=Finding, relation_path="ooi [is Hostname]"), # findings on hostnames - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HTTPResource].website.hostname" - ), # findings on resources - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HTTPHeader].resource.website.hostname" - ), # findings on headers - BitParameterDefinition(ooi_type=Finding, relation_path="ooi [is Website].hostname"), # findings on websites - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HostnameHTTPURL].netloc" - ), # findings on weburls - BitParameterDefinition(ooi_type=Website, relation_path="hostname"), # only websites have to comply - ], - module="bits.internetnl.internetnl", -) diff --git a/octopoes/bits/internetnl/internetnl.py b/octopoes/bits/internetnl/internetnl.py deleted file mode 100644 index a0058b22fc8..00000000000 --- a/octopoes/bits/internetnl/internetnl.py +++ /dev/null @@ -1,50 +0,0 @@ -from collections.abc import Iterator -from typing import Any - -from octopoes.models import OOI -from octopoes.models.ooi.dns.zone import Hostname -from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.web import Website - - -def run(input_ooi: Hostname, additional_oois: list[Finding | Website], config: dict[str, Any]) -> Iterator[OOI]: - # only websites have to comply with the internetnl rules - websites = [websites for websites in additional_oois if isinstance(websites, Website)] - if not websites: - return - - finding_ids = [finding.finding_type.tokenized.id for finding in additional_oois if isinstance(finding, Finding)] - - result = "" - internetnl_findings = { - "KAT-WEBSERVER-NO-IPV6": "This webserver does not have an IPv6 address", - "KAT-NAMESERVER-NO-TWO-IPV6": "This webserver does not have at least two nameservers with an IPv6 address", - "KAT-NO-DNSSEC": "This webserver is not DNSSEC signed", - "KAT-INVALID-DNSSEC": "The DNSSEC signature of this webserver is not valid", - "KAT-NO-HSTS": "This website has at least one webpage with a missing Strict-Transport-Policy header", - "KAT-NO-CSP": "This website has at least one webpage with a missing Content-Security-Policy header", - "KAT-NO-X-FRAME-OPTIONS": "This website has at least one webpage with a missing X-Frame-Options header", - "KAT-NO-X-CONTENT-TYPE-OPTIONS": ( - "This website has at least one webpage with a missing X-Content-Type-Options header" - ), - "KAT-CSP-VULNERABILITIES": "This website has at least one webpage with a mis-configured CSP header", - "KAT-HSTS-VULNERABILITIES": "This website has at least one webpage with a mis-configured HSTS header", - "KAT-NO-CERTIFICATE": "This website does not have an SSL certificate", - "KAT-HTTPS-NOT-AVAILABLE": "HTTPS is not available for this website", - "KAT-SSL-CERT-HOSTNAME-MISMATCH": "The SSL certificate of this website does not match the hostname", - "KAT-HTTPS-REDIRECT": "This website has at least one HTTP URL that does not redirect to HTTPS", - } - - for finding, description in internetnl_findings.items(): - if finding in finding_ids: - result += f"{description}\n" - - if result: - ft = KATFindingType(id="KAT-INTERNETNL") - yield ft - f = Finding( - finding_type=ft.reference, - ooi=input_ooi.reference, - description=f"This hostname has at least one website with the following finding(s): {result}", - ) - yield f diff --git a/octopoes/nibbles/definitions.py b/octopoes/nibbles/definitions.py index 78775c36057..dda5825e132 100644 --- a/octopoes/nibbles/definitions.py +++ b/octopoes/nibbles/definitions.py @@ -19,9 +19,10 @@ class NibbleParameter(BaseModel): - object_type: type[Any] + object_type: Any parser: str = "[]" optional: bool = False + additional: set[type[OOI]] = set() def __eq__(self, other): if isinstance(other, NibbleParameter): @@ -31,12 +32,20 @@ def __eq__(self, other): else: return False + @property + def triggers(self) -> set[type[OOI]]: + if isinstance(self.object_type, type) and issubclass(self.object_type, OOI): + return {self.object_type} | self.additional + else: + return self.additional + class NibbleDefinition(BaseModel): id: str signature: list[NibbleParameter] query: str | Callable[[list[Reference | None]], str] | None = None enabled: bool = True + additional: set[type[OOI]] = set() _payload: MethodType | None = None _checksum: str | None = None @@ -55,7 +64,7 @@ def _ini(self) -> dict[str, Any]: @property def triggers(self) -> set[type[OOI]]: - return {sgn.object_type for sgn in self.signature if issubclass(sgn.object_type, OOI)} + return set.union(*[sgn.triggers for sgn in self.signature]) | self.additional def get_nibble_definitions() -> dict[str, NibbleDefinition]: diff --git a/octopoes/bits/internetnl/__init__.py b/octopoes/nibbles/internetnl/__init__.py similarity index 100% rename from octopoes/bits/internetnl/__init__.py rename to octopoes/nibbles/internetnl/__init__.py diff --git a/octopoes/nibbles/internetnl/internetnl.py b/octopoes/nibbles/internetnl/internetnl.py new file mode 100644 index 00000000000..85e24a274a9 --- /dev/null +++ b/octopoes/nibbles/internetnl/internetnl.py @@ -0,0 +1,18 @@ +from collections.abc import Iterator + +from octopoes.models import OOI +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType + + +def nibble(hostname: Hostname, findings: list[Finding]) -> Iterator[OOI]: + result = "\n".join([str(finding.description) for finding in findings]) + + if result: + ft = KATFindingType(id="KAT-INTERNETNL") + yield ft + yield Finding( + finding_type=ft.reference, + ooi=hostname.reference, + description=f"This hostname has at least one website with the following finding(s): {result}", + ) diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py new file mode 100644 index 00000000000..58c6cac1d7a --- /dev/null +++ b/octopoes/nibbles/internetnl/nibble.py @@ -0,0 +1,130 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.dns.zone import Hostname, Network +from octopoes.models.ooi.findings import Finding +from octopoes.models.ooi.web import Website + +finding_types = [ + "KAT-WEBSERVER-NO-IPV6", + "KAT-NAMESERVER-NO-TWO-IPV6", + "KAT-NO-DNSSEC", + "KAT-INVALID-DNSSEC", + "KAT-NO-HSTS", + "KAT-NO-CSP", + "KAT-NO-X-FRAME-OPTIONS", + "KAT-NO-X-CONTENT-TYPE-OPTIONS", + "KAT-CSP-VULNERABILITIES", + "KAT-HSTS-VULNERABILITIES", + "KAT-NO-CERTIFICATE", + "KAT-HTTPS-NOT-AVAILABLE", + "KAT-SSL-CERT-HOSTNAME-MISMATCH", + "KAT-HTTPS-REDIRECT", +] + + +def or_finding_types() -> str: + clauses = "".join([f'[?finding :Finding/finding_type "{ft}"]' for ft in finding_types]) + return f"(or {clauses})" + + +def query(targets: list[Reference | None]) -> str: + def pull(statements: list[str]) -> str: + return f""" + {{ + :query {{ + :find [(pull ?hostname [*]) (pull ?finding [*])] + :where [ + {" ".join(statements)} + ] + }} + }} + """ + + base_query = [ + """ + [?website :Website/hostname ?hostname] + [?finding :Finding/ooi ?ooi] + (or + (or + (and + [?ooi :Hostname/primary_key ?hostname] + ) + (and + [(identity nil) ?ooi] + ) + ) + (or + (and + [?ooi :HTTPResource/website ?website] + [?website :Website/hostname ?hostname] + ) + (and + [(identity nil) ?ooi] + [(identity nil) ?website] + ) + ) + (or + (and + [?ooi :HTTPHeader/website ?resource] + [?resource :HTTPResource/website ?website] + [?website :Website/hostname ?hostname] + ) + (and + [(identity nil) ?ooi] + [(identity nil) ?resource] + [(identity nil) ?website] + ) + ) + (or + (and + [?ooi :Website/hostname ?hostname] + ) + (and + [(identity nil) ?ooi] + ) + ) + (or + (and + [?ooi :HostnameHTTPURL/hostname ?hostname] + ) + (and + [(identity nil) ?ooi] + ) + ) + ) + """ + ] + + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + ref_query = ["[?hostname :Hostname/primary_key]"] + if sgn == "100": + ref_query = [f'[?hostname :Hostname/primary_key "{str(targets[0])}"]'] + elif sgn == "010": + ref_query = [f'[?hostname :Hostname/primary_key "{str(targets[1]).split("|")[-1]}"]'] + elif sgn == "001": + tokens = str(targets[2]).split("|")[1:-1] + target_reference = Reference.from_str("|".join(tokens)) + if tokens[0] == "Hostname": + hostname = target_reference.tokenized + elif tokens[0] == "HTTPResource": + hostname = target_reference.tokenized.website.hostname + elif tokens[0] == "HTTPHeader": + hostname = target_reference.tokenized.resource.website.hostname + elif tokens[0] in {"Website", "HostnameHTTPURL"}: + hostname = target_reference.tokenized.hostname + else: + raise ValueError() + hostname_pk = Hostname(name=hostname.name, network=Network(name=hostname.network.name).reference).reference + ref_query = [f'[?hostname :Hostname/primary_key "{str(hostname_pk)}"]'] + return pull(ref_query + base_query) + + +NIBBLE = NibbleDefinition( + id="internet_nl", + signature=[ + NibbleParameter(object_type=Hostname, parser="[*][?object_type == 'IPPort'][]"), + NibbleParameter(object_type=list[Website], parser="[[*][?object_type == 'Website'][]]", additional={Website}), + NibbleParameter(object_type=list[Finding], parser="[[*][?object_type == 'Findings'][]]", additional={Finding}), + ], + query=query, +) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index c171c42c19a..d2739acd8e7 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -919,13 +919,14 @@ def nibble_query( first = True arguments = [ ooi.reference - if sgn.object_type == type_by_name(ooi.get_ooi_type()) and (first and not (first := False)) + if type_by_name(ooi.get_ooi_type()) in sgn.triggers and (first and not (first := False)) else None for sgn in nibble.signature ] else: arguments = [None for _ in nibble.signature] query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) + breakpoint() data = self.session.client.query(query, valid_time) objects = [ {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} diff --git a/octopoes/tests/integration/test_internetnl_nibble.py b/octopoes/tests/integration/test_internetnl_nibble.py new file mode 100644 index 00000000000..ab437f563df --- /dev/null +++ b/octopoes/tests/integration/test_internetnl_nibble.py @@ -0,0 +1,69 @@ +import os +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.internetnl.nibble import NIBBLE as internetnl + +from octopoes.core.service import OctopoesService +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType +from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol +from octopoes.models.ooi.service import IPService, Service +from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + +STATIC_IP = ".".join((4 * "1 ").split()) + + +def test_internetnl_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {internetnl.id: internetnl} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + web_url = HostnameHTTPURL( + network=network.reference, netloc=hostname.reference, port=443, path="/", scheme=WebScheme.HTTP + ) + xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) + + service = Service(name="https") + xtdb_octopoes_service.ooi_repository.save(service, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=80, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + ip_service = IPService(ip_port=port.reference, service=service.reference) + xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) + + website = Website(ip_service=ip_service.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(website, valid_time) + + resource = HTTPResource(website=website.reference, web_url=web_url.reference) + xtdb_octopoes_service.ooi_repository.save(resource, valid_time) + + header = HTTPHeader( + resource=resource.reference, key="strict-transport-security", value="max-age=21536000; includeSubDomains" + ) + xtdb_octopoes_service.ooi_repository.save(header, valid_time) + + ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") + xtdb_octopoes_service.ooi_repository.save(ft, valid_time) + + finding = Finding( + ooi=website.reference, finding_type=ft.reference, description="HTTP port is open, but HTTPS port is not open" + ) + xtdb_octopoes_service.ooi_repository.save(finding, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 2 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 2 diff --git a/octopoes/tools/xtdb-cli.py b/octopoes/tools/xtdb-cli.py index 31a63730733..0aa0575d97c 100755 --- a/octopoes/tools/xtdb-cli.py +++ b/octopoes/tools/xtdb-cli.py @@ -59,9 +59,9 @@ def query( client: XTDBClient = ctx.obj["client"] if edn: - click.echo(json.dumps(client.query(edn, valid_time, tx_time, tx_id))) + click.echo(client.query(edn, valid_time, tx_time, tx_id)) else: - click.echo(json.dumps(client.query(valid_time=valid_time, tx_time=tx_time, tx_id=tx_id))) + click.echo(client.query(valid_time=valid_time, tx_time=tx_time, tx_id=tx_id)) @cli.command(help="List all keys in node") diff --git a/octopoes/tools/xtdb_client.py b/octopoes/tools/xtdb_client.py index ac599c75937..f4e2ac90573 100644 --- a/octopoes/tools/xtdb_client.py +++ b/octopoes/tools/xtdb_client.py @@ -41,7 +41,7 @@ def query( valid_time: datetime.datetime | None = None, tx_time: datetime.datetime | None = None, tx_id: int | None = None, - ) -> JsonValue: + ) -> str: params = {} if valid_time is not None: params["valid-time"] = valid_time.isoformat() @@ -52,6 +52,7 @@ def query( res = self._client.post("/query", params=params, content=query, headers={"Content-Type": "application/edn"}) + return res.text return res.json() def entity( From 681d050358117dc60dbcbe7a2ac9f42c385b3fa7 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 10:54:27 +0100 Subject: [PATCH 08/12] Integrate feedback --- octopoes/nibbles/runner.py | 4 +--- .../tests/integration/test_https_availability_nibble.py | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py index 96882691161..73192cb12c3 100644 --- a/octopoes/nibbles/runner.py +++ b/octopoes/nibbles/runner.py @@ -179,9 +179,7 @@ def _run(self, ooi: OOI, valid_time: datetime) -> dict[str, dict[tuple[Any, ...] nibblet_nibbles = {self.nibbles[nibblet.method] for nibblet in nibblets if nibblet.method in self.nibbles} for nibble in filter( - lambda nibbly: nibbly.enabled - and nibbly not in nibblet_nibbles - and any(isinstance(ooi, t) for t in nibbly.triggers), + lambda x: x.enabled and x not in nibblet_nibbles and any(isinstance(ooi, t) for t in x.triggers), self.nibbles.values(), ): if len(nibble.signature) > 1: diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py index de2f01c8938..314078d806a 100644 --- a/octopoes/tests/integration/test_https_availability_nibble.py +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -58,7 +58,15 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_ event_manager.complete_process_events(xtdb_octopoes_service) assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert ( + xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).items[0].description + == "HTTP port is open, but HTTPS port is not open" + ) assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 1 + assert ( + xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).items[0].id + == "KAT-HTTPS-NOT-AVAILABLE" + ) port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) xtdb_octopoes_service.ooi_repository.save(port443, valid_time) From 3e6c87e764ecb3ddf5daa4f9eab9427c1182e715 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 11:11:51 +0100 Subject: [PATCH 09/12] Fix nibble --- octopoes/nibbles/https_availability/https_availability.py | 3 +-- octopoes/nibbles/https_availability/nibble.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py index 99909cf9cd8..e1c5b9f4f31 100644 --- a/octopoes/nibbles/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -9,8 +9,7 @@ def nibble(ipaddress: IPAddress, ipport80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: _ = ipaddress _ = ipport80 - # The Null in the XTDB query is counted for one, hence any port443 object starts at > 1 - if port443s < 2: + if port443s < 1: ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") yield ft yield Finding( diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index 91f4617f293..09f2f99413c 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -9,7 +9,7 @@ def pull(statements: list[str]) -> str: return f""" {{ :query {{ - :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?website [*]) (count ?ipport443)] + :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?website [*]) (- (count ?ipport443) 1)] :where [ {" ".join(statements)} ] From 99cc00c5fc91c0a37b3b291831f4902588ade4dd Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 11:17:09 +0100 Subject: [PATCH 10/12] Fix nibble test --- octopoes/tests/test_bits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octopoes/tests/test_bits.py b/octopoes/tests/test_bits.py index a998565eb8f..c0f4605657f 100644 --- a/octopoes/tests/test_bits.py +++ b/octopoes/tests/test_bits.py @@ -42,7 +42,7 @@ def test_url_extracted_by_oois_in_headers_relative_path(http_resource_https): def test_finding_generated_when_443_not_open_and_80_is_open(): port_80 = IPPort(address="fake", protocol="tcp", port=80) website = Website(ip_service="fake", hostname="fake") - results = list(run_https_availability(None, port_80, website, 1)) + results = list(run_https_availability(None, port_80, website, 0)) finding = [result for result in results if isinstance(result, Finding)][0] assert finding.description == "HTTP port is open, but HTTPS port is not open" katfindingtype = [result for result in results if not isinstance(result, Finding)][0] From 2ca65dd35920b49fa3444be546cc19d76a42bc26 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 14:08:13 +0100 Subject: [PATCH 11/12] or-join ftw --- octopoes/nibbles/internetnl/nibble.py | 69 ++++++++------------------- 1 file changed, 21 insertions(+), 48 deletions(-) diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py index 58c6cac1d7a..b35fe6f6630 100644 --- a/octopoes/nibbles/internetnl/nibble.py +++ b/octopoes/nibbles/internetnl/nibble.py @@ -32,7 +32,11 @@ def pull(statements: list[str]) -> str: return f""" {{ :query {{ - :find [(pull ?hostname [*]) (pull ?finding [*])] + :find [ + (pull ?hostname [*]) + (pull ?website [*]) + (pull ?finding [*]) + ] :where [ {" ".join(statements)} ] @@ -42,54 +46,23 @@ def pull(statements: list[str]) -> str: base_query = [ """ + [?hostname :object_type "Hostname"] [?website :Website/hostname ?hostname] - [?finding :Finding/ooi ?ooi] - (or - (or - (and - [?ooi :Hostname/primary_key ?hostname] - ) - (and - [(identity nil) ?ooi] - ) + (or-join [?finding] + [?finding :Finding/ooi ?hostname] + (and + [?hostnamehttpurl :HostnameHTTPURL/netloc ?hostname] + [?finding :Finding/ooi ?hostnamehttpurl] ) - (or - (and - [?ooi :HTTPResource/website ?website] - [?website :Website/hostname ?hostname] - ) - (and - [(identity nil) ?ooi] - [(identity nil) ?website] - ) + [?finding :Finding/ooi ?website] + (and + [?resource :HTTPResource/website ?website] + [?finding :Finding/ooi ?resource] ) - (or - (and - [?ooi :HTTPHeader/website ?resource] - [?resource :HTTPResource/website ?website] - [?website :Website/hostname ?hostname] - ) - (and - [(identity nil) ?ooi] - [(identity nil) ?resource] - [(identity nil) ?website] - ) - ) - (or - (and - [?ooi :Website/hostname ?hostname] - ) - (and - [(identity nil) ?ooi] - ) - ) - (or - (and - [?ooi :HostnameHTTPURL/hostname ?hostname] - ) - (and - [(identity nil) ?ooi] - ) + (and + [?header :HTTPHeader/resource ?resource] + [?resource :HTTPResource/website ?website] + [?finding :Finding/ooi ?header] ) ) """ @@ -106,12 +79,12 @@ def pull(statements: list[str]) -> str: target_reference = Reference.from_str("|".join(tokens)) if tokens[0] == "Hostname": hostname = target_reference.tokenized + elif tokens[0] in {"HostnameHTTPURL", "Website"}: + hostname = target_reference.tokenized.hostname elif tokens[0] == "HTTPResource": hostname = target_reference.tokenized.website.hostname elif tokens[0] == "HTTPHeader": hostname = target_reference.tokenized.resource.website.hostname - elif tokens[0] in {"Website", "HostnameHTTPURL"}: - hostname = target_reference.tokenized.hostname else: raise ValueError() hostname_pk = Hostname(name=hostname.name, network=Network(name=hostname.network.name).reference).reference From ff021ea2900981290cf4a7514db40a7293d03b44 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 15:03:23 +0100 Subject: [PATCH 12/12] Nail internetnl (almost) --- octopoes/nibbles/internetnl/nibble.py | 6 ++---- octopoes/octopoes/repositories/ooi_repository.py | 4 +++- octopoes/tests/test_ooi_repository.py | 4 ++++ octopoes/tools/xtdb-cli.py | 4 ++-- octopoes/tools/xtdb_client.py | 3 +-- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py index b35fe6f6630..cfc14d1c979 100644 --- a/octopoes/nibbles/internetnl/nibble.py +++ b/octopoes/nibbles/internetnl/nibble.py @@ -2,7 +2,6 @@ from octopoes.models import Reference from octopoes.models.ooi.dns.zone import Hostname, Network from octopoes.models.ooi.findings import Finding -from octopoes.models.ooi.web import Website finding_types = [ "KAT-WEBSERVER-NO-IPV6", @@ -95,9 +94,8 @@ def pull(statements: list[str]) -> str: NIBBLE = NibbleDefinition( id="internet_nl", signature=[ - NibbleParameter(object_type=Hostname, parser="[*][?object_type == 'IPPort'][]"), - NibbleParameter(object_type=list[Website], parser="[[*][?object_type == 'Website'][]]", additional={Website}), - NibbleParameter(object_type=list[Finding], parser="[[*][?object_type == 'Findings'][]]", additional={Finding}), + NibbleParameter(object_type=Hostname, parser="[*][?object_type == 'Hostname'][]"), + NibbleParameter(object_type=list[Finding], parser="[[*][?object_type == 'Finding'][]]", additional={Finding}), ], query=query, ) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index d2739acd8e7..ed71b696665 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -284,6 +284,9 @@ def parse_as(cls, type_: type | list[type], obj: dict | list | set | Any) -> tup # list --> tuple if isinstance(type_, list): return tuple(cls.parse_as(type_t, o) for o, type_t in zip(obj, type_)) + elif hasattr(type_, "__origin__") and hasattr(type_, "__args__") and type_.__origin__ is list: + t = [type_.__args__[0]] * len(obj) + return tuple(cls.parse_as(t, o) for o, t in zip(obj, t)) else: return tuple(cls.parse_as(type_, o) for o in obj) else: @@ -926,7 +929,6 @@ def nibble_query( else: arguments = [None for _ in nibble.signature] query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) - breakpoint() data = self.session.client.query(query, valid_time) objects = [ {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} diff --git a/octopoes/tests/test_ooi_repository.py b/octopoes/tests/test_ooi_repository.py index 1c4c6e131de..0497973d682 100644 --- a/octopoes/tests/test_ooi_repository.py +++ b/octopoes/tests/test_ooi_repository.py @@ -164,3 +164,7 @@ def pull(ooi: OOI): [network, url, network] ) assert self.repository.parse_as(Network, [pull(network), pull(url), pull(url)]) == tuple([network, url, url]) + + assert self.repository.parse_as(list[Network], [pull(network), pull(network), pull(network)]) == tuple( + [network, network, network] + ) diff --git a/octopoes/tools/xtdb-cli.py b/octopoes/tools/xtdb-cli.py index 0aa0575d97c..31a63730733 100755 --- a/octopoes/tools/xtdb-cli.py +++ b/octopoes/tools/xtdb-cli.py @@ -59,9 +59,9 @@ def query( client: XTDBClient = ctx.obj["client"] if edn: - click.echo(client.query(edn, valid_time, tx_time, tx_id)) + click.echo(json.dumps(client.query(edn, valid_time, tx_time, tx_id))) else: - click.echo(client.query(valid_time=valid_time, tx_time=tx_time, tx_id=tx_id)) + click.echo(json.dumps(client.query(valid_time=valid_time, tx_time=tx_time, tx_id=tx_id))) @cli.command(help="List all keys in node") diff --git a/octopoes/tools/xtdb_client.py b/octopoes/tools/xtdb_client.py index f4e2ac90573..ac599c75937 100644 --- a/octopoes/tools/xtdb_client.py +++ b/octopoes/tools/xtdb_client.py @@ -41,7 +41,7 @@ def query( valid_time: datetime.datetime | None = None, tx_time: datetime.datetime | None = None, tx_id: int | None = None, - ) -> str: + ) -> JsonValue: params = {} if valid_time is not None: params["valid-time"] = valid_time.isoformat() @@ -52,7 +52,6 @@ def query( res = self._client.post("/query", params=params, content=query, headers={"Content-Type": "application/edn"}) - return res.text return res.json() def entity(