diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml index 954a15e6..5e440f56 100644 --- a/.github/workflows/code-style.yml +++ b/.github/workflows/code-style.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.9' - name: Install Kestrel package run: | python -m pip install --upgrade pip diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index f95e94c0..49ae7704 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.9' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 842e299d..4812a96d 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -35,5 +35,6 @@ jobs: python -m pip install --upgrade setuptools python -m pip install pytest python -m pip install . + python -m pip install stix-shifter-modules-stix_bundle - name: Unit testing run: pytest -vv diff --git a/.github/workflows/unused-import.yml b/.github/workflows/unused-import.yml index 22da3492..d6228cd6 100644 --- a/.github/workflows/unused-import.yml +++ b/.github/workflows/unused-import.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.9' - name: Install Kestrel package run: | python -m pip install --upgrade pip diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 46cdc33e..0f5faf61 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,20 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog`_. +1.1.4 (2021-10-27) +================== + +Added +----- + +- multi-data source support +- detailed error message from stix-shifter + +Fixed +----- + +- Limit Python<=3.9 since numpy is not ready for 3.10 + 1.1.3 (2021-10-08) ================== diff --git a/README.rst b/README.rst index ecfa3689..a6e08569 100644 --- a/README.rst +++ b/README.rst @@ -149,36 +149,53 @@ Hunting In The Real World Find more at `Kestrel documentation hub`_ and `Kestrel blogs at OCA`_. -RSA Presentation And Demo -========================= - -Kestrel is introduced at RSA Conference 2021 with its goal of an `efficient -cyberthreat hunting symbiosis`_, its design concepts like `entity-based -reasoning`_ and `composable hunt flow`_, as well as a cross-host hunting demo -with TTP pattern matching, provenance tracking, TI-enrichment, machine learning -analytics, and more. Watch our session `The Game of Cyber Threat Hunting: The -Return of the Fun`_ (30 minutes with demo) or the `demo`_ alone (15 -minutes). - Kestrel Hunting Blogs ===================== #. `Building a Huntbook to Discover Persistent Threats from Scheduled Windows Tasks`_ #. `Practicing Backward And Forward Tracking Hunts on A Windows Host`_ +#. `Building Your Own Kestrel Analytics and Sharing With the Community`_ -Kestrel Huntbook/Analytics Repo -=============================== +Learning/Sharing With the Community +=================================== - `Kestrel huntbook repo`_ - `Kestrel analytics repo`_ +Talks And Demos +=============== + +Kestrel was debuted at RSA Conference 2021 with its goal of an `efficient +cyberthreat hunting symbiosis`_, its key design concepts `entity-based +reasoning`_ and `composable hunt flow`_, as well as a small-enterprise APT +hunting demo with TTP pattern matching, cross-host provenance tracking, +TI-enrichment, machine learning analytics, and more. Watch our session `The +Game of Cyber Threat Hunting: The Return of the Fun`_ (30 minutes with demo) or +the `demo`_ alone (15 minutes). + +Kestrel was further introduced to the threat hunting community at `SANS Threat +Hunting Summit 2021`_ in session `Compose Your Hunts With Reusable Knowledge +and Share Your Huntbook With the Community`_ to facilitate huntbook +composition, sharing, and reuse. The session started from 3 simple hunt step +demos---TTP pattern matching, provenance tracking, and data visualization +analytics---then went into comprehensive hunt flow composition to convey the +idea of hunting knowledge composition and reuse. The recording is currently +available at SANS library and will be published by SANS. + +Kestrel will be presented as part of the open hunting stack for hybrid cloud in +Black Hat Europe Arsenal 2021 session: `An Open Stack for Threat Hunting in +Hybrid Cloud With Connected Observability`_. We will hunt an APT in a hybrid +cloud that is a variant of a typical supply chain attack yet implemented in a +more stealthy manner. The open stack consisting of Kestrel, `SysFlow`_, and +other open-source projects will be presented. + Connecting With The Community ============================= Quick questions? Like to meet other users? Want to contribute? -Join our *Kestrel slack channel* at `Open Cybersecurity Alliance slack -workspace`_. +Get a `slack invitation`_ to `Open Cybersecurity +Alliance workspace`_ and join our *kestrel* channel. .. _Kestrel documentation hub: https://kestrel.readthedocs.io/ .. _Kestrel blogs at OCA: https://opencybersecurityalliance.org/posts/ @@ -186,7 +203,8 @@ workspace`_. .. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ .. _Python virtual environment: https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/ .. _Jupyter Notebook: https://jupyter.org/ -.. _Open Cybersecurity Alliance slack workspace: https://open-cybersecurity.slack.com/ +.. _slack invitation: https://docs.google.com/forms/d/1vEAqg9SKBF3UMtmbJJ9qqLarrXN5zeVG3_obedA3DKs/viewform?edit_requested=true +.. _Open Cybersecurity Alliance workspace: https://open-cybersecurity.slack.com/ .. _efficient cyberthreat hunting symbiosis: https://kestrel.readthedocs.io/en/latest/overview.html#human-machine .. _demo: https://www.youtube.com/watch?v=tASFWZfD7l8 .. _entity-based reasoning: https://kestrel.readthedocs.io/en/latest/language.html#entity-based-reasoning @@ -194,5 +212,10 @@ workspace`_. .. _The Game of Cyber Threat Hunting\: The Return of the Fun: https://www.rsaconference.com/Library/presentation/USA/2021/The%20Game%20of%20Cyber%20Threat%20Hunting%20The%20Return%20of%20the%20Fun .. _Building a Huntbook to Discover Persistent Threats from Scheduled Windows Tasks: https://opencybersecurityalliance.org/posts/kestrel-2021-07-26/ .. _Practicing Backward And Forward Tracking Hunts on A Windows Host: https://opencybersecurityalliance.org/posts/kestrel-2021-08-16/ +.. _Building Your Own Kestrel Analytics and Sharing With the Community: https://opencybersecurityalliance.org/posts/kestrel-custom-analytics/ .. _Kestrel huntbook repo: https://github.com/opencybersecurityalliance/kestrel-huntbook .. _Kestrel analytics repo: https://github.com/opencybersecurityalliance/kestrel-analytics +.. _SANS Threat Hunting Summit 2021: https://www.sans.org/cyber-security-summit/ +.. _Compose Your Hunts With Reusable Knowledge and Share Your Huntbook With the Community: https://www.sans.org/blog/a-visual-summary-of-sans-threat-hunting-summit-2021/ +.. _An Open Stack for Threat Hunting in Hybrid Cloud With Connected Observability: https://www.blackhat.com/eu-21/arsenal/schedule/index.html#an-open-stack-for-threat-hunting-in-hybrid-cloud-with-connected-observability-25112 +.. _SysFlow: https://github.com/sysflow-telemetry diff --git a/setup.cfg b/setup.cfg index eef5dd95..79961e81 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = kestrel-lang -version = 1.1.3 +version = 1.1.4 description = Kestrel Threat Hunting Language long_description = file:README.rst long_description_content_type = text/x-rst diff --git a/src/kestrel/exceptions.py b/src/kestrel/exceptions.py index 95239b41..18367dc8 100644 --- a/src/kestrel/exceptions.py +++ b/src/kestrel/exceptions.py @@ -171,7 +171,8 @@ def __init__(self, uri, itf, msg=""): class DataSourceError(KestrelException): def __init__(self, error): super().__init__( - f"data source internal error: {error}", "please test data source manually" + f"data source internal error: {error}", + "please check data source config or test the query manually", ) diff --git a/src/kestrel/syntax/kestrel.lark b/src/kestrel/syntax/kestrel.lark index f9f9a41a..42fe8961 100644 --- a/src/kestrel/syntax/kestrel.lark +++ b/src/kestrel/syntax/kestrel.lark @@ -77,7 +77,7 @@ ANALYTICS: (LETTER|DIGIT|/[-_.:\/]/)+ STIXPATH: (LETTER|DIGIT|/[-_.:']/)+ STIXPATHS: STIXPATH (/\s*,\s*/ STIXPATH)* PATH: (LETTER|DIGIT|/[-_.:\/]/)+ -DATASRC: (PATH|ESCAPED_STRING) +DATASRC: (PATH ("," PATH)* |ESCAPED_STRING) DUMPPATH: PATH ASC: "asc"i DESC: "desc"i diff --git a/src/kestrel_datasource_stixbundle/interface.py b/src/kestrel_datasource_stixbundle/interface.py index 33c10153..a51d5a4b 100644 --- a/src/kestrel_datasource_stixbundle/interface.py +++ b/src/kestrel_datasource_stixbundle/interface.py @@ -40,41 +40,47 @@ def list_data_sources(): @staticmethod def query(uri, pattern, session_id=None): - scheme, _, data_path = uri.rpartition("://") + scheme, _, data_paths = uri.rpartition("://") + data_paths = data_paths.split(",") pattern = fixup_pattern(pattern) ingestdir = _make_query_dir(uri) - ingestfile = ingestdir / "data.json" - - # TODO: keep files in LRU cache? - - if scheme == "file": - try: - with open(data_path, "r") as f: - bundle_in = json.load(f) - except Exception: - raise DataSourceConnectionError(uri) - elif scheme == "http" or scheme == "https": - try: - bundle_in = requests.get(uri).json() - except requests.exceptions.ConnectionError: - raise DataSourceConnectionError(uri) - else: - raise DataSourceManagerInternalError( - f"interface {__package__} should not process scheme {scheme}" - ) - - bundle_out = {} - for prop, val in bundle_in.items(): - if prop == "objects": - bundle_out[prop] = [] - for obj in val: - if obj["type"] != "observed-data" or match(pattern, [obj], False): - bundle_out[prop].append(obj) + bundles = [] + for i, data_path in enumerate(data_paths): + data_path_striped = "".join(filter(str.isalnum, data_path)) + ingestfile = ingestdir / f"{i}_{data_path_striped}.json" + + # TODO: keep files in LRU cache? + if scheme == "file": + try: + with open(data_path, "r") as f: + bundle_in = json.load(f) + except Exception: + raise DataSourceConnectionError(uri) + elif scheme == "http" or scheme == "https": + try: + bundle_in = requests.get(f"{scheme}://{data_path}").json() + except requests.exceptions.ConnectionError: + raise DataSourceConnectionError(uri) else: - bundle_out[prop] = val - - with ingestfile.open("w") as f: - json.dump(bundle_out, f) - - return ReturnFromFile(ingestdir.name, [str(ingestfile.resolve())]) + raise DataSourceManagerInternalError( + f"interface {__package__} should not process scheme {scheme}" + ) + + bundle_out = {} + for prop, val in bundle_in.items(): + if prop == "objects": + bundle_out[prop] = [] + for obj in val: + if obj["type"] != "observed-data" or match( + pattern, [obj], False + ): + bundle_out[prop].append(obj) + else: + bundle_out[prop] = val + + with ingestfile.open("w") as f: + json.dump(bundle_out, f) + bundles.append(str(ingestfile.resolve())) + + return ReturnFromFile(ingestdir.name, bundles) diff --git a/src/kestrel_datasource_stixshifter/interface.py b/src/kestrel_datasource_stixshifter/interface.py index 01022a9e..94403c53 100644 --- a/src/kestrel_datasource_stixshifter/interface.py +++ b/src/kestrel_datasource_stixshifter/interface.py @@ -67,88 +67,117 @@ def list_data_sources(): def query(uri, pattern, session_id=None): """Query a stixshifter data source.""" scheme, _, profile = uri.rpartition("://") + profiles = profile.split(",") if scheme != "stixshifter": raise DataSourceManagerInternalError( f"interface {__package__} should not process scheme {scheme}" ) - ( - connector_name, - connection_dict, - configuration_dict, - ) = StixShifterInterface._get_stixshifter_config(profile) - ingestdir = mkdtemp() - ingestfile = ingestdir / "data.json" - query_id = ingestdir.name - query_metadata = json.dumps( - {"id": "identity--" + query_id, "name": connector_name} - ) - - translation = stix_translation.StixTranslation() - transmission = stix_transmission.StixTransmission( - connector_name, connection_dict, configuration_dict - ) - - dsl = translation.translate( - connector_name, "query", query_metadata, pattern, {} - ) - - if "error" in dsl: - raise DataSourceError("STIX-shifter translation failed") - - # query results should be put together; when translated to STIX, the relation between them will remain - connector_results = [] - for query in dsl["queries"]: - search_meta_result = transmission.query(query) - if search_meta_result["success"]: - search_id = search_meta_result["search_id"] - if transmission.is_async(): - time.sleep(1) - status = transmission.status(search_id) - if status["success"]: - while ( - status["progress"] < 100 and status["status"] == "RUNNING" - ): - status = transmission.status(search_id) - else: - raise DataSourceError( - "STIX-shifter transmission.status() failed" - ) + bundles = [] + for i, profile in enumerate(profiles): + ( + connector_name, + connection_dict, + configuration_dict, + ) = StixShifterInterface._get_stixshifter_config(profile) + + data_path_striped = "".join(filter(str.isalnum, profile)) + ingestfile = ingestdir / f"{i}_{data_path_striped}.json" + + query_metadata = json.dumps( + {"id": "identity--" + query_id, "name": connector_name} + ) - result_retrieval_offset = 0 - has_remaining_results = True - while has_remaining_results: - result_batch = transmission.results( - search_id, result_retrieval_offset, RETRIEVAL_BATCH_SIZE - ) - if result_batch["success"]: - new_entries = result_batch["data"] - if new_entries: - connector_results += new_entries - result_retrieval_offset += RETRIEVAL_BATCH_SIZE - if len(new_entries) < RETRIEVAL_BATCH_SIZE: - has_remaining_results = False + translation = stix_translation.StixTranslation() + transmission = stix_transmission.StixTransmission( + connector_name, connection_dict, configuration_dict + ) + + dsl = translation.translate( + connector_name, "query", query_metadata, pattern, {} + ) + + if "error" in dsl: + raise DataSourceError( + f"STIX-shifter translation failed with message: {dsl['error']}" + ) + + # query results should be put together; when translated to STIX, the relation between them will remain + connector_results = [] + for query in dsl["queries"]: + search_meta_result = transmission.query(query) + if search_meta_result["success"]: + search_id = search_meta_result["search_id"] + if transmission.is_async(): + time.sleep(1) + status = transmission.status(search_id) + if status["success"]: + while ( + status["progress"] < 100 + and status["status"] == "RUNNING" + ): + status = transmission.status(search_id) else: - has_remaining_results = False - else: - raise DataSourceError( - "STIX-shifter transmission.results() failed" + stix_shifter_error_msg = ( + status["error"] + if "error" in status + else "details not avaliable" + ) + raise DataSourceError( + f"STIX-shifter transmission.status() failed with message: {stix_shifter_error_msg}" + ) + + result_retrieval_offset = 0 + has_remaining_results = True + while has_remaining_results: + result_batch = transmission.results( + search_id, result_retrieval_offset, RETRIEVAL_BATCH_SIZE ) + if result_batch["success"]: + new_entries = result_batch["data"] + if new_entries: + connector_results += new_entries + result_retrieval_offset += RETRIEVAL_BATCH_SIZE + if len(new_entries) < RETRIEVAL_BATCH_SIZE: + has_remaining_results = False + else: + has_remaining_results = False + else: + stix_shifter_error_msg = ( + result_batch["error"] + if "error" in result_batch + else "details not avaliable" + ) + raise DataSourceError( + f"STIX-shifter transmission.results() failed with message: {stix_shifter_error_msg}" + ) + + else: + stix_shifter_error_msg = ( + search_meta_result["error"] + if "error" in search_meta_result + else "details not avaliable" + ) + raise DataSourceError( + f"STIX-shifter transmission.query() failed with message: {stix_shifter_error_msg}" + ) - else: - raise DataSourceError("STIX-shifter transmission.query() failed") - - stixbundle = translation.translate( - connector_name, "results", query_metadata, json.dumps(connector_results), {} - ) + stixbundle = translation.translate( + connector_name, + "results", + query_metadata, + json.dumps(connector_results), + {}, + ) - with ingestfile.open("w") as ingest: - json.dump(stixbundle, ingest, indent=4) + with ingestfile.open("w") as ingest: + json.dump(stixbundle, ingest, indent=4) + bundles.append(str(ingestfile.resolve())) - return ReturnFromFile(query_id, [str(ingestfile.resolve())]) + return ReturnFromFile(query_id, bundles) @staticmethod def _get_stixshifter_config(profile_name): @@ -198,7 +227,7 @@ def _get_stixshifter_config(profile_name): f'invalid {env_conn_name} environment variable: no "host" field', ) - if "port" not in connection: + if "port" not in connection and connector_name != "stix_bundle": raise InvalidDataSource( profile_name, "stixshifter", diff --git a/tests/test_bundle_4.json b/tests/test_bundle_4.json new file mode 100644 index 00000000..79123931 --- /dev/null +++ b/tests/test_bundle_4.json @@ -0,0 +1,164 @@ +{ + "type": "bundle", + "id": "bundle--11e77454-448c-4229-9927-624b43826ad3", + "spec_version": "2.0", + "objects": [ + { + "type": "identity", + "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "name": "CbCloud" + }, + { + "id": "observed-data--522acfb0-2b9f-4695-9dbf-37e2572e7139", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "created": "2021-08-05T13:03:25.633Z", + "modified": "2021-08-05T13:03:25.633Z", + "objects": { + "0": { + "type": "x-cbcloud", + "childproc_count": 0, + "crossproc_count": 0, + "device_external_ip": "192.168.100.212", + "device_group_id": 0, + "device_id": 27969132, + "device_name": "windows10-lab2", + "device_os": "WINDOWS", + "device_timestamp": "2021-02-26T08:14:22.562Z", + "filemod_count": 0, + "modload_count": 0, + "netconn_count": 0, + "org_id": "MYORGIDX", + "regmod_count": 0, + "scriptload_count": 0 + }, + "1": { + "type": "process", + "x_unique_id": "MYORGIDX-01aac66c-00000820-00000000-1d70c280e79cd04", + "name": "compattelrunner.exe", + "binary_ref": "2", + "pid": 2080 + }, + "2": { + "type": "file", + "name": "compattelrunner.exe", + "hashes": { + "SHA-256": "c0a5986a4dd6d7cacf09c5a980df634c44ff73028206d99cb561e64a74a0958a" + }, + "parent_directory_ref": "7" + }, + "3": { + "type": "process", + "parent_ref": "1", + "command_line": "powershell.exe -ExecutionPolicy Restricted -Command $Res = 0; if((Get-WindowsOptionalFeature -Online -FeatureName IIS-WebServerRole).State -eq 'Enabled') { $Path = $env:windir + '\\system32\\inetsrv\\config\\applicationHost.Config'; if (Test-Path -Path $Path) { try { [XML]$Xml = Get-Content $Path } catch { $Res = 1 } }; } Write-Host 'Final result:',$Res", + "x_unique_id": "MYORGIDX-01aac66c-00001170-00000000-1d70c2820408601", + "name": "powershell.exe", + "binary_ref": "4", + "pid": 4464, + "created": "2021-02-26T09:14:15.304Z", + "creator_user_ref": "5" + }, + "4": { + "type": "file", + "name": "powershell.exe", + "hashes": { + "MD5": "7353f60b1739074eb17c5f4dddefe239", + "SHA-256": "de96a6e69944335375dc1ac238336066889d9ffc7d73628ef4fe1b1b160ab32c" + }, + "parent_directory_ref": "6" + }, + "5": { + "type": "user-account", + "user_id": "NT AUTHORITY\\SYSTEM" + }, + "6": { + "type": "directory", + "path": "c:\\windows\\system32\\windowspowershell\\v1.0" + }, + "7": { + "type": "directory", + "path": "c:\\windows\\system32" + } + }, + "first_observed": "2021-02-26T08:14:22.562Z", + "last_observed": "2021-02-26T08:14:22.562Z", + "number_observed": 1 + }, + { + "id": "observed-data--de6ca397-e7b6-4e48-8bfa-f85999fc26e5", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "created": "2021-08-05T13:03:25.634Z", + "modified": "2021-08-05T13:03:25.634Z", + "objects": { + "0": { + "type": "x-cbcloud", + "childproc_count": 0, + "crossproc_count": 0, + "device_external_ip": "192.168.100.212", + "device_group_id": 0, + "device_id": 27969132, + "device_name": "windows10-lab2", + "device_os": "WINDOWS", + "device_timestamp": "2021-03-19T15:16:28.196Z", + "filemod_count": 0, + "modload_count": 0, + "netconn_count": 0, + "org_id": "MYORGIDX", + "regmod_count": 0, + "scriptload_count": 0 + }, + "1": { + "type": "process", + "x_unique_id": "MYORGIDX-01aac66c-00001a58-00000000-1d71ce346740e1e", + "name": "compattelrunner.exe", + "binary_ref": "2", + "pid": 6744 + }, + "2": { + "type": "file", + "name": "compattelrunner.exe", + "hashes": { + "SHA-256": "c0a5986a4dd6d7cacf09c5a980df634c44ff73028206d99cb561e64a74a0958a" + }, + "parent_directory_ref": "7" + }, + "3": { + "type": "process", + "parent_ref": "1", + "command_line": "powershell.exe -ExecutionPolicy Restricted -Command $Res = 0; if((Get-WindowsOptionalFeature -Online -FeatureName IIS-WebServerRole).State -eq 'Enabled') { $Path = $env:windir + '\\system32\\inetsrv\\config\\applicationHost.Config'; if (Test-Path -Path $Path) { try { [XML]$Xml = Get-Content $Path } catch { $Res = 1 } }; } Write-Host 'Final result:',$Res", + "x_unique_id": "MYORGIDX-01aac66c-00001c6c-00000000-1d71ce37d053a33", + "name": "powershell.exe", + "binary_ref": "4", + "pid": 7276, + "created": "2021-03-19T16:16:04.136Z", + "creator_user_ref": "5" + }, + "4": { + "type": "file", + "name": "powershell.exe", + "hashes": { + "MD5": "7353f60b1739074eb17c5f4dddefe239", + "SHA-256": "de96a6e69944335375dc1ac238336066889d9ffc7d73628ef4fe1b1b160ab32c" + }, + "parent_directory_ref": "6" + }, + "5": { + "type": "user-account", + "user_id": "NT AUTHORITY\\SYSTEM" + }, + "6": { + "type": "directory", + "path": "c:\\windows\\system32\\windowspowershell\\v1.0" + }, + "7": { + "type": "directory", + "path": "c:\\windows\\system32" + } + }, + "first_observed": "2021-03-19T15:16:28.196Z", + "last_observed": "2021-03-19T15:16:28.196Z", + "number_observed": 1 + } + ] +} diff --git a/tests/test_bundle_5.json b/tests/test_bundle_5.json new file mode 100644 index 00000000..e986a385 --- /dev/null +++ b/tests/test_bundle_5.json @@ -0,0 +1,241 @@ +{ + "type": "bundle", + "id": "bundle--11e77454-448c-4229-9927-624b43826ad3", + "spec_version": "2.0", + "objects": [ + { + "type": "identity", + "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "name": "CbCloud" + }, + { + "id": "observed-data--7b5a131e-c9b9-4f32-b5fa-b5f48f9208f8", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "created": "2021-08-05T13:03:25.634Z", + "modified": "2021-08-05T13:03:25.634Z", + "objects": { + "0": { + "type": "x-cbcloud", + "childproc_count": 0, + "crossproc_count": 0, + "device_external_ip": "192.168.100.212", + "device_group_id": 0, + "device_id": 27969112, + "device_name": "windows10-lab1", + "device_os": "WINDOWS", + "device_timestamp": "2021-03-19T16:19:19.138Z", + "filemod_count": 0, + "modload_count": 0, + "netconn_count": 0, + "org_id": "MYORGIDX", + "regmod_count": 0, + "scriptload_count": 0 + }, + "1": { + "type": "process", + "x_unique_id": "MYORGIDX-01aac658-00001b7c-00000000-1d71cec2d7f188e", + "name": "compattelrunner.exe", + "binary_ref": "2", + "pid": 7036 + }, + "2": { + "type": "file", + "name": "compattelrunner.exe", + "hashes": { + "SHA-256": "c0a5986a4dd6d7cacf09c5a980df634c44ff73028206d99cb561e64a74a0958a" + }, + "parent_directory_ref": "7" + }, + "3": { + "type": "process", + "parent_ref": "1", + "command_line": "powershell.exe -ExecutionPolicy Restricted -Command $Res = 0; if((Get-WindowsOptionalFeature -Online -FeatureName IIS-WebServerRole).State -eq 'Enabled') { $Path = $env:windir + '\\system32\\inetsrv\\config\\applicationHost.Config'; if (Test-Path -Path $Path) { try { [XML]$Xml = Get-Content $Path } catch { $Res = 1 } }; } Write-Host 'Final result:',$Res", + "x_unique_id": "MYORGIDX-01aac658-00001370-00000000-1d71cec444d7279", + "name": "powershell.exe", + "binary_ref": "4", + "pid": 4976, + "created": "2021-03-19T17:18:55.150Z", + "creator_user_ref": "5" + }, + "4": { + "type": "file", + "name": "powershell.exe", + "hashes": { + "MD5": "7353f60b1739074eb17c5f4dddefe239", + "SHA-256": "de96a6e69944335375dc1ac238336066889d9ffc7d73628ef4fe1b1b160ab32c" + }, + "parent_directory_ref": "6" + }, + "5": { + "type": "user-account", + "user_id": "NT AUTHORITY\\SYSTEM" + }, + "6": { + "type": "directory", + "path": "c:\\windows\\system32\\windowspowershell\\v1.0" + }, + "7": { + "type": "directory", + "path": "c:\\windows\\system32" + } + }, + "first_observed": "2021-03-19T16:19:19.138Z", + "last_observed": "2021-03-19T16:19:19.138Z", + "number_observed": 1 + }, + { + "id": "observed-data--292d681f-df32-47b7-a8b3-8aaae2b61618", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "created": "2021-08-05T13:03:25.635Z", + "modified": "2021-08-05T13:03:25.635Z", + "objects": { + "0": { + "type": "x-cbcloud", + "childproc_count": 0, + "crossproc_count": 0, + "device_external_ip": "192.168.100.212", + "device_group_id": 0, + "device_id": 27969132, + "device_name": "windows10-lab2", + "device_os": "WINDOWS", + "device_timestamp": "2021-03-19T19:03:54.637Z", + "filemod_count": 0, + "modload_count": 0, + "netconn_count": 0, + "org_id": "MYORGIDX", + "regmod_count": 0, + "scriptload_count": 0 + }, + "1": { + "type": "process", + "x_unique_id": "MYORGIDX-01aac66c-00001644-00000000-1d71cf28cfa1765", + "name": "compattelrunner.exe", + "binary_ref": "2", + "pid": 5700 + }, + "2": { + "type": "file", + "name": "compattelrunner.exe", + "hashes": { + "SHA-256": "c0a5986a4dd6d7cacf09c5a980df634c44ff73028206d99cb561e64a74a0958a" + }, + "parent_directory_ref": "7" + }, + "3": { + "type": "process", + "parent_ref": "1", + "command_line": "powershell.exe -ExecutionPolicy Restricted -Command $Res = 0; if((Get-WindowsOptionalFeature -Online -FeatureName IIS-WebServerRole).State -eq 'Enabled') { $Path = $env:windir + '\\system32\\inetsrv\\config\\applicationHost.Config'; if (Test-Path -Path $Path) { try { [XML]$Xml = Get-Content $Path } catch { $Res = 1 } }; } Write-Host 'Final result:',$Res", + "x_unique_id": "MYORGIDX-01aac66c-00001790-00000000-1d71cf29b2fc9c3", + "name": "powershell.exe", + "binary_ref": "4", + "pid": 6032, + "created": "2021-03-19T19:03:54.198Z", + "creator_user_ref": "5" + }, + "4": { + "type": "file", + "name": "powershell.exe", + "hashes": { + "MD5": "7353f60b1739074eb17c5f4dddefe239", + "SHA-256": "de96a6e69944335375dc1ac238336066889d9ffc7d73628ef4fe1b1b160ab32c" + }, + "parent_directory_ref": "6" + }, + "5": { + "type": "user-account", + "user_id": "NT AUTHORITY\\SYSTEM" + }, + "6": { + "type": "directory", + "path": "c:\\windows\\system32\\windowspowershell\\v1.0" + }, + "7": { + "type": "directory", + "path": "c:\\windows\\system32" + } + }, + "first_observed": "2021-03-19T19:03:54.637Z", + "last_observed": "2021-03-19T19:03:54.637Z", + "number_observed": 1 + }, + { + "id": "observed-data--fbe8bf8b-fc2e-4268-b325-9a12b90f2b2c", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "created": "2021-08-05T13:03:25.657Z", + "modified": "2021-08-05T13:03:25.657Z", + "objects": { + "0": { + "type": "x-cbcloud", + "childproc_count": 0, + "crossproc_count": 0, + "device_external_ip": "192.168.100.212", + "device_group_id": 0, + "device_id": 40017686, + "device_internal_ip": "10.0.0.139", + "device_name": "windows10-lab1", + "device_os": "WINDOWS", + "device_timestamp": "2021-03-19T22:39:10.117Z", + "filemod_count": 0, + "modload_count": 0, + "netconn_count": 0, + "org_id": "MYORGIDX", + "regmod_count": 0, + "scriptload_count": 0 + }, + "1": { + "type": "process", + "x_unique_id": "MYORGIDX-02629f16-00000608-00000000-1d71d10a09cc7c4", + "name": "compattelrunner.exe", + "binary_ref": "2", + "pid": 1544 + }, + "2": { + "type": "file", + "name": "compattelrunner.exe", + "hashes": { + "SHA-256": "c0a5986a4dd6d7cacf09c5a980df634c44ff73028206d99cb561e64a74a0958a" + }, + "parent_directory_ref": "7" + }, + "3": { + "type": "process", + "parent_ref": "1", + "command_line": "powershell.exe -ExecutionPolicy Restricted -Command $Res = 0; if((Get-WindowsOptionalFeature -Online -FeatureName IIS-WebServerRole).State -eq 'Enabled') { $Path = $env:windir + '\\system32\\inetsrv\\config\\applicationHost.Config'; if (Test-Path -Path $Path) { try { [XML]$Xml = Get-Content $Path } catch { $Res = 1 } }; } Write-Host 'Final result:',$Res", + "x_unique_id": "MYORGIDX-02629f16-00000ad8-00000000-1d71d10acbe8a66", + "name": "powershell.exe", + "binary_ref": "4", + "pid": 2776, + "created": "2021-03-19T22:39:08.556Z", + "creator_user_ref": "5" + }, + "4": { + "type": "file", + "name": "powershell.exe", + "hashes": { + "MD5": "7353f60b1739074eb17c5f4dddefe239", + "SHA-256": "de96a6e69944335375dc1ac238336066889d9ffc7d73628ef4fe1b1b160ab32c" + }, + "parent_directory_ref": "6" + }, + "5": { + "type": "user-account", + "user_id": "NT AUTHORITY\\SYSTEM" + }, + "6": { + "type": "directory", + "path": "c:\\windows\\system32\\windowspowershell\\v1.0" + }, + "7": { + "type": "directory", + "path": "c:\\windows\\system32" + } + }, + "first_observed": "2021-03-19T22:39:10.117Z", + "last_observed": "2021-03-19T22:39:10.117Z", + "number_observed": 1 + } + ] +} diff --git a/tests/test_command_get.py b/tests/test_command_get.py new file mode 100644 index 00000000..665c9768 --- /dev/null +++ b/tests/test_command_get.py @@ -0,0 +1,81 @@ +import pytest +import json +import os + +from kestrel.session import Session + + +@pytest.fixture() +def file_stix_bundles(): + cwd = os.path.dirname(os.path.abspath(__file__)) + return [os.path.join(cwd, "test_bundle_4.json"), + os.path.join(cwd, "test_bundle_5.json")] + + +@pytest.fixture() +def set_stixshifter_stix_bundles(): + cfg = '{"auth": {"username": "","password": ""}}' + connector = 'stix_bundle' + stixshifter_data_url = 'https://raw.githubusercontent.com/opencybersecurityalliance/stix-shifter/develop/data/cybox' + host1 = f"{stixshifter_data_url}/carbon_black/cb_observed_156.json" + host2 = f"{stixshifter_data_url}/qradar/qradar_custom_process_observable.json" + + os.environ['STIXSHIFTER_HOST1_CONNECTION'] = json.dumps({"host": host1}) + os.environ['STIXSHIFTER_HOST1_CONNECTOR'] = connector + os.environ['STIXSHIFTER_HOST1_CONFIG'] = cfg + os.environ['STIXSHIFTER_HOST2_CONNECTION'] = json.dumps({"host": host2}) + os.environ['STIXSHIFTER_HOST2_CONNECTOR'] = connector + os.environ['STIXSHIFTER_HOST2_CONFIG'] = cfg + + +def test_get_single_file(file_stix_bundles): + with Session() as s: + stmt = f"var = GET process FROM file://{file_stix_bundles[0]} WHERE [process:name='compattelrunner.exe']" + + s.execute(stmt) + v = s.get_variable("var") + assert len(v) == 2 + assert v[0]["type"] == "process" + assert v[0]["name"] == "compattelrunner.exe" + + +def test_get_multiple_file_stix_bundles(file_stix_bundles): + with Session() as s: + file_bundles = ','.join(file_stix_bundles) + stmt = f"var = GET process FROM file://{file_bundles} WHERE [process:name='compattelrunner.exe']" + + s.execute(stmt) + v = s.get_variable("var") + assert len(v) == 5 + assert v[0]["type"] == "process" + assert v[0]["name"] == "compattelrunner.exe" + + +def test_get_single_stixshifter_stix_bundle(set_stixshifter_stix_bundles): + with Session() as s: + # default data source schema is stixshifter + stmt = "var = GET process FROM HOST2 WHERE [ipv4-addr:value = '127.0.0.1']" + + s.execute(stmt) + v = s.get_variable("var") + assert len(v) == 6 + for i in range(len(v)): + assert v[i]["type"] == "process" + assert v[i]["name"] == "powershell.exe" + + +def test_get_multiple_stixshifter_stix_bundles(set_stixshifter_stix_bundles): + with Session() as s: + # default data source schema is stixshifter + stmt = "var = GET process FROM HOST1,HOST2 WHERE [ipv4-addr:value = '127.0.0.1']" + + s.execute(stmt) + v = s.get_variable("var") + assert len(v) == 240 + for i in range(len(v)): + assert v[i]["type"] == "process" + assert v[i]["name"] in [ + "powershell.exe", "(unknown)", "explorer.exe", "firefox.exe", "ntoskrnl.exe", + "teamviewer_service.exe", "teamviewer.exe", "vmware.exe", "dashost.exe", + "applemobiledeviceservice.exe", "svctest.exe", "vmware-hostd.exe"] +