From b1e568efd2a70e4e502556b73d79b1d0e1befb47 Mon Sep 17 00:00:00 2001 From: yshamai Date: Mon, 20 Jan 2025 11:49:16 +0200 Subject: [PATCH 01/15] start --- Packs/Dynarace/.pack-ignore | 0 Packs/Dynarace/.secrets-ignore | 0 Packs/Dynarace/Author_image.png | 0 .../Integrations/Dynatrace/Dynatrace.py | 226 ++++++++++++++++++ .../Integrations/Dynatrace/Dynatrace.yml | 135 +++++++++++ .../Dynatrace/Dynatrace_description.md | 8 + .../Dynatrace/Dynatrace_image.png | Bin 0 -> 2507 bytes .../Integrations/Dynatrace/Dynatrace_test.py | 41 ++++ .../Dynarace/Integrations/Dynatrace/README.md | 5 + .../Integrations/Dynatrace/command_examples | 0 Packs/Dynarace/README.md | 0 Packs/Dynarace/pack_metadata.json | 18 ++ 12 files changed, 433 insertions(+) create mode 100644 Packs/Dynarace/.pack-ignore create mode 100644 Packs/Dynarace/.secrets-ignore create mode 100644 Packs/Dynarace/Author_image.png create mode 100644 Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py create mode 100644 Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml create mode 100644 Packs/Dynarace/Integrations/Dynatrace/Dynatrace_description.md create mode 100644 Packs/Dynarace/Integrations/Dynatrace/Dynatrace_image.png create mode 100644 Packs/Dynarace/Integrations/Dynatrace/Dynatrace_test.py create mode 100644 Packs/Dynarace/Integrations/Dynatrace/README.md create mode 100644 Packs/Dynarace/Integrations/Dynatrace/command_examples create mode 100644 Packs/Dynarace/README.md create mode 100644 Packs/Dynarace/pack_metadata.json diff --git a/Packs/Dynarace/.pack-ignore b/Packs/Dynarace/.pack-ignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/Dynarace/.secrets-ignore b/Packs/Dynarace/.secrets-ignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/Dynarace/Author_image.png b/Packs/Dynarace/Author_image.png new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py new file mode 100644 index 000000000000..9ee06b0a5b17 --- /dev/null +++ b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py @@ -0,0 +1,226 @@ +from typing import Any, Dict, Optional +import demistomock as demisto +import urllib3 +from CommonServerPython import * # noqa # pylint: disable=unused-wildcard-import +from CommonServerUserPython import * # noqa +# Disable insecure warnings +urllib3.disable_warnings() + +""" CONSTANTS """ + +DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # ISO8601 format with UTC, default in XSOAR +VENDOR = "Dynatrace" +PRODUCT = "Platform" +EVENTS_TYPE_DICT = {"Vulnerability": ("securityProblems", "vul_limit"), "Audit logs": ("auditLogs", "audit_limit"), "APM": ("events", "apm_limit")} + + +""" CLIENT CLASS """ + +class Client(BaseClient): + def init(self, base_url: str, client_id: str, client_secret: str, uuid: str, token: str, events_to_fetch: List[str], verify: bool, proxy): + super().__init__(base_url=base_url, verify=verify, proxy=proxy) + self.client_id = client_id, + self.client_secret = client_secret, + self.uuid = uuid, + self.token = token + self.personal_token = None + if not self.token: # We are using OAuth2 authentication + self.personal_token = self.create_personal_token(events_to_fetch) + self._headers = {"Authorization": f"Api-Token {self.personal_token}" if self.personal_token else f"Bearer {self.token}"} + def create_personal_token(self, events_to_fetch): + scopes = [] + if "Vulnerability" in events_to_fetch: + scopes.append("securityProblems.read") + if "Audit logs" in events_to_fetch: + scopes.append("auditLogs.read") + if "APM" in events_to_fetch: + scopes.append("events.read") + params = assign_params( + grant_type = "client_credentials", + client_id = self.client_id, + client_secret = self.client_secret, + scope = " ".join(scopes), + resource = f"urn:dtaccount:{self.uuid}" + ) + raw_response = self._http_request( + method='POST', + url_suffix="https://sso.dynatrace.com/sso/oauth2/token", + json_data=params, + headers=self._headers + ) + return raw_response # TODO test how response returns and return the token within it + + + def get_vulnerability_events(self, params: dict): + return self._http_request("GET", "/api/v2/securityProblems", json_data=params, headers=self._headers) + + + def get_audit_logs_events(self, params: dict): + return self._http_request("GET", "/api/v2/auditlogs", json_data=params, headers=self._headers) + + + def get_APM_events(self, params: dict): + return self._http_request("GET", "/api/v2/events", json_data=params, headers=self._headers) + + +""" HELPER FUNCTIONS """ + +def validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, vul_max, audit_max, apm_max): + + if not ((client_id and client_secret and uuid and not token) or (token and not client_id and not client_secret and not uuid)): + raise DemistoException("When using OAuth 2, ensure to specify the client ID, client secret, and Account UUID. When using a personal access token, make sure to specify the access token. It's important to include only the required parameters for each type and avoid including any extra parameters.") + if not events_to_fetch: + raise DemistoException("Please specify at least one event type to fetch.") + if not vul_max > 0 and not vul_max <= 2500: + raise DemistoException("Thee maximum number of vulnerability events per fetch needs to be grater than 0 and not more than 2500") + if not audit_max > 0 and not audit_max <= 2500: + raise DemistoException("The maximum number of audit logs events per fetch needs to be grater then 0 and not more then then 25000") + if not apm_max > 0 and not apm_max <= 25000: + raise DemistoException("The maximum number of APM events per fetch needs to be grater then 0 and not more then then 5000") + + +""" COMMAND FUNCTIONS """ + +def fetch_events(client: Client, events_to_fetch: list, vul_limit: int, audit_limit: int, apm_limit: int): + + integration_context = demisto.getIntegrationContext() + + # first fetch + if not integration_context: + pass + # TODO implement first run, need to specify from for each type and no next page. Maybe the defaulf from of the api is good for us. + + integration_context_to_save = {} + events_to_return = [] + vul_count, audit_count, apm_count = 0, 0, 0 + + for _ in range(0, 5): + + # TODO In the first time integration_context will be empty and there won't be a next page value in it, in this case the query function will not send an empty arg in the json body, the api has a default from date that can be used in the first fetch, need to see what happens when I send a nextPage, the api adds a from date? which one does it look at? + + args = {"vul_limit": min(vul_limit-vul_count, 500), "audit_limit": min(audit_limit-audit_count, 5000), "apm_limit": min(apm_limit-apm_count,1000), + "Vulnerability_next_page": integration_context.get("Vulnerability_next_page"), + "Audit logs_next_page": integration_context.get("Audit logs_next_page"), + "APM_next_page": integration_context.get("APM_next_page")} + + for event_type in events_to_fetch: + if args[EVENTS_TYPE_DICT[event_type][1]] != 0: + response = events_query(client, args, event_type) + events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type[0]]], event_type) + events_to_return.extend(events) + integration_context_to_save[event_type+"_next_page"] = response["nextPageKey"] # TODO what happens when there are no more events? Do we still gat a nextPageKey? If not we need to go out of the loop, need to check this use case + + set_integration_context(integration_context_to_save) + + return events_to_return + + +def add_fields_to_events(events, event_type): + + # TODO ask sara if we need the word 'events' in the end of the type, I don't think we usually do so. + + field_mapping = { + "Vulnerability": {"SOURCE_LOG_TYPE": "Vulnerability events", "_time": "firstSeenTimestamp"}, + "Audit logs": {"SOURCE_LOG_TYPE": "Audit logs events", "_time": "timestamp"}, + "APM events": {"SOURCE_LOG_TYPE": "APM events", "_time": "startTime"} + } + + for event in events: + for key, value in field_mapping[event_type].items(): + event[key] = event[value] + + return events + + +def get_events_command(client: Client, args: dict): + + events_types = argToList(args.get("events_types_to_get")) + events_to_return = [] + + for event_type in events_types: + response = events_query(client, args, event_type) + events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type[0]]], event_type) + events_to_return.extend(events) + # send_events_to_xsiam(events_to_return, vendor=VENDOR, product=PRODUCT) move this to main + + return events_to_return # Make a human readable + + +def test_module(client: Client, events_to_fetch: List[str]) -> str: + + # TODO change this function to use query function + + try: + if "Vulnerability" in events_to_fetch: + client.get_vulnerability_events({"pageSize": 1}) + if "Audit logs" in events_to_fetch: + client.get_audit_logs_events({"pageSize": 1}) + if "APM" in events_to_fetch: + client.get_APM_events({"pageSize": 1}) + + except Exception as e: + raise DemistoException (str(e).lower()) + + return "ok" + + +def events_query(client: Client, args: dict, event_type: str): + # + if event_type == "Vulnerability": + events = client.get_vulnerability_events({key: value for key, value in {"pageSize": args.get("vul_limit"), "from": args.get("vul_from"), "nextPageKey": args.get("Vulnerability_next_page")}.items() if value}) + elif event_type == "Audit logs": + events = client.get_audit_logs_events({key: value for key, value in {"pageSize": args.get("audit_limit"), "from": args.get("audit_from"), "nextPageKey": args.get("Audit logs_next_page")}.items() if value}) + elif event_type == "APM": + events = client.get_audit_logs_events({key: value for key, value in {"pageSize": args.get("apm_limit"), "from": args.get("apm_from"), "nextPageKey": args.get("APM_next_page")}.items() if value}) + # TODO add needed fields for every event with calling a new function + return events + + +def main(): + + """main function, parses params and runs command functions""" + + params = demisto.params() + url = params.get("url") + client_id = params.get('client_id') + client_secret = params.get('client_secret') + uuid = params.get('uuid') + token = params.get('token') + events_to_fetch = argToList(params.get('events_to_fetch')) + vul_limit = arg_to_number(params.get('vul_limit')) or 1000 + audit_limit = arg_to_number(params.get('audit_limit')) or 25000 + apm_limit = arg_to_number(params.get('apm_limit')) or 25000 + validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, vul_limit, audit_limit, apm_limit) + + verify = not argToBoolean(params.get("insecure", False)) + proxy = argToBoolean(params.get("proxy", False)) + + command = demisto.command() + + demisto.debug(f"Command being called is {command}") + + try: + + client = Client(base_url=url, client_id=client_id, client_secret=client_secret, uuid=uuid, token=token, events_to_fetch=events_to_fetch, verify=verify, proxy=proxy) + + args = demisto.args() + + if command == "test-module": + # This is the call made when pressing the integration Test button. + result = test_module(client, events_to_fetch) + elif command == "dynatrace-get-events": + result = get_events_command(client, args) + elif command == "fetch-events": + result = fetch_events(client, events_to_fetch, vul_limit, audit_limit, apm_limit) + else: + raise NotImplementedError(f"Command {command} is not implemented") + return_results( + result + ) # Returns either str, CommandResults and a list of CommandResults + + # Log exceptions and return errors + except Exception as e: + return_error(f"Failed to execute {command} command.\nError:\n{str(e)}") + +if __name__ in ("__main__", "__builtin__", "builtins"): # pragma: no cover + main() \ No newline at end of file diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml new file mode 100644 index 000000000000..05600a59a499 --- /dev/null +++ b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml @@ -0,0 +1,135 @@ +category: Analytics & SIEM +commonfields: + id: Dynatrace + version: -1 +configuration: +- display: Server URL + name: url + required: true + type: 0 +- display: Client ID + additionalinfo: Required If using OAuth 2 as authentication method. + name: client_id + required: false + type: 4 +- display: Client Secret + additionalinfo: Required If using OAuth 2 as authentication method. + name: client_secret + required: false + type: 4 +- display: Account UUID + additionalinfo: Required If using OAuth 2 as authentication method. + name: uuid + required: false + type: 4 +- display: Access Token + additionalinfo: Required If using Personal Access Token. + name: token + required: false + type: 4 +- display: Fetch Events + name: isFetchEvents + type: 8 + required: false + section: Collect + defaultvalue: true +- display: Events Fetch Interval + name: eventFetchInterval + type: 19 + required: false + section: Collect + defaultvalue: '1' + advanced: true +- display: Event types to fetch + name: events_to_fetch + type: 16 + defaultvalue: Vulnerability,Audit logs,APM + required: true + section: Collect + advanced: true + options: + - Vulnerability + - Audit logs + - APM +- display: The maximum number of vulnerability events per fetch + section: Collect + advanced: true + default: 2500 + type: 0 + name: vul_limit +- display: The maximum number of audit logs events per fetch + section: Collect + advanced: true + default: 5000 + type: 0 + name: audit_limit +- display: The maximum number of APM events per fetch + section: Collect + advanced: true + default: 5000 # Need to ask sara about the default limit in all event types + type: 0 + name: apm_limit +- display: Trust any certificate (not secure) + name: insecure + required: false + type: 8 +- display: Use system proxy settings + name: proxy + required: false + type: 8 +description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.]' +display: Dynatrace +name: Dynatrace +script: + commands: + arguments: + - name: events_types_to_get + description: comma separated list of events types to get + auto: PREDEFINED + predefined: + - Vulnerability + - Audit logs + - APM + required: true + - name: vul_from + description: The start date for searching vulnerability events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + - name: vul_to + description: The end date for retrieving vulnerability events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + - name: audit_logs_from + description: The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + - name: audit_logs_to + description: The end date for retrieving audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + - name: apm_from + description: The start date for searching apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + - name: apm_to + description: The end date for retrieving apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + - name: vul_limit + required: true + description: Number of vulnerability events to fetch from each event type that was chosen. + defaultValue: "1" + - name: audit_limit + required: true + description: Number of audit_logs events to fetch from each event type that was chosen. + defaultValue: "1" + - name: apm_limit + required: true + description: Number of apm events to fetch from each event type that was chosen. + defaultValue: "1" + - auto: PREDEFINED + defaultValue: "False" + description: Set this argument to True in order to create events, otherwise the command will only display them. + name: should_push_events + predefined: + - "True" + - "False" + required: true + description: Manual command to fetch events and display them. + name: dynatrace-get-events + type: python + subtype: python3 + dockerimage: demisto/python3:3.9.8.24399 +fromversion: 6.1.0 +marketplaces: +- marketplacev2 +tests: +- No tests (auto formatted) \ No newline at end of file diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_description.md b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_description.md new file mode 100644 index 000000000000..51bd561c623f --- /dev/null +++ b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_description.md @@ -0,0 +1,8 @@ +## BaseIntegration Help + +Markdown file for integration configuration help snippet. In this file add: + +- Brief information about how to retrieve the API key of your product +- Other useful information on how to configure your integration in XSOAR + +Since this is a Markdown file, we encourage you to use MD formatting for sections, sub-sections, lists, etc. diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_image.png b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_image.png new file mode 100644 index 0000000000000000000000000000000000000000..772c6af21bebd9230ea48f23a29ad2260f386e71 GIT binary patch literal 2507 zcmV;+2{iVJP)NJU1VgcOS!Wr4 zO{q;AyQYAIs;$+wHZ~Sp+*+hq)D%gf ztHY9woe3{0IJoIrpgCQC7`DJ3m>Rb4IVaSF>acO==CH3?2YW9zeW_Dx~*%o3bFhWN8-vqxH&=>B0u zuaCmH%$&2TPhhFWSb?U_&pUZ|*bx4bY2lAeKf4Fz( zmQcC1G(7!QIj=-*@_K6Xu1z?%2l)e_654|gT&Yu996XHs=&CTZ`w-v}%L}p|&0V{i z+>M}fRVtn9z|!)NwJa=JU&b?XP1tkblQ4D3yx@~^KJ{ql)!hryGQsnWH2d0+H@nGX zw4$doc(6Jj$F{`H#bN9f?}g#LF3mfF;nU`awLA8cKM!>FeJ~$VrQ(!~W*_628V=UH z#L*3oZlPbNGeW+jEG_4Q0)Dht@52W-_{z?+TAyKZtwufpGKm1!42gYaP3o=Hp5 zpENDeNw=&g(wishi@^u5KHBy z0B5lAQOWLBab_QSK09fjTO zC@3Ws@CkVee3)foO2$1e+uh|0=G5tv6Q@D!mTg_+}$dIq?{bj!*uvDc%`2UX?@FJXeYlO zql@{RdZcteM0jQY4X%XKp#!*g{X6jLzX!TOK_|!pF2zT3WJB`(jkNkL_}X+Gl)`YB z2EHqQ4+=WY7VrZ@LmX(C`XO?ji^Ej+Opo`g5xWAGl^6a3G? z&p{{f9xG+wdu=iJnzREvZxZ-gR8YYJ1q&1`P_RJ30tE{cEO1OM;46U-Cb=BbMjBDs z)ttK)h8|@daa>LHgTYT3?|-25D93$R@gid10e^0qi+Lx5PNNfk+3|QZisSOhx*-=W$J3%r+sPEn@)0ekD#*uCOYo^?Wv^h40bS-pMXKv)%JM$A&pY;mgh?FhkU+QE8BVmEWZwZ1Os3?tOv{bg-ZM9Yuh@kKT3Z*1=`auTt9(gP*;L= zYM0sIh3fe&AlwBta3|cJr8h@40S(p`YfCjh$JYPeoo}f{m1KXYlPCUyd701;zn`qs(u>dr$7Yn+zbza=YJN;K&Ss5w9P+3!^a6@TgF#|cJ-WZz!Y%D%ukcHbs5yv zHpv-V`0+!O}lnF#l)S zyxg`;wvOoF*cT^l)Di+ZR7D$U3#BdW&v=a=gNtBJyf z9dok((ZwPiqQALjyh|Hg0izuz@yL&?uKKjIP+TF!L!4*?Gxd3lZ5vK1_C>o# z8FoUss^nF&!V#46z7aKpzyxRwec=Uo1iYV=g7SPDa*1iIT!y{~=fU;h4D~9}AgV|0 zA^e%+Z7>n;gBsA;&qHJMLH6XYaP*3u3#(upYy+3X!SF}W;opKW@FaNUB(GkgpHdkq z-nYBJbznRyx9xXgM)nx{@@?W(Pyt>9lfd!UtGmdxP%h5M{V*0(KR5utHiBFledzcY zeh!}hB~Z@J$(c~?pZ7TYD-oYVcx57P`NiPUVm(VCE=x|PWnkOUD7O&S36?e8iI=N5 zuCCdPsT|`y&7@J_`Ij0&CGZNE<`vowoXoF-Y4Rq-3HM?m-Mlq$2dLb(OmijJ2G*_Z z-T=#~y4udI!;8#GW?S0biQWoSyazjRR5@7g0V9o(_ai&kSn1$Dfh$5ZAi+awM#yxJ zQHFJ1bb3s0?6C_KIIg5R5J{%>CcYC~YWu~r>$8V_Z3FpKSKGN|JCP6_o11si z#&fUuleVMTj(wG*SJ92&!{Pu~4vl>dpo(;ACd2wc6B?Sg4ex(($=#5RpLMd0;eS43 V(U=mQ{m=ja002ovPDHLkV1nG2;bs5; literal 0 HcmV?d00001 diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_test.py b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_test.py new file mode 100644 index 000000000000..c42f4e3a8b39 --- /dev/null +++ b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_test.py @@ -0,0 +1,41 @@ +"""Base Integration for Cortex XSOAR - Unit Tests file + +Pytest Unit Tests: all funcion names must start with "test_" + +More details: https://xsoar.pan.dev/docs/integrations/unit-testing + +MAKE SURE YOU REVIEW/REPLACE ALL THE COMMENTS MARKED AS "TODO" + +You must add at least a Unit Test function for every XSOAR command +you are implementing with your integration +""" + +from demisto_sdk.commands.common.handlers import JSON_Handler + +import json + + +def util_load_json(path): + with open(path, encoding="utf-8") as f: + return json.loads(f.read()) + + +# TODO: REMOVE the following dummy unit test function +def test_baseintegration_dummy(): + """Tests helloworld-say-hello command function. + + Checks the output of the command function with the expected output. + + No mock is needed here because the say_hello_command does not call + any external API. + """ + from BaseIntegration import Client, baseintegration_dummy_command + + client = Client(base_url="some_mock_url", verify=False) + args = {"dummy": "this is a dummy response", "dummy2": "a dummy value"} + response = baseintegration_dummy_command(client, args) + + assert response.outputs == args + + +# TODO: ADD HERE unit tests for every command diff --git a/Packs/Dynarace/Integrations/Dynatrace/README.md b/Packs/Dynarace/Integrations/Dynatrace/README.md new file mode 100644 index 000000000000..5d6d1dc2bf52 --- /dev/null +++ b/Packs/Dynarace/Integrations/Dynatrace/README.md @@ -0,0 +1,5 @@ +This README contains the full documentation for your integration. + +You auto-generate this README file from your integration YML file using the `demisto-sdk generate-docs` command. + +For more information see the [integration documentation](https://xsoar.pan.dev/docs/integrations/integration-docs). diff --git a/Packs/Dynarace/Integrations/Dynatrace/command_examples b/Packs/Dynarace/Integrations/Dynatrace/command_examples new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/Dynarace/README.md b/Packs/Dynarace/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/Dynarace/pack_metadata.json b/Packs/Dynarace/pack_metadata.json new file mode 100644 index 000000000000..59ee5d5afec7 --- /dev/null +++ b/Packs/Dynarace/pack_metadata.json @@ -0,0 +1,18 @@ +{ + "name": "Dynatrace", + "description": "## FILL MANDATORY FIELD ##", + "support": "xsoar", + "currentVersion": "1.0.0", + "author": "Cortex XSOAR", + "url": "https://www.paloaltonetworks.com/cortex", + "email": "", + "categories": [ + "Cloud Services" + ], + "tags": [], + "useCases": [], + "keywords": [], + "marketplaces": [ + "marketplacev2." + ] +} \ No newline at end of file From 92385b4e96fb134e6b22e5cec21093a75cb95582 Mon Sep 17 00:00:00 2001 From: yshamai Date: Mon, 20 Jan 2025 14:32:36 +0200 Subject: [PATCH 02/15] improve --- Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py | 9 +++++++++ Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml | 2 +- Packs/Dynarace/pack_metadata.json | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py index 9ee06b0a5b17..8e7e4d3f006e 100644 --- a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py @@ -19,22 +19,29 @@ class Client(BaseClient): def init(self, base_url: str, client_id: str, client_secret: str, uuid: str, token: str, events_to_fetch: List[str], verify: bool, proxy): super().__init__(base_url=base_url, verify=verify, proxy=proxy) + self.client_id = client_id, self.client_secret = client_secret, self.uuid = uuid, self.token = token self.personal_token = None + if not self.token: # We are using OAuth2 authentication self.personal_token = self.create_personal_token(events_to_fetch) + self._headers = {"Authorization": f"Api-Token {self.personal_token}" if self.personal_token else f"Bearer {self.token}"} + def create_personal_token(self, events_to_fetch): + scopes = [] + if "Vulnerability" in events_to_fetch: scopes.append("securityProblems.read") if "Audit logs" in events_to_fetch: scopes.append("auditLogs.read") if "APM" in events_to_fetch: scopes.append("events.read") + params = assign_params( grant_type = "client_credentials", client_id = self.client_id, @@ -42,12 +49,14 @@ def create_personal_token(self, events_to_fetch): scope = " ".join(scopes), resource = f"urn:dtaccount:{self.uuid}" ) + raw_response = self._http_request( method='POST', url_suffix="https://sso.dynatrace.com/sso/oauth2/token", json_data=params, headers=self._headers ) + return raw_response # TODO test how response returns and return the token within it diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml index 05600a59a499..491a71c7c08c 100644 --- a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml +++ b/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml @@ -1,4 +1,4 @@ -category: Analytics & SIEM +category: Cloud Services commonfields: id: Dynatrace version: -1 diff --git a/Packs/Dynarace/pack_metadata.json b/Packs/Dynarace/pack_metadata.json index 59ee5d5afec7..734643ea3f8a 100644 --- a/Packs/Dynarace/pack_metadata.json +++ b/Packs/Dynarace/pack_metadata.json @@ -13,6 +13,6 @@ "useCases": [], "keywords": [], "marketplaces": [ - "marketplacev2." + "marketplacev2" ] } \ No newline at end of file From b1a21bf0ce966d440a5e5510129c3b8ea9a0a339 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 12:38:40 +0200 Subject: [PATCH 03/15] change pack name --- Packs/{Dynarace => Dynatrace}/.pack-ignore | 0 Packs/{Dynarace => Dynatrace}/.secrets-ignore | 0 .../{Dynarace => Dynatrace}/Author_image.png | 0 .../Integrations/Dynatrace/Dynatrace.py | 38 +++++------------- .../Integrations/Dynatrace/Dynatrace.yml | 22 ++-------- .../Dynatrace/Dynatrace_description.md | 0 .../Dynatrace/Dynatrace_image.png | Bin .../Integrations/Dynatrace/Dynatrace_test.py | 0 .../Integrations/Dynatrace/README.md | 0 .../Integrations/Dynatrace/command_examples | 0 Packs/{Dynarace => Dynatrace}/README.md | 0 .../pack_metadata.json | 0 12 files changed, 12 insertions(+), 48 deletions(-) rename Packs/{Dynarace => Dynatrace}/.pack-ignore (100%) rename Packs/{Dynarace => Dynatrace}/.secrets-ignore (100%) rename Packs/{Dynarace => Dynatrace}/Author_image.png (100%) rename Packs/{Dynarace => Dynatrace}/Integrations/Dynatrace/Dynatrace.py (80%) rename Packs/{Dynarace => Dynatrace}/Integrations/Dynatrace/Dynatrace.yml (74%) rename Packs/{Dynarace => Dynatrace}/Integrations/Dynatrace/Dynatrace_description.md (100%) rename Packs/{Dynarace => Dynatrace}/Integrations/Dynatrace/Dynatrace_image.png (100%) rename Packs/{Dynarace => Dynatrace}/Integrations/Dynatrace/Dynatrace_test.py (100%) rename Packs/{Dynarace => Dynatrace}/Integrations/Dynatrace/README.md (100%) rename Packs/{Dynarace => Dynatrace}/Integrations/Dynatrace/command_examples (100%) rename Packs/{Dynarace => Dynatrace}/README.md (100%) rename Packs/{Dynarace => Dynatrace}/pack_metadata.json (100%) diff --git a/Packs/Dynarace/.pack-ignore b/Packs/Dynatrace/.pack-ignore similarity index 100% rename from Packs/Dynarace/.pack-ignore rename to Packs/Dynatrace/.pack-ignore diff --git a/Packs/Dynarace/.secrets-ignore b/Packs/Dynatrace/.secrets-ignore similarity index 100% rename from Packs/Dynarace/.secrets-ignore rename to Packs/Dynatrace/.secrets-ignore diff --git a/Packs/Dynarace/Author_image.png b/Packs/Dynatrace/Author_image.png similarity index 100% rename from Packs/Dynarace/Author_image.png rename to Packs/Dynatrace/Author_image.png diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py similarity index 80% rename from Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py rename to Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index 8e7e4d3f006e..f87a9435c4e8 100644 --- a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -11,7 +11,7 @@ DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # ISO8601 format with UTC, default in XSOAR VENDOR = "Dynatrace" PRODUCT = "Platform" -EVENTS_TYPE_DICT = {"Vulnerability": ("securityProblems", "vul_limit"), "Audit logs": ("auditLogs", "audit_limit"), "APM": ("events", "apm_limit")} +EVENTS_TYPE_DICT = {"Audit logs": ("auditLogs", "audit_limit"), "APM": ("events", "apm_limit")} """ CLIENT CLASS """ @@ -35,8 +35,6 @@ def create_personal_token(self, events_to_fetch): scopes = [] - if "Vulnerability" in events_to_fetch: - scopes.append("securityProblems.read") if "Audit logs" in events_to_fetch: scopes.append("auditLogs.read") if "APM" in events_to_fetch: @@ -60,10 +58,6 @@ def create_personal_token(self, events_to_fetch): return raw_response # TODO test how response returns and return the token within it - def get_vulnerability_events(self, params: dict): - return self._http_request("GET", "/api/v2/securityProblems", json_data=params, headers=self._headers) - - def get_audit_logs_events(self, params: dict): return self._http_request("GET", "/api/v2/auditlogs", json_data=params, headers=self._headers) @@ -74,14 +68,12 @@ def get_APM_events(self, params: dict): """ HELPER FUNCTIONS """ -def validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, vul_max, audit_max, apm_max): +def validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, audit_max, apm_max): if not ((client_id and client_secret and uuid and not token) or (token and not client_id and not client_secret and not uuid)): raise DemistoException("When using OAuth 2, ensure to specify the client ID, client secret, and Account UUID. When using a personal access token, make sure to specify the access token. It's important to include only the required parameters for each type and avoid including any extra parameters.") if not events_to_fetch: raise DemistoException("Please specify at least one event type to fetch.") - if not vul_max > 0 and not vul_max <= 2500: - raise DemistoException("Thee maximum number of vulnerability events per fetch needs to be grater than 0 and not more than 2500") if not audit_max > 0 and not audit_max <= 2500: raise DemistoException("The maximum number of audit logs events per fetch needs to be grater then 0 and not more then then 25000") if not apm_max > 0 and not apm_max <= 25000: @@ -90,7 +82,7 @@ def validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, """ COMMAND FUNCTIONS """ -def fetch_events(client: Client, events_to_fetch: list, vul_limit: int, audit_limit: int, apm_limit: int): +def fetch_events(client: Client, events_to_fetch: list, audit_limit: int, apm_limit: int): integration_context = demisto.getIntegrationContext() @@ -101,14 +93,13 @@ def fetch_events(client: Client, events_to_fetch: list, vul_limit: int, audit_li integration_context_to_save = {} events_to_return = [] - vul_count, audit_count, apm_count = 0, 0, 0 + audit_count, apm_count = 0, 0 for _ in range(0, 5): # TODO In the first time integration_context will be empty and there won't be a next page value in it, in this case the query function will not send an empty arg in the json body, the api has a default from date that can be used in the first fetch, need to see what happens when I send a nextPage, the api adds a from date? which one does it look at? - args = {"vul_limit": min(vul_limit-vul_count, 500), "audit_limit": min(audit_limit-audit_count, 5000), "apm_limit": min(apm_limit-apm_count,1000), - "Vulnerability_next_page": integration_context.get("Vulnerability_next_page"), + args = {"audit_limit": min(audit_limit-audit_count, 5000), "apm_limit": min(apm_limit-apm_count,1000), "Audit logs_next_page": integration_context.get("Audit logs_next_page"), "APM_next_page": integration_context.get("APM_next_page")} @@ -129,7 +120,6 @@ def add_fields_to_events(events, event_type): # TODO ask sara if we need the word 'events' in the end of the type, I don't think we usually do so. field_mapping = { - "Vulnerability": {"SOURCE_LOG_TYPE": "Vulnerability events", "_time": "firstSeenTimestamp"}, "Audit logs": {"SOURCE_LOG_TYPE": "Audit logs events", "_time": "timestamp"}, "APM events": {"SOURCE_LOG_TYPE": "APM events", "_time": "startTime"} } @@ -150,18 +140,13 @@ def get_events_command(client: Client, args: dict): response = events_query(client, args, event_type) events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type[0]]], event_type) events_to_return.extend(events) - # send_events_to_xsiam(events_to_return, vendor=VENDOR, product=PRODUCT) move this to main return events_to_return # Make a human readable def test_module(client: Client, events_to_fetch: List[str]) -> str: - - # TODO change this function to use query function - + try: - if "Vulnerability" in events_to_fetch: - client.get_vulnerability_events({"pageSize": 1}) if "Audit logs" in events_to_fetch: client.get_audit_logs_events({"pageSize": 1}) if "APM" in events_to_fetch: @@ -174,14 +159,10 @@ def test_module(client: Client, events_to_fetch: List[str]) -> str: def events_query(client: Client, args: dict, event_type: str): - # - if event_type == "Vulnerability": - events = client.get_vulnerability_events({key: value for key, value in {"pageSize": args.get("vul_limit"), "from": args.get("vul_from"), "nextPageKey": args.get("Vulnerability_next_page")}.items() if value}) - elif event_type == "Audit logs": + if event_type == "Audit logs": events = client.get_audit_logs_events({key: value for key, value in {"pageSize": args.get("audit_limit"), "from": args.get("audit_from"), "nextPageKey": args.get("Audit logs_next_page")}.items() if value}) elif event_type == "APM": events = client.get_audit_logs_events({key: value for key, value in {"pageSize": args.get("apm_limit"), "from": args.get("apm_from"), "nextPageKey": args.get("APM_next_page")}.items() if value}) - # TODO add needed fields for every event with calling a new function return events @@ -196,7 +177,6 @@ def main(): uuid = params.get('uuid') token = params.get('token') events_to_fetch = argToList(params.get('events_to_fetch')) - vul_limit = arg_to_number(params.get('vul_limit')) or 1000 audit_limit = arg_to_number(params.get('audit_limit')) or 25000 apm_limit = arg_to_number(params.get('apm_limit')) or 25000 validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, vul_limit, audit_limit, apm_limit) @@ -220,12 +200,12 @@ def main(): elif command == "dynatrace-get-events": result = get_events_command(client, args) elif command == "fetch-events": - result = fetch_events(client, events_to_fetch, vul_limit, audit_limit, apm_limit) + result = fetch_events(client, events_to_fetch, audit_limit, apm_limit) else: raise NotImplementedError(f"Command {command} is not implemented") return_results( result - ) # Returns either str, CommandResults and a list of CommandResults + ) # Log exceptions and return errors except Exception as e: diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml similarity index 74% rename from Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml rename to Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml index 491a71c7c08c..74a8df01a3aa 100644 --- a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace.yml +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml @@ -43,20 +43,13 @@ configuration: - display: Event types to fetch name: events_to_fetch type: 16 - defaultvalue: Vulnerability,Audit logs,APM + defaultvalue: Audit logs,APM required: true section: Collect advanced: true options: - - Vulnerability - Audit logs - APM -- display: The maximum number of vulnerability events per fetch - section: Collect - advanced: true - default: 2500 - type: 0 - name: vul_limit - display: The maximum number of audit logs events per fetch section: Collect advanced: true @@ -87,14 +80,9 @@ script: description: comma separated list of events types to get auto: PREDEFINED predefined: - - Vulnerability - Audit logs - APM required: true - - name: vul_from - description: The start date for searching vulnerability events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - - name: vul_to - description: The end date for retrieving vulnerability events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - name: audit_logs_from description: The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - name: audit_logs_to @@ -103,17 +91,13 @@ script: description: The start date for searching apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - name: apm_to description: The end date for retrieving apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - - name: vul_limit - required: true - description: Number of vulnerability events to fetch from each event type that was chosen. - defaultValue: "1" - name: audit_limit required: true - description: Number of audit_logs events to fetch from each event type that was chosen. + description: Number of audit_logs events to fetch. defaultValue: "1" - name: apm_limit required: true - description: Number of apm events to fetch from each event type that was chosen. + description: Number of apm events to fetch. defaultValue: "1" - auto: PREDEFINED defaultValue: "False" diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_description.md b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md similarity index 100% rename from Packs/Dynarace/Integrations/Dynatrace/Dynatrace_description.md rename to Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_image.png b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_image.png similarity index 100% rename from Packs/Dynarace/Integrations/Dynatrace/Dynatrace_image.png rename to Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_image.png diff --git a/Packs/Dynarace/Integrations/Dynatrace/Dynatrace_test.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py similarity index 100% rename from Packs/Dynarace/Integrations/Dynatrace/Dynatrace_test.py rename to Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py diff --git a/Packs/Dynarace/Integrations/Dynatrace/README.md b/Packs/Dynatrace/Integrations/Dynatrace/README.md similarity index 100% rename from Packs/Dynarace/Integrations/Dynatrace/README.md rename to Packs/Dynatrace/Integrations/Dynatrace/README.md diff --git a/Packs/Dynarace/Integrations/Dynatrace/command_examples b/Packs/Dynatrace/Integrations/Dynatrace/command_examples similarity index 100% rename from Packs/Dynarace/Integrations/Dynatrace/command_examples rename to Packs/Dynatrace/Integrations/Dynatrace/command_examples diff --git a/Packs/Dynarace/README.md b/Packs/Dynatrace/README.md similarity index 100% rename from Packs/Dynarace/README.md rename to Packs/Dynatrace/README.md diff --git a/Packs/Dynarace/pack_metadata.json b/Packs/Dynatrace/pack_metadata.json similarity index 100% rename from Packs/Dynarace/pack_metadata.json rename to Packs/Dynatrace/pack_metadata.json From 11993a0028b0d47ecc4168713386ce82dbf9c886 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 13:02:36 +0200 Subject: [PATCH 04/15] yml improvements --- .../Integrations/Dynatrace/Dynatrace.yml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml index 74a8df01a3aa..19c78d66c36b 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml @@ -1,32 +1,43 @@ -category: Cloud Services commonfields: id: Dynatrace version: -1 +sectionOrder: +- Connect +- Collect +name: Dynatrace +display: Dynatrace +category: Cloud Services +description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.]' configuration: - display: Server URL name: url required: true type: 0 + section: Connect - display: Client ID additionalinfo: Required If using OAuth 2 as authentication method. name: client_id required: false type: 4 + section: Connect - display: Client Secret additionalinfo: Required If using OAuth 2 as authentication method. name: client_secret required: false type: 4 + section: Connect - display: Account UUID additionalinfo: Required If using OAuth 2 as authentication method. name: uuid required: false type: 4 + section: Connect - display: Access Token additionalinfo: Required If using Personal Access Token. name: token required: false type: 4 + section: Connect - display: Fetch Events name: isFetchEvents type: 8 @@ -66,14 +77,15 @@ configuration: name: insecure required: false type: 8 + section: Connect - display: Use system proxy settings name: proxy required: false type: 8 -description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.]' -display: Dynatrace -name: Dynatrace + section: Connect script: + script: "" + type: python commands: arguments: - name: events_types_to_get @@ -109,7 +121,6 @@ script: required: true description: Manual command to fetch events and display them. name: dynatrace-get-events - type: python subtype: python3 dockerimage: demisto/python3:3.9.8.24399 fromversion: 6.1.0 From 99eedeca948ffd87e11bd332a4205cad19772d3b Mon Sep 17 00:00:00 2001 From: yshamai Date: Thu, 23 Jan 2025 15:23:34 +0200 Subject: [PATCH 05/15] continue --- .../Integrations/Dynatrace/Dynatrace.py | 238 +++++++++++++----- .../Integrations/Dynatrace/Dynatrace.yml | 32 ++- 2 files changed, 192 insertions(+), 78 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index f87a9435c4e8..4b9d6de032f8 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -11,27 +11,35 @@ DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # ISO8601 format with UTC, default in XSOAR VENDOR = "Dynatrace" PRODUCT = "Platform" -EVENTS_TYPE_DICT = {"Audit logs": ("auditLogs", "audit_limit"), "APM": ("events", "apm_limit")} +EVENTS_TYPE_DICT = {"Audit logs": ("auditLogs", "audit"), "APM": ("events", "apm")} +FIELD_MAPPING = { + "Audit logs": ["Audit logs events", "timestamp"], + "APM": ["APM events", "startTime"] + } """ CLIENT CLASS """ -class Client(BaseClient): - def init(self, base_url: str, client_id: str, client_secret: str, uuid: str, token: str, events_to_fetch: List[str], verify: bool, proxy): - super().__init__(base_url=base_url, verify=verify, proxy=proxy) - - self.client_id = client_id, - self.client_secret = client_secret, - self.uuid = uuid, +class DynatraceClient(BaseClient): + def __init__(self, base_url, client_id, client_secret, uuid, + token, events_to_fetch, verify, proxy): + super().__init__(proxy=proxy, base_url=base_url, verify=verify) + self.client_id = client_id + self.client_secret = client_secret + self.uuid = uuid self.token = token - self.personal_token = None + self.auth2_token = None if not self.token: # We are using OAuth2 authentication - self.personal_token = self.create_personal_token(events_to_fetch) + self.auth2_token = self.create_auth2_token(events_to_fetch) + + if self.auth2_token: + self._headers = {"Authorization": f"Bearer {self.auth2_token}"} + else: + self._headers = {"Authorization": f"Api-Token {self.token}"} - self._headers = {"Authorization": f"Api-Token {self.personal_token}" if self.personal_token else f"Bearer {self.token}"} - - def create_personal_token(self, events_to_fetch): + + def create_auth2_token(self, events_to_fetch): scopes = [] @@ -58,12 +66,13 @@ def create_personal_token(self, events_to_fetch): return raw_response # TODO test how response returns and return the token within it - def get_audit_logs_events(self, params: dict): - return self._http_request("GET", "/api/v2/auditlogs", json_data=params, headers=self._headers) + def get_audit_logs_events(self, query: str=""): + url = "/api/v2/auditlogs"+query + return self._http_request("GET", url, headers=self._headers) - def get_APM_events(self, params: dict): - return self._http_request("GET", "/api/v2/events", json_data=params, headers=self._headers) + def get_APM_events(self, query: str=""): + return self._http_request("GET", "/api/v2/events"+query, headers=self._headers) """ HELPER FUNCTIONS """ @@ -82,75 +91,164 @@ def validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, """ COMMAND FUNCTIONS """ -def fetch_events(client: Client, events_to_fetch: list, audit_limit: int, apm_limit: int): + +def fetch_apm_events(client, limit): + # last_apm_run should be None or a {"nextPageKey": val, "last_timestamp": val} + integration_cnx = demisto.getIntegrationContext() + last_run = integration_cnx.get("last_apm_run") or {} + + last_run_to_save = {} + events_to_return = [] + events_count = 0 + args = {} + fetch_start_time = datetime.now() - integration_context = demisto.getIntegrationContext() + for _ in range(5): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch + args["apm_limit"] = min(limit-events_count, 1000) + + if args["apm_limit"] != 0: # We didn't get to the limit needed, need to fetch more events + + if not last_run: # First time fetching + args["apm_from"] = "now-1w" # Change to "now" after I finish testing + + else: # Not first fetch + if last_run["nextPageKey"]: + args["apm_next_page_key"] = last_run["nextPageKey"] + else: + args["apm_from"] = last_run["last_timestamp"]#+one mili second # Need to implement this + + response = events_query(client, args, "APM") + # TODO need to see what happens if we get no events is respone.get("events") empty or None? + if response.get("nextPageKey"): + last_run_to_save["nextPageKey"] = response["nextPageKey"] + last_run_to_save["last_timestamp"] = None + else: + last_run_to_save["last_timestamp"] = response.get("events")[0]["startTime"] or last_run.get("last_timestamp") or fetch_start_time#-1 mili second # Need to implement + last_run_to_save["nextPageKey"] = None + + events = response.get("events") + events = add_fields_to_events(events, "APM") + events_to_return.extend(events) + + last_run["last_apm_run"] = last_run_to_save - # first fetch - if not integration_context: - pass - # TODO implement first run, need to specify from for each type and no next page. Maybe the defaulf from of the api is good for us. + integration_cnx["last_apm_run"] = last_run_to_save + set_integration_context(integration_cnx) - integration_context_to_save = {} - events_to_return = [] - audit_count, apm_count = 0, 0 + return events_to_return + + + +def fetch_events(client: DynatraceClient, events_to_fetch: list, audit_limit: int, apm_limit: int): + events_to_send = [] + if "APM" in events_to_fetch: + events = fetch_apm_events(client, apm_limit) + events_to_send.extend(events) + if "Audit Logs" in events_to_fetch: + events = fetch_audit_events(client, apm_limit) + events_to_send.extend(events) + return events_to_send - for _ in range(0, 5): - # TODO In the first time integration_context will be empty and there won't be a next page value in it, in this case the query function will not send an empty arg in the json body, the api has a default from date that can be used in the first fetch, need to see what happens when I send a nextPage, the api adds a from date? which one does it look at? + + # integration_context = demisto.getIntegrationContext() + + # integration_context_to_save = {} + # events_to_return = [] + # audit_count, apm_count = 0, 0 + + # args = {} + + # for _ in range(0, 5): - args = {"audit_limit": min(audit_limit-audit_count, 5000), "apm_limit": min(apm_limit-apm_count,1000), - "Audit logs_next_page": integration_context.get("Audit logs_next_page"), - "APM_next_page": integration_context.get("APM_next_page")} + # args["audit_limit"] = min(audit_limit-audit_count, 5000) + # args["apm_limit"] = min(apm_limit-apm_count, 1000) - for event_type in events_to_fetch: - if args[EVENTS_TYPE_DICT[event_type][1]] != 0: - response = events_query(client, args, event_type) - events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type[0]]], event_type) - events_to_return.extend(events) - integration_context_to_save[event_type+"_next_page"] = response["nextPageKey"] # TODO what happens when there are no more events? Do we still gat a nextPageKey? If not we need to go out of the loop, need to check this use case + # for event_type in events_to_fetch: + + # # set args + # if not integration_context.get(f"{events_to_fetch[0]}_last_timestamp"): # first fetch + # args[EVENTS_TYPE_DICT[event_type][1]+"_from"] = "now-1d" + # else: + # args[EVENTS_TYPE_DICT[event_type][1]+"_from"] = integration_context.get(EVENTS_TYPE_DICT[event_type][1]+"_last_timestamp") + + + # if args[EVENTS_TYPE_DICT[event_type][1]+"_limit"] != 0: + # response = events_query(client, args, event_type) + + # #dedup + # counter = 0 + # for i in range(len(response[EVENTS_TYPE_DICT[event_type][0]])): + # if response[EVENTS_TYPE_DICT[event_type][0]][i]["entityId"] in integration_context.get(f"{events_to_fetch[0]}_last_events_ids"): + # continue + # else: + # counter = i+1 + # break + + # while counter == len(response[EVENTS_TYPE_DICT[event_type][0]]): + # if response.get("nextPageKey"): + # integration_context_to_save[EVENTS_TYPE_DICT[event_type][1]+"_last_run"] = (response.get("nextPageKey"), None) + # else: + # integration_context_to_save[EVENTS_TYPE_DICT[event_type][1]+"_last_timestamp"] = (None, response[EVENTS_TYPE_DICT[event_type][0]][0][FIELD_MAPPING[event_type][1]]) + # response = events_query(client, args, event_type) + + # #dedup + # counter = 0 + # for i in range(len(response[EVENTS_TYPE_DICT[event_type][0]])): + # if response[EVENTS_TYPE_DICT[event_type][0]][i]["entityId"] in integration_context.get(f"{events_to_fetch[0]}_last_events_ids"): + # continue + # else: + # counter = i+1 + # break + + # events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type][0]], event_type) + # events_to_return.extend(events) + # if event_type == "APM": + # apm_count += len(events) + # else: + # audit_count += len(events) + # integration_context_to_save[EVENTS_TYPE_DICT[event_type][1]+"_next_page"] = response["nextPageKey"] # TODO what happens when there are no more events? Do we still gat a nextPageKey? If not we need to go out of the loop, need to check this use case - set_integration_context(integration_context_to_save) + # set_integration_context(integration_context_to_save) - return events_to_return + # return events_to_return def add_fields_to_events(events, event_type): - + # Need to convert the dates to the right format # TODO ask sara if we need the word 'events' in the end of the type, I don't think we usually do so. - field_mapping = { - "Audit logs": {"SOURCE_LOG_TYPE": "Audit logs events", "_time": "timestamp"}, - "APM events": {"SOURCE_LOG_TYPE": "APM events", "_time": "startTime"} - } - for event in events: - for key, value in field_mapping[event_type].items(): - event[key] = event[value] + event["SOURCE_LOG_TYPE"] = FIELD_MAPPING[event_type][0] + event["_time"] = event[FIELD_MAPPING[event_type][1]] return events -def get_events_command(client: Client, args: dict): +def get_events_command(client: DynatraceClient, args: dict): events_types = argToList(args.get("events_types_to_get")) events_to_return = [] - + for event_type in events_types: response = events_query(client, args, event_type) - events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type[0]]], event_type) + events = response[EVENTS_TYPE_DICT[event_type][0]] + events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type][0]], event_type) events_to_return.extend(events) - return events_to_return # Make a human readable + if args.get("should_push_events"): + send_events_to_xsiam(events=events_to_return, vendor=VENDOR, product=PRODUCT) + + return CommandResults(readable_output=tableToMarkdown(name='Events', t=events_to_return)) -def test_module(client: Client, events_to_fetch: List[str]) -> str: +def test_module(client: DynatraceClient, events_to_fetch: List[str]) -> str: try: if "Audit logs" in events_to_fetch: - client.get_audit_logs_events({"pageSize": 1}) + client.get_audit_logs_events("?pageSize=1") if "APM" in events_to_fetch: - client.get_APM_events({"pageSize": 1}) + client.get_APM_events("?pageSize=1") except Exception as e: raise DemistoException (str(e).lower()) @@ -158,12 +256,32 @@ def test_module(client: Client, events_to_fetch: List[str]) -> str: return "ok" -def events_query(client: Client, args: dict, event_type: str): +def events_query(client: DynatraceClient, args: dict, event_type: str): + query_lst = [] + query = "" + #query = "?from=now-1w&pageSize=2" if event_type == "Audit logs": - events = client.get_audit_logs_events({key: value for key, value in {"pageSize": args.get("audit_limit"), "from": args.get("audit_from"), "nextPageKey": args.get("Audit logs_next_page")}.items() if value}) + audit_limit = args.get("audit_limit") + if audit_limit: + query_lst.append(f"pageSize={audit_limit}") + audit_from = args.get("audit_from") + if audit_from: + query_lst.append(f"from={audit_from}") + if query_lst: + query="?"+"&".join(query_lst) + response = client.get_audit_logs_events(query) #"nextPageKey": args.get("Audit logs_next_page") + elif event_type == "APM": - events = client.get_audit_logs_events({key: value for key, value in {"pageSize": args.get("apm_limit"), "from": args.get("apm_from"), "nextPageKey": args.get("APM_next_page")}.items() if value}) - return events + apm_limit = args.get("apm_limit") + if apm_limit: + query_lst.append(f"pageSize={apm_limit}") + apm_from = args.get("apm_from") + if apm_from: + query_lst.append(f"from={apm_from}") + if query_lst: + query="?"+"&".join(query_lst) + response = client.get_APM_events(query) + return response def main(): @@ -179,7 +297,7 @@ def main(): events_to_fetch = argToList(params.get('events_to_fetch')) audit_limit = arg_to_number(params.get('audit_limit')) or 25000 apm_limit = arg_to_number(params.get('apm_limit')) or 25000 - validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, vul_limit, audit_limit, apm_limit) + validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, audit_limit, apm_limit) verify = not argToBoolean(params.get("insecure", False)) proxy = argToBoolean(params.get("proxy", False)) @@ -190,7 +308,7 @@ def main(): try: - client = Client(base_url=url, client_id=client_id, client_secret=client_secret, uuid=uuid, token=token, events_to_fetch=events_to_fetch, verify=verify, proxy=proxy) + client = DynatraceClient(url, client_id, client_secret, uuid, token, events_to_fetch, verify, proxy) args = demisto.args() diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml index 19c78d66c36b..d03ba062cafb 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml @@ -7,7 +7,7 @@ sectionOrder: name: Dynatrace display: Dynatrace category: Cloud Services -description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.]' +description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.].' configuration: - display: Server URL name: url @@ -43,7 +43,7 @@ configuration: type: 8 required: false section: Collect - defaultvalue: true + defaultvalue: 'true' - display: Events Fetch Interval name: eventFetchInterval type: 19 @@ -64,13 +64,13 @@ configuration: - display: The maximum number of audit logs events per fetch section: Collect advanced: true - default: 5000 + defaultvalue: "5000" type: 0 name: audit_limit - display: The maximum number of APM events per fetch section: Collect advanced: true - default: 5000 # Need to ask sara about the default limit in all event types + defaultvalue: "5000" # Need to ask sara about the default limit in all event types type: 0 name: apm_limit - display: Trust any certificate (not secure) @@ -87,28 +87,24 @@ script: script: "" type: python commands: - arguments: + - arguments: - name: events_types_to_get - description: comma separated list of events types to get + description: comma separated list of events types to get. auto: PREDEFINED predefined: - Audit logs - APM required: true - - name: audit_logs_from - description: The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - - name: audit_logs_to - description: The end date for retrieving audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + - name: audit_from + description: The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to # TODO https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - name: apm_from - description: The start date for searching apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - - name: apm_to - description: The end date for retrieving apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + description: The start date for searching apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to #TODO https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. - name: audit_limit - required: true + required: false description: Number of audit_logs events to fetch. defaultValue: "1" - name: apm_limit - required: true + required: false description: Number of apm events to fetch. defaultValue: "1" - auto: PREDEFINED @@ -118,13 +114,13 @@ script: predefined: - "True" - "False" - required: true + required: false description: Manual command to fetch events and display them. name: dynatrace-get-events subtype: python3 dockerimage: demisto/python3:3.9.8.24399 -fromversion: 6.1.0 marketplaces: - marketplacev2 +fromversion: 6.9.0 tests: -- No tests (auto formatted) \ No newline at end of file +- No tests From d032d2f83960c83eedf09e0cbbcba7ecc3c30d93 Mon Sep 17 00:00:00 2001 From: yshamai Date: Sun, 26 Jan 2025 14:25:53 +0200 Subject: [PATCH 06/15] integration ready and some unit tests --- .../Integrations/Dynatrace/Dynatrace.py | 259 +++++++++++----- .../Integrations/Dynatrace/Dynatrace.yml | 14 +- .../Integrations/Dynatrace/Dynatrace_test.py | 287 ++++++++++++++++-- 3 files changed, 435 insertions(+), 125 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index 4b9d6de032f8..90f43dd5b51f 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -1,3 +1,5 @@ +from datetime import datetime, timezone, timedelta +import pytz from typing import Any, Dict, Optional import demistomock as demisto import urllib3 @@ -11,11 +13,7 @@ DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # ISO8601 format with UTC, default in XSOAR VENDOR = "Dynatrace" PRODUCT = "Platform" -EVENTS_TYPE_DICT = {"Audit logs": ("auditLogs", "audit"), "APM": ("events", "apm")} -FIELD_MAPPING = { - "Audit logs": ["Audit logs events", "timestamp"], - "APM": ["APM events", "startTime"] - } +EVENTS_TYPE_DICT = {"Audit logs": "auditLogs", "APM": "events"} """ CLIENT CLASS """ @@ -39,7 +37,7 @@ def __init__(self, base_url, client_id, client_secret, uuid, self._headers = {"Authorization": f"Api-Token {self.token}"} - def create_auth2_token(self, events_to_fetch): + def create_auth2_token(self, events_to_fetch): # Needs to be modified when costumer will give us more information scopes = [] @@ -67,8 +65,7 @@ def create_auth2_token(self, events_to_fetch): def get_audit_logs_events(self, query: str=""): - url = "/api/v2/auditlogs"+query - return self._http_request("GET", url, headers=self._headers) + return self._http_request("GET", "/api/v2/auditlogs"+query, headers=self._headers) def get_APM_events(self, query: str=""): @@ -83,16 +80,65 @@ def validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, raise DemistoException("When using OAuth 2, ensure to specify the client ID, client secret, and Account UUID. When using a personal access token, make sure to specify the access token. It's important to include only the required parameters for each type and avoid including any extra parameters.") if not events_to_fetch: raise DemistoException("Please specify at least one event type to fetch.") - if not audit_max > 0 and not audit_max <= 2500: - raise DemistoException("The maximum number of audit logs events per fetch needs to be grater then 0 and not more then then 25000") - if not apm_max > 0 and not apm_max <= 25000: - raise DemistoException("The maximum number of APM events per fetch needs to be grater then 0 and not more then then 5000") + if audit_max < 1 or audit_max > 25000: + raise DemistoException("The maximum number of audit logs events per fetch needs to be grater then 0 and not more then then 2500") + if apm_max < 1 or apm_max > 5000: + raise DemistoException("The maximum number of APM events per fetch needs to be grater then 0 and not more then then 25000") -""" COMMAND FUNCTIONS """ +# def convert_date(date): # This function gets a date that looks like this: 1737656746001 +# seconds = timestamp // 1000 +# milliseconds = timestamp % 1000 + +# # Convert seconds to a datetime object in UTC +# dt = datetime.fromtimestamp(seconds, tz=timezone.utc) + +def add_fields_to_events(events, event_type): + + # TODO ask sara if we need the word 'events' in the end of the type, I don't think we usually do so. + + field_mapping = { + "Audit logs": ["Audit logs events", "timestamp"], + "APM": ["APM events", "startTime"] + + } + for event in events: + event["SOURCE_LOG_TYPE"] = field_mapping[event_type][0] + event["_time"] = event[field_mapping[event_type][1]] + + return events + + +def events_query(client: DynatraceClient, args: dict, event_type: str): + query_lst = [] + query = "" + + if event_type == "Audit logs": + audit_limit = args.get("audit_limit") + if audit_limit: + query_lst.append(f"pageSize={audit_limit}") + audit_from = args.get("audit_from") + if audit_from: + query_lst.append(f"from={audit_from}") + if query_lst: + query="?"+"&".join(query_lst) + response = client.get_audit_logs_events(query) + + elif event_type == "APM": + apm_limit = args.get("apm_limit") + if apm_limit: + query_lst.append(f"pageSize={apm_limit}") + apm_from = args.get("apm_from") + if apm_from: + query_lst.append(f"from={apm_from}") + if query_lst: + query="?"+"&".join(query_lst) + response = client.get_APM_events(query) + return response -def fetch_apm_events(client, limit): +def fetch_apm_events(client, limit, fetch_start_time): + # last_apm_run should be None or a {"nextPageKey": val, "last_timestamp": val} integration_cnx = demisto.getIntegrationContext() last_run = integration_cnx.get("last_apm_run") or {} @@ -101,36 +147,52 @@ def fetch_apm_events(client, limit): events_to_return = [] events_count = 0 args = {} - fetch_start_time = datetime.now() for _ in range(5): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch args["apm_limit"] = min(limit-events_count, 1000) if args["apm_limit"] != 0: # We didn't get to the limit needed, need to fetch more events - if not last_run: # First time fetching - args["apm_from"] = "now-1w" # Change to "now" after I finish testing - - else: # Not first fetch - if last_run["nextPageKey"]: + # First time fetching + if last_run == {}: + args["apm_from"] = fetch_start_time # Change to "now" after I finish testing + + else: + if last_run.get("nextPageKey"): args["apm_next_page_key"] = last_run["nextPageKey"] else: - args["apm_from"] = last_run["last_timestamp"]#+one mili second # Need to implement this + # If the previous run did not return a nextPageKey, it indicates there are no more events + # with the same timestamp as the last_timestamp from the previous run. + # Therefore, we query for events starting from last_timestamp + 1 millisecond + # to avoid retrieving the same events as the previous run. + # This approach eliminates the need for deduplication. + args["apm_from"] = last_run["last_timestamp"] + 1 + demisto.debug(f"Dynatrace calling query with {args=}") response = events_query(client, args, "APM") - # TODO need to see what happens if we get no events is respone.get("events") empty or None? + num_events = len(response.get("events")) + demisto.debug(f"Dynatrace got {num_events} events") + + # TODO need to see what happens if we get no events is response.get("events") empty or None? if response.get("nextPageKey"): + demisto.debug("Dynatrace setting last run with nextPageKey") last_run_to_save["nextPageKey"] = response["nextPageKey"] - last_run_to_save["last_timestamp"] = None + last_run_to_save["last_timestamp"] = None # This timestamp won't be relevant at the next run. else: - last_run_to_save["last_timestamp"] = response.get("events")[0]["startTime"] or last_run.get("last_timestamp") or fetch_start_time#-1 mili second # Need to implement + demisto.debug("Dynatrace setting last run with timestamp") + # If events were retrieved during this run (which might not always happen), + # we save the last timestamp from this run. + # If no events were retrieved, we retain the same last_timestamp as before, + # In cases where no events were retrieved and this is the first run (i.e., no last_run_timestamp exists), + # the query will use start_fetch_time again in the next execution. + last_run_to_save["last_timestamp"] = response.get("events")[0]["startTime"] if response["totalCount"] != 0 else (last_run.get("last_timestamp") or fetch_start_time) # What happens when no events are retrieved? last_run_to_save["nextPageKey"] = None events = response.get("events") events = add_fields_to_events(events, "APM") events_to_return.extend(events) - last_run["last_apm_run"] = last_run_to_save + last_run = last_run_to_save integration_cnx["last_apm_run"] = last_run_to_save set_integration_context(integration_cnx) @@ -138,17 +200,66 @@ def fetch_apm_events(client, limit): return events_to_return +def fetch_audit_log_events(client, limit, fetch_start_time): + + # last_apm_run should be None or a {"nextPageKey": val, "last_timestamp": val} + integration_cnx = demisto.getIntegrationContext() + last_run = integration_cnx.get("last_audit_run") or {} -def fetch_events(client: DynatraceClient, events_to_fetch: list, audit_limit: int, apm_limit: int): - events_to_send = [] - if "APM" in events_to_fetch: - events = fetch_apm_events(client, apm_limit) - events_to_send.extend(events) - if "Audit Logs" in events_to_fetch: - events = fetch_audit_events(client, apm_limit) - events_to_send.extend(events) - return events_to_send + last_run_to_save = {} + events_to_return = [] + events_count = 0 + args = {} + + for _ in range(5): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch + args["audit_limit"] = min(limit-events_count, 5000) + + if args["audit_limit"] != 0: # We didn't get to the limit needed, need to fetch more events + + # First time fetching + if last_run == {}: + args["audit_from"] = fetch_start_time + + else: + if last_run.get("nextPageKey"): + args["audit_next_page_key"] = last_run["nextPageKey"] + else: + # If the previous run did not return a nextPageKey, it indicates there are no more events + # with the same timestamp as the last_timestamp from the previous run. + # Therefore, we query for events starting from last_timestamp + 1 millisecond + # to avoid retrieving the same events as the previous run. + # This approach eliminates the need for deduplication. + args["audit_from"] = last_run["last_timestamp"] + 1 + + demisto.debug(f"Dynatrace calling query with {args=}") + response = events_query(client, args, "Audit logs") + num_events = len(response.get("auditLogs")) + demisto.debug(f"Dynatrace got {num_events} events") + + if response.get("nextPageKey"): + demisto.debug("Dynatrace setting last run with nextPageKey") + last_run_to_save["nextPageKey"] = response["nextPageKey"] + last_run_to_save["last_timestamp"] = None # This timestamp won't be relevant at the next run. + else: + demisto.debug("Dynatrace setting last run with timestamp") + # If events were retrieved during this run (which might not always happen), + # we save the last timestamp from this run. + # If no events were retrieved, we retain the same last_timestamp as before, + # In cases where no events were retrieved and this is the first run (i.e., no last_run_timestamp exists), + # the query will use start_fetch_time again in the next execution. + last_run_to_save["last_timestamp"] = response.get("auditLogs")[0]["timestamp"] if response["totalCount"] != 0 else (last_run.get("last_timestamp") or fetch_start_time) + last_run_to_save["nextPageKey"] = None # This nextPageKey won't be relevant at the next run. + + events = response.get("auditLogs") + events = add_fields_to_events(events, "Audit logs") + events_to_return.extend(events) + + last_run = last_run_to_save + + integration_cnx["last_audit_run"] = last_run_to_save + set_integration_context(integration_cnx) + return events_to_return # integration_context = demisto.getIntegrationContext() @@ -214,15 +325,25 @@ def fetch_events(client: DynatraceClient, events_to_fetch: list, audit_limit: in # return events_to_return -def add_fields_to_events(events, event_type): - # Need to convert the dates to the right format - # TODO ask sara if we need the word 'events' in the end of the type, I don't think we usually do so. +""" COMMAND FUNCTIONS """ + +def fetch_events(client: DynatraceClient, events_to_fetch: list, audit_limit: int, apm_limit: int): - for event in events: - event["SOURCE_LOG_TYPE"] = FIELD_MAPPING[event_type][0] - event["_time"] = event[FIELD_MAPPING[event_type][1]] - - return events + fetch_start_time = int(datetime.now().timestamp() * 1000) # We want this timestamp to look like this: 1737656746001 + demisto.debug(f"Dynatrace fetch Audit Logs events start time is {fetch_start_time}") + + events_to_send = [] + if "APM" in events_to_fetch: + events = fetch_apm_events(client, apm_limit, fetch_start_time) + events = add_fields_to_events(events, "APM") + events_to_send.extend(events) + if "Audit logs" in events_to_fetch: + events = fetch_audit_log_events(client, audit_limit, fetch_start_time) + events = add_fields_to_events(events, "Audit logs") + events_to_send.extend(events) + + demisto.debug(f"Dynatrace sending {len(events_to_send)} to xsiam") + send_events_to_xsiam(events_to_send, VENDOR, PRODUCT) def get_events_command(client: DynatraceClient, args: dict): @@ -232,14 +353,19 @@ def get_events_command(client: DynatraceClient, args: dict): for event_type in events_types: response = events_query(client, args, event_type) - events = response[EVENTS_TYPE_DICT[event_type][0]] - events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type][0]], event_type) + events = response[EVENTS_TYPE_DICT[event_type]] + demisto.debug(f"Dynatrace got {len(events)} events of type {event_type}") + events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type]], event_type) events_to_return.extend(events) - if args.get("should_push_events"): + if args["should_push_events"]: + demisto.debug("Dynatrace sending events to xsiam") send_events_to_xsiam(events=events_to_return, vendor=VENDOR, product=PRODUCT) - return CommandResults(readable_output=tableToMarkdown(name='Events', t=events_to_return)) + if events_to_return!=[]: + return CommandResults(readable_output=tableToMarkdown(name='Events', t=events_to_return)) + else: + return CommandResults(readable_output="No events were received") def test_module(client: DynatraceClient, events_to_fetch: List[str]) -> str: @@ -256,35 +382,7 @@ def test_module(client: DynatraceClient, events_to_fetch: List[str]) -> str: return "ok" -def events_query(client: DynatraceClient, args: dict, event_type: str): - query_lst = [] - query = "" - #query = "?from=now-1w&pageSize=2" - if event_type == "Audit logs": - audit_limit = args.get("audit_limit") - if audit_limit: - query_lst.append(f"pageSize={audit_limit}") - audit_from = args.get("audit_from") - if audit_from: - query_lst.append(f"from={audit_from}") - if query_lst: - query="?"+"&".join(query_lst) - response = client.get_audit_logs_events(query) #"nextPageKey": args.get("Audit logs_next_page") - - elif event_type == "APM": - apm_limit = args.get("apm_limit") - if apm_limit: - query_lst.append(f"pageSize={apm_limit}") - apm_from = args.get("apm_from") - if apm_from: - query_lst.append(f"from={apm_from}") - if query_lst: - query="?"+"&".join(query_lst) - response = client.get_APM_events(query) - return response - - -def main(): +def main(): # pragma: no cover """main function, parses params and runs command functions""" @@ -297,6 +395,7 @@ def main(): events_to_fetch = argToList(params.get('events_to_fetch')) audit_limit = arg_to_number(params.get('audit_limit')) or 25000 apm_limit = arg_to_number(params.get('apm_limit')) or 25000 + validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, audit_limit, apm_limit) verify = not argToBoolean(params.get("insecure", False)) @@ -313,18 +412,16 @@ def main(): args = demisto.args() if command == "test-module": - # This is the call made when pressing the integration Test button. result = test_module(client, events_to_fetch) + return_results(result) elif command == "dynatrace-get-events": result = get_events_command(client, args) + return_results(result) elif command == "fetch-events": - result = fetch_events(client, events_to_fetch, audit_limit, apm_limit) + fetch_events(client, events_to_fetch, audit_limit, apm_limit) else: raise NotImplementedError(f"Command {command} is not implemented") - return_results( - result - ) - + # Log exceptions and return errors except Exception as e: return_error(f"Failed to execute {command} command.\nError:\n{str(e)}") diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml index d03ba062cafb..460aad08489b 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml @@ -38,19 +38,6 @@ configuration: required: false type: 4 section: Connect -- display: Fetch Events - name: isFetchEvents - type: 8 - required: false - section: Collect - defaultvalue: 'true' -- display: Events Fetch Interval - name: eventFetchInterval - type: 19 - required: false - section: Collect - defaultvalue: '1' - advanced: true - display: Event types to fetch name: events_to_fetch type: 16 @@ -117,6 +104,7 @@ script: required: false description: Manual command to fetch events and display them. name: dynatrace-get-events + isfetchevents: true subtype: python3 dockerimage: demisto/python3:3.9.8.24399 marketplaces: diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py index c42f4e3a8b39..f90988a52513 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py @@ -1,41 +1,266 @@ -"""Base Integration for Cortex XSOAR - Unit Tests file - -Pytest Unit Tests: all funcion names must start with "test_" - -More details: https://xsoar.pan.dev/docs/integrations/unit-testing - -MAKE SURE YOU REVIEW/REPLACE ALL THE COMMENTS MARKED AS "TODO" - -You must add at least a Unit Test function for every XSOAR command -you are implementing with your integration -""" - from demisto_sdk.commands.common.handlers import JSON_Handler - import json +import pytest +import Dynatrace as dyn +import demistomock as demisto +from CommonServerPython import * -def util_load_json(path): - with open(path, encoding="utf-8") as f: - return json.loads(f.read()) - +# def util_load_json(path): +# with open(path, encoding="utf-8") as f: +# return json.loads(f.read()) -# TODO: REMOVE the following dummy unit test function -def test_baseintegration_dummy(): - """Tests helloworld-say-hello command function. +AUDIT_CLIENT = dyn.DynatraceClient( + base_url="https://AAAAA.dynatrace.com", + client_id=None, + client_secret=None, + uuid=None, + token="AAAAAAAAA.AAAAAAA.AAAAAAA", + events_to_fetch=["Audit logs"], + verify=True, + proxy=None + ) - Checks the output of the command function with the expected output. +APM_CLIENT = dyn.DynatraceClient( + base_url="https://AAAAA.dynatrace.com", + client_id=None, + client_secret=None, + uuid=None, + token="AAAAAAAAA.AAAAAAA.AAAAAAA", + events_to_fetch=["APM"], + verify=True, + proxy=None + ) - No mock is needed here because the say_hello_command does not call - any external API. - """ - from BaseIntegration import Client, baseintegration_dummy_command - - client = Client(base_url="some_mock_url", verify=False) - args = {"dummy": "this is a dummy response", "dummy2": "a dummy value"} - response = baseintegration_dummy_command(client, args) +APM_AUDIT_CLIENT = dyn.DynatraceClient( + base_url="https://AAAAA.dynatrace.com", + client_id=None, + client_secret=None, + uuid=None, + token="AAAAAAAAA.AAAAAAA.AAAAAAA", + events_to_fetch=["APM", "Audit logs"], + verify=True, + proxy=None + ) - assert response.outputs == args +def test_get_audit_logs_events(mocker): + """ + Given: A query + When: calling get_audit_logs_events function + Then: the http request is called with "GET" arg and with the right url. + """ + http_request = mocker.patch.object(AUDIT_CLIENT, '_http_request', return_value=[]) + AUDIT_CLIENT.get_audit_logs_events(query="?querykey=queryarg") + assert http_request.call_args[0][0] == "GET" + assert http_request.call_args[0][1] == "/api/v2/auditlogs?querykey=queryarg" + + +def test_get_APM_events(mocker): + """ + Given: A query + When: calling get_APM_events function + Then: the http request is called with "GET" arg and with the right url. + """ + http_request = mocker.patch.object(APM_CLIENT, '_http_request', return_value=[]) + APM_CLIENT.get_APM_events(query="?querykey=queryarg") + assert http_request.call_args[0][0] == "GET" + assert http_request.call_args[0][1] == "/api/v2/events?querykey=queryarg" + + +@pytest.mark.parametrize( + "client_id, client_secret, uuid, token, events_to_fetch, audit_max, apm_max, expected_exception, expected_message", + [ + # Valid OAuth2 configuration + ("client_id", "client_secret", "uuid", None, ["APM"], 25000, 5000, None, None), + + # Valid token configuration + (None, None, None, "token", ["APM"], 25000, 5000, None, None), + + # Invalid: Both OAuth2 and token parameters provided + ("client_id", "client_secret", "uuid", "token", ["APM"], 25000, 5000, DemistoException, "When using OAuth 2"), + + # Invalid: Missing both OAuth2 and token parameters + (None, None, None, None, ["APM"], 25000, 5000, DemistoException, "When using OAuth 2"), + + # Invalid: No event types specified + ("client_id", "client_secret", "uuid", None, [], 25000, 5000, DemistoException, "Please specify at least one event type"), + + # Invalid: audit_max out of range (too high) + ("client_id", "client_secret", "uuid", None, ["Audit logs"], 30000, 5000, DemistoException, "The maximum number of audit logs events"), + + # Invalid: audit_max out of range (negative) + ("client_id", "client_secret", "uuid", None, ["Audit logs"], -1, 5000, DemistoException, "The maximum number of audit logs events"), + + # Invalid: apm_max out of range (too high) + ("client_id", "client_secret", "uuid", None, ["APM"], 25000, 6000, DemistoException, "The maximum number of APM events"), + + # Invalid: apm_max out of range (negative) + ("client_id", "client_secret", "uuid", None, ["APM"], 25000, -1, DemistoException, "The maximum number of APM events"), + ], +) +def test_validate_params(client_id, client_secret, uuid, token, events_to_fetch, audit_max, apm_max, expected_exception, expected_message): + """ + Given: all instance params + When: Calling validate_params function + Then: The function doesn't raise an error when params are valid and raises the right exception when params are invalid + """ + from Dynatrace import validate_params + if expected_exception: + with pytest.raises(expected_exception) as excinfo: + validate_params( + url="http://example.com", + client_id=client_id, + client_secret=client_secret, + uuid=uuid, + token=token, + events_to_fetch=events_to_fetch, + audit_max=audit_max, + apm_max=apm_max, + ) + assert expected_message in str(excinfo.value) + else: + validate_params( + url="http://example.com", + client_id=client_id, + client_secret=client_secret, + uuid=uuid, + token=token, + events_to_fetch=events_to_fetch, + audit_max=audit_max, + apm_max=apm_max, + ) + + +@pytest.mark.parametrize( + "events, event_type, expected", + [ + # Test case 1: Audit logs + ( + [{"timestamp": 1640995200000}], + "Audit logs", + [{"timestamp": 1640995200000, "SOURCE_LOG_TYPE": "Audit logs events", "_time": 1640995200000}], + ), + # Test case 2: APM + ( + [{"startTime": 1640995200000}], + "APM", + [{"startTime": 1640995200000, "SOURCE_LOG_TYPE": "APM events", "_time": 1640995200000}], + ), + # Test case 3: Multiple Audit logs events + ( + [ + {"timestamp": 1640995200000}, + {"timestamp": 1640995300000}, + ], + "Audit logs", + [ + {"timestamp": 1640995200000, "SOURCE_LOG_TYPE": "Audit logs events", "_time": 1640995200000}, + {"timestamp": 1640995300000, "SOURCE_LOG_TYPE": "Audit logs events", "_time": 1640995300000}, + ], + ), + # Test case 4: Empty events + ( + [], + "APM", + [], + ), + ], +) +def test_add_fields_to_events(events, event_type, expected): + """ + Given: events and event_type + When: Calling add_fields_to_events function + Then: The function retrieves the events with the expected added fields. + """ + from Dynatrace import add_fields_to_events + assert add_fields_to_events(events, event_type) == expected + + +def test_events_query__APM(mocker): + """ + Given: args and event_type=APM + When: Calling the events_query function + Then: The get_APM_events is called with the right query. + """ + from Dynatrace import events_query + request = mocker.patch.object(APM_CLIENT, 'get_APM_events', return_value=[]) + events_query(APM_CLIENT, {"apm_limit": "100", "apm_from": "1640995200000"}, "APM") + assert request.call_args[0][0] == '?pageSize=100&from=1640995200000' + + +def test_events_query__audit(mocker): + """ + Given: args and event_type=Audit logs + When: Calling the events_query function + Then: The get_audit_events is called with the right query. + """ + from Dynatrace import events_query + request = mocker.patch.object(AUDIT_CLIENT, 'get_audit_logs_events', return_value=[]) + events_query(AUDIT_CLIENT, {"audit_limit": "100", "audit_from": "1640995200000"}, "Audit logs") + assert request.call_args[0][0] == '?pageSize=100&from=1640995200000' + -# TODO: ADD HERE unit tests for every command +def test_fetch_events(mocker): + """ + Given: events_to_fetch=["APM", "Audit logs"], audit_limit and apm_limit + When: Running fetch-events command + Then: The fetch_events function calls all expected functions to be called with the right arguments. + """ + from Dynatrace import fetch_events + apm_mock = mocker.patch("Dynatrace.fetch_apm_events", return_value=[]) + audit_mock = mocker.patch("Dynatrace.fetch_audit_log_events", return_value=[]) + add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", return_value=[]) + send_events_to_xsiam_mock = mocker.patch("Dynatrace.send_events_to_xsiam") + + fetch_events(APM_AUDIT_CLIENT, ["APM", "Audit logs"], 200, 100) + + assert apm_mock.call_args.args[1] == 100 + assert audit_mock.call_args.args[1] == 200 + assert add_fields_to_events_mock.call_count == 2 + send_events_to_xsiam_mock.assert_called_once_with([], "Dynatrace", "Platform") + + +def test_get_events_command__APM(mocker): + """ + Given: args = {"events_types_to_get": "APM", "should_push_events": True} and no events are received. + When: executing the dynatrace-get-events command + Then: + - events_query function and send_events_to_xsiam are called once with the right arguments. + - The human readable returned is "No events were received". + """ + from Dynatrace import get_events_command + events_query_mock = mocker.patch("Dynatrace.events_query", return_value={"events": []}) + add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", return_value=[]) + send_events_to_xsiam_mock = mocker.patch("Dynatrace.send_events_to_xsiam") + + res = get_events_command(APM_CLIENT, {"events_types_to_get": "APM", "should_push_events": True}) + + events_query_mock.assert_called_once_with(APM_CLIENT, {"events_types_to_get": "APM", "should_push_events": True}, "APM") + add_fields_to_events_mock.assert_called_once_with([], "APM") + send_events_to_xsiam_mock.assert_called_once_with(events=[], vendor="Dynatrace", product="Platform") + assert "No events were received" in res.readable_output + + +def test_get_events_command__Audit_logs(mocker): + """ + Given: args = {"events_types_to_get": "Audit logs", "should_push_events": False} and events are received. + When: executing the dynatrace-get-events command + Then: + - events_query function is called with the right arguments. + - send_events_to_xsiam function is not called. + - The human readable includes the received events. + """ + from Dynatrace import get_events_command + events_query_mock = mocker.patch("Dynatrace.events_query", return_value={"auditLogs": [{"timestamp": 1640995200000}]}) + add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", + return_value=[{"timestamp": 1640995200000, "SOURCE_LOG_TYPE": "Audit logs events"}]) + send_events_to_xsiam_mock = mocker.patch("Dynatrace.send_events_to_xsiam") + + res = get_events_command(AUDIT_CLIENT, {"events_types_to_get": "Audit logs", "should_push_events": False}) + + events_query_mock.assert_called_once_with(AUDIT_CLIENT, + {"events_types_to_get": "Audit logs", "should_push_events": False}, "Audit logs") + add_fields_to_events_mock.assert_called_once_with([{"timestamp": 1640995200000}], "Audit logs") + send_events_to_xsiam_mock.assert_not_called() + assert "1640995200000" in res.readable_output \ No newline at end of file From dec1083bcc0a0e01652b3945642aa116431ae9fe Mon Sep 17 00:00:00 2001 From: yshamai Date: Mon, 27 Jan 2025 13:19:47 +0200 Subject: [PATCH 07/15] remove auth2, docs and code improvements --- .../Integrations/Dynatrace/README.md | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/README.md b/Packs/Dynatrace/Integrations/Dynatrace/README.md index 5d6d1dc2bf52..2e9696755b74 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/README.md +++ b/Packs/Dynatrace/Integrations/Dynatrace/README.md @@ -1,5 +1,44 @@ -This README contains the full documentation for your integration. +Fetch Audit logs and APM events from Dynatrace Platform +This integration was integrated and tested with version xx of Dynatrace. -You auto-generate this README file from your integration YML file using the `demisto-sdk generate-docs` command. +## Configure Dynatrace in Cortex -For more information see the [integration documentation](https://xsoar.pan.dev/docs/integrations/integration-docs). + +| **Parameter** | **Required** | +| --- | --- | +| Server URL | True | +| Access Token | False | +| Event types to fetch | True | +| The maximum number of audit logs events per fetch | | +| The maximum number of APM events per fetch | | +| Trust any certificate (not secure) | False | +| Use system proxy settings | False | + +## Commands + +You can execute these commands from the CLI, as part of an automation, or in a playbook. +After you successfully execute a command, a DBot message appears in the War Room with the command details. + +### dynatrace-get-events + +*** +Manual command to fetch events and display them. + +#### Base Command + +`dynatrace-get-events` + +#### Input + +| **Argument Name** | **Description** | **Required** | +| --- | --- | --- | +| events_types_to_get | comma separated list of events types to get. Possible values are: Audit logs, APM. | Required | +| audit_from | The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/events-v2/get-events fro more information. | Optional | +| apm_from | The start date for searching apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/audit-logs/get-log for more information. | Optional | +| audit_limit | Number of audit_logs events to fetch. Default is 1. | Optional | +| apm_limit | Number of apm events to fetch. Default is 1. | Optional | +| should_push_events | Set this argument to True in order to create events, otherwise the command will only display them. Possible values are: True, False. Default is False. | Optional | + +#### Context Output + +There is no context output for this command. From ec207126de8d047b76138b22fd86d66bbdd1c293 Mon Sep 17 00:00:00 2001 From: yshamai Date: Mon, 27 Jan 2025 13:20:25 +0200 Subject: [PATCH 08/15] code improvements --- Packs/Dynatrace/Author_image.png | 0 .../Integrations/Dynatrace/Dynatrace.py | 162 +++--------------- .../Integrations/Dynatrace/Dynatrace.yml | 29 +--- .../Dynatrace/Dynatrace_description.md | 31 +++- .../Dynatrace/Dynatrace_image.png | Bin 2507 -> 1002 bytes .../Integrations/Dynatrace/Dynatrace_test.py | 91 +++------- .../Integrations/Dynatrace/README.md | 2 +- Packs/Dynatrace/README.md | 1 + Packs/Dynatrace/pack_metadata.json | 2 +- 9 files changed, 80 insertions(+), 238 deletions(-) delete mode 100644 Packs/Dynatrace/Author_image.png diff --git a/Packs/Dynatrace/Author_image.png b/Packs/Dynatrace/Author_image.png deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index 90f43dd5b51f..dd89a7776a80 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -19,51 +19,10 @@ """ CLIENT CLASS """ class DynatraceClient(BaseClient): - def __init__(self, base_url, client_id, client_secret, uuid, - token, events_to_fetch, verify, proxy): - super().__init__(proxy=proxy, base_url=base_url, verify=verify) - self.client_id = client_id - self.client_secret = client_secret - self.uuid = uuid - self.token = token - self.auth2_token = None - - if not self.token: # We are using OAuth2 authentication - self.auth2_token = self.create_auth2_token(events_to_fetch) - - if self.auth2_token: - self._headers = {"Authorization": f"Bearer {self.auth2_token}"} - else: - self._headers = {"Authorization": f"Api-Token {self.token}"} - - - def create_auth2_token(self, events_to_fetch): # Needs to be modified when costumer will give us more information - - scopes = [] - - if "Audit logs" in events_to_fetch: - scopes.append("auditLogs.read") - if "APM" in events_to_fetch: - scopes.append("events.read") - - params = assign_params( - grant_type = "client_credentials", - client_id = self.client_id, - client_secret = self.client_secret, - scope = " ".join(scopes), - resource = f"urn:dtaccount:{self.uuid}" - ) - - raw_response = self._http_request( - method='POST', - url_suffix="https://sso.dynatrace.com/sso/oauth2/token", - json_data=params, - headers=self._headers - ) - - return raw_response # TODO test how response returns and return the token within it - - + def __init__(self, base_url, token, verify, proxy): + super().__init__(proxy=proxy, base_url=base_url, verify=verify, headers={"Authorization": f"Api-Token {token}"}) + + def get_audit_logs_events(self, query: str=""): return self._http_request("GET", "/api/v2/auditlogs"+query, headers=self._headers) @@ -74,24 +33,15 @@ def get_APM_events(self, query: str=""): """ HELPER FUNCTIONS """ -def validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, audit_max, apm_max): +def validate_params(url, token, events_to_fetch, audit_max, apm_max): - if not ((client_id and client_secret and uuid and not token) or (token and not client_id and not client_secret and not uuid)): - raise DemistoException("When using OAuth 2, ensure to specify the client ID, client secret, and Account UUID. When using a personal access token, make sure to specify the access token. It's important to include only the required parameters for each type and avoid including any extra parameters.") if not events_to_fetch: raise DemistoException("Please specify at least one event type to fetch.") if audit_max < 1 or audit_max > 25000: raise DemistoException("The maximum number of audit logs events per fetch needs to be grater then 0 and not more then then 2500") if apm_max < 1 or apm_max > 5000: - raise DemistoException("The maximum number of APM events per fetch needs to be grater then 0 and not more then then 25000") - + raise DemistoException("The maximum number of APM events per fetch needs to be grater then 0 and not more then then 5000") -# def convert_date(date): # This function gets a date that looks like this: 1737656746001 -# seconds = timestamp // 1000 -# milliseconds = timestamp % 1000 - -# # Convert seconds to a datetime object in UTC -# dt = datetime.fromtimestamp(seconds, tz=timezone.utc) def add_fields_to_events(events, event_type): @@ -148,14 +98,14 @@ def fetch_apm_events(client, limit, fetch_start_time): events_count = 0 args = {} - for _ in range(5): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch - args["apm_limit"] = min(limit-events_count, 1000) + for i in range(5): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch + args["apm_limit"] = min(limit-events_count, 1000) # The api can bring up to 1000 events per call if args["apm_limit"] != 0: # We didn't get to the limit needed, need to fetch more events # First time fetching if last_run == {}: - args["apm_from"] = fetch_start_time # Change to "now" after I finish testing + args["apm_from"] = fetch_start_time else: if last_run.get("nextPageKey"): @@ -168,18 +118,18 @@ def fetch_apm_events(client, limit, fetch_start_time): # This approach eliminates the need for deduplication. args["apm_from"] = last_run["last_timestamp"] + 1 - demisto.debug(f"Dynatrace calling query with {args=}") + demisto.debug(f"Dynatrace fetch APM {i} times in loop. calling query with {args=}") response = events_query(client, args, "APM") num_events = len(response.get("events")) - demisto.debug(f"Dynatrace got {num_events} events") + demisto.debug(f"Dynatrace fetch APM {i} times in loop. got {num_events} events") # TODO need to see what happens if we get no events is response.get("events") empty or None? if response.get("nextPageKey"): - demisto.debug("Dynatrace setting last run with nextPageKey") + demisto.debug(f"Dynatrace fetch APM {i} times in loop. setting last run with nextPageKey") last_run_to_save["nextPageKey"] = response["nextPageKey"] last_run_to_save["last_timestamp"] = None # This timestamp won't be relevant at the next run. else: - demisto.debug("Dynatrace setting last run with timestamp") + demisto.debug(f"Dynatrace fetch APM {i} times in loop. setting last run with timestamp") # If events were retrieved during this run (which might not always happen), # we save the last timestamp from this run. # If no events were retrieved, we retain the same last_timestamp as before, @@ -194,6 +144,7 @@ def fetch_apm_events(client, limit, fetch_start_time): last_run = last_run_to_save + demisto.debug(f"Dynatrace fetch APM ou of loop. setting last_apm_run to {last_run_to_save}") integration_cnx["last_apm_run"] = last_run_to_save set_integration_context(integration_cnx) @@ -202,7 +153,7 @@ def fetch_apm_events(client, limit, fetch_start_time): def fetch_audit_log_events(client, limit, fetch_start_time): - # last_apm_run should be None or a {"nextPageKey": val, "last_timestamp": val} + # last_audit_run should be None or a {"nextPageKey": val, "last_timestamp": val} integration_cnx = demisto.getIntegrationContext() last_run = integration_cnx.get("last_audit_run") or {} @@ -211,8 +162,8 @@ def fetch_audit_log_events(client, limit, fetch_start_time): events_count = 0 args = {} - for _ in range(5): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch - args["audit_limit"] = min(limit-events_count, 5000) + for i in range(5): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch + args["audit_limit"] = min(limit-events_count, 5000) # The api can return up to 5000 events per call if args["audit_limit"] != 0: # We didn't get to the limit needed, need to fetch more events @@ -231,17 +182,17 @@ def fetch_audit_log_events(client, limit, fetch_start_time): # This approach eliminates the need for deduplication. args["audit_from"] = last_run["last_timestamp"] + 1 - demisto.debug(f"Dynatrace calling query with {args=}") + demisto.debug(f"Dynatrace fetch audit logs {i} times in loop. calling query with {args=}") response = events_query(client, args, "Audit logs") num_events = len(response.get("auditLogs")) - demisto.debug(f"Dynatrace got {num_events} events") + demisto.debug(f"Dynatrace fetch audit logs {i} times in loop. got {num_events} events") if response.get("nextPageKey"): - demisto.debug("Dynatrace setting last run with nextPageKey") + demisto.debug(f"Dynatrace fetch audit logs {i} times in loop. setting last run with nextPageKey") last_run_to_save["nextPageKey"] = response["nextPageKey"] last_run_to_save["last_timestamp"] = None # This timestamp won't be relevant at the next run. else: - demisto.debug("Dynatrace setting last run with timestamp") + demisto.debug(f"Dynatrace fetch audit logs {i} times in loop. setting last run with timestamp") # If events were retrieved during this run (which might not always happen), # we save the last timestamp from this run. # If no events were retrieved, we retain the same last_timestamp as before, @@ -256,73 +207,11 @@ def fetch_audit_log_events(client, limit, fetch_start_time): last_run = last_run_to_save + demisto.debug(f"Dynatrace fetch Audit logs out of loop. setting last_audit_run to {last_run_to_save}") integration_cnx["last_audit_run"] = last_run_to_save set_integration_context(integration_cnx) return events_to_return - - - # integration_context = demisto.getIntegrationContext() - - # integration_context_to_save = {} - # events_to_return = [] - # audit_count, apm_count = 0, 0 - - # args = {} - - # for _ in range(0, 5): - - # args["audit_limit"] = min(audit_limit-audit_count, 5000) - # args["apm_limit"] = min(apm_limit-apm_count, 1000) - - # for event_type in events_to_fetch: - - # # set args - # if not integration_context.get(f"{events_to_fetch[0]}_last_timestamp"): # first fetch - # args[EVENTS_TYPE_DICT[event_type][1]+"_from"] = "now-1d" - # else: - # args[EVENTS_TYPE_DICT[event_type][1]+"_from"] = integration_context.get(EVENTS_TYPE_DICT[event_type][1]+"_last_timestamp") - - - # if args[EVENTS_TYPE_DICT[event_type][1]+"_limit"] != 0: - # response = events_query(client, args, event_type) - - # #dedup - # counter = 0 - # for i in range(len(response[EVENTS_TYPE_DICT[event_type][0]])): - # if response[EVENTS_TYPE_DICT[event_type][0]][i]["entityId"] in integration_context.get(f"{events_to_fetch[0]}_last_events_ids"): - # continue - # else: - # counter = i+1 - # break - - # while counter == len(response[EVENTS_TYPE_DICT[event_type][0]]): - # if response.get("nextPageKey"): - # integration_context_to_save[EVENTS_TYPE_DICT[event_type][1]+"_last_run"] = (response.get("nextPageKey"), None) - # else: - # integration_context_to_save[EVENTS_TYPE_DICT[event_type][1]+"_last_timestamp"] = (None, response[EVENTS_TYPE_DICT[event_type][0]][0][FIELD_MAPPING[event_type][1]]) - # response = events_query(client, args, event_type) - - # #dedup - # counter = 0 - # for i in range(len(response[EVENTS_TYPE_DICT[event_type][0]])): - # if response[EVENTS_TYPE_DICT[event_type][0]][i]["entityId"] in integration_context.get(f"{events_to_fetch[0]}_last_events_ids"): - # continue - # else: - # counter = i+1 - # break - - # events = add_fields_to_events(response[EVENTS_TYPE_DICT[event_type][0]], event_type) - # events_to_return.extend(events) - # if event_type == "APM": - # apm_count += len(events) - # else: - # audit_count += len(events) - # integration_context_to_save[EVENTS_TYPE_DICT[event_type][1]+"_next_page"] = response["nextPageKey"] # TODO what happens when there are no more events? Do we still gat a nextPageKey? If not we need to go out of the loop, need to check this use case - - # set_integration_context(integration_context_to_save) - - # return events_to_return """ COMMAND FUNCTIONS """ @@ -388,15 +277,12 @@ def main(): # pragma: no cover params = demisto.params() url = params.get("url") - client_id = params.get('client_id') - client_secret = params.get('client_secret') - uuid = params.get('uuid') token = params.get('token') events_to_fetch = argToList(params.get('events_to_fetch')) audit_limit = arg_to_number(params.get('audit_limit')) or 25000 apm_limit = arg_to_number(params.get('apm_limit')) or 25000 - validate_params(url, client_id, client_secret, uuid, token, events_to_fetch, audit_limit, apm_limit) + validate_params(url, token, events_to_fetch, audit_limit, apm_limit) verify = not argToBoolean(params.get("insecure", False)) proxy = argToBoolean(params.get("proxy", False)) @@ -407,7 +293,7 @@ def main(): # pragma: no cover try: - client = DynatraceClient(url, client_id, client_secret, uuid, token, events_to_fetch, verify, proxy) + client = DynatraceClient(url, token, verify, proxy) args = demisto.args() diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml index 460aad08489b..2c173dcd4f1e 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml @@ -7,33 +7,14 @@ sectionOrder: name: Dynatrace display: Dynatrace category: Cloud Services -description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.].' +description: 'Fetch Audit logs and APM events from Dynatrace Platform' configuration: - display: Server URL name: url required: true type: 0 section: Connect -- display: Client ID - additionalinfo: Required If using OAuth 2 as authentication method. - name: client_id - required: false - type: 4 - section: Connect -- display: Client Secret - additionalinfo: Required If using OAuth 2 as authentication method. - name: client_secret - required: false - type: 4 - section: Connect -- display: Account UUID - additionalinfo: Required If using OAuth 2 as authentication method. - name: uuid - required: false - type: 4 - section: Connect - display: Access Token - additionalinfo: Required If using Personal Access Token. name: token required: false type: 4 @@ -51,13 +32,13 @@ configuration: - display: The maximum number of audit logs events per fetch section: Collect advanced: true - defaultvalue: "5000" + defaultvalue: "25000" type: 0 name: audit_limit - display: The maximum number of APM events per fetch section: Collect advanced: true - defaultvalue: "5000" # Need to ask sara about the default limit in all event types + defaultvalue: "5000" type: 0 name: apm_limit - display: Trust any certificate (not secure) @@ -83,9 +64,9 @@ script: - APM required: true - name: audit_from - description: The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to # TODO https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + description: The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/events-v2/get-events for more information. - name: apm_from - description: The start date for searching apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to #TODO https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/application-security/vulnerabilities/get-vulnerabilities fpr more information. + description: The start date for searching apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/audit-logs/get-log for more information. - name: audit_limit required: false description: Number of audit_logs events to fetch. diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md index 51bd561c623f..4b7a92d5d36e 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md @@ -1,8 +1,29 @@ -## BaseIntegration Help +## Dynatrace Help -Markdown file for integration configuration help snippet. In this file add: +### How to Create a Personal Access Token (Classic Access Token): +Generate an access token +To generate an access token +1. Go to Access Tokens. +2. Select Generate new token. +3. Enter a name for your token. +Dynatrace doesn't enforce unique token names. You can create multiple tokens with the same name. Be sure to provide a meaningful name for each token you generate. Proper naming helps you to efficiently manage your tokens and perhaps delete them when they're no longer needed. +4. Select the required scopes for the token. +5. Select Generate token. +6. Copy the generated token to the clipboard. Store the token in a password manager for future use. +You can only access your token once upon creation. You can't reveal it afterward. + +### Required scopes: +​​https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/basics/dynatrace-api-authentication + +Audit logs API - To execute this request, you need an access token with auditLogs.read scope. +List Events API - To execute this request, you need an access token with events.read scope. + +### Server URL +Make sure to include the correct url: + +For SaaS: https://{your-environment-id}.live.dynatrace.com + +For ActiveGate Cluster: +https://{your-activegate-domain}:9999/e/{your-environment-id} -- Brief information about how to retrieve the API key of your product -- Other useful information on how to configure your integration in XSOAR -Since this is a Markdown file, we encourage you to use MD formatting for sections, sub-sections, lists, etc. diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_image.png b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_image.png index 772c6af21bebd9230ea48f23a29ad2260f386e71..4b7b1e5fe656dab2b833ff4e755def307564e1cb 100644 GIT binary patch delta 991 zcmV<510ej%6Y2+$8Gi!+005wQ@9F>m0QOK!R7C&)05&!@H8nLjI5;&mH8nLgI5;>p zH8nIeG&D3cH8nLeGcz?cH8eCdHa0dkHa0XgG_~6RZ7!v>-2iVbs5Uk>6qWxpG&FL) zD0IIlbG|6F-2k-Q05vr=6_x+9+W>OED{wHUaK9+G-vD#IDSs7~{}h-1a4o2Fz9$ru z`*1F&v)cd^mj4u%|FYZwwch}6Ft2Yds5CS*MWv~Bx>{&tX>-0PHa0dCnEw@)|8Om+ zYJXF;-2igEC~z*QwA%o4zbSLPDR3;Oa4@Q`-vBi=HFLfwwA=s`mj7=ps1%U-WhR|` zzEESZb!Kl}Y=4YNZ=gJ&(H&jrw+jFO07Z0CPE!DYzaZaWK+iyMpAhd*FyD}1VDHKrj%r>>ua= z00Le~L_t(o!|j!4bJIW&hWGYPmR0Q($El8~tiB8{ zc(Jwo*Tkm_Q}5i$?hO)pky45cMHv-nr4b_Q5y+HOg?Ok`LX==Tb|?_(ssWG!QQqa< zoO%~FmwzV{5o@-FnFof(X+lCgD8tmo2owWrG{hluLfWJVa$*qL4crK00gw(+NYvuw z+xhvYyK}C4Gvavxzh=(|^)*o=f!!7X8iszeVMV%>VED2>aqx_|Fs*b-lR50=V$$BFy4=fqOyROqz} z11Bk#I>oUetE8xR>K0$A#6oksyktm&hBz3Nr`c*A0F(-vQ^iC4PBr;R=2xegyCS_QCUc@I<`N#AS+pqmV#`bE1Zr*?n&kJkeB~xaY+7 zaUpb_x>>?ZZ!ew+A3wvpOhAXW7@nZ$m)f8y&a5exhiEJtGl+%QGB7ryWN6Lw2ZUL* z#YfI7;xEn%@(2UbFYD2u@GD)e$dqH%Em&u$@_sc-O7cWXok=a3?9JNZeh^ZpvzXjoNJU1VgcOS!Wr4 zO{q;AyQYAIs;$+wHZ~Sp+*+hq)D%gf ztHY9woe3{0IJoIrpgCQC7`DJ3m>Rb4IVaSF>acO==CH3?2YW9zeW_Dx~*%o3bFhWN8-vqxH&=>B0u zuaCmH%$&2TPhhFWSb?U_&pUZ|*bx4bY2lAeKf4Fz( zmQcC1G(7!QIj=-*@_K6Xu1z?%2l)e_654|gT&Yu996XHs=&CTZ`w-v}%L}p|&0V{i z+>M}fRVtn9z|!)NwJa=JU&b?XP1tkblQ4D3yx@~^KJ{ql)!hryGQsnWH2d0+H@nGX zw4$doc(6Jj$F{`H#bN9f?}g#LF3mfF;nU`awLA8cKM!>FeJ~$VrQ(!~W*_628V=UH z#L*3oZlPbNGeW+jEG_4Q0)Dht@52W-_{z?+TAyKZtwufpGKm1!42gYaP3o=Hp5 zpENDeNw=&g(wishi@^u5KHBy z0B5lAQOWLBab_QSK09fjTO zC@3Ws@CkVee3)foO2$1e+uh|0=G5tv6Q@D!mTg_+}$dIq?{bj!*uvDc%`2UX?@FJXeYlO zql@{RdZcteM0jQY4X%XKp#!*g{X6jLzX!TOK_|!pF2zT3WJB`(jkNkL_}X+Gl)`YB z2EHqQ4+=WY7VrZ@LmX(C`XO?ji^Ej+Opo`g5xWAGl^6a3G? z&p{{f9xG+wdu=iJnzREvZxZ-gR8YYJ1q&1`P_RJ30tE{cEO1OM;46U-Cb=BbMjBDs z)ttK)h8|@daa>LHgTYT3?|-25D93$R@gid10e^0qi+Lx5PNNfk+3|QZisSOhx*-=W$J3%r+sPEn@)0ekD#*uCOYo^?Wv^h40bS-pMXKv)%JM$A&pY;mgh?FhkU+QE8BVmEWZwZ1Os3?tOv{bg-ZM9Yuh@kKT3Z*1=`auTt9(gP*;L= zYM0sIh3fe&AlwBta3|cJr8h@40S(p`YfCjh$JYPeoo}f{m1KXYlPCUyd701;zn`qs(u>dr$7Yn+zbza=YJN;K&Ss5w9P+3!^a6@TgF#|cJ-WZz!Y%D%ukcHbs5yv zHpv-V`0+!O}lnF#l)S zyxg`;wvOoF*cT^l)Di+ZR7D$U3#BdW&v=a=gNtBJyf z9dok((ZwPiqQALjyh|Hg0izuz@yL&?uKKjIP+TF!L!4*?Gxd3lZ5vK1_C>o# z8FoUss^nF&!V#46z7aKpzyxRwec=Uo1iYV=g7SPDa*1iIT!y{~=fU;h4D~9}AgV|0 zA^e%+Z7>n;gBsA;&qHJMLH6XYaP*3u3#(upYy+3X!SF}W;opKW@FaNUB(GkgpHdkq z-nYBJbznRyx9xXgM)nx{@@?W(Pyt>9lfd!UtGmdxP%h5M{V*0(KR5utHiBFledzcY zeh!}hB~Z@J$(c~?pZ7TYD-oYVcx57P`NiPUVm(VCE=x|PWnkOUD7O&S36?e8iI=N5 zuCCdPsT|`y&7@J_`Ij0&CGZNE<`vowoXoF-Y4Rq-3HM?m-Mlq$2dLb(OmijJ2G*_Z z-T=#~y4udI!;8#GW?S0biQWoSyazjRR5@7g0V9o(_ai&kSn1$Dfh$5ZAi+awM#yxJ zQHFJ1bb3s0?6C_KIIg5R5J{%>CcYC~YWu~r>$8V_Z3FpKSKGN|JCP6_o11si z#&fUuleVMTj(wG*SJ92&!{Pu~4vl>dpo(;ACd2wc6B?Sg4ex(($=#5RpLMd0;eS43 V(U=mQ{m=ja002ovPDHLkV1nG2;bs5; diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py index f90988a52513..758767b2f304 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py @@ -1,44 +1,12 @@ from demisto_sdk.commands.common.handlers import JSON_Handler -import json - import pytest import Dynatrace as dyn -import demistomock as demisto from CommonServerPython import * -# def util_load_json(path): -# with open(path, encoding="utf-8") as f: -# return json.loads(f.read()) - -AUDIT_CLIENT = dyn.DynatraceClient( - base_url="https://AAAAA.dynatrace.com", - client_id=None, - client_secret=None, - uuid=None, - token="AAAAAAAAA.AAAAAAA.AAAAAAA", - events_to_fetch=["Audit logs"], - verify=True, - proxy=None - ) - -APM_CLIENT = dyn.DynatraceClient( - base_url="https://AAAAA.dynatrace.com", - client_id=None, - client_secret=None, - uuid=None, - token="AAAAAAAAA.AAAAAAA.AAAAAAA", - events_to_fetch=["APM"], - verify=True, - proxy=None - ) -APM_AUDIT_CLIENT = dyn.DynatraceClient( +CLIENT = dyn.DynatraceClient( base_url="https://AAAAA.dynatrace.com", - client_id=None, - client_secret=None, - uuid=None, token="AAAAAAAAA.AAAAAAA.AAAAAAA", - events_to_fetch=["APM", "Audit logs"], verify=True, proxy=None ) @@ -50,8 +18,8 @@ def test_get_audit_logs_events(mocker): When: calling get_audit_logs_events function Then: the http request is called with "GET" arg and with the right url. """ - http_request = mocker.patch.object(AUDIT_CLIENT, '_http_request', return_value=[]) - AUDIT_CLIENT.get_audit_logs_events(query="?querykey=queryarg") + http_request = mocker.patch.object(CLIENT, '_http_request', return_value=[]) + CLIENT.get_audit_logs_events(query="?querykey=queryarg") assert http_request.call_args[0][0] == "GET" assert http_request.call_args[0][1] == "/api/v2/auditlogs?querykey=queryarg" @@ -62,44 +30,35 @@ def test_get_APM_events(mocker): When: calling get_APM_events function Then: the http request is called with "GET" arg and with the right url. """ - http_request = mocker.patch.object(APM_CLIENT, '_http_request', return_value=[]) - APM_CLIENT.get_APM_events(query="?querykey=queryarg") + http_request = mocker.patch.object(CLIENT, '_http_request', return_value=[]) + CLIENT.get_APM_events(query="?querykey=queryarg") assert http_request.call_args[0][0] == "GET" assert http_request.call_args[0][1] == "/api/v2/events?querykey=queryarg" @pytest.mark.parametrize( - "client_id, client_secret, uuid, token, events_to_fetch, audit_max, apm_max, expected_exception, expected_message", + "token, events_to_fetch, audit_max, apm_max, expected_exception, expected_message", [ - # Valid OAuth2 configuration - ("client_id", "client_secret", "uuid", None, ["APM"], 25000, 5000, None, None), - # Valid token configuration - (None, None, None, "token", ["APM"], 25000, 5000, None, None), - - # Invalid: Both OAuth2 and token parameters provided - ("client_id", "client_secret", "uuid", "token", ["APM"], 25000, 5000, DemistoException, "When using OAuth 2"), - - # Invalid: Missing both OAuth2 and token parameters - (None, None, None, None, ["APM"], 25000, 5000, DemistoException, "When using OAuth 2"), + ("token", ["APM"], 25000, 5000, None, None), # Invalid: No event types specified - ("client_id", "client_secret", "uuid", None, [], 25000, 5000, DemistoException, "Please specify at least one event type"), + (None, [], 25000, 5000, DemistoException, "Please specify at least one event type"), # Invalid: audit_max out of range (too high) - ("client_id", "client_secret", "uuid", None, ["Audit logs"], 30000, 5000, DemistoException, "The maximum number of audit logs events"), + (None, ["Audit logs"], 30000, 5000, DemistoException, "The maximum number of audit logs events"), # Invalid: audit_max out of range (negative) - ("client_id", "client_secret", "uuid", None, ["Audit logs"], -1, 5000, DemistoException, "The maximum number of audit logs events"), + (None, ["Audit logs"], -1, 5000, DemistoException, "The maximum number of audit logs events"), # Invalid: apm_max out of range (too high) - ("client_id", "client_secret", "uuid", None, ["APM"], 25000, 6000, DemistoException, "The maximum number of APM events"), + (None, ["APM"], 25000, 6000, DemistoException, "The maximum number of APM events"), # Invalid: apm_max out of range (negative) - ("client_id", "client_secret", "uuid", None, ["APM"], 25000, -1, DemistoException, "The maximum number of APM events"), + (None, ["APM"], 25000, -1, DemistoException, "The maximum number of APM events"), ], ) -def test_validate_params(client_id, client_secret, uuid, token, events_to_fetch, audit_max, apm_max, expected_exception, expected_message): +def test_validate_params(token, events_to_fetch, audit_max, apm_max, expected_exception, expected_message): """ Given: all instance params When: Calling validate_params function @@ -110,9 +69,6 @@ def test_validate_params(client_id, client_secret, uuid, token, events_to_fetch, with pytest.raises(expected_exception) as excinfo: validate_params( url="http://example.com", - client_id=client_id, - client_secret=client_secret, - uuid=uuid, token=token, events_to_fetch=events_to_fetch, audit_max=audit_max, @@ -122,9 +78,6 @@ def test_validate_params(client_id, client_secret, uuid, token, events_to_fetch, else: validate_params( url="http://example.com", - client_id=client_id, - client_secret=client_secret, - uuid=uuid, token=token, events_to_fetch=events_to_fetch, audit_max=audit_max, @@ -184,8 +137,8 @@ def test_events_query__APM(mocker): Then: The get_APM_events is called with the right query. """ from Dynatrace import events_query - request = mocker.patch.object(APM_CLIENT, 'get_APM_events', return_value=[]) - events_query(APM_CLIENT, {"apm_limit": "100", "apm_from": "1640995200000"}, "APM") + request = mocker.patch.object(CLIENT, 'get_APM_events', return_value=[]) + events_query(CLIENT, {"apm_limit": "100", "apm_from": "1640995200000"}, "APM") assert request.call_args[0][0] == '?pageSize=100&from=1640995200000' @@ -196,8 +149,8 @@ def test_events_query__audit(mocker): Then: The get_audit_events is called with the right query. """ from Dynatrace import events_query - request = mocker.patch.object(AUDIT_CLIENT, 'get_audit_logs_events', return_value=[]) - events_query(AUDIT_CLIENT, {"audit_limit": "100", "audit_from": "1640995200000"}, "Audit logs") + request = mocker.patch.object(CLIENT, 'get_audit_logs_events', return_value=[]) + events_query(CLIENT, {"audit_limit": "100", "audit_from": "1640995200000"}, "Audit logs") assert request.call_args[0][0] == '?pageSize=100&from=1640995200000' @@ -213,7 +166,7 @@ def test_fetch_events(mocker): add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", return_value=[]) send_events_to_xsiam_mock = mocker.patch("Dynatrace.send_events_to_xsiam") - fetch_events(APM_AUDIT_CLIENT, ["APM", "Audit logs"], 200, 100) + fetch_events(CLIENT, ["APM", "Audit logs"], 200, 100) assert apm_mock.call_args.args[1] == 100 assert audit_mock.call_args.args[1] == 200 @@ -234,9 +187,9 @@ def test_get_events_command__APM(mocker): add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", return_value=[]) send_events_to_xsiam_mock = mocker.patch("Dynatrace.send_events_to_xsiam") - res = get_events_command(APM_CLIENT, {"events_types_to_get": "APM", "should_push_events": True}) + res = get_events_command(CLIENT, {"events_types_to_get": "APM", "should_push_events": True}) - events_query_mock.assert_called_once_with(APM_CLIENT, {"events_types_to_get": "APM", "should_push_events": True}, "APM") + events_query_mock.assert_called_once_with(CLIENT, {"events_types_to_get": "APM", "should_push_events": True}, "APM") add_fields_to_events_mock.assert_called_once_with([], "APM") send_events_to_xsiam_mock.assert_called_once_with(events=[], vendor="Dynatrace", product="Platform") assert "No events were received" in res.readable_output @@ -257,9 +210,9 @@ def test_get_events_command__Audit_logs(mocker): return_value=[{"timestamp": 1640995200000, "SOURCE_LOG_TYPE": "Audit logs events"}]) send_events_to_xsiam_mock = mocker.patch("Dynatrace.send_events_to_xsiam") - res = get_events_command(AUDIT_CLIENT, {"events_types_to_get": "Audit logs", "should_push_events": False}) + res = get_events_command(CLIENT, {"events_types_to_get": "Audit logs", "should_push_events": False}) - events_query_mock.assert_called_once_with(AUDIT_CLIENT, + events_query_mock.assert_called_once_with(CLIENT, {"events_types_to_get": "Audit logs", "should_push_events": False}, "Audit logs") add_fields_to_events_mock.assert_called_once_with([{"timestamp": 1640995200000}], "Audit logs") send_events_to_xsiam_mock.assert_not_called() diff --git a/Packs/Dynatrace/Integrations/Dynatrace/README.md b/Packs/Dynatrace/Integrations/Dynatrace/README.md index 2e9696755b74..2ddbbaba069a 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/README.md +++ b/Packs/Dynatrace/Integrations/Dynatrace/README.md @@ -33,7 +33,7 @@ Manual command to fetch events and display them. | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | events_types_to_get | comma separated list of events types to get. Possible values are: Audit logs, APM. | Required | -| audit_from | The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/events-v2/get-events fro more information. | Optional | +| audit_from | The start date for searching audit_logs events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/events-v2/get-events for more information. | Optional | | apm_from | The start date for searching apm events. The date can be provided in three formats- Timestamp in UTC milliseconds, Human-readable format in the following format- 2021-01-25T05:57:01.123+01:00 or relative timeframe using the format now-NU/A. For more information, please refer to https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/environment-api/audit-logs/get-log for more information. | Optional | | audit_limit | Number of audit_logs events to fetch. Default is 1. | Optional | | apm_limit | Number of apm events to fetch. Default is 1. | Optional | diff --git a/Packs/Dynatrace/README.md b/Packs/Dynatrace/README.md index e69de29bb2d1..c736001ace6d 100644 --- a/Packs/Dynatrace/README.md +++ b/Packs/Dynatrace/README.md @@ -0,0 +1 @@ +Fetch Audit logs and APM events from Dynatrace Platform. \ No newline at end of file diff --git a/Packs/Dynatrace/pack_metadata.json b/Packs/Dynatrace/pack_metadata.json index 734643ea3f8a..0acb0def7b50 100644 --- a/Packs/Dynatrace/pack_metadata.json +++ b/Packs/Dynatrace/pack_metadata.json @@ -1,6 +1,6 @@ { "name": "Dynatrace", - "description": "## FILL MANDATORY FIELD ##", + "description": "Dynatrace is a revolutionary platform that delivers analytics and automation for unified observability and security.", "support": "xsoar", "currentVersion": "1.0.0", "author": "Cortex XSOAR", From f0fa5dec568d460793d73dac25e5ec7d7bef1025 Mon Sep 17 00:00:00 2001 From: yshamai Date: Tue, 28 Jan 2025 12:36:43 +0200 Subject: [PATCH 09/15] fetch tests and code fixes --- .../Integrations/Dynatrace/Dynatrace.py | 24 +-- .../Integrations/Dynatrace/Dynatrace_test.py | 152 +++++++++++++++++- 2 files changed, 164 insertions(+), 12 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index dd89a7776a80..b7ed0242aa98 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -38,7 +38,7 @@ def validate_params(url, token, events_to_fetch, audit_max, apm_max): if not events_to_fetch: raise DemistoException("Please specify at least one event type to fetch.") if audit_max < 1 or audit_max > 25000: - raise DemistoException("The maximum number of audit logs events per fetch needs to be grater then 0 and not more then then 2500") + raise DemistoException("The maximum number of audit logs events per fetch needs to be grater then 0 and not more then then 25000") if apm_max < 1 or apm_max > 5000: raise DemistoException("The maximum number of APM events per fetch needs to be grater then 0 and not more then then 5000") @@ -118,24 +118,25 @@ def fetch_apm_events(client, limit, fetch_start_time): # This approach eliminates the need for deduplication. args["apm_from"] = last_run["last_timestamp"] + 1 - demisto.debug(f"Dynatrace fetch APM {i} times in loop. calling query with {args=}") + demisto.debug(f"Dynatrace fetch APM {i+1} times in loop. calling query with {args=}") response = events_query(client, args, "APM") num_events = len(response.get("events")) - demisto.debug(f"Dynatrace fetch APM {i} times in loop. got {num_events} events") + events_count += num_events + demisto.debug(f"Dynatrace fetch APM {i+1} times in loop. got {num_events} events") # TODO need to see what happens if we get no events is response.get("events") empty or None? if response.get("nextPageKey"): - demisto.debug(f"Dynatrace fetch APM {i} times in loop. setting last run with nextPageKey") + demisto.debug(f"Dynatrace fetch APM {i+1} times in loop. setting last run with nextPageKey") last_run_to_save["nextPageKey"] = response["nextPageKey"] last_run_to_save["last_timestamp"] = None # This timestamp won't be relevant at the next run. else: - demisto.debug(f"Dynatrace fetch APM {i} times in loop. setting last run with timestamp") + demisto.debug(f"Dynatrace fetch APM {i+1} times in loop. setting last run with timestamp") # If events were retrieved during this run (which might not always happen), # we save the last timestamp from this run. # If no events were retrieved, we retain the same last_timestamp as before, # In cases where no events were retrieved and this is the first run (i.e., no last_run_timestamp exists), # the query will use start_fetch_time again in the next execution. - last_run_to_save["last_timestamp"] = response.get("events")[0]["startTime"] if response["totalCount"] != 0 else (last_run.get("last_timestamp") or fetch_start_time) # What happens when no events are retrieved? + last_run_to_save["last_timestamp"] = response.get("events")[0]["startTime"] if response["totalCount"] != 0 else (last_run.get("last_timestamp") or fetch_start_time) last_run_to_save["nextPageKey"] = None events = response.get("events") @@ -143,6 +144,7 @@ def fetch_apm_events(client, limit, fetch_start_time): events_to_return.extend(events) last_run = last_run_to_save + args = {} demisto.debug(f"Dynatrace fetch APM ou of loop. setting last_apm_run to {last_run_to_save}") integration_cnx["last_apm_run"] = last_run_to_save @@ -182,17 +184,18 @@ def fetch_audit_log_events(client, limit, fetch_start_time): # This approach eliminates the need for deduplication. args["audit_from"] = last_run["last_timestamp"] + 1 - demisto.debug(f"Dynatrace fetch audit logs {i} times in loop. calling query with {args=}") + demisto.debug(f"Dynatrace fetch audit logs {i+1} times in loop. calling query with {args=}") response = events_query(client, args, "Audit logs") num_events = len(response.get("auditLogs")) - demisto.debug(f"Dynatrace fetch audit logs {i} times in loop. got {num_events} events") + events_count += num_events + demisto.debug(f"Dynatrace fetch audit logs {i+1} times in loop. got {num_events} events") if response.get("nextPageKey"): - demisto.debug(f"Dynatrace fetch audit logs {i} times in loop. setting last run with nextPageKey") + demisto.debug(f"Dynatrace fetch audit logs {i+1} times in loop. setting last run with nextPageKey") last_run_to_save["nextPageKey"] = response["nextPageKey"] last_run_to_save["last_timestamp"] = None # This timestamp won't be relevant at the next run. else: - demisto.debug(f"Dynatrace fetch audit logs {i} times in loop. setting last run with timestamp") + demisto.debug(f"Dynatrace fetch audit logs {i+1} times in loop. setting last run with timestamp") # If events were retrieved during this run (which might not always happen), # we save the last timestamp from this run. # If no events were retrieved, we retain the same last_timestamp as before, @@ -206,6 +209,7 @@ def fetch_audit_log_events(client, limit, fetch_start_time): events_to_return.extend(events) last_run = last_run_to_save + args = {} demisto.debug(f"Dynatrace fetch Audit logs out of loop. setting last_audit_run to {last_run_to_save}") integration_cnx["last_audit_run"] = last_run_to_save diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py index 758767b2f304..c52b46316f33 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py @@ -1,8 +1,10 @@ +from unittest.mock import Mock from demisto_sdk.commands.common.handlers import JSON_Handler import pytest import Dynatrace as dyn from CommonServerPython import * - +from unittest.mock import call +import demistomock as demisto CLIENT = dyn.DynatraceClient( base_url="https://AAAAA.dynatrace.com", @@ -216,4 +218,150 @@ def test_get_events_command__Audit_logs(mocker): {"events_types_to_get": "Audit logs", "should_push_events": False}, "Audit logs") add_fields_to_events_mock.assert_called_once_with([{"timestamp": 1640995200000}], "Audit logs") send_events_to_xsiam_mock.assert_not_called() - assert "1640995200000" in res.readable_output \ No newline at end of file + assert "1640995200000" in res.readable_output + + +events_query_expected_calls_apm = [ + (CLIENT, {"apm_limit": 100, "apm_from": 1000}, "APM"), + (CLIENT, {"apm_limit": 100, "apm_from": 1001}, "APM"), + (CLIENT, {"apm_limit": 97, "apm_from": 1002}, "APM"), + (CLIENT, {"apm_limit": 94, "apm_next_page_key": "AAAA"}, "APM"), + (CLIENT, {"apm_limit": 91, "apm_next_page_key": "BBBB"}, "APM") +] +events_query_responses_apm = [ + {"events": [], "totalCount": 0}, # No events returned + {"events": [{"startTime": 1001}, {"startTime": 1000}, {"startTime": 1000}], "totalCount": 3}, # No next page key + {"events": [{"startTime": 1002}, {"startTime": 1002}, {"startTime": 1002}], "totalCount": 3, "nextPageKey": "AAAA"}, # NextPageKey exists + {"events": [{"startTime": 1004}, {"startTime": 1003}, {"startTime": 1002}], "totalCount": 3, "nextPageKey": "BBBB"}, # NextPageKey exists + {"events": [{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}], "totalCount": 3} # No nextPageKey +] +add_fields_to_events_expected_calls_apm = [ + ([], "APM"), + ([{"startTime": 1001}, {"startTime": 1000}, {"startTime": 1000}], "APM"), + ([{"startTime": 1002}, {"startTime": 1002}, {"startTime": 1002}], "APM"), + ([{"startTime": 1004}, {"startTime": 1003}, {"startTime": 1002}], "APM"), + ([{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}], "APM") +] +add_fields_to_events_responses_apm = [ + [], + [{"startTime": 1001}, {"startTime": 1000}, {"startTime": 1000}], + [{"startTime": 1002}, {"startTime": 1002}, {"startTime": 1002}], + [{"startTime": 1004}, {"startTime": 1003}, {"startTime": 1002}], + [{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}] +] +def test_fetch_apm_events(mocker): + """ + Given: A client, a higher limit then events to be returned and a fetch_start_time + When: calling fetch_apm_events function + Then: + - The function receives exactly all the relevant events. + - The events_query function is called 5 times every time with the right arguments + - the add_fields_to_events function is called 5 times every time with the right arguments + - The set_integration_context function is called with the right cnx to set. + + This test checks these use cases: (next test will check other use cases) + - First time fetching (the first loop iteration will have no last_apm_run) + - Limit of events isn't reached + - First time calling api to get events returns no events + - A case where response has a next page key in one of the middle loop times + - A case when there is no nextPage key in one of the middle loops + - the last iteration of the loop receives a response with no nextPageKey + """ + from Dynatrace import fetch_apm_events + mocker.patch("Dynatrace.demisto.getIntegrationContext", return_value={}) + events_query_mock = mocker.patch("Dynatrace.events_query", side_effect=events_query_responses_apm) + add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", side_effect=add_fields_to_events_responses_apm) + set_integration_context_mock = mocker.patch("Dynatrace.set_integration_context") + fetch_apm_events(CLIENT, 100, 1000) + assert events_query_mock.call_count == 5 + assert [events_query_mock.call_args_list[i][0] for i in range(5)] == events_query_expected_calls_apm + assert add_fields_to_events_mock.call_count == 5 + assert [add_fields_to_events_mock.call_args_list[i][0] for i in range(5)] == add_fields_to_events_expected_calls_apm + set_integration_context_mock.assert_called_with({'last_apm_run': {'last_timestamp': 2000, 'nextPageKey': None}}) + + +# Need to check use case when limit is reached reached and excactly reached +# use case where we get events in the first time and we get nextPageKey +events_query_expected_calls_audit = [ #"audit_next_page_key" + (CLIENT, {"audit_limit": 100, "audit_from": 1000}, "Audit logs"), + (CLIENT, {"audit_limit": 97, "audit_from": 1001}, "Audit logs"), + (CLIENT, {"audit_limit": 97, "audit_from": 1001}, "Audit logs"), + (CLIENT, {"audit_limit": 94, "audit_next_page_key": "AAAA"}, "Audit logs"), + (CLIENT, {"audit_limit": 91, "audit_from": 2001}, "Audit logs") +] +events_query_responses_audit = [ + {"auditLogs": [{"timestamp": 1000}, {"timestamp": 1000}, {"timestamp": 1000}], "totalCount": 3}, # Events are returned, no nextPageKey + {"auditLogs": [], "totalCount": 0}, # No events are returned + {"auditLogs": [{"timestamp": 2000}, {"timestamp": 2000}, {"timestamp": 2000}], "totalCount": 3, "nextPageKey": "AAAA"}, # NextPageKey exists + {"auditLogs": [{"timestamp": 2000}, {"timestamp": 2000}, {"timestamp": 2000}], "totalCount": 3,}, # no NextPageKey + {"auditLogs": [{"timestamp": 3000}, {"timestamp": 3000}, {"timestamp": 3000}], "totalCount": 3, "nextPageKey": "BBBB"} # NextPageKey exists +] +add_fields_to_events_responses_audit = [ + [{"timestamp": 1000}, {"timestamp": 1000}, {"timestamp": 1000}], + [], + [{"timestamp": 2000}, {"timestamp": 2000}, {"timestamp": 2000}], + [{"timestamp": 2000}, {"timestamp": 2000}, {"timestamp": 2000}], + [{"timestamp": 3000}, {"timestamp": 3000}, {"timestamp": 3000}] +] +add_fields_to_events_expected_calls_audit = [ + ([{"timestamp": 1000}, {"timestamp": 1000}, {"timestamp": 1000}], "Audit logs"), + ([], "Audit logs"), + ([{"timestamp": 2000}, {"timestamp": 2000}, {"timestamp": 2000}], "Audit logs"), + ([{"timestamp": 2000}, {"timestamp": 2000}, {"timestamp": 2000}], "Audit logs"), + ([{"timestamp": 3000}, {"timestamp": 3000}, {"timestamp": 3000}], "Audit logs") +] +def test_fetch_audit_log_events(mocker): + """ + Given: A client, a higher limit then events to be returned and a fetch_start_time + When: calling fetch_audit_log_events function + Then: + - The function receives exactly all the relevant events. + - The events_query function is called 5 times every time with the right arguments + - the add_fields_to_events function is called 5 times every time with the right arguments + - The set_integration_context function is called with the right cnx to set. + + This test checks these use cases: (apm test test will check other use cases) + - No events returned in one of the middle loop iterations. + - No nextPageKey in the first time + - Last iteration has a nextPageKey + """ + from Dynatrace import fetch_audit_log_events + mocker.patch("Dynatrace.demisto.getIntegrationContext", return_value={}) + events_query_mock = mocker.patch("Dynatrace.events_query", side_effect=events_query_responses_audit) + add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", side_effect=add_fields_to_events_responses_audit) + set_integration_context_mock = mocker.patch("Dynatrace.set_integration_context") + fetch_audit_log_events(CLIENT, 100, 1000) + assert events_query_mock.call_count == 5 + assert [events_query_mock.call_args_list[i][0] for i in range(5)] == events_query_expected_calls_audit + assert add_fields_to_events_mock.call_count == 5 + assert [add_fields_to_events_mock.call_args_list[i][0] for i in range(5)] == add_fields_to_events_expected_calls_audit + set_integration_context_mock.assert_called_with({'last_audit_run': {'last_timestamp': None, 'nextPageKey': "BBBB"}}) + +add_fields_return_value = [ + {"timestamp": 1000}, + {"timestamp": 1000}, + {"timestamp": 1000} + ] +events_query_mock_return_value = { + "auditLogs": + [ + {"timestamp": 1000}, + {"timestamp": 1000}, + {"timestamp": 1000} + ], + "totalCount": 3 + } +def test_fetch_events__limit_is_reached(mocker): + """ + Given: A limit + When: fetching audit logs events and the first response returns amount of events as the limit + Then: The events_query function is called only once and len(events returned) == given limit + """ + from Dynatrace import fetch_audit_log_events + mocker.patch("Dynatrace.demisto.getIntegrationContext", return_value={}) + events_query_mock = mocker.patch("Dynatrace.events_query", return_value=events_query_mock_return_value) + mocker.patch("Dynatrace.add_fields_to_events", return_value=add_fields_return_value) + mocker.patch("Dynatrace.set_integration_context") + res = fetch_audit_log_events(CLIENT, 3, 1000) + events_query_mock.assert_called_once() + assert len(res) == 3 \ No newline at end of file From 66952027f01e7dd000726301b10cb9235ec36985 Mon Sep 17 00:00:00 2001 From: yshamai Date: Tue, 28 Jan 2025 12:53:03 +0200 Subject: [PATCH 10/15] code docs --- .../Integrations/Dynatrace/Dynatrace.py | 52 ++++++++++++++++--- .../Integrations/Dynatrace/Dynatrace_test.py | 20 +++---- 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index b7ed0242aa98..e7719878996e 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -33,8 +33,16 @@ def get_APM_events(self, query: str=""): """ HELPER FUNCTIONS """ -def validate_params(url, token, events_to_fetch, audit_max, apm_max): - +def validate_params(events_to_fetch, audit_max, apm_max): + """ + Validates the integration parameters. + + 1. `events_to_fetch` must contain at least one event type. + 2. `audit_max` must not exceed 25,000. + 3. `apm_max` must not exceed 5,000. + + If any of the parameters are invalid, the function raises a `ValueError` with a descriptive error message. + """ if not events_to_fetch: raise DemistoException("Please specify at least one event type to fetch.") if audit_max < 1 or audit_max > 25000: @@ -44,8 +52,15 @@ def validate_params(url, token, events_to_fetch, audit_max, apm_max): def add_fields_to_events(events, event_type): - - # TODO ask sara if we need the word 'events' in the end of the type, I don't think we usually do so. + """Adds SOURCE_LOG_TYPE and _time field to each event. + + Args: + events (List): list of events. + event_type (str): "APM" if events are apm type or "Audit logs" if events are audit logs type. + + Returns: + list or events with the added fields. + """ field_mapping = { "Audit logs": ["Audit logs events", "timestamp"], @@ -60,6 +75,16 @@ def add_fields_to_events(events, event_type): def events_query(client: DynatraceClient, args: dict, event_type: str): + """Calls the right api to get events of event_type type according to the args + + Args: + client (DynatraceClient): client + args (dict): A dictionary containing the arguments such as amp_limit or apm_from so we can call the api with the right query. + event_type (str): "APM" or "Audit logs". + + Returns: + The response from the api. + """ query_lst = [] query = "" @@ -88,7 +113,8 @@ def events_query(client: DynatraceClient, args: dict, event_type: str): def fetch_apm_events(client, limit, fetch_start_time): - + """Fetches events of APM type from fetch_start_time and not more than the limit given. + """ # last_apm_run should be None or a {"nextPageKey": val, "last_timestamp": val} integration_cnx = demisto.getIntegrationContext() last_run = integration_cnx.get("last_apm_run") or {} @@ -154,6 +180,8 @@ def fetch_apm_events(client, limit, fetch_start_time): def fetch_audit_log_events(client, limit, fetch_start_time): + """Fetches events of Audit logs type from fetch_start_time and not more than the limit given. + """ # last_audit_run should be None or a {"nextPageKey": val, "last_timestamp": val} integration_cnx = demisto.getIntegrationContext() @@ -221,7 +249,14 @@ def fetch_audit_log_events(client, limit, fetch_start_time): """ COMMAND FUNCTIONS """ def fetch_events(client: DynatraceClient, events_to_fetch: list, audit_limit: int, apm_limit: int): - + """Gets events from the fetching functions, adds the events the relevant fields and sends the events to XSIAM. + + Args: + client (DynatraceClient): client + events_to_fetch (list): list of events types to fetch + audit_limit (int): limit of Audit logs events to fetch + apm_limit (int): limit of APM events to fetch + """ fetch_start_time = int(datetime.now().timestamp() * 1000) # We want this timestamp to look like this: 1737656746001 demisto.debug(f"Dynatrace fetch Audit Logs events start time is {fetch_start_time}") @@ -240,7 +275,8 @@ def fetch_events(client: DynatraceClient, events_to_fetch: list, audit_limit: in def get_events_command(client: DynatraceClient, args: dict): - + """Gets Dynatrace events according to the arguments given. + """ events_types = argToList(args.get("events_types_to_get")) events_to_return = [] @@ -286,7 +322,7 @@ def main(): # pragma: no cover audit_limit = arg_to_number(params.get('audit_limit')) or 25000 apm_limit = arg_to_number(params.get('apm_limit')) or 25000 - validate_params(url, token, events_to_fetch, audit_limit, apm_limit) + validate_params(events_to_fetch, audit_limit, apm_limit) verify = not argToBoolean(params.get("insecure", False)) proxy = argToBoolean(params.get("proxy", False)) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py index c52b46316f33..ece0a733880e 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py @@ -39,28 +39,28 @@ def test_get_APM_events(mocker): @pytest.mark.parametrize( - "token, events_to_fetch, audit_max, apm_max, expected_exception, expected_message", + "events_to_fetch, audit_max, apm_max, expected_exception, expected_message", [ # Valid token configuration - ("token", ["APM"], 25000, 5000, None, None), + (["APM"], 25000, 5000, None, None), # Invalid: No event types specified - (None, [], 25000, 5000, DemistoException, "Please specify at least one event type"), + ([], 25000, 5000, DemistoException, "Please specify at least one event type"), # Invalid: audit_max out of range (too high) - (None, ["Audit logs"], 30000, 5000, DemistoException, "The maximum number of audit logs events"), + (["Audit logs"], 30000, 5000, DemistoException, "The maximum number of audit logs events"), # Invalid: audit_max out of range (negative) - (None, ["Audit logs"], -1, 5000, DemistoException, "The maximum number of audit logs events"), + (["Audit logs"], -1, 5000, DemistoException, "The maximum number of audit logs events"), # Invalid: apm_max out of range (too high) - (None, ["APM"], 25000, 6000, DemistoException, "The maximum number of APM events"), + (["APM"], 25000, 6000, DemistoException, "The maximum number of APM events"), # Invalid: apm_max out of range (negative) - (None, ["APM"], 25000, -1, DemistoException, "The maximum number of APM events"), + (["APM"], 25000, -1, DemistoException, "The maximum number of APM events"), ], ) -def test_validate_params(token, events_to_fetch, audit_max, apm_max, expected_exception, expected_message): +def test_validate_params(events_to_fetch, audit_max, apm_max, expected_exception, expected_message): """ Given: all instance params When: Calling validate_params function @@ -70,8 +70,6 @@ def test_validate_params(token, events_to_fetch, audit_max, apm_max, expected_ex if expected_exception: with pytest.raises(expected_exception) as excinfo: validate_params( - url="http://example.com", - token=token, events_to_fetch=events_to_fetch, audit_max=audit_max, apm_max=apm_max, @@ -79,8 +77,6 @@ def test_validate_params(token, events_to_fetch, audit_max, apm_max, expected_ex assert expected_message in str(excinfo.value) else: validate_params( - url="http://example.com", - token=token, events_to_fetch=events_to_fetch, audit_max=audit_max, apm_max=apm_max, From 0de3f3a2e8dfaeb02daf7ef8e165514b6d09ff89 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 29 Jan 2025 09:13:14 +0200 Subject: [PATCH 11/15] change debug --- Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index e7719878996e..2249d12032c0 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -258,7 +258,7 @@ def fetch_events(client: DynatraceClient, events_to_fetch: list, audit_limit: in apm_limit (int): limit of APM events to fetch """ fetch_start_time = int(datetime.now().timestamp() * 1000) # We want this timestamp to look like this: 1737656746001 - demisto.debug(f"Dynatrace fetch Audit Logs events start time is {fetch_start_time}") + demisto.debug(f"Dynatrace fetch start time is {fetch_start_time}") events_to_send = [] if "APM" in events_to_fetch: From da261b93b46301febd69bc81766281478ed73992 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 29 Jan 2025 10:42:43 +0200 Subject: [PATCH 12/15] demo fixes --- Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py | 12 ++++++++---- Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml | 6 +++--- .../Integrations/Dynatrace/Dynatrace_description.md | 7 ++++--- Packs/Dynatrace/Integrations/Dynatrace/README.md | 1 - Packs/Dynatrace/README.md | 1 - 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index 2249d12032c0..71e6e34d9aa3 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -14,6 +14,7 @@ VENDOR = "Dynatrace" PRODUCT = "Platform" EVENTS_TYPE_DICT = {"Audit logs": "auditLogs", "APM": "events"} +EVENT_TYPES = ["APM", "Audit logs"] """ CLIENT CLASS """ @@ -45,9 +46,12 @@ def validate_params(events_to_fetch, audit_max, apm_max): """ if not events_to_fetch: raise DemistoException("Please specify at least one event type to fetch.") + for events_type in events_to_fetch: + if events_type not in EVENT_TYPES: + raise DemistoException("Events types to fetch can only include 'APM' or 'Audit logs'.") if audit_max < 1 or audit_max > 25000: raise DemistoException("The maximum number of audit logs events per fetch needs to be grater then 0 and not more then then 25000") - if apm_max < 1 or apm_max > 5000: + if apm_max < 1 or apm_max > 7000: raise DemistoException("The maximum number of APM events per fetch needs to be grater then 0 and not more then then 5000") @@ -63,8 +67,8 @@ def add_fields_to_events(events, event_type): """ field_mapping = { - "Audit logs": ["Audit logs events", "timestamp"], - "APM": ["APM events", "startTime"] + "Audit logs": ["Audit", "timestamp"], + "APM": ["APM", "startTime"] } for event in events: @@ -124,7 +128,7 @@ def fetch_apm_events(client, limit, fetch_start_time): events_count = 0 args = {} - for i in range(5): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch + for i in range(7): # Design says we will do at most five calls every fetch_interval so we can get more events per fetch args["apm_limit"] = min(limit-events_count, 1000) # The api can bring up to 1000 events per call if args["apm_limit"] != 0: # We didn't get to the limit needed, need to fetch more events diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml index 2c173dcd4f1e..9293e09db183 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml @@ -7,7 +7,7 @@ sectionOrder: name: Dynatrace display: Dynatrace category: Cloud Services -description: 'Fetch Audit logs and APM events from Dynatrace Platform' +description: 'Dynatrace is a revolutionary platform that delivers analytics and automation for unified observability and security.' configuration: - display: Server URL name: url @@ -16,7 +16,7 @@ configuration: section: Connect - display: Access Token name: token - required: false + required: true type: 4 section: Connect - display: Event types to fetch @@ -38,7 +38,7 @@ configuration: - display: The maximum number of APM events per fetch section: Collect advanced: true - defaultvalue: "5000" + defaultvalue: "7000" type: 0 name: apm_limit - display: Trust any certificate (not secure) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md index 4b7a92d5d36e..ef1b32ca8c15 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md @@ -13,10 +13,11 @@ Dynatrace doesn't enforce unique token names. You can create multiple tokens wit You can only access your token once upon creation. You can't reveal it afterward. ### Required scopes: -​​https://docs.dynatrace.com/docs/discover-dynatrace/references/dynatrace-api/basics/dynatrace-api-authentication +For each event type to fetch the according scope needs to be added to the token: -Audit logs API - To execute this request, you need an access token with auditLogs.read scope. -List Events API - To execute this request, you need an access token with events.read scope. +Audit logs events- auditLogs.read scope. + +APM events- events.read scope. ### Server URL Make sure to include the correct url: diff --git a/Packs/Dynatrace/Integrations/Dynatrace/README.md b/Packs/Dynatrace/Integrations/Dynatrace/README.md index 2ddbbaba069a..ce30e9551172 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/README.md +++ b/Packs/Dynatrace/Integrations/Dynatrace/README.md @@ -1,5 +1,4 @@ Fetch Audit logs and APM events from Dynatrace Platform -This integration was integrated and tested with version xx of Dynatrace. ## Configure Dynatrace in Cortex diff --git a/Packs/Dynatrace/README.md b/Packs/Dynatrace/README.md index c736001ace6d..e69de29bb2d1 100644 --- a/Packs/Dynatrace/README.md +++ b/Packs/Dynatrace/README.md @@ -1 +0,0 @@ -Fetch Audit logs and APM events from Dynatrace Platform. \ No newline at end of file From b82ee30f0ac156f3fd0d4a41d6bb7b7e43e25894 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 29 Jan 2025 11:08:02 +0200 Subject: [PATCH 13/15] fix unit tests --- .../Integrations/Dynatrace/Dynatrace_test.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py index ece0a733880e..fc180b9dd5dc 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py @@ -54,7 +54,7 @@ def test_get_APM_events(mocker): (["Audit logs"], -1, 5000, DemistoException, "The maximum number of audit logs events"), # Invalid: apm_max out of range (too high) - (["APM"], 25000, 6000, DemistoException, "The maximum number of APM events"), + (["APM"], 25000, 8000, DemistoException, "The maximum number of APM events"), # Invalid: apm_max out of range (negative) (["APM"], 25000, -1, DemistoException, "The maximum number of APM events"), @@ -90,13 +90,13 @@ def test_validate_params(events_to_fetch, audit_max, apm_max, expected_exception ( [{"timestamp": 1640995200000}], "Audit logs", - [{"timestamp": 1640995200000, "SOURCE_LOG_TYPE": "Audit logs events", "_time": 1640995200000}], + [{"timestamp": 1640995200000, "SOURCE_LOG_TYPE": "Audit", "_time": 1640995200000}], ), # Test case 2: APM ( [{"startTime": 1640995200000}], "APM", - [{"startTime": 1640995200000, "SOURCE_LOG_TYPE": "APM events", "_time": 1640995200000}], + [{"startTime": 1640995200000, "SOURCE_LOG_TYPE": "APM", "_time": 1640995200000}], ), # Test case 3: Multiple Audit logs events ( @@ -106,8 +106,8 @@ def test_validate_params(events_to_fetch, audit_max, apm_max, expected_exception ], "Audit logs", [ - {"timestamp": 1640995200000, "SOURCE_LOG_TYPE": "Audit logs events", "_time": 1640995200000}, - {"timestamp": 1640995300000, "SOURCE_LOG_TYPE": "Audit logs events", "_time": 1640995300000}, + {"timestamp": 1640995200000, "SOURCE_LOG_TYPE": "Audit", "_time": 1640995200000}, + {"timestamp": 1640995300000, "SOURCE_LOG_TYPE": "Audit", "_time": 1640995300000}, ], ), # Test case 4: Empty events @@ -222,28 +222,36 @@ def test_get_events_command__Audit_logs(mocker): (CLIENT, {"apm_limit": 100, "apm_from": 1001}, "APM"), (CLIENT, {"apm_limit": 97, "apm_from": 1002}, "APM"), (CLIENT, {"apm_limit": 94, "apm_next_page_key": "AAAA"}, "APM"), - (CLIENT, {"apm_limit": 91, "apm_next_page_key": "BBBB"}, "APM") + (CLIENT, {"apm_limit": 91, "apm_next_page_key": "BBBB"}, "APM"), + (CLIENT, {"apm_limit": 88, "apm_from": 2001}, "APM"), + (CLIENT, {"apm_limit": 85, "apm_from": 2002}, "APM"), ] events_query_responses_apm = [ {"events": [], "totalCount": 0}, # No events returned {"events": [{"startTime": 1001}, {"startTime": 1000}, {"startTime": 1000}], "totalCount": 3}, # No next page key {"events": [{"startTime": 1002}, {"startTime": 1002}, {"startTime": 1002}], "totalCount": 3, "nextPageKey": "AAAA"}, # NextPageKey exists {"events": [{"startTime": 1004}, {"startTime": 1003}, {"startTime": 1002}], "totalCount": 3, "nextPageKey": "BBBB"}, # NextPageKey exists - {"events": [{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}], "totalCount": 3} # No nextPageKey + {"events": [{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}], "totalCount": 3}, # No nextPageKey + {"events": [{"startTime": 2001}, {"startTime": 2001}, {"startTime": 2001}], "totalCount": 3}, # No nextPageKey + {"events": [{"startTime": 2002}, {"startTime": 2002}, {"startTime": 2002}], "totalCount": 3}, # No nextPageKey ] add_fields_to_events_expected_calls_apm = [ ([], "APM"), ([{"startTime": 1001}, {"startTime": 1000}, {"startTime": 1000}], "APM"), ([{"startTime": 1002}, {"startTime": 1002}, {"startTime": 1002}], "APM"), ([{"startTime": 1004}, {"startTime": 1003}, {"startTime": 1002}], "APM"), - ([{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}], "APM") + ([{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}], "APM"), + ([{"startTime": 2001}, {"startTime": 2001}, {"startTime": 2001}], "APM"), + ([{"startTime": 2002}, {"startTime": 2002}, {"startTime": 2002}], "APM") ] add_fields_to_events_responses_apm = [ [], [{"startTime": 1001}, {"startTime": 1000}, {"startTime": 1000}], [{"startTime": 1002}, {"startTime": 1002}, {"startTime": 1002}], [{"startTime": 1004}, {"startTime": 1003}, {"startTime": 1002}], - [{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}] + [{"startTime": 2000}, {"startTime": 2000}, {"startTime": 2000}], + [{"startTime": 2001}, {"startTime": 2001}, {"startTime": 2001}], + [{"startTime": 2002}, {"startTime": 2002}, {"startTime": 2002}] ] def test_fetch_apm_events(mocker): """ @@ -269,15 +277,13 @@ def test_fetch_apm_events(mocker): add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", side_effect=add_fields_to_events_responses_apm) set_integration_context_mock = mocker.patch("Dynatrace.set_integration_context") fetch_apm_events(CLIENT, 100, 1000) - assert events_query_mock.call_count == 5 - assert [events_query_mock.call_args_list[i][0] for i in range(5)] == events_query_expected_calls_apm - assert add_fields_to_events_mock.call_count == 5 - assert [add_fields_to_events_mock.call_args_list[i][0] for i in range(5)] == add_fields_to_events_expected_calls_apm - set_integration_context_mock.assert_called_with({'last_apm_run': {'last_timestamp': 2000, 'nextPageKey': None}}) + assert events_query_mock.call_count == 7 + assert [events_query_mock.call_args_list[i][0] for i in range(7)] == events_query_expected_calls_apm + assert add_fields_to_events_mock.call_count == 7 + assert [add_fields_to_events_mock.call_args_list[i][0] for i in range(7)] == add_fields_to_events_expected_calls_apm + set_integration_context_mock.assert_called_with({'last_apm_run': {'last_timestamp': 2002, 'nextPageKey': None}}) -# Need to check use case when limit is reached reached and excactly reached -# use case where we get events in the first time and we get nextPageKey events_query_expected_calls_audit = [ #"audit_next_page_key" (CLIENT, {"audit_limit": 100, "audit_from": 1000}, "Audit logs"), (CLIENT, {"audit_limit": 97, "audit_from": 1001}, "Audit logs"), From 13facf69dfbe9c3cd9edc1c8484907e227f08371 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 29 Jan 2025 11:24:12 +0200 Subject: [PATCH 14/15] code review fixes --- .../Integrations/Dynatrace/Dynatrace.py | 33 +++++++++---------- .../Integrations/Dynatrace/Dynatrace.yml | 2 +- .../Dynatrace/Dynatrace_description.md | 17 ++++------ .../Integrations/Dynatrace/README.md | 2 +- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py index 71e6e34d9aa3..42bb435978c7 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.py @@ -79,7 +79,7 @@ def add_fields_to_events(events, event_type): def events_query(client: DynatraceClient, args: dict, event_type: str): - """Calls the right api to get events of event_type type according to the args + """Calls the relevant api to get events of event_type type according to the args Args: client (DynatraceClient): client @@ -252,27 +252,25 @@ def fetch_audit_log_events(client, limit, fetch_start_time): """ COMMAND FUNCTIONS """ -def fetch_events(client: DynatraceClient, events_to_fetch: list, audit_limit: int, apm_limit: int): +def fetch_events(client: DynatraceClient, events_to_fetch: list, events_limits: dict[str, int]): """Gets events from the fetching functions, adds the events the relevant fields and sends the events to XSIAM. Args: client (DynatraceClient): client events_to_fetch (list): list of events types to fetch - audit_limit (int): limit of Audit logs events to fetch - apm_limit (int): limit of APM events to fetch + events_limit: dict[str, int]: limit of events to fetch by event type """ fetch_start_time = int(datetime.now().timestamp() * 1000) # We want this timestamp to look like this: 1737656746001 demisto.debug(f"Dynatrace fetch start time is {fetch_start_time}") events_to_send = [] - if "APM" in events_to_fetch: - events = fetch_apm_events(client, apm_limit, fetch_start_time) - events = add_fields_to_events(events, "APM") - events_to_send.extend(events) - if "Audit logs" in events_to_fetch: - events = fetch_audit_log_events(client, audit_limit, fetch_start_time) - events = add_fields_to_events(events, "Audit logs") - events_to_send.extend(events) + events_fetch_function = {"APM": fetch_apm_events, "Audit logs": fetch_audit_log_events} + for event_type in events_fetch_function: + demisto.debug(f"Fetching: {event_type} with limit {events_limits[event_type]}") + events = events_fetch_function[event_type](client, events_limits[event_type], fetch_start_time) + events = add_fields_to_events(events, event_type) + events_to_send.extend(events) + demisto.debug(f"Dynatrace sending {len(events_to_send)} to xsiam") send_events_to_xsiam(events_to_send, VENDOR, PRODUCT) @@ -285,6 +283,7 @@ def get_events_command(client: DynatraceClient, args: dict): events_to_return = [] for event_type in events_types: + demisto.debug(f"Dynatrace calling {event_type} api with {args=}") response = events_query(client, args, event_type) events = response[EVENTS_TYPE_DICT[event_type]] demisto.debug(f"Dynatrace got {len(events)} events of type {event_type}") @@ -301,8 +300,10 @@ def get_events_command(client: DynatraceClient, args: dict): return CommandResults(readable_output="No events were received") -def test_module(client: DynatraceClient, events_to_fetch: List[str]) -> str: - +def test_module(client: DynatraceClient, events_to_fetch: List[str], audit_limit, apm_limit) -> str: + + validate_params(events_to_fetch, audit_limit, apm_limit) + try: if "Audit logs" in events_to_fetch: client.get_audit_logs_events("?pageSize=1") @@ -326,8 +327,6 @@ def main(): # pragma: no cover audit_limit = arg_to_number(params.get('audit_limit')) or 25000 apm_limit = arg_to_number(params.get('apm_limit')) or 25000 - validate_params(events_to_fetch, audit_limit, apm_limit) - verify = not argToBoolean(params.get("insecure", False)) proxy = argToBoolean(params.get("proxy", False)) @@ -342,7 +341,7 @@ def main(): # pragma: no cover args = demisto.args() if command == "test-module": - result = test_module(client, events_to_fetch) + result = test_module(client, events_to_fetch, audit_limit, apm_limit) return_results(result) elif command == "dynatrace-get-events": result = get_events_command(client, args) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml index 9293e09db183..a63c0874055d 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace.yml @@ -83,7 +83,7 @@ script: - "True" - "False" required: false - description: Manual command to fetch events and display them. + description: Manual command to fetch events and display them. This command is used for developing/debugging and is to be used with caution, as it can create events, leading to events duplication and exceeding the API request limitation. name: dynatrace-get-events isfetchevents: true subtype: python3 diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md index ef1b32ca8c15..ecfcec83023d 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_description.md @@ -1,16 +1,13 @@ ## Dynatrace Help ### How to Create a Personal Access Token (Classic Access Token): -Generate an access token -To generate an access token -1. Go to Access Tokens. -2. Select Generate new token. -3. Enter a name for your token. -Dynatrace doesn't enforce unique token names. You can create multiple tokens with the same name. Be sure to provide a meaningful name for each token you generate. Proper naming helps you to efficiently manage your tokens and perhaps delete them when they're no longer needed. -4. Select the required scopes for the token. -5. Select Generate token. -6. Copy the generated token to the clipboard. Store the token in a password manager for future use. -You can only access your token once upon creation. You can't reveal it afterward. +Generate an access token: +1. In Dynatrace, go to Access Tokens -> `Generate new token`. +2. Enter a name for your token. +Note that Dynatrace doesn't enforce unique token names. You can create multiple tokens with the same name. Be sure to provide a meaningful name for each token you generate. Proper naming helps you to efficiently manage your tokens and perhaps delete them when they're no longer needed. +3. Select the required scopes for the token. +4. Click on `Generate token`. +5. Copy the generated token to the Collector's instance. Make sure to store the token in a password manager for future use, as you will not be able to access it later. ### Required scopes: For each event type to fetch the according scope needs to be added to the token: diff --git a/Packs/Dynatrace/Integrations/Dynatrace/README.md b/Packs/Dynatrace/Integrations/Dynatrace/README.md index ce30e9551172..b1544a996935 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/README.md +++ b/Packs/Dynatrace/Integrations/Dynatrace/README.md @@ -21,7 +21,7 @@ After you successfully execute a command, a DBot message appears in the War Room ### dynatrace-get-events *** -Manual command to fetch events and display them. +Manual command to fetch events and display them. This command is used for developing/debugging and is to be used with caution, as it can create events, leading to events duplication and exceeding the API request limitation. #### Base Command From 03a3ae7bf6a3b1cbf7fc446461bfb3147b0d5ee0 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 29 Jan 2025 11:26:15 +0200 Subject: [PATCH 15/15] fix test --- Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py index fc180b9dd5dc..aec366f04dff 100644 --- a/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py +++ b/Packs/Dynatrace/Integrations/Dynatrace/Dynatrace_test.py @@ -164,7 +164,7 @@ def test_fetch_events(mocker): add_fields_to_events_mock = mocker.patch("Dynatrace.add_fields_to_events", return_value=[]) send_events_to_xsiam_mock = mocker.patch("Dynatrace.send_events_to_xsiam") - fetch_events(CLIENT, ["APM", "Audit logs"], 200, 100) + fetch_events(CLIENT, ["APM", "Audit logs"], {"APM": 100, "Audit logs": 200}) assert apm_mock.call_args.args[1] == 100 assert audit_mock.call_args.args[1] == 200