From b041d6bf8902e3d675dcfd11aa8b76d4b0c5312a Mon Sep 17 00:00:00 2001 From: yshamai Date: Tue, 17 Dec 2024 14:16:21 +0200 Subject: [PATCH 01/31] generate new pack --- Packs/RetarusSecureEmailGateway/.pack-ignore | 0 .../RetarusSecureEmailGateway/.secrets-ignore | 0 .../Author_image.png | 0 .../RetarusSecureEmailGateway/README.md | 5 + .../RetarusSecureEmailGateway.py | 179 ++++++++++++++++++ .../RetarusSecureEmailGateway.yml | 50 +++++ .../RetarusSecureEmailGateway_description.md | 8 + .../RetarusSecureEmailGateway_image.png | Bin 0 -> 2507 bytes .../RetarusSecureEmailGateway_test.py | 42 ++++ .../command_examples | 0 .../test_data/baseintegration-dummy.json | 3 + Packs/RetarusSecureEmailGateway/README.md | 0 .../pack_metadata.json | 18 ++ .../SymantecEndpointSecurity.py | 12 +- 14 files changed, 309 insertions(+), 8 deletions(-) create mode 100644 Packs/RetarusSecureEmailGateway/.pack-ignore create mode 100644 Packs/RetarusSecureEmailGateway/.secrets-ignore create mode 100644 Packs/RetarusSecureEmailGateway/Author_image.png create mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md create mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py create mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml create mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md create mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_image.png create mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py create mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples create mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/test_data/baseintegration-dummy.json create mode 100644 Packs/RetarusSecureEmailGateway/README.md create mode 100644 Packs/RetarusSecureEmailGateway/pack_metadata.json diff --git a/Packs/RetarusSecureEmailGateway/.pack-ignore b/Packs/RetarusSecureEmailGateway/.pack-ignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/RetarusSecureEmailGateway/.secrets-ignore b/Packs/RetarusSecureEmailGateway/.secrets-ignore new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/RetarusSecureEmailGateway/Author_image.png b/Packs/RetarusSecureEmailGateway/Author_image.png new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md new file mode 100644 index 000000000000..5d6d1dc2bf52 --- /dev/null +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -0,0 +1,5 @@ +This README contains the full documentation for your integration. + +You auto-generate this README file from your integration YML file using the `demisto-sdk generate-docs` command. + +For more information see the [integration documentation](https://xsoar.pan.dev/docs/integrations/integration-docs). diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py new file mode 100644 index 000000000000..1739bfbbe344 --- /dev/null +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -0,0 +1,179 @@ +import demistomock as demisto # noqa: F401 +from CommonServerPython import * # noqa: F401 +"""Base Integration for Cortex XSOAR (aka Demisto) + +This is an empty Integration with some basic structure according +to the code conventions. + +MAKE SURE YOU REVIEW/REPLACE ALL THE COMMENTS MARKED AS "TODO" + +Developer Documentation: https://xsoar.pan.dev/docs/welcome +Code Conventions: https://xsoar.pan.dev/docs/integrations/code-conventions +Linting: https://xsoar.pan.dev/docs/integrations/linting + +This is an empty structure file. Check an example at; +https://github.com/demisto/content/blob/master/Packs/HelloWorld/Integrations/HelloWorld/HelloWorld.py + +""" + +from CommonServerUserPython import * # noqa + +import urllib3 +from typing import Dict, Any + +# Disable insecure warnings +urllib3.disable_warnings() + + +''' CONSTANTS ''' + +VENDOR = "Retarus" +PRODUCT = "Secure Email Gateway" +DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # ISO8601 format with UTC, default in XSOAR + +''' CLIENT CLASS ''' + + +class Client(BaseClient): + def __init__( + self, + base_url: str, + token: str, + stream_id: str, + channel_id: str, + verify: bool, + proxy: bool, + ) -> None: + + self.headers: dict[str, str] = {} + self.token = token + self.stream_id = stream_id + self.channel_id = channel_id + + super().__init__( + base_url=base_url, + verify=verify, + proxy=proxy, + timeout=180, + ) + + # TODO: ADD HERE THE FUNCTIONS TO INTERACT WITH YOUR PRODUCT API + + +''' HELPER FUNCTIONS ''' +def push_events(events: list[dict]): + """ + Push events to XSIAM. + """ + demisto.debug(f"Pushing {len(events)} to XSIAM") + send_events_to_xsiam(events=events, vendor=VENDOR, product=PRODUCT) + demisto.debug(f"Pushed {len(events)} to XSIAM successfully") + + +''' COMMAND FUNCTIONS ''' + + +def test_module(client: Client) -> str: + """Tests API connectivity and authentication' + + Returning 'ok' indicates that the integration works like it is supposed to. + Connection to the service is successful. + Raises exceptions if something goes wrong. + + :type client: ``Client`` + :param Client: client to use + + :return: 'ok' if test passed, anything else will fail the test. + :rtype: ``str`` + """ + + message: str = '' + try: + # TODO: ADD HERE some code to test connectivity and authentication to your service. + # This should validate all the inputs given in the integration configuration panel, + # either manually or by using an API that uses them. + message = 'ok' + except DemistoException as e: + if 'Forbidden' in str(e) or 'Authorization' in str(e): # TODO: make sure you capture authentication errors + message = 'Authorization Error: make sure API Key is correctly set' + else: + raise e + return message + + +# TODO: REMOVE the following dummy command function +def get_command(client: Client, args: Dict[str, Any]) -> CommandResults: + + dummy = args.get('dummy', None) + if not dummy: + raise ValueError('dummy not specified') + + # Call the Client function and get the raw response + result = client.baseintegration_dummy(dummy) + + return CommandResults( + outputs_prefix='BaseIntegration', + outputs_key_field='', + outputs=result, + ) +# TODO: ADD additional command functions that translate XSOAR inputs/outputs to Client + + +''' MAIN FUNCTION ''' + + +def main() -> None: + """main function, parses params and runs command functions + + :return: + :rtype: + """ + + # TODO: make sure you properly handle authentication + # api_key = demisto.params().get('credentials', {}).get('password') + + # get the service API url + base_url = urljoin(demisto.params()['url'], '/api/v1') + + # if your Client class inherits from BaseClient, SSL verification is + # handled out of the box by it, just pass ``verify_certificate`` to + # the Client constructor + verify_certificate = not demisto.params().get('insecure', False) + + # if your Client class inherits from BaseClient, system proxy is handled + # out of the box by it, just pass ``proxy`` to the Client constructor + proxy = demisto.params().get('proxy', False) + + demisto.debug(f'Command being called is {demisto.command()}') + try: + + # TODO: Make sure you add the proper headers for authentication + # (i.e. "Authorization": {api key}) + headers: Dict = {} + + client = Client( + base_url=base_url, + verify=verify_certificate, + headers=headers, + proxy=proxy) + + if demisto.command() == 'test-module': + # This is the call made when pressing the integration Test button. + result = test_module(client) + return_results(result) + + # TODO: REMOVE the following dummy command case: + elif demisto.command() == 'baseintegration-dummy': + return_results(baseintegration_dummy_command(client, demisto.args())) + # TODO: ADD command cases for the commands you will implement + + # Log exceptions and return errors + except Exception as e: + return_error(f'Failed to execute {demisto.command()} command.\nError:\n{str(e)}') + + +''' ENTRY POINT ''' + + +if __name__ in ('__main__', '__builtin__', 'builtins'): + main() diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml new file mode 100644 index 000000000000..bc3a5c8c477b --- /dev/null +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -0,0 +1,50 @@ +category: Email +commonfields: + id: RetarusSecureEmailGateway + version: -1 +configuration: +- defaultvalue: https://events.retarus.com + display: Server URL + name: url + required: true + type: 0 + section: Connect +- displaypassword: Token ID + additionalinfo: The Token ID to use for connection + name: credentials + required: true + hiddenusername: true + type: 9 +- display: Trust any certificate (not secure) + name: insecure + type: 8 + required: false +- display: Use system proxy settings + name: proxy + type: 8 + required: false +- display: Fetch events + name: is_fetch + type: 8 + required: false +description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.]' #TODO +display: RetarusSecureEmailGateway +name: RetarusSecureEmailGateway +script: + commands: + - arguments: + - description: '[Enter a description of the argument, including any important information users need to know, for example, default values.]' + name: Retarus-seg-get-events + required: true + description: '[Enter a description of the command, including any important information users need to know, for example required permissions.]' + name: baseintegration-dummy + outputs: + - contextPath: BaseIntegration.Output + description: '[Enter a description of the data returned in this output.]' + type: String + type: python + subtype: python3 + dockerimage: demisto/python3:3.11.10.115186 +fromversion: 5.5.0 +tests: +- No tests (auto formatted) #TODO diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md new file mode 100644 index 000000000000..51bd561c623f --- /dev/null +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md @@ -0,0 +1,8 @@ +## BaseIntegration Help + +Markdown file for integration configuration help snippet. In this file add: + +- Brief information about how to retrieve the API key of your product +- Other useful information on how to configure your integration in XSOAR + +Since this is a Markdown file, we encourage you to use MD formatting for sections, sub-sections, lists, etc. diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_image.png b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_image.png new file mode 100644 index 0000000000000000000000000000000000000000..772c6af21bebd9230ea48f23a29ad2260f386e71 GIT binary patch literal 2507 zcmV;+2{iVJP)NJU1VgcOS!Wr4 zO{q;AyQYAIs;$+wHZ~Sp+*+hq)D%gf ztHY9woe3{0IJoIrpgCQC7`DJ3m>Rb4IVaSF>acO==CH3?2YW9zeW_Dx~*%o3bFhWN8-vqxH&=>B0u zuaCmH%$&2TPhhFWSb?U_&pUZ|*bx4bY2lAeKf4Fz( zmQcC1G(7!QIj=-*@_K6Xu1z?%2l)e_654|gT&Yu996XHs=&CTZ`w-v}%L}p|&0V{i z+>M}fRVtn9z|!)NwJa=JU&b?XP1tkblQ4D3yx@~^KJ{ql)!hryGQsnWH2d0+H@nGX zw4$doc(6Jj$F{`H#bN9f?}g#LF3mfF;nU`awLA8cKM!>FeJ~$VrQ(!~W*_628V=UH z#L*3oZlPbNGeW+jEG_4Q0)Dht@52W-_{z?+TAyKZtwufpGKm1!42gYaP3o=Hp5 zpENDeNw=&g(wishi@^u5KHBy z0B5lAQOWLBab_QSK09fjTO zC@3Ws@CkVee3)foO2$1e+uh|0=G5tv6Q@D!mTg_+}$dIq?{bj!*uvDc%`2UX?@FJXeYlO zql@{RdZcteM0jQY4X%XKp#!*g{X6jLzX!TOK_|!pF2zT3WJB`(jkNkL_}X+Gl)`YB z2EHqQ4+=WY7VrZ@LmX(C`XO?ji^Ej+Opo`g5xWAGl^6a3G? z&p{{f9xG+wdu=iJnzREvZxZ-gR8YYJ1q&1`P_RJ30tE{cEO1OM;46U-Cb=BbMjBDs z)ttK)h8|@daa>LHgTYT3?|-25D93$R@gid10e^0qi+Lx5PNNfk+3|QZisSOhx*-=W$J3%r+sPEn@)0ekD#*uCOYo^?Wv^h40bS-pMXKv)%JM$A&pY;mgh?FhkU+QE8BVmEWZwZ1Os3?tOv{bg-ZM9Yuh@kKT3Z*1=`auTt9(gP*;L= zYM0sIh3fe&AlwBta3|cJr8h@40S(p`YfCjh$JYPeoo}f{m1KXYlPCUyd701;zn`qs(u>dr$7Yn+zbza=YJN;K&Ss5w9P+3!^a6@TgF#|cJ-WZz!Y%D%ukcHbs5yv zHpv-V`0+!O}lnF#l)S zyxg`;wvOoF*cT^l)Di+ZR7D$U3#BdW&v=a=gNtBJyf z9dok((ZwPiqQALjyh|Hg0izuz@yL&?uKKjIP+TF!L!4*?Gxd3lZ5vK1_C>o# z8FoUss^nF&!V#46z7aKpzyxRwec=Uo1iYV=g7SPDa*1iIT!y{~=fU;h4D~9}AgV|0 zA^e%+Z7>n;gBsA;&qHJMLH6XYaP*3u3#(upYy+3X!SF}W;opKW@FaNUB(GkgpHdkq z-nYBJbznRyx9xXgM)nx{@@?W(Pyt>9lfd!UtGmdxP%h5M{V*0(KR5utHiBFledzcY zeh!}hB~Z@J$(c~?pZ7TYD-oYVcx57P`NiPUVm(VCE=x|PWnkOUD7O&S36?e8iI=N5 zuCCdPsT|`y&7@J_`Ij0&CGZNE<`vowoXoF-Y4Rq-3HM?m-Mlq$2dLb(OmijJ2G*_Z z-T=#~y4udI!;8#GW?S0biQWoSyazjRR5@7g0V9o(_ai&kSn1$Dfh$5ZAi+awM#yxJ zQHFJ1bb3s0?6C_KIIg5R5J{%>CcYC~YWu~r>$8V_Z3FpKSKGN|JCP6_o11si z#&fUuleVMTj(wG*SJ92&!{Pu~4vl>dpo(;ACd2wc6B?Sg4ex(($=#5RpLMd0;eS43 V(U=mQ{m=ja002ovPDHLkV1nG2;bs5; literal 0 HcmV?d00001 diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py new file mode 100644 index 000000000000..6a48b6e932e3 --- /dev/null +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py @@ -0,0 +1,42 @@ +"""Base Integration for Cortex XSOAR - Unit Tests file + +Pytest Unit Tests: all funcion names must start with "test_" + +More details: https://xsoar.pan.dev/docs/integrations/unit-testing + +MAKE SURE YOU REVIEW/REPLACE ALL THE COMMENTS MARKED AS "TODO" + +You must add at least a Unit Test function for every XSOAR command +you are implementing with your integration +""" + +import json +import io + + +def util_load_json(path): + with io.open(path, mode='r', encoding='utf-8') as f: + return json.loads(f.read()) + + +# TODO: REMOVE the following dummy unit test function +def test_baseintegration_dummy(): + """Tests helloworld-say-hello command function. + + Checks the output of the command function with the expected output. + + No mock is needed here because the say_hello_command does not call + any external API. + """ + from BaseIntegration import Client, baseintegration_dummy_command + + client = Client(base_url='some_mock_url', verify=False) + args = { + 'dummy': 'this is a dummy response' + } + response = baseintegration_dummy_command(client, args) + + mock_response = util_load_json('test_data/baseintegration-dummy.json') + + assert response.outputs == mock_response +# TODO: ADD HERE unit tests for every command diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/test_data/baseintegration-dummy.json b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/test_data/baseintegration-dummy.json new file mode 100644 index 000000000000..37fa47b18cd0 --- /dev/null +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/test_data/baseintegration-dummy.json @@ -0,0 +1,3 @@ +{ + "dummy": "this is a dummy response" +} \ No newline at end of file diff --git a/Packs/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Packs/RetarusSecureEmailGateway/pack_metadata.json b/Packs/RetarusSecureEmailGateway/pack_metadata.json new file mode 100644 index 000000000000..8061fad746a2 --- /dev/null +++ b/Packs/RetarusSecureEmailGateway/pack_metadata.json @@ -0,0 +1,18 @@ +{ + "name": "Retarus Secure Email Gateway", + "description": "## FILL MANDATORY FIELD ##", + "support": "xsoar", + "currentVersion": "1.0.0", + "author": "Cortex XSOAR", + "url": "https://www.paloaltonetworks.com/cortex", + "email": "", + "categories": [ + "Email" + ], + "tags": [], + "useCases": [], + "keywords": [], + "marketplaces": [ + "marketplacev2." + ] +} \ No newline at end of file diff --git a/Packs/SymantecEndpointSecurity/Integrations/SymantecEndpointSecurity/SymantecEndpointSecurity.py b/Packs/SymantecEndpointSecurity/Integrations/SymantecEndpointSecurity/SymantecEndpointSecurity.py index 160fc0a00d21..0a9ac60f075b 100644 --- a/Packs/SymantecEndpointSecurity/Integrations/SymantecEndpointSecurity/SymantecEndpointSecurity.py +++ b/Packs/SymantecEndpointSecurity/Integrations/SymantecEndpointSecurity/SymantecEndpointSecurity.py @@ -41,21 +41,17 @@ class NextPointingNotAvailable(Exception): class Client(BaseClient): def __init__( self, - base_url: str, - token: str, - stream_id: str, - channel_id: str, + url: str, + token_id: str, verify: bool, proxy: bool, ) -> None: self.headers: dict[str, str] = {} - self.token = token - self.stream_id = stream_id - self.channel_id = channel_id + self.token_id = token_id super().__init__( - base_url=base_url, + base_url=url, verify=verify, proxy=proxy, timeout=180, From ddfc67346379a7ea9086bb3e2155fb55ae5aac68 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 18 Dec 2024 12:56:11 +0200 Subject: [PATCH 02/31] small fix --- .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml | 4 ++-- Packs/RetarusSecureEmailGateway/pack_metadata.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index bc3a5c8c477b..4051a6c5a8c1 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -44,7 +44,7 @@ script: type: String type: python subtype: python3 - dockerimage: demisto/python3:3.11.10.115186 -fromversion: 5.5.0 + dockerimage: demisto/python3:3.11.10.116949 +fromversion: 6.10.0 tests: - No tests (auto formatted) #TODO diff --git a/Packs/RetarusSecureEmailGateway/pack_metadata.json b/Packs/RetarusSecureEmailGateway/pack_metadata.json index 8061fad746a2..4dadfc3e43f9 100644 --- a/Packs/RetarusSecureEmailGateway/pack_metadata.json +++ b/Packs/RetarusSecureEmailGateway/pack_metadata.json @@ -13,6 +13,6 @@ "useCases": [], "keywords": [], "marketplaces": [ - "marketplacev2." + "marketplacev2" ] } \ No newline at end of file From 558946af6c5d740273ae60705b00533902ccb29e Mon Sep 17 00:00:00 2001 From: yshamai Date: Sun, 29 Dec 2024 09:33:39 +0200 Subject: [PATCH 03/31] continue --- .../RetarusSecureEmailGateway.py | 160 +++++------------- .../RetarusSecureEmailGateway.yml | 4 + 2 files changed, 50 insertions(+), 114 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 1739bfbbe344..c461e2d93a71 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -20,6 +20,13 @@ import urllib3 from typing import Dict, Any +from websockets import Data +from CommonServerPython import * # noqa: F401 +from websockets.sync.client import connect +from websockets.sync.connection import Connection +from websockets.exceptions import InvalidStatus +from dateutil import tz +import traceback # Disable insecure warnings urllib3.disable_warnings() @@ -29,33 +36,50 @@ VENDOR = "Retarus" PRODUCT = "Secure Email Gateway" +FETCH_INTERVAL_IN_SECONDS = 60 +SERVER_IDLE_TIMEOUT = 60 # Retarus closes connection after 60 sec of inactivation +DEFAULT_CHANNEL = "default" +URL = "wss://events.retarus.com/email/siem/v1/websocket?channel={channel}" + DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # ISO8601 format with UTC, default in XSOAR ''' CLIENT CLASS ''' -class Client(BaseClient): - def __init__( - self, - base_url: str, - token: str, - stream_id: str, - channel_id: str, - verify: bool, - proxy: bool, - ) -> None: - - self.headers: dict[str, str] = {} - self.token = token - self.stream_id = stream_id - self.channel_id = channel_id - - super().__init__( - base_url=base_url, - verify=verify, - proxy=proxy, - timeout=180, - ) +class EventConnection: + def __init__(self, connection: Connection, fetch_interval: int = FETCH_INTERVAL_IN_SECONDS, + idle_timeout: int = SERVER_IDLE_TIMEOUT): + self.connection = connection + self.lock = threading.Lock() + self.idle_timeout = idle_timeout + self.fetch_interval = fetch_interval + + def recv(self, timeout: float | None = None) -> Data: + """ + Receive the next message from the connection + + Args: + timeout (float): Block until timeout seconds have elapsed or a message is received. If None, waits indefinitely. + If timeout passes, raises TimeoutError + + Returns: + Data: Next event received from the connection + """ + with self.lock: + event = self.connection.recv(timeout=timeout) + return event + + + def heartbeat(self): + """ + Heartbeat thread function to periodically send keep-alives to the server. + For the sake of simplicity and error prevention, keep-alives are sent regardless of the actual connection activity. + """ + while True: + with self.lock: + self.connection.pong() + time.sleep(self.idle_timeout) + # TODO: ADD HERE THE FUNCTIONS TO INTERACT WITH YOUR PRODUCT API @@ -73,106 +97,14 @@ def push_events(events: list[dict]): ''' COMMAND FUNCTIONS ''' -def test_module(client: Client) -> str: - """Tests API connectivity and authentication' - - Returning 'ok' indicates that the integration works like it is supposed to. - Connection to the service is successful. - Raises exceptions if something goes wrong. - - :type client: ``Client`` - :param Client: client to use - - :return: 'ok' if test passed, anything else will fail the test. - :rtype: ``str`` - """ - - message: str = '' - try: - # TODO: ADD HERE some code to test connectivity and authentication to your service. - # This should validate all the inputs given in the integration configuration panel, - # either manually or by using an API that uses them. - message = 'ok' - except DemistoException as e: - if 'Forbidden' in str(e) or 'Authorization' in str(e): # TODO: make sure you capture authentication errors - message = 'Authorization Error: make sure API Key is correctly set' - else: - raise e - return message - - -# TODO: REMOVE the following dummy command function -def get_command(client: Client, args: Dict[str, Any]) -> CommandResults: - - dummy = args.get('dummy', None) - if not dummy: - raise ValueError('dummy not specified') - - # Call the Client function and get the raw response - result = client.baseintegration_dummy(dummy) - - return CommandResults( - outputs_prefix='BaseIntegration', - outputs_key_field='', - outputs=result, - ) -# TODO: ADD additional command functions that translate XSOAR inputs/outputs to Client - ''' MAIN FUNCTION ''' def main() -> None: - """main function, parses params and runs command functions - - :return: - :rtype: - """ - - # TODO: make sure you properly handle authentication - # api_key = demisto.params().get('credentials', {}).get('password') - - # get the service API url - base_url = urljoin(demisto.params()['url'], '/api/v1') - - # if your Client class inherits from BaseClient, SSL verification is - # handled out of the box by it, just pass ``verify_certificate`` to - # the Client constructor - verify_certificate = not demisto.params().get('insecure', False) - - # if your Client class inherits from BaseClient, system proxy is handled - # out of the box by it, just pass ``proxy`` to the Client constructor - proxy = demisto.params().get('proxy', False) - - demisto.debug(f'Command being called is {demisto.command()}') - try: - - # TODO: Make sure you add the proper headers for authentication - # (i.e. "Authorization": {api key}) - headers: Dict = {} - - client = Client( - base_url=base_url, - verify=verify_certificate, - headers=headers, - proxy=proxy) - - if demisto.command() == 'test-module': - # This is the call made when pressing the integration Test button. - result = test_module(client) - return_results(result) - - # TODO: REMOVE the following dummy command case: - elif demisto.command() == 'baseintegration-dummy': - return_results(baseintegration_dummy_command(client, demisto.args())) - # TODO: ADD command cases for the commands you will implement - # Log exceptions and return errors - except Exception as e: - return_error(f'Failed to execute {demisto.command()} command.\nError:\n{str(e)}') -''' ENTRY POINT ''' if __name__ in ('__main__', '__builtin__', 'builtins'): diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 4051a6c5a8c1..3435ed77abb0 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -15,6 +15,10 @@ configuration: required: true hiddenusername: true type: 9 +- defaultvalue: default + display: Channel name + section: Collect + type: 0 - display: Trust any certificate (not secure) name: insecure type: 8 From 9b381f1d1cbe4a5f1c744cc34926a18e0208dfb2 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 1 Jan 2025 13:28:49 +0200 Subject: [PATCH 04/31] continue- skeleton is ready --- .../RetarusSecureEmailGateway.py | 151 +++++++++++++++++- 1 file changed, 147 insertions(+), 4 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index c461e2d93a71..ad4309d71ec6 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -21,12 +21,17 @@ import urllib3 from typing import Dict, Any from websockets import Data +import websockets from CommonServerPython import * # noqa: F401 from websockets.sync.client import connect from websockets.sync.connection import Connection from websockets.exceptions import InvalidStatus from dateutil import tz import traceback +import threading +from contextlib import contextmanager + + # Disable insecure warnings urllib3.disable_warnings() @@ -37,9 +42,9 @@ VENDOR = "Retarus" PRODUCT = "Secure Email Gateway" FETCH_INTERVAL_IN_SECONDS = 60 -SERVER_IDLE_TIMEOUT = 60 # Retarus closes connection after 60 sec of inactivation +FETCH_SLEEP = 5 +SERVER_IDLE_TIMEOUT = 60 DEFAULT_CHANNEL = "default" -URL = "wss://events.retarus.com/email/siem/v1/websocket?channel={channel}" DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # ISO8601 format with UTC, default in XSOAR @@ -92,17 +97,155 @@ def push_events(events: list[dict]): demisto.debug(f"Pushing {len(events)} to XSIAM") send_events_to_xsiam(events=events, vendor=VENDOR, product=PRODUCT) demisto.debug(f"Pushed {len(events)} to XSIAM successfully") + + +@contextmanager +def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: str, verify_ssl: bool): + extra_headers = {"Authorization": f"Bearer {token_id}"} + + context = ssl.create_default_context() + if not verify_ssl: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + with connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", additional_headers=extra_headers) as ws: + connection = EventConnection( + connection=ws, + fetch_interval=fetch_interval + ) + yield connection + +def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: + """This function checks if the given interval has passed since the given start time + + Args: + fetch_start_time (datetime): The start time of the interval + fetch_interval (int): The interval in seconds + + Returns: + bool: True if the interval has passed, False otherwise + """ + return fetch_start_time + timedelta(seconds=fetch_interval) < datetime.utcnow() + + +def perform_long_running_loop(connection: EventConnection, fetch_interval: int): + integration_context = demisto.getIntegrationContext() + events_to_send = [] + events = fetch_events(connection, fetch_interval) + events.extend(integration_context.get("events", [])) + integration_context["events"] = events # update events in context in case of a failure. + demisto.debug(f'Adding {len(events)} Events to XSIAM') + events_to_send.extend(events) + + # Send the events to the XSIAM, with events from the context + # Need to add the option that if we have more then one failure of sending the events to xsiam then we stop fetching. consult with Meital and Dima # TODO + try: + send_events_to_xsiam(events_to_send, vendor=VENDOR, product=PRODUCT) + # clear the context after sending the events + demisto.setIntegrationContext({}) + except DemistoException: + demisto.error(f"Failed to send events to XSIAM. Error: {traceback.format_exc()}") + # save the events to the context so we can send them again in the next execution + demisto.setIntegrationContext(integration_context) ''' COMMAND FUNCTIONS ''' +def long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl): + """ + Performs the long running execution loop. + Opens a connection to Retarus. + Heartbeat thread is opened for the connection to send keepalives if the connection is idle for too long. + + Args: + url (str): URL for the websocket connection. + token_id (str): Retarus token_id to connect to. + channel (str): channel to connect with. + fetch_interval (int): Total time allocated per fetch cycle. + """ + with websocket_connection(url, token_id, fetch_interval, channel, verify_ssl) as connection: + demisto.info("Connected to websocket") + # Retarus will keep connections with no traffic open for at most 5 minutes. + # It is highly recommended that the client sends a PING control frame every 60 seconds to keep the connection open. + # (sentence taken from Retarus docs) + # Setting up heartbeat daemon threads to send keep-alives if needed + threading.Thread(target=connection.heartbeat, daemon=True).start() + while True: + perform_long_running_loop(connection, fetch_interval) + # sleep for a bit to not throttle the CPU + time.sleep(FETCH_SLEEP) -''' MAIN FUNCTION ''' +def test_module(url, token_id): + if not url: + raise DemistoException("Missing url parameter.") + if not token_id: + raise DemistoException("Missing token id parameter.") + return 'ok' + + +def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: int = 10) -> list[dict]: + """ + This function fetches events from the given connection, for the given fetch interval + + Args: + connection (EventConnection): the connection to the event type + fetch_interval (int): Total time to keep fetching before stopping + recv_timeout (int): The timeout for the receive function in the socket connection + Returns: + list[dict]: A list of events + """ + events: list[dict] = [] + event_ids = set() + fetch_start_time = datetime.utcnow() + demisto.debug(f'Starting to fetch events at {fetch_start_time}') + while not is_interval_passed(fetch_start_time, fetch_interval): + try: + event = json.loads(connection.recv(timeout=recv_timeout)) + except TimeoutError: + # if we didn't receive an event for `fetch_interval` seconds, finish fetching + continue + event_id = None # TODO we don't get an id from Retarus + event_ts = event.get("ts") + if not event_ts: + # if timestamp is not in the response, use the current time + demisto.debug(f"Event {event_id} does not have a timestamp, using current time") + event_ts = datetime.utcnow().isoformat() + date = dateparser.parse(event_ts) + if not date: + demisto.debug(f"Event {event_id} has an invalid timestamp, using current time") + # if timestamp is not in correct format, use the current time + date = datetime.utcnow() + event["_time"] = date + event["event_type"] = event.get("type") + events.append(event) + event_ids.add(event_id) + demisto.debug(f"Fetched {len(events)} events") + demisto.debug("The fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) + return events + +''' MAIN FUNCTION ''' -def main() -> None: +def main(): # pragma: no cover + command = demisto.command() + params = demisto.params() + url = params.get("url", "events.retarus.com") + token_id = params.get("credentials", {}).get("password", "") + fetch_interval = int(params.get("fetch_interval", FETCH_INTERVAL_IN_SECONDS)) + verify_ssl = argToBoolean(not params.get("insecure", False)) + channel = params.get("channel", DEFAULT_CHANNEL) + + try: + if command == "long-running-execution": + return_results(long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl)) + elif command == "test-module": + return_results(test_module(url, token_id)) + else: + raise NotImplementedError(f"Command {command} is not implemented.") + except Exception: + return_error(f'Failed to execute {command} command.\nError:\n{traceback.format_exc()}') From ff82656c78df0ad62f44ae1ffa497ccf2e988622 Mon Sep 17 00:00:00 2001 From: yshamai Date: Sun, 5 Jan 2025 11:41:39 +0200 Subject: [PATCH 05/31] more --- .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.py | 3 ++- .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index ad4309d71ec6..72ab1ed9f5bd 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -206,7 +206,7 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: except TimeoutError: # if we didn't receive an event for `fetch_interval` seconds, finish fetching continue - event_id = None # TODO we don't get an id from Retarus + event_id = event_id = event.get("rmxId", event.get("mimeId")) event_ts = event.get("ts") if not event_ts: # if timestamp is not in the response, use the current time @@ -217,6 +217,7 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: demisto.debug(f"Event {event_id} has an invalid timestamp, using current time") # if timestamp is not in correct format, use the current time date = datetime.utcnow() + event["id"] = event_id # TODO not sure I am supposed to do it, maybe it's for @bavly event["_time"] = date event["event_type"] = event.get("type") events.append(event) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 3435ed77abb0..9cb8cab54ed0 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -3,7 +3,7 @@ commonfields: id: RetarusSecureEmailGateway version: -1 configuration: -- defaultvalue: https://events.retarus.com +- defaultvalue: events.retarus.com display: Server URL name: url required: true @@ -48,7 +48,7 @@ script: type: String type: python subtype: python3 - dockerimage: demisto/python3:3.11.10.116949 + dockerimage: demisto/netutils:1.0.0.115452 fromversion: 6.10.0 tests: - No tests (auto formatted) #TODO From 863ad5aabd241b41c585cc25439cf5cd52f0fded Mon Sep 17 00:00:00 2001 From: yshamai Date: Sun, 5 Jan 2025 11:41:51 +0200 Subject: [PATCH 06/31] more --- .../RetarusSecureEmailGateway.py | 19 ++++++++------ .../RetarusSecureEmailGateway.yml | 26 ++++++++----------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 72ab1ed9f5bd..6999e00ed8ff 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -45,6 +45,7 @@ FETCH_SLEEP = 5 SERVER_IDLE_TIMEOUT = 60 DEFAULT_CHANNEL = "default" +LOG_PREFIX = "Retarus" DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # ISO8601 format with UTC, default in XSOAR @@ -94,9 +95,9 @@ def push_events(events: list[dict]): """ Push events to XSIAM. """ - demisto.debug(f"Pushing {len(events)} to XSIAM") + demisto.debug(f"{LOG_PREFIX} Pushing {len(events)} to XSIAM") send_events_to_xsiam(events=events, vendor=VENDOR, product=PRODUCT) - demisto.debug(f"Pushed {len(events)} to XSIAM successfully") + demisto.debug(f"{LOG_PREFIX} Pushed {len(events)} to XSIAM successfully") @contextmanager @@ -134,7 +135,7 @@ def perform_long_running_loop(connection: EventConnection, fetch_interval: int): events = fetch_events(connection, fetch_interval) events.extend(integration_context.get("events", [])) integration_context["events"] = events # update events in context in case of a failure. - demisto.debug(f'Adding {len(events)} Events to XSIAM') + demisto.debug(f'{LOG_PREFIX} Adding {len(events)} Events to XSIAM') events_to_send.extend(events) # Send the events to the XSIAM, with events from the context @@ -170,6 +171,7 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif # (sentence taken from Retarus docs) # Setting up heartbeat daemon threads to send keep-alives if needed threading.Thread(target=connection.heartbeat, daemon=True).start() + demisto.debug(f"{LOG_PREFIX} Created connection and created heartbeat") while True: perform_long_running_loop(connection, fetch_interval) @@ -199,7 +201,7 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: events: list[dict] = [] event_ids = set() fetch_start_time = datetime.utcnow() - demisto.debug(f'Starting to fetch events at {fetch_start_time}') + demisto.debug(f'{LOG_PREFIX} Starting to fetch events at {fetch_start_time}') while not is_interval_passed(fetch_start_time, fetch_interval): try: event = json.loads(connection.recv(timeout=recv_timeout)) @@ -210,11 +212,11 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: event_ts = event.get("ts") if not event_ts: # if timestamp is not in the response, use the current time - demisto.debug(f"Event {event_id} does not have a timestamp, using current time") + demisto.debug(f"{LOG_PREFIX} Event {event_id} does not have a timestamp, using current time") event_ts = datetime.utcnow().isoformat() date = dateparser.parse(event_ts) if not date: - demisto.debug(f"Event {event_id} has an invalid timestamp, using current time") + demisto.debug(f"{LOG_PREFIX} Event {event_id} has an invalid timestamp, using current time") # if timestamp is not in correct format, use the current time date = datetime.utcnow() event["id"] = event_id # TODO not sure I am supposed to do it, maybe it's for @bavly @@ -222,8 +224,8 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: event["event_type"] = event.get("type") events.append(event) event_ids.add(event_id) - demisto.debug(f"Fetched {len(events)} events") - demisto.debug("The fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) + demisto.debug(f"{LOG_PREFIX} Fetched {len(events)} events") + demisto.debug(f"{LOG_PREFIX}vThe fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) return events ''' MAIN FUNCTION ''' @@ -240,6 +242,7 @@ def main(): # pragma: no cover try: if command == "long-running-execution": + demisto.error(f"{LOG_PREFIX} Entering long-rinning-execution command") return_results(long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl)) elif command == "test-module": return_results(test_module(url, token_id)) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 9cb8cab54ed0..f51a301fdb58 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -18,6 +18,7 @@ configuration: - defaultvalue: default display: Channel name section: Collect + name: channel type: 0 - display: Trust any certificate (not secure) name: insecure @@ -27,28 +28,23 @@ configuration: name: proxy type: 8 required: false -- display: Fetch events - name: is_fetch +- defaultvalue: 'true' + display: Long Running Instance + hidden: true + name: longRunning type: 8 - required: false + section: Connect description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.]' #TODO display: RetarusSecureEmailGateway name: RetarusSecureEmailGateway script: - commands: - - arguments: - - description: '[Enter a description of the argument, including any important information users need to know, for example, default values.]' - name: Retarus-seg-get-events - required: true - description: '[Enter a description of the command, including any important information users need to know, for example required permissions.]' - name: baseintegration-dummy - outputs: - - contextPath: BaseIntegration.Output - description: '[Enter a description of the data returned in this output.]' - type: String type: python + commands: [] subtype: python3 dockerimage: demisto/netutils:1.0.0.115452 + longRunning: true +marketplaces: +- marketplacev2 fromversion: 6.10.0 tests: -- No tests (auto formatted) #TODO +- No tests From 090dab370129e185c5d6414067800e1a2baaef22 Mon Sep 17 00:00:00 2001 From: yshamai Date: Tue, 7 Jan 2025 15:39:55 +0200 Subject: [PATCH 07/31] working version with insecure implementation, no proxy implementation --- .../RetarusSecureEmailGateway.py | 103 ++++++++++++++---- .../RetarusSecureEmailGateway.yml | 30 ++--- 2 files changed, 97 insertions(+), 36 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 6999e00ed8ff..ff528d359175 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -17,7 +17,8 @@ """ from CommonServerUserPython import * # noqa - +import aiohttp +from aiohttp import ClientWebSocketResponse import urllib3 from typing import Dict, Any from websockets import Data @@ -45,7 +46,7 @@ FETCH_SLEEP = 5 SERVER_IDLE_TIMEOUT = 60 DEFAULT_CHANNEL = "default" -LOG_PREFIX = "Retarus" +LOG_PREFIX = "Retarus- My logs" DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # ISO8601 format with UTC, default in XSOAR @@ -53,7 +54,7 @@ class EventConnection: - def __init__(self, connection: Connection, fetch_interval: int = FETCH_INTERVAL_IN_SECONDS, + def __init__(self, connection:Connection, fetch_interval: int = FETCH_INTERVAL_IN_SECONDS, idle_timeout: int = SERVER_IDLE_TIMEOUT): self.connection = connection self.lock = threading.Lock() @@ -102,19 +103,79 @@ def push_events(events: list[dict]): @contextmanager def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: str, verify_ssl: bool): + extra_headers = {"Authorization": f"Bearer {token_id}"} - context = ssl.create_default_context() + ssl_context = ssl.create_default_context() if not verify_ssl: - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE - with connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", additional_headers=extra_headers) as ws: - connection = EventConnection( + #proxies, _ = handle_proxy_for_long_running() + #proxy = proxies.get("https", "") if is_proxy else None + + try: + with connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", + additional_headers=extra_headers, + ssl=ssl_context) as ws: + connection = EventConnection( connection=ws, fetch_interval=fetch_interval - ) - yield connection + ) + # async with aiohttp.ClientSession() as session: + # async with session.ws_connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", + # headers=extra_headers, + # ssl=ssl_context) as ws: + # connection = EventConnection( + # connection=ws, + # fetch_interval=fetch_interval + # ) + yield connection + + except Exception as e: + raise DemistoException(f"{str(e)}\nAuthentication failed. Please check the URL, channel name and access token.") + + +# sync def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: str, verify_ssl: bool): + +# uri = '' + +# if not verify_ssl: +# ssl_context = ssl.create_default_context() +# ssl_context.check_hostname = False +# ssl_context.verify_mode = ssl.CERT_NONE +# if 'http://' in url: +# uri = url.replace("http://", "ws://", 1) +# elif "wss://" in url: +# uri = url.replace("wss://", "ws://", 1) + +# else: +# # Use default SSL context +# ssl_context = None +# if 'https://' in url: +# uri = url.replace("https://", "wss://", 1) +# elif 'ws://' in url: +# uri = url.replace("ws://", "wss://", 1) + +# if "ws://" not in uri and "wss://" not in uri: +# uri = "ws://"+uri if not verify_ssl else "wss://"+uri + +# extra_headers = {"Authorization": f"Bearer {token_id}"} + +# try: +# async with aiohttp.ClientSession() as session: +# with session.ws_connect( +# uri= uri+f"/email/siem/v1/websocket?channel={channel}", +# ssl=ssl_context, +# headers = extra_headers +# ) as ws: + +# # with connect(uri+f"/email/siem/v1/websocket?channel={channel}", additional_headers=extra_headers) as ws: +# connection = EventConnection( +# connection=ws, +# fetch_interval=fetch_interval +# ) +# yield connection def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: """This function checks if the given interval has passed since the given start time @@ -131,17 +192,15 @@ def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: def perform_long_running_loop(connection: EventConnection, fetch_interval: int): integration_context = demisto.getIntegrationContext() - events_to_send = [] events = fetch_events(connection, fetch_interval) events.extend(integration_context.get("events", [])) integration_context["events"] = events # update events in context in case of a failure. demisto.debug(f'{LOG_PREFIX} Adding {len(events)} Events to XSIAM') - events_to_send.extend(events) # Send the events to the XSIAM, with events from the context # Need to add the option that if we have more then one failure of sending the events to xsiam then we stop fetching. consult with Meital and Dima # TODO try: - send_events_to_xsiam(events_to_send, vendor=VENDOR, product=PRODUCT) + send_events_to_xsiam(events, vendor=VENDOR, product=PRODUCT) # clear the context after sending the events demisto.setIntegrationContext({}) except DemistoException: @@ -151,7 +210,7 @@ def perform_long_running_loop(connection: EventConnection, fetch_interval: int): ''' COMMAND FUNCTIONS ''' -def long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl): +def long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl, is_proxy): """ Performs the long running execution loop. Opens a connection to Retarus. @@ -164,11 +223,11 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif fetch_interval (int): Total time allocated per fetch cycle. """ with websocket_connection(url, token_id, fetch_interval, channel, verify_ssl) as connection: - demisto.info("Connected to websocket") + demisto.info(f"{LOG_PREFIX} Connected to websocket") # Retarus will keep connections with no traffic open for at most 5 minutes. # It is highly recommended that the client sends a PING control frame every 60 seconds to keep the connection open. - # (sentence taken from Retarus docs) + # (sentence taken from Retarus API docs) # Setting up heartbeat daemon threads to send keep-alives if needed threading.Thread(target=connection.heartbeat, daemon=True).start() demisto.debug(f"{LOG_PREFIX} Created connection and created heartbeat") @@ -208,7 +267,7 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: except TimeoutError: # if we didn't receive an event for `fetch_interval` seconds, finish fetching continue - event_id = event_id = event.get("rmxId", event.get("mimeId")) + event_id = event.get("_id", event.get("rmxId")) event_ts = event.get("ts") if not event_ts: # if timestamp is not in the response, use the current time @@ -220,12 +279,12 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: # if timestamp is not in correct format, use the current time date = datetime.utcnow() event["id"] = event_id # TODO not sure I am supposed to do it, maybe it's for @bavly - event["_time"] = date + event["_time"] = date.astimezone(tz.tzutc()).isoformat() event["event_type"] = event.get("type") events.append(event) event_ids.add(event_id) demisto.debug(f"{LOG_PREFIX} Fetched {len(events)} events") - demisto.debug(f"{LOG_PREFIX}vThe fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) + demisto.debug(f"{LOG_PREFIX} The fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) return events ''' MAIN FUNCTION ''' @@ -236,14 +295,14 @@ def main(): # pragma: no cover params = demisto.params() url = params.get("url", "events.retarus.com") token_id = params.get("credentials", {}).get("password", "") - fetch_interval = int(params.get("fetch_interval", FETCH_INTERVAL_IN_SECONDS)) + fetch_interval = arg_to_number(params.get("fetch_interval", FETCH_INTERVAL_IN_SECONDS)) verify_ssl = argToBoolean(not params.get("insecure", False)) + is_proxy = argToBoolean(params.get("proxy", False)) channel = params.get("channel", DEFAULT_CHANNEL) try: if command == "long-running-execution": - demisto.error(f"{LOG_PREFIX} Entering long-rinning-execution command") - return_results(long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl)) + return_results(long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl, is_proxy)) elif command == "test-module": return_results(test_module(url, token_id)) else: diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index f51a301fdb58..62b8d1fa8fb9 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -1,7 +1,10 @@ -category: Email commonfields: - id: RetarusSecureEmailGateway + id: Retarus Secure Email Gateway version: -1 +name: Retarus Secure Email Gateway +display: Retarus Secure Email Gateway +category: Email +description: Collects events for Retarus Secure Email Gateway using the streaming API. configuration: - defaultvalue: events.retarus.com display: Server URL @@ -20,24 +23,23 @@ configuration: section: Collect name: channel type: 0 -- display: Trust any certificate (not secure) - name: insecure - type: 8 - required: false -- display: Use system proxy settings - name: proxy - type: 8 - required: false - defaultvalue: 'true' display: Long Running Instance hidden: true name: longRunning type: 8 section: Connect -description: '[Enter a comprehensive, yet concise, description of what the integration does, what use cases it is designed for, etc.]' #TODO -display: RetarusSecureEmailGateway -name: RetarusSecureEmailGateway +- display: Use system proxy settings + name: proxy + type: 8 + required: false + section: Collect +- display: Trust any certificate (not secure) + name: insecure + type: 8 + required: false script: + script: "" type: python commands: [] subtype: python3 @@ -45,6 +47,6 @@ script: longRunning: true marketplaces: - marketplacev2 -fromversion: 6.10.0 +fromversion: 6.9.0 tests: - No tests From 10b842d6278e7873088f597f145529fc68dcc51b Mon Sep 17 00:00:00 2001 From: yshamai Date: Thu, 9 Jan 2025 10:13:03 +0200 Subject: [PATCH 08/31] remove proxy parameter and code improvements --- .../RetarusSecureEmailGateway/README.md | 19 ++++- .../RetarusSecureEmailGateway.py | 67 ++---------------- .../RetarusSecureEmailGateway.yml | 14 ++-- .../RetarusSecureEmailGateway_image.png | Bin 2507 -> 1115 bytes .../test_data/baseintegration-dummy.json | 3 - Packs/RetarusSecureEmailGateway/README.md | 7 ++ 6 files changed, 36 insertions(+), 74 deletions(-) delete mode 100644 Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/test_data/baseintegration-dummy.json diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index 5d6d1dc2bf52..0678477d5df5 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -1,5 +1,18 @@ -This README contains the full documentation for your integration. +Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email security. +This integration was integrated and tested with RetarusSecureEmailGateway. -You auto-generate this README file from your integration YML file using the `demisto-sdk generate-docs` command. +## Configure RetarusSecureEmailGateway Event Collector on Cortex XSIAM -For more information see the [integration documentation](https://xsoar.pan.dev/docs/integrations/integration-docs). +1. Navigate to **Settings** > **Integrations** > **Servers & Services**. +2. Search for RetarusSecureEmailGateway. +3. Click **Add instance** to create and configure a new integration instance. + + | **Parameter** | **Required** | + | --- | --- | + | Server URL | True | + | Token ID | True | + | Channel name | False | + | Fetch interval in seconds | True | + | Trust any certificate (not secure) | False | + +5. No test button option available. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. \ No newline at end of file diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index ff528d359175..8db648c88c7a 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -111,71 +111,17 @@ def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE - #proxies, _ = handle_proxy_for_long_running() - #proxy = proxies.get("https", "") if is_proxy else None - try: - with connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", - additional_headers=extra_headers, - ssl=ssl_context) as ws: + with connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", additional_headers=extra_headers, ssl=ssl_context) as ws: connection = EventConnection( connection=ws, fetch_interval=fetch_interval - ) - # async with aiohttp.ClientSession() as session: - # async with session.ws_connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", - # headers=extra_headers, - # ssl=ssl_context) as ws: - # connection = EventConnection( - # connection=ws, - # fetch_interval=fetch_interval - # ) + ) yield connection - + except Exception as e: raise DemistoException(f"{str(e)}\nAuthentication failed. Please check the URL, channel name and access token.") - -# sync def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: str, verify_ssl: bool): - -# uri = '' - -# if not verify_ssl: -# ssl_context = ssl.create_default_context() -# ssl_context.check_hostname = False -# ssl_context.verify_mode = ssl.CERT_NONE -# if 'http://' in url: -# uri = url.replace("http://", "ws://", 1) -# elif "wss://" in url: -# uri = url.replace("wss://", "ws://", 1) - -# else: -# # Use default SSL context -# ssl_context = None -# if 'https://' in url: -# uri = url.replace("https://", "wss://", 1) -# elif 'ws://' in url: -# uri = url.replace("ws://", "wss://", 1) - -# if "ws://" not in uri and "wss://" not in uri: -# uri = "ws://"+uri if not verify_ssl else "wss://"+uri - -# extra_headers = {"Authorization": f"Bearer {token_id}"} - -# try: -# async with aiohttp.ClientSession() as session: -# with session.ws_connect( -# uri= uri+f"/email/siem/v1/websocket?channel={channel}", -# ssl=ssl_context, -# headers = extra_headers -# ) as ws: - -# # with connect(uri+f"/email/siem/v1/websocket?channel={channel}", additional_headers=extra_headers) as ws: -# connection = EventConnection( -# connection=ws, -# fetch_interval=fetch_interval -# ) -# yield connection def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: """This function checks if the given interval has passed since the given start time @@ -238,11 +184,8 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif time.sleep(FETCH_SLEEP) def test_module(url, token_id): - if not url: - raise DemistoException("Missing url parameter.") - if not token_id: - raise DemistoException("Missing token id parameter.") - return 'ok' + raise DemistoException("No test option available. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration.") + def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: int = 10) -> list[dict]: diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 62b8d1fa8fb9..0b35d05a7750 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -4,7 +4,7 @@ commonfields: name: Retarus Secure Email Gateway display: Retarus Secure Email Gateway category: Email -description: Collects events for Retarus Secure Email Gateway using the streaming API. +description: Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email security. configuration: - defaultvalue: events.retarus.com display: Server URL @@ -23,17 +23,19 @@ configuration: section: Collect name: channel type: 0 +- display: Fetch interval in seconds + name: fetch_interval + type: 0 + defaultvalue: 60 + required: true + section: Collect + advanced: true - defaultvalue: 'true' display: Long Running Instance hidden: true name: longRunning type: 8 section: Connect -- display: Use system proxy settings - name: proxy - type: 8 - required: false - section: Collect - display: Trust any certificate (not secure) name: insecure type: 8 diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_image.png b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_image.png index 772c6af21bebd9230ea48f23a29ad2260f386e71..e5887a242c185fd2837f5d66a60a3c6a315e1d9d 100644 GIT binary patch literal 1115 zcmeAS@N?(olHy`uVBq!ia0vp^6+mpn!3-o9RK9-&q`n3Cgt!9f_V)IUjt(H%*Vk89 zR|jN6xX28kED!)iy1To9jE05=AiK4-wYIhvS(&%D_vFcw;UZv7Ct38-ILKd=ziq{(Z~TZMpnsPycNy_dT=YmiCv>jREWLGA-Xa z(IX^Q`%BTmFl;tL6x5@XV+bvv^h3hhxz{Hge8jeYZGhRQEgeoLaH&*=?81h|Z0bw#*R)0lQe;mTp}C>Wop} z=I!UFIrJ=wI-inyOQ71pIMd4QeV5ZeySHN7y==5y*cNnVZx4vbUsu&BalAE!Eoo{n zpZSqUlZ@Wqvk%|5K6=++4U=PVN#e}gyLK_xh&@{5vAt`l%HAhU@yq8v`fKoo!Sr!3 zLr>|(o8MmlTxzk~dkynS$0QB)_>SchlXxdEF;x1_nLR}}uuo`v_lJo=yPTTZUw-*$ z`e<>8N17_%?pyJhE?qBUigkZfh3$$`qO)nY9-p^zlGZ|t&-;Iz zciu81t)YCaMb^?8oV#lZ;)LFRPdVbmKRd}R?K3%kMRm`hh z&P!fcS%;lC{xa|K%9p!z%H~zP+;c5w{l2dI@8)T!fk7Jge+Kg_^Vqh{yPXTnD-52l KelF{r5}E*%%L#}8 literal 2507 zcmV;+2{iVJP)NJU1VgcOS!Wr4 zO{q;AyQYAIs;$+wHZ~Sp+*+hq)D%gf ztHY9woe3{0IJoIrpgCQC7`DJ3m>Rb4IVaSF>acO==CH3?2YW9zeW_Dx~*%o3bFhWN8-vqxH&=>B0u zuaCmH%$&2TPhhFWSb?U_&pUZ|*bx4bY2lAeKf4Fz( zmQcC1G(7!QIj=-*@_K6Xu1z?%2l)e_654|gT&Yu996XHs=&CTZ`w-v}%L}p|&0V{i z+>M}fRVtn9z|!)NwJa=JU&b?XP1tkblQ4D3yx@~^KJ{ql)!hryGQsnWH2d0+H@nGX zw4$doc(6Jj$F{`H#bN9f?}g#LF3mfF;nU`awLA8cKM!>FeJ~$VrQ(!~W*_628V=UH z#L*3oZlPbNGeW+jEG_4Q0)Dht@52W-_{z?+TAyKZtwufpGKm1!42gYaP3o=Hp5 zpENDeNw=&g(wishi@^u5KHBy z0B5lAQOWLBab_QSK09fjTO zC@3Ws@CkVee3)foO2$1e+uh|0=G5tv6Q@D!mTg_+}$dIq?{bj!*uvDc%`2UX?@FJXeYlO zql@{RdZcteM0jQY4X%XKp#!*g{X6jLzX!TOK_|!pF2zT3WJB`(jkNkL_}X+Gl)`YB z2EHqQ4+=WY7VrZ@LmX(C`XO?ji^Ej+Opo`g5xWAGl^6a3G? z&p{{f9xG+wdu=iJnzREvZxZ-gR8YYJ1q&1`P_RJ30tE{cEO1OM;46U-Cb=BbMjBDs z)ttK)h8|@daa>LHgTYT3?|-25D93$R@gid10e^0qi+Lx5PNNfk+3|QZisSOhx*-=W$J3%r+sPEn@)0ekD#*uCOYo^?Wv^h40bS-pMXKv)%JM$A&pY;mgh?FhkU+QE8BVmEWZwZ1Os3?tOv{bg-ZM9Yuh@kKT3Z*1=`auTt9(gP*;L= zYM0sIh3fe&AlwBta3|cJr8h@40S(p`YfCjh$JYPeoo}f{m1KXYlPCUyd701;zn`qs(u>dr$7Yn+zbza=YJN;K&Ss5w9P+3!^a6@TgF#|cJ-WZz!Y%D%ukcHbs5yv zHpv-V`0+!O}lnF#l)S zyxg`;wvOoF*cT^l)Di+ZR7D$U3#BdW&v=a=gNtBJyf z9dok((ZwPiqQALjyh|Hg0izuz@yL&?uKKjIP+TF!L!4*?Gxd3lZ5vK1_C>o# z8FoUss^nF&!V#46z7aKpzyxRwec=Uo1iYV=g7SPDa*1iIT!y{~=fU;h4D~9}AgV|0 zA^e%+Z7>n;gBsA;&qHJMLH6XYaP*3u3#(upYy+3X!SF}W;opKW@FaNUB(GkgpHdkq z-nYBJbznRyx9xXgM)nx{@@?W(Pyt>9lfd!UtGmdxP%h5M{V*0(KR5utHiBFledzcY zeh!}hB~Z@J$(c~?pZ7TYD-oYVcx57P`NiPUVm(VCE=x|PWnkOUD7O&S36?e8iI=N5 zuCCdPsT|`y&7@J_`Ij0&CGZNE<`vowoXoF-Y4Rq-3HM?m-Mlq$2dLb(OmijJ2G*_Z z-T=#~y4udI!;8#GW?S0biQWoSyazjRR5@7g0V9o(_ai&kSn1$Dfh$5ZAi+awM#yxJ zQHFJ1bb3s0?6C_KIIg5R5J{%>CcYC~YWu~r>$8V_Z3FpKSKGN|JCP6_o11si z#&fUuleVMTj(wG*SJ92&!{Pu~4vl>dpo(;ACd2wc6B?Sg4ex(($=#5RpLMd0;eS43 V(U=mQ{m=ja002ovPDHLkV1nG2;bs5; diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/test_data/baseintegration-dummy.json b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/test_data/baseintegration-dummy.json deleted file mode 100644 index 37fa47b18cd0..000000000000 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/test_data/baseintegration-dummy.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "dummy": "this is a dummy response" -} \ No newline at end of file diff --git a/Packs/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/README.md index e69de29bb2d1..99c3f780fc4e 100644 --- a/Packs/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/README.md @@ -0,0 +1,7 @@ +<~XSIAM> +# Retarus Secure Email Gateway +This pack includes Cortex XSIAM content. + +Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email security. + + \ No newline at end of file From b4aadd503e8c09aaa3b2471fcd14fdc39b2685cf Mon Sep 17 00:00:00 2001 From: yshamai Date: Thu, 9 Jan 2025 10:32:27 +0200 Subject: [PATCH 09/31] more code improvements --- .../RetarusSecureEmailGateway.py | 75 ++++++++----------- 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 8db648c88c7a..db64228e2ad2 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -1,45 +1,20 @@ import demistomock as demisto # noqa: F401 from CommonServerPython import * # noqa: F401 -"""Base Integration for Cortex XSOAR (aka Demisto) - -This is an empty Integration with some basic structure according -to the code conventions. - -MAKE SURE YOU REVIEW/REPLACE ALL THE COMMENTS MARKED AS "TODO" - -Developer Documentation: https://xsoar.pan.dev/docs/welcome -Code Conventions: https://xsoar.pan.dev/docs/integrations/code-conventions -Linting: https://xsoar.pan.dev/docs/integrations/linting - -This is an empty structure file. Check an example at; -https://github.com/demisto/content/blob/master/Packs/HelloWorld/Integrations/HelloWorld/HelloWorld.py - -""" - from CommonServerUserPython import * # noqa -import aiohttp -from aiohttp import ClientWebSocketResponse import urllib3 -from typing import Dict, Any from websockets import Data -import websockets -from CommonServerPython import * # noqa: F401 from websockets.sync.client import connect from websockets.sync.connection import Connection -from websockets.exceptions import InvalidStatus from dateutil import tz import traceback import threading from contextlib import contextmanager - - # Disable insecure warnings urllib3.disable_warnings() ''' CONSTANTS ''' - VENDOR = "Retarus" PRODUCT = "Secure Email Gateway" FETCH_INTERVAL_IN_SECONDS = 60 @@ -47,12 +22,10 @@ SERVER_IDLE_TIMEOUT = 60 DEFAULT_CHANNEL = "default" LOG_PREFIX = "Retarus- My logs" - DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # ISO8601 format with UTC, default in XSOAR -''' CLIENT CLASS ''' - +''' CLIENT CLASS ''' class EventConnection: def __init__(self, connection:Connection, fetch_interval: int = FETCH_INTERVAL_IN_SECONDS, idle_timeout: int = SERVER_IDLE_TIMEOUT): @@ -88,22 +61,30 @@ def heartbeat(self): time.sleep(self.idle_timeout) - # TODO: ADD HERE THE FUNCTIONS TO INTERACT WITH YOUR PRODUCT API - - ''' HELPER FUNCTIONS ''' def push_events(events: list[dict]): """ Push events to XSIAM. """ - demisto.debug(f"{LOG_PREFIX} Pushing {len(events)} to XSIAM") send_events_to_xsiam(events=events, vendor=VENDOR, product=PRODUCT) demisto.debug(f"{LOG_PREFIX} Pushed {len(events)} to XSIAM successfully") @contextmanager def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: str, verify_ssl: bool): - + """ + Create a connection to the api. + + Args: + url (str): host URL for the websocket connection. + channel (str): Retarus channel to connect through. + token_id (str): Retarus token id. + fetch_interval (int): Time between fetch iterations. + verify_ssl (bool): Whether to verify ssl when connecting. + + Yields: + EventConnection: eventConnection to receive events from. + """ extra_headers = {"Authorization": f"Bearer {token_id}"} ssl_context = ssl.create_default_context() @@ -112,7 +93,9 @@ def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: ssl_context.verify_mode = ssl.CERT_NONE try: - with connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", additional_headers=extra_headers, ssl=ssl_context) as ws: + with connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", + additional_headers=extra_headers, + ssl=ssl_context) as ws: connection = EventConnection( connection=ws, fetch_interval=fetch_interval @@ -137,6 +120,13 @@ def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: def perform_long_running_loop(connection: EventConnection, fetch_interval: int): + """ + Long running loop iteration function. Fetches events from the connection and sends them to XSIAM. + + Args: + connection (EventConnection): A connection object to fetch events from. + fetch_interval (int): Fetch time for this fetching events cycle. + """ integration_context = demisto.getIntegrationContext() events = fetch_events(connection, fetch_interval) events.extend(integration_context.get("events", [])) @@ -156,7 +146,7 @@ def perform_long_running_loop(connection: EventConnection, fetch_interval: int): ''' COMMAND FUNCTIONS ''' -def long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl, is_proxy): +def long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl): """ Performs the long running execution loop. Opens a connection to Retarus. @@ -167,6 +157,7 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif token_id (str): Retarus token_id to connect to. channel (str): channel to connect with. fetch_interval (int): Total time allocated per fetch cycle. + verify_ssl (bool): Whether to verify ssl when opening the connection. """ with websocket_connection(url, token_id, fetch_interval, channel, verify_ssl) as connection: demisto.info(f"{LOG_PREFIX} Connected to websocket") @@ -183,9 +174,9 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif # sleep for a bit to not throttle the CPU time.sleep(FETCH_SLEEP) -def test_module(url, token_id): - raise DemistoException("No test option available. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration.") +def test_module(): + raise DemistoException("No test option available. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration.") def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: int = 10) -> list[dict]: @@ -230,9 +221,8 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: demisto.debug(f"{LOG_PREFIX} The fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) return events -''' MAIN FUNCTION ''' - +''' MAIN FUNCTION ''' def main(): # pragma: no cover command = demisto.command() params = demisto.params() @@ -240,21 +230,18 @@ def main(): # pragma: no cover token_id = params.get("credentials", {}).get("password", "") fetch_interval = arg_to_number(params.get("fetch_interval", FETCH_INTERVAL_IN_SECONDS)) verify_ssl = argToBoolean(not params.get("insecure", False)) - is_proxy = argToBoolean(params.get("proxy", False)) channel = params.get("channel", DEFAULT_CHANNEL) try: if command == "long-running-execution": - return_results(long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl, is_proxy)) + return_results(long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl)) elif command == "test-module": - return_results(test_module(url, token_id)) + return_results(test_module()) else: raise NotImplementedError(f"Command {command} is not implemented.") except Exception: return_error(f'Failed to execute {command} command.\nError:\n{traceback.format_exc()}') - - if __name__ in ('__main__', '__builtin__', 'builtins'): main() From a1ed84a2d293b7820a24c43edda518553c3fcaf5 Mon Sep 17 00:00:00 2001 From: yshamai Date: Thu, 9 Jan 2025 10:52:30 +0200 Subject: [PATCH 10/31] BaseIntegration Help --- .../RetarusSecureEmailGateway_description.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md index 51bd561c623f..4c00697d2600 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md @@ -1,8 +1,3 @@ ## BaseIntegration Help -Markdown file for integration configuration help snippet. In this file add: - -- Brief information about how to retrieve the API key of your product -- Other useful information on how to configure your integration in XSOAR - -Since this is a Markdown file, we encourage you to use MD formatting for sections, sub-sections, lists, etc. +No test button option available. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. From d19f3c306df924284e09a0ee0e9377deed815c51 Mon Sep 17 00:00:00 2001 From: yshamai Date: Thu, 9 Jan 2025 14:30:02 +0200 Subject: [PATCH 11/31] start unit tests --- .../RetarusSecureEmailGateway_test.py | 118 ++++++++++++++---- 1 file changed, 91 insertions(+), 27 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py index 6a48b6e932e3..41e5e896cc0c 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py @@ -1,42 +1,106 @@ -"""Base Integration for Cortex XSOAR - Unit Tests file +import json +import io +import uuid +import pytest +from contextlib import contextmanager -Pytest Unit Tests: all funcion names must start with "test_" +from RetarusSecureEmailGateway import ( + fetch_events, + json, + demisto, + datetime, + timedelta, + websocket_connection, + perform_long_running_loop, + DemistoException, + Connection, + EventConnection, + long_running_execution_command, +) +import RetarusSecureEmailGateway -More details: https://xsoar.pan.dev/docs/integrations/unit-testing +CURRENT_TIME: datetime | None = None -MAKE SURE YOU REVIEW/REPLACE ALL THE COMMENTS MARKED AS "TODO" +EVENTS = [ + {"ts": "2023-08-16T13:24:12.147573+0100", "type": "type_1", "_id": "1"}, + {"ts": "2023-08-14T13:24:12.147573+0200", "type": "type_2", "_id": "2"}, + {"ts": "2023-08-12T13:24:11.147573+0000", "type": "type_3", "_id": "3"} +] -You must add at least a Unit Test function for every XSOAR command -you are implementing with your integration -""" +def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: + global CURRENT_TIME + if not CURRENT_TIME: + CURRENT_TIME = fetch_start_time + return fetch_start_time + timedelta(seconds=fetch_interval) < CURRENT_TIME + +@pytest.fixture +def connection(): + # Set up a mock connection + return MockConnection() -import json -import io +class MockConnection(Connection): + def __init__( + self, + ): + global CURRENT_TIME + self.id = uuid.uuid4() + self.events = EVENTS + self.index = 0 + self.pongs = 0 + self.create_time = datetime.now() -def util_load_json(path): - with io.open(path, mode='r', encoding='utf-8') as f: - return json.loads(f.read()) + def recv(self, timeout): + global CURRENT_TIME + # pretend to sleep for 4 seconds + assert CURRENT_TIME + CURRENT_TIME += timedelta(seconds=4) + if self.index >= len(self.events): + raise TimeoutError + event = self.events[self.index] + self.index += 1 + return json.dumps(event) -# TODO: REMOVE the following dummy unit test function -def test_baseintegration_dummy(): - """Tests helloworld-say-hello command function. + def pong(self): + self.pongs += 1 + + +def test_heartbeat(mocker, connection): + """ + Given: + - A connection object with scarce messages + + When: + - The long running execution loop runs - Checks the output of the command function with the expected output. + Then: + - Periodic keep-alive messages (pongs) are sent to the websocket connection to prevent it from closing. - No mock is needed here because the say_hello_command does not call - any external API. """ - from BaseIntegration import Client, baseintegration_dummy_command + from RetarusSecureEmailGateway import long_running_execution_command + idle_timeout = 3 + + @contextmanager + def mock_websocket_connection(host, cluster_id, api_key, since_time=None, to_time=None, fetch_interval=60): + yield EventConnection(connection, fetch_interval, idle_timeout) + + def mock_perform_long_running_loop(connection, interval): + # This mock will raise exceptions to stop the long running loop + # StopIteration exception marks success + if connection.connection.pongs: + raise StopIteration(f'Sent {connection.connection.pongs} pongs') + if datetime.now() > connection.create_time + timedelta(seconds=idle_timeout + 2): + # Heartbeat should've been sent already + raise TimeoutError(f'No heartbeat sent within {idle_timeout} seconds') + + mocker.patch.object(RetarusSecureEmailGateway, 'websocket_connection', + side_effect=mock_websocket_connection) + mocker.patch.object(RetarusSecureEmailGateway, 'perform_long_running_loop', + side_effect=mock_perform_long_running_loop) - client = Client(base_url='some_mock_url', verify=False) - args = { - 'dummy': 'this is a dummy response' - } - response = baseintegration_dummy_command(client, args) + with pytest.raises(StopIteration): + long_running_execution_command(url='url', token_id='token_id', fetch_interval=60, channel='channel', verify_ssl=False) - mock_response = util_load_json('test_data/baseintegration-dummy.json') + assert connection.pongs > 0 - assert response.outputs == mock_response -# TODO: ADD HERE unit tests for every command From e4457ebfe612bfe101cd36a8ba6b7119b12af91d Mon Sep 17 00:00:00 2001 From: yshamai Date: Thu, 9 Jan 2025 15:19:38 +0200 Subject: [PATCH 12/31] changed datetime calls to not use the deprecated function --- .../RetarusSecureEmailGateway.py | 10 ++--- .../RetarusSecureEmailGateway_test.py | 38 ++++++++++++++++++- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index db64228e2ad2..21894ad38244 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -116,7 +116,7 @@ def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: Returns: bool: True if the interval has passed, False otherwise """ - return fetch_start_time + timedelta(seconds=fetch_interval) < datetime.utcnow() + return fetch_start_time + timedelta(seconds=fetch_interval) < datetime.now().astimezone(timezone.utc) def perform_long_running_loop(connection: EventConnection, fetch_interval: int): @@ -193,7 +193,7 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: """ events: list[dict] = [] event_ids = set() - fetch_start_time = datetime.utcnow() + fetch_start_time = datetime.now().astimezone(timezone.utc) demisto.debug(f'{LOG_PREFIX} Starting to fetch events at {fetch_start_time}') while not is_interval_passed(fetch_start_time, fetch_interval): try: @@ -206,14 +206,14 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: if not event_ts: # if timestamp is not in the response, use the current time demisto.debug(f"{LOG_PREFIX} Event {event_id} does not have a timestamp, using current time") - event_ts = datetime.utcnow().isoformat() + event_ts = datetime.now().isoformat() date = dateparser.parse(event_ts) if not date: demisto.debug(f"{LOG_PREFIX} Event {event_id} has an invalid timestamp, using current time") # if timestamp is not in correct format, use the current time - date = datetime.utcnow() + date = datetime.now() event["id"] = event_id # TODO not sure I am supposed to do it, maybe it's for @bavly - event["_time"] = date.astimezone(tz.tzutc()).isoformat() + event["_time"] = date.astimezone(timezone.utc).isoformat() event["event_type"] = event.get("type") events.append(event) event_ids.add(event_id) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py index 41e5e896cc0c..cb20fab4e7c1 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py @@ -65,7 +65,44 @@ def recv(self, timeout): def pong(self): self.pongs += 1 + +def test_handle_failures_of_send_events(mocker, capfd): + """ + Given: + - A connection to the websocket, and events are fetched from the socket + + When: + - Sending events to XSIAM are failing. + + Then: + - Add the failing events to the context, and try again in the next run. + """ + def fetch_events_mock(connection: EventConnection, fetch_interval: int): + return EVENTS + + def sends_events_to_xsiam_mock(events, **kwargs): + raise DemistoException("Message") + + mocker.patch.object(RetarusSecureEmailGateway, "fetch_events", side_effect=fetch_events_mock) + mocker.patch.object(RetarusSecureEmailGateway, "send_events_to_xsiam", side_effect=sends_events_to_xsiam_mock) + with capfd.disabled(): + perform_long_running_loop(EventConnection(MockConnection()), 60) + + context = demisto.getIntegrationContext() + assert context["events"] == EVENTS + + second_try_send_events_mock = mocker.patch.object(RetarusSecureEmailGateway, "send_events_to_xsiam") + with capfd.disabled(): + perform_long_running_loop(EventConnection(MockConnection()), 60) + context = demisto.getIntegrationContext() + # check the the context is cleared + assert not context + # check that the events failed events were sent to xsiam + for event in EVENTS: + assert event in second_try_send_events_mock.call_args_list[0][0][0] + + def test_heartbeat(mocker, connection): """ Given: @@ -78,7 +115,6 @@ def test_heartbeat(mocker, connection): - Periodic keep-alive messages (pongs) are sent to the websocket connection to prevent it from closing. """ - from RetarusSecureEmailGateway import long_running_execution_command idle_timeout = 3 @contextmanager From bf749d32821f2f2683bfca9de9da410374ca1f8f Mon Sep 17 00:00:00 2001 From: yshamai Date: Sun, 12 Jan 2025 10:53:46 +0200 Subject: [PATCH 13/31] more unit tests --- .../RetarusSecureEmailGateway.py | 3 +- .../RetarusSecureEmailGateway_test.py | 42 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 21894ad38244..0a7ff31027ae 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -21,7 +21,7 @@ FETCH_SLEEP = 5 SERVER_IDLE_TIMEOUT = 60 DEFAULT_CHANNEL = "default" -LOG_PREFIX = "Retarus- My logs" +LOG_PREFIX = "Retarus-logs" DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # ISO8601 format with UTC, default in XSOAR @@ -34,6 +34,7 @@ def __init__(self, connection:Connection, fetch_interval: int = FETCH_INTERVAL_I self.idle_timeout = idle_timeout self.fetch_interval = fetch_interval + def recv(self, timeout: float | None = None) -> Data: """ Receive the next message from the connection diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py index cb20fab4e7c1..d6048bc56d20 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py @@ -1,9 +1,11 @@ import json import io +from unittest.mock import Mock, patch import uuid import pytest from contextlib import contextmanager - +import RetarusSecureEmailGateway +import CommonServerPython from RetarusSecureEmailGateway import ( fetch_events, json, @@ -17,14 +19,14 @@ EventConnection, long_running_execution_command, ) -import RetarusSecureEmailGateway +from CommonServerPython import * CURRENT_TIME: datetime | None = None EVENTS = [ - {"ts": "2023-08-16T13:24:12.147573+0100", "type": "type_1", "_id": "1"}, - {"ts": "2023-08-14T13:24:12.147573+0200", "type": "type_2", "_id": "2"}, - {"ts": "2023-08-12T13:24:11.147573+0000", "type": "type_3", "_id": "3"} + {"ts": "2023-08-16T13:24:12.147573+0100", "_id": "1"}, + {"ts": "2023-08-14T13:24:12.147573+0200", "_id": "2"}, + {"ts": "2023-08-12T13:24:11.147573+0000", "_id": "3"} ] def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: @@ -33,6 +35,7 @@ def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: CURRENT_TIME = fetch_start_time return fetch_start_time + timedelta(seconds=fetch_interval) < CURRENT_TIME + @pytest.fixture def connection(): # Set up a mock connection @@ -140,3 +143,32 @@ def mock_perform_long_running_loop(connection, interval): assert connection.pongs > 0 + +def test_fetch_events(mocker, connection): + """ + Given: + A connection to the websocket + + When: + Calling fetch_events function to get events from the websocket connection + + Then: + - Ensure that the function returns the events from the websocket connection + - Ensure that the function converts the timestamp to UTC + - Ensure that the function returns the events collected in the interval. + """ + + # We set fetch_interval to 7 to get this first two events (as we "wait" 4 seconds between each event) + fetch_interval = 7 + event_connection = EventConnection(connection=connection) + mocker.patch.object(RetarusSecureEmailGateway, "is_interval_passed", side_effect=is_interval_passed) + debug_logs = mocker.patch.object(demisto, "debug") + events = fetch_events(connection=event_connection, fetch_interval=fetch_interval) + + assert len(events) == 2 + assert events[0]["_time"] == "2023-08-16T12:24:12.147573+00:00" + assert events[0]["id"] == "1" + assert events[1]["_time"] == "2023-08-14T11:24:12.147573+00:00" + assert events[1]["id"] == "2" + + debug_logs.assert_any_call("Retarus-logs Fetched 2 events") From e2dd4b98ff01105b2fc687750e4c8d6d5c7b92d2 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 15 Jan 2025 08:18:15 +0200 Subject: [PATCH 14/31] another unit test --- .../RetarusSecureEmailGateway_test.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py index d6048bc56d20..42152fd99f90 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py @@ -172,3 +172,27 @@ def test_fetch_events(mocker, connection): assert events[1]["id"] == "2" debug_logs.assert_any_call("Retarus-logs Fetched 2 events") + + +# def test_websocket_connection(mocker): +# """ +# Given: url, token_id, fetch_interval, channel, verify_ssl +# When: Calling websocket_connection func +# Then: -connect function is called with the right arguments +# -EventConnection object is set with the right fields +# """ +# from unittest.mock import patch +# mock_event_connection = mocker.patch('RetarusSecureEmailGateway.EventConnection', side_effect = EventConnection(None, 10, 5)) +# with mocker.patch('RetarusSecureEmailGateway.connect') as mock_connect: + +# mock_event_connection.return_value = Mock() + +# with websocket_connection("url_1", "token_id_1", 10, "channel_1", True): +# pass + +# mock_connect.assert_called_with( +# "wss://url_1/email/siem/v1/websocket?channel=channel_1", +# additional_headers={"Authorization: Bearer token_id_1"}, +# ssl=ssl.create_default_context() +# ) + From 2b63642206f992f98271a518641ae23c4c2327b6 Mon Sep 17 00:00:00 2001 From: yshamai Date: Mon, 20 Jan 2025 11:38:03 +0200 Subject: [PATCH 15/31] some code improvements --- .../Integrations/RetarusSecureEmailGateway/README.md | 2 +- .../RetarusSecureEmailGateway.py | 3 ++- .../RetarusSecureEmailGateway_description.md | 6 ++++-- .../SymantecEndpointSecurity.py | 12 ++++++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index 0678477d5df5..e5cb89ab3ae5 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -15,4 +15,4 @@ This integration was integrated and tested with RetarusSecureEmailGateway. | Fetch interval in seconds | True | | Trust any certificate (not secure) | False | -5. No test button option available. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. \ No newline at end of file +5. No test button option available due to API limitation. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 0a7ff31027ae..87a25009546b 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -47,6 +47,7 @@ def recv(self, timeout: float | None = None) -> Data: Data: Next event received from the connection """ with self.lock: + demisto.debug("Locked the thread to recv a message") event = self.connection.recv(timeout=timeout) return event @@ -227,7 +228,7 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: def main(): # pragma: no cover command = demisto.command() params = demisto.params() - url = params.get("url", "events.retarus.com") + url = params.get["url"] token_id = params.get("credentials", {}).get("password", "") fetch_interval = arg_to_number(params.get("fetch_interval", FETCH_INTERVAL_IN_SECONDS)) verify_ssl = argToBoolean(not params.get("insecure", False)) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md index 4c00697d2600..a861d66c78aa 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md @@ -1,3 +1,5 @@ -## BaseIntegration Help +## Retarus Integration help -No test button option available. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. +write here about connection (If we take of the heartbeat we need to write her that fetch interval should not be more that 4 minutes and what happens if connection closes because fetch took too long and there was no traffic in the connection) If we dont take the heartbeat of we don't need to explain this. + +one sentence about the api key \ No newline at end of file diff --git a/Packs/SymantecEndpointSecurity/Integrations/SymantecEndpointSecurity/SymantecEndpointSecurity.py b/Packs/SymantecEndpointSecurity/Integrations/SymantecEndpointSecurity/SymantecEndpointSecurity.py index 0a9ac60f075b..160fc0a00d21 100644 --- a/Packs/SymantecEndpointSecurity/Integrations/SymantecEndpointSecurity/SymantecEndpointSecurity.py +++ b/Packs/SymantecEndpointSecurity/Integrations/SymantecEndpointSecurity/SymantecEndpointSecurity.py @@ -41,17 +41,21 @@ class NextPointingNotAvailable(Exception): class Client(BaseClient): def __init__( self, - url: str, - token_id: str, + base_url: str, + token: str, + stream_id: str, + channel_id: str, verify: bool, proxy: bool, ) -> None: self.headers: dict[str, str] = {} - self.token_id = token_id + self.token = token + self.stream_id = stream_id + self.channel_id = channel_id super().__init__( - base_url=url, + base_url=base_url, verify=verify, proxy=proxy, timeout=180, From 1b77816e6fe4490829818ebecdba0e3ef0b7a684 Mon Sep 17 00:00:00 2001 From: yshamai Date: Mon, 20 Jan 2025 12:38:20 +0200 Subject: [PATCH 16/31] fix needed fields for events --- .../RetarusSecureEmailGateway.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 87a25009546b..e840bc02dd49 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -197,13 +197,14 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: event_ids = set() fetch_start_time = datetime.now().astimezone(timezone.utc) demisto.debug(f'{LOG_PREFIX} Starting to fetch events at {fetch_start_time}') + while not is_interval_passed(fetch_start_time, fetch_interval): try: event = json.loads(connection.recv(timeout=recv_timeout)) except TimeoutError: # if we didn't receive an event for `fetch_interval` seconds, finish fetching continue - event_id = event.get("_id", event.get("rmxId")) + event_id = event.get("rmxId") event_ts = event.get("ts") if not event_ts: # if timestamp is not in the response, use the current time @@ -214,13 +215,14 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: demisto.debug(f"{LOG_PREFIX} Event {event_id} has an invalid timestamp, using current time") # if timestamp is not in correct format, use the current time date = datetime.now() - event["id"] = event_id # TODO not sure I am supposed to do it, maybe it's for @bavly event["_time"] = date.astimezone(timezone.utc).isoformat() - event["event_type"] = event.get("type") + event["SOURCE_LOG_TYPE"] = event.get("type") events.append(event) event_ids.add(event_id) + demisto.debug(f"{LOG_PREFIX} Fetched {len(events)} events") demisto.debug(f"{LOG_PREFIX} The fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) + return events @@ -228,7 +230,7 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: def main(): # pragma: no cover command = demisto.command() params = demisto.params() - url = params.get["url"] + url = params["url"] token_id = params.get("credentials", {}).get("password", "") fetch_interval = arg_to_number(params.get("fetch_interval", FETCH_INTERVAL_IN_SECONDS)) verify_ssl = argToBoolean(not params.get("insecure", False)) From a984cd9778aae4c62b8358843099b625fbb0d707 Mon Sep 17 00:00:00 2001 From: yshamai Date: Mon, 20 Jan 2025 13:49:15 +0200 Subject: [PATCH 17/31] docs improvements --- .../RetarusSecureEmailGateway/README.md | 13 ++++++------- .../RetarusSecureEmailGateway.yml | 2 +- .../RetarusSecureEmailGateway_description.md | 5 ++--- Packs/RetarusSecureEmailGateway/README.md | 2 +- Packs/RetarusSecureEmailGateway/pack_metadata.json | 2 +- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index e5cb89ab3ae5..7feba09000e7 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -1,5 +1,4 @@ Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email security. -This integration was integrated and tested with RetarusSecureEmailGateway. ## Configure RetarusSecureEmailGateway Event Collector on Cortex XSIAM @@ -7,12 +6,12 @@ This integration was integrated and tested with RetarusSecureEmailGateway. 2. Search for RetarusSecureEmailGateway. 3. Click **Add instance** to create and configure a new integration instance. - | **Parameter** | **Required** | - | --- | --- | - | Server URL | True | - | Token ID | True | - | Channel name | False | - | Fetch interval in seconds | True | + | **Parameter** | **Required** | **Description** | + | --- | --- | --- | + | Server URL | True | | + | Token ID | True | | + | Channel name | False | The channel to fetch events from | + | Fetch interval in seconds | True | | | Trust any certificate (not secure) | False | 5. No test button option available due to API limitation. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 0b35d05a7750..2c67423ebb0a 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -4,7 +4,7 @@ commonfields: name: Retarus Secure Email Gateway display: Retarus Secure Email Gateway category: Email -description: Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email security. +description: Integrate RetarusSecureEmailGateway to seamlessly fetch events from SecureEmailGateway by Retarus and enhance email security. configuration: - defaultvalue: events.retarus.com display: Server URL diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md index a861d66c78aa..aad7937a5a5e 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md @@ -1,5 +1,4 @@ ## Retarus Integration help -write here about connection (If we take of the heartbeat we need to write her that fetch interval should not be more that 4 minutes and what happens if connection closes because fetch took too long and there was no traffic in the connection) If we dont take the heartbeat of we don't need to explain this. - -one sentence about the api key \ No newline at end of file +Provide the api token that Retarus has provided to you. +Provide the channel you created or use the default channel. \ No newline at end of file diff --git a/Packs/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/README.md index 99c3f780fc4e..18958139bc1f 100644 --- a/Packs/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/README.md @@ -2,6 +2,6 @@ # Retarus Secure Email Gateway This pack includes Cortex XSIAM content. -Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email security. +Integrate RetarusSecureEmailGateway to seamlessly fetch events from SecureEmailGateway by Retarus and enhance email security. \ No newline at end of file diff --git a/Packs/RetarusSecureEmailGateway/pack_metadata.json b/Packs/RetarusSecureEmailGateway/pack_metadata.json index 4dadfc3e43f9..2c3d4262a43c 100644 --- a/Packs/RetarusSecureEmailGateway/pack_metadata.json +++ b/Packs/RetarusSecureEmailGateway/pack_metadata.json @@ -1,6 +1,6 @@ { "name": "Retarus Secure Email Gateway", - "description": "## FILL MANDATORY FIELD ##", + "description": "The Retarus Secure Email Platform provides comprehensive security and advanced email routing. It offers features such as Advanced Threat Protection and Email Archiving.", "support": "xsoar", "currentVersion": "1.0.0", "author": "Cortex XSOAR", From 8e26117679b0e04c3da1c5e367d4abddc85ce696 Mon Sep 17 00:00:00 2001 From: yshamai Date: Tue, 21 Jan 2025 12:47:52 +0200 Subject: [PATCH 18/31] add get-last-run-results command --- .../RetarusSecureEmailGateway.py | 39 ++++++++++++++++--- .../RetarusSecureEmailGateway.yml | 4 +- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index e840bc02dd49..811c39780a8f 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -102,12 +102,23 @@ def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: connection=ws, fetch_interval=fetch_interval ) + set_the_integration_context("last_run_results", f"Opened a connection successfully at {datetime.now().astimezone(timezone.utc)}") yield connection except Exception as e: - raise DemistoException(f"{str(e)}\nAuthentication failed. Please check the URL, channel name and access token.") + set_the_integration_context("last_run_results", + f"{str(e)} \n This error happened at {datetime.now().astimezone(timezone.utc)}") + raise DemistoException(f"{str(e)}\n") +def set_the_integration_context(key: str, val: str): + """Given a key and value the functions adds them to the integration context dict. + """ + cnx = get_integration_context() + cnx[key] = val + set_integration_context(cnx) + + def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: """This function checks if the given interval has passed since the given start time @@ -178,7 +189,13 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif def test_module(): - raise DemistoException("No test option available. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration.") + raise DemistoException("No test option available do to api limitation.") + + +def get_last_run_results_command(): + last_run_results = get_integration_context()["last_run_results"] + return CommandResults(readable_output=last_run_results) + def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: int = 10) -> list[dict]: @@ -202,26 +219,36 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: try: event = json.loads(connection.recv(timeout=recv_timeout)) except TimeoutError: - # if we didn't receive an event for `fetch_interval` seconds, finish fetching continue + except Exception as e: + set_the_integration_context("last_run_results", + f"{str(e)} \n This error happened at {datetime.now().astimezone(timezone.utc)}") + raise DemistoException(str(e)) + event_id = event.get("rmxId") event_ts = event.get("ts") if not event_ts: # if timestamp is not in the response, use the current time demisto.debug(f"{LOG_PREFIX} Event {event_id} does not have a timestamp, using current time") event_ts = datetime.now().isoformat() + date = dateparser.parse(event_ts) if not date: demisto.debug(f"{LOG_PREFIX} Event {event_id} has an invalid timestamp, using current time") # if timestamp is not in correct format, use the current time date = datetime.now() + event["_time"] = date.astimezone(timezone.utc).isoformat() event["SOURCE_LOG_TYPE"] = event.get("type") + events.append(event) event_ids.add(event_id) - - demisto.debug(f"{LOG_PREFIX} Fetched {len(events)} events") + + num_events = len(events) + demisto.debug(f"{LOG_PREFIX} Fetched {num_events} events") demisto.debug(f"{LOG_PREFIX} The fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) + set_the_integration_context("last_run_results", + f"Got from connection {num_events} events starting at {str(fetch_start_time)} untill {datetime.now().astimezone(timezone.utc)}") return events @@ -239,6 +266,8 @@ def main(): # pragma: no cover try: if command == "long-running-execution": return_results(long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl)) + elif command == "retarus-get-last-run-results": + return_results(get_last_run_results_command()) elif command == "test-module": return_results(test_module()) else: diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 2c67423ebb0a..aa4782f57ba7 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -43,7 +43,9 @@ configuration: script: script: "" type: python - commands: [] + commands: + - name: "retarus-get-last-run-results" + description: Retrieves the results of a connection attempt to Retarus, indicating whether it was successful or failed and why. If event fetching has been initiated, this command provides the results of the most recent fetch attempt. subtype: python3 dockerimage: demisto/netutils:1.0.0.115452 longRunning: true From 3fe8479016584cbf1e89d18c20f6174ca369fffa Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 08:01:51 +0200 Subject: [PATCH 19/31] code review fixes and more tests --- .../RetarusSecureEmailGateway.py | 45 +++++------ .../RetarusSecureEmailGateway_test.py | 79 ++++--------------- 2 files changed, 38 insertions(+), 86 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 811c39780a8f..d98d4bfb7d1b 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -35,7 +35,7 @@ def __init__(self, connection:Connection, fetch_interval: int = FETCH_INTERVAL_I self.fetch_interval = fetch_interval - def recv(self, timeout: float | None = None) -> Data: + def recv(self, timeout: float | None = None) -> Data: # pragma: no cover """ Receive the next message from the connection @@ -52,19 +52,20 @@ def recv(self, timeout: float | None = None) -> Data: return event - def heartbeat(self): + def heartbeat(self): # pragma: no cover """ Heartbeat thread function to periodically send keep-alives to the server. For the sake of simplicity and error prevention, keep-alives are sent regardless of the actual connection activity. """ while True: with self.lock: + demisto.debug("Locked the thread to pong the connection") self.connection.pong() time.sleep(self.idle_timeout) ''' HELPER FUNCTIONS ''' -def push_events(events: list[dict]): +def push_events(events: list[dict]): # pragma: no cover """ Push events to XSIAM. """ @@ -111,17 +112,19 @@ def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: raise DemistoException(f"{str(e)}\n") -def set_the_integration_context(key: str, val: str): - """Given a key and value the functions adds them to the integration context dict. +def set_the_integration_context(key: str, val): # pragma: no cover + """Adds a key-value pair to the integration context dictionary. + If the key already exists in the integration context, the function will overwrite the existing value with the new one. """ - cnx = get_integration_context() + cnx = demisto.getIntegrationContext() cnx[key] = val - set_integration_context(cnx) + demisto.setIntegrationContext(cnx) def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: - """This function checks if the given interval has passed since the given start time - + """Checks if the specified interval has passed since the given start time. + This function is used within the fetch_events function to determine if the time to fetch events is over or not. + Args: fetch_start_time (datetime): The start time of the interval fetch_interval (int): The interval in seconds @@ -140,22 +143,16 @@ def perform_long_running_loop(connection: EventConnection, fetch_interval: int): connection (EventConnection): A connection object to fetch events from. fetch_interval (int): Fetch time for this fetching events cycle. """ - integration_context = demisto.getIntegrationContext() events = fetch_events(connection, fetch_interval) - events.extend(integration_context.get("events", [])) - integration_context["events"] = events # update events in context in case of a failure. demisto.debug(f'{LOG_PREFIX} Adding {len(events)} Events to XSIAM') # Send the events to the XSIAM, with events from the context # Need to add the option that if we have more then one failure of sending the events to xsiam then we stop fetching. consult with Meital and Dima # TODO try: send_events_to_xsiam(events, vendor=VENDOR, product=PRODUCT) - # clear the context after sending the events - demisto.setIntegrationContext({}) except DemistoException: demisto.error(f"Failed to send events to XSIAM. Error: {traceback.format_exc()}") # save the events to the context so we can send them again in the next execution - demisto.setIntegrationContext(integration_context) ''' COMMAND FUNCTIONS ''' @@ -180,7 +177,7 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif # (sentence taken from Retarus API docs) # Setting up heartbeat daemon threads to send keep-alives if needed threading.Thread(target=connection.heartbeat, daemon=True).start() - demisto.debug(f"{LOG_PREFIX} Created connection and created heartbeat") + demisto.debug(f"{LOG_PREFIX} Created heartbeat") while True: perform_long_running_loop(connection, fetch_interval) @@ -188,16 +185,20 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif time.sleep(FETCH_SLEEP) -def test_module(): - raise DemistoException("No test option available do to api limitation.") +def test_module(): # pragma: no cover + raise DemistoException( + "No test option is available due to API limitations. To verify the configuration, run the retarus-get-last-run-results command and ensure it returns no errors.") def get_last_run_results_command(): - last_run_results = get_integration_context()["last_run_results"] - return CommandResults(readable_output=last_run_results) + last_run_results = demisto.getIntegrationContext().get("last_run_results") + if last_run_results: + return CommandResults(readable_output=last_run_results) + else: + return CommandResults(readable_output="No results from the last run yet. Ensure that a Retarus instance is configured and enabled. If it is, please wait one minute and try running the command again.") + - def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: int = 10) -> list[dict]: """ This function fetches events from the given connection, for the given fetch interval @@ -247,9 +248,9 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: num_events = len(events) demisto.debug(f"{LOG_PREFIX} Fetched {num_events} events") demisto.debug(f"{LOG_PREFIX} The fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) + set_the_integration_context("last_run_results", f"Got from connection {num_events} events starting at {str(fetch_start_time)} untill {datetime.now().astimezone(timezone.utc)}") - return events diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py index 42152fd99f90..ae51c085c388 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py @@ -3,6 +3,7 @@ from unittest.mock import Mock, patch import uuid import pytest +import demistomock as demisto from contextlib import contextmanager import RetarusSecureEmailGateway import CommonServerPython @@ -24,9 +25,9 @@ CURRENT_TIME: datetime | None = None EVENTS = [ - {"ts": "2023-08-16T13:24:12.147573+0100", "_id": "1"}, - {"ts": "2023-08-14T13:24:12.147573+0200", "_id": "2"}, - {"ts": "2023-08-12T13:24:11.147573+0000", "_id": "3"} + {"ts": "2023-08-16T13:24:12.147573+0100", "id": "1"}, + {"ts": "2023-08-14T13:24:12.147573+0200", "id": "2"}, + {"ts": "2023-08-12T13:24:11.147573+0000", "id": "3"} ] def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: @@ -68,43 +69,6 @@ def recv(self, timeout): def pong(self): self.pongs += 1 - -def test_handle_failures_of_send_events(mocker, capfd): - """ - Given: - - A connection to the websocket, and events are fetched from the socket - - When: - - Sending events to XSIAM are failing. - - Then: - - Add the failing events to the context, and try again in the next run. - """ - def fetch_events_mock(connection: EventConnection, fetch_interval: int): - return EVENTS - - def sends_events_to_xsiam_mock(events, **kwargs): - raise DemistoException("Message") - - mocker.patch.object(RetarusSecureEmailGateway, "fetch_events", side_effect=fetch_events_mock) - mocker.patch.object(RetarusSecureEmailGateway, "send_events_to_xsiam", side_effect=sends_events_to_xsiam_mock) - with capfd.disabled(): - perform_long_running_loop(EventConnection(MockConnection()), 60) - - context = demisto.getIntegrationContext() - assert context["events"] == EVENTS - - second_try_send_events_mock = mocker.patch.object(RetarusSecureEmailGateway, "send_events_to_xsiam") - with capfd.disabled(): - perform_long_running_loop(EventConnection(MockConnection()), 60) - - context = demisto.getIntegrationContext() - # check the the context is cleared - assert not context - # check that the events failed events were sent to xsiam - for event in EVENTS: - assert event in second_try_send_events_mock.call_args_list[0][0][0] - def test_heartbeat(mocker, connection): """ @@ -167,32 +131,19 @@ def test_fetch_events(mocker, connection): assert len(events) == 2 assert events[0]["_time"] == "2023-08-16T12:24:12.147573+00:00" - assert events[0]["id"] == "1" assert events[1]["_time"] == "2023-08-14T11:24:12.147573+00:00" - assert events[1]["id"] == "2" debug_logs.assert_any_call("Retarus-logs Fetched 2 events") -# def test_websocket_connection(mocker): -# """ -# Given: url, token_id, fetch_interval, channel, verify_ssl -# When: Calling websocket_connection func -# Then: -connect function is called with the right arguments -# -EventConnection object is set with the right fields -# """ -# from unittest.mock import patch -# mock_event_connection = mocker.patch('RetarusSecureEmailGateway.EventConnection', side_effect = EventConnection(None, 10, 5)) -# with mocker.patch('RetarusSecureEmailGateway.connect') as mock_connect: - -# mock_event_connection.return_value = Mock() - -# with websocket_connection("url_1", "token_id_1", 10, "channel_1", True): -# pass - -# mock_connect.assert_called_with( -# "wss://url_1/email/siem/v1/websocket?channel=channel_1", -# additional_headers={"Authorization: Bearer token_id_1"}, -# ssl=ssl.create_default_context() -# ) - +def test_get_last_run_results_command__with_results(mocker): + cnx = {"last_run_results": "results"} + mocker.patch.object(demisto, "getIntegrationContext", return_value=cnx) + res = RetarusSecureEmailGateway.get_last_run_results_command() + assert res.readable_output == "results" + + +def test_get_last_run_results_command__no_results(mocker): + mocker.patch.object(demisto, "getIntegrationContext", return_value={}) + res = RetarusSecureEmailGateway.get_last_run_results_command() + assert res.readable_output == "No results from the last run yet. Ensure that a Retarus instance is configured and enabled. If it is, please wait one minute and try running the command again." \ No newline at end of file From cd55bb7d2c71597b0fc835b4306bcbec276b0ecf Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 08:19:11 +0200 Subject: [PATCH 20/31] add troubleshooting section to readme --- .../RetarusSecureEmailGateway/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index 7feba09000e7..169d9649f71b 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -15,3 +15,17 @@ Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email | Trust any certificate (not secure) | False | 5. No test button option available due to API limitation. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. + +### Troubleshooting + +## Integration issues +If you encounter any issues with this integration, follow these steps to troubleshoot: + +Run the retarus-get-last-run-results command to obtain detailed information about the errors and problems you are facing. This command provides insights into the last execution of the integration and helps you understand the root cause of the issues. + +If you receive an HTTP 400 or HTTP 401 status code when running the command, verify the token provided in the instance configuration. + +When opening a support case, include the results you obtained from running the retarus-get-last-run-results command. + +## Only one instance can be configured on the same token and channel +Due to the Retarus API limitation, only one instance can be configured for each token and channel. It is important to note that while two instances with the same token but different channels are allowed, configuring two instances with the same token and channel may result in encountering errors and/or missing events. \ No newline at end of file From c54a8aca0249209dfe56d2d4801f4bbb40dfba6c Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 08:22:22 +0200 Subject: [PATCH 21/31] . --- .../Integrations/RetarusSecureEmailGateway/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index 169d9649f71b..b7dc84c2399f 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -12,7 +12,7 @@ Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email | Token ID | True | | | Channel name | False | The channel to fetch events from | | Fetch interval in seconds | True | | - | Trust any certificate (not secure) | False | + | Trust any certificate (not secure) | False | | 5. No test button option available due to API limitation. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. From 09ea746c271f072ad746576733b1da4d0b98b06d Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 08:23:16 +0200 Subject: [PATCH 22/31] readme improvements --- .../Integrations/RetarusSecureEmailGateway/README.md | 2 +- Packs/RetarusSecureEmailGateway/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index b7dc84c2399f..8d4a8e09fe7f 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -1,4 +1,4 @@ -Integrate RetarusSecureEmailGateway to seamlessly fetch events and enhance email security. +Integrate Retarus Secure Email Gateway to seamlessly fetch events and enhance email security. ## Configure RetarusSecureEmailGateway Event Collector on Cortex XSIAM diff --git a/Packs/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/README.md index 18958139bc1f..fa658aa52324 100644 --- a/Packs/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/README.md @@ -2,6 +2,6 @@ # Retarus Secure Email Gateway This pack includes Cortex XSIAM content. -Integrate RetarusSecureEmailGateway to seamlessly fetch events from SecureEmailGateway by Retarus and enhance email security. +Integrate Retarus Secure Email Gateway to seamlessly fetch events from Secure Email Gateway by Retarus and enhance email security. \ No newline at end of file From 6f361c13a990dceab93f145377aeb87f840f52e8 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 11:06:08 +0200 Subject: [PATCH 23/31] run pre commit --- .../RetarusSecureEmailGateway.py | 78 ++++++++++--------- .../RetarusSecureEmailGateway.yml | 3 + .../RetarusSecureEmailGateway_test.py | 17 ++-- 3 files changed, 51 insertions(+), 47 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index d98d4bfb7d1b..26de38c69cd9 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -5,7 +5,6 @@ from websockets import Data from websockets.sync.client import connect from websockets.sync.connection import Connection -from dateutil import tz import traceback import threading from contextlib import contextmanager @@ -26,16 +25,17 @@ ''' CLIENT CLASS ''' + + class EventConnection: - def __init__(self, connection:Connection, fetch_interval: int = FETCH_INTERVAL_IN_SECONDS, + def __init__(self, connection: Connection, fetch_interval: int = FETCH_INTERVAL_IN_SECONDS, idle_timeout: int = SERVER_IDLE_TIMEOUT): self.connection = connection self.lock = threading.Lock() self.idle_timeout = idle_timeout self.fetch_interval = fetch_interval - - def recv(self, timeout: float | None = None) -> Data: # pragma: no cover + def recv(self, timeout: float | None = None) -> Data: # pragma: no cover """ Receive the next message from the connection @@ -50,9 +50,8 @@ def recv(self, timeout: float | None = None) -> Data: # pragma: no cover demisto.debug("Locked the thread to recv a message") event = self.connection.recv(timeout=timeout) return event - - - def heartbeat(self): # pragma: no cover + + def heartbeat(self): # pragma: no cover """ Heartbeat thread function to periodically send keep-alives to the server. For the sake of simplicity and error prevention, keep-alives are sent regardless of the actual connection activity. @@ -65,13 +64,15 @@ def heartbeat(self): # pragma: no cover ''' HELPER FUNCTIONS ''' -def push_events(events: list[dict]): # pragma: no cover + + +def push_events(events: list[dict]): # pragma: no cover """ Push events to XSIAM. """ send_events_to_xsiam(events=events, vendor=VENDOR, product=PRODUCT) demisto.debug(f"{LOG_PREFIX} Pushed {len(events)} to XSIAM successfully") - + @contextmanager def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: str, verify_ssl: bool): @@ -89,42 +90,43 @@ def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: EventConnection: eventConnection to receive events from. """ extra_headers = {"Authorization": f"Bearer {token_id}"} - + ssl_context = ssl.create_default_context() if not verify_ssl: ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE - + try: - with connect("wss://"+url+f"/email/siem/v1/websocket?channel={channel}", + with connect("wss://" + url + f"/email/siem/v1/websocket?channel={channel}", additional_headers=extra_headers, ssl=ssl_context) as ws: connection = EventConnection( connection=ws, fetch_interval=fetch_interval ) - set_the_integration_context("last_run_results", f"Opened a connection successfully at {datetime.now().astimezone(timezone.utc)}") + set_the_integration_context( + "last_run_results", f"Opened a connection successfully at {datetime.now().astimezone(timezone.utc)}") yield connection except Exception as e: set_the_integration_context("last_run_results", - f"{str(e)} \n This error happened at {datetime.now().astimezone(timezone.utc)}") + f"{str(e)} \n This error happened at {datetime.now().astimezone(timezone.utc)}") raise DemistoException(f"{str(e)}\n") -def set_the_integration_context(key: str, val): # pragma: no cover +def set_the_integration_context(key: str, val): # pragma: no cover """Adds a key-value pair to the integration context dictionary. If the key already exists in the integration context, the function will overwrite the existing value with the new one. """ cnx = demisto.getIntegrationContext() cnx[key] = val demisto.setIntegrationContext(cnx) - - + + def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: """Checks if the specified interval has passed since the given start time. This function is used within the fetch_events function to determine if the time to fetch events is over or not. - + Args: fetch_start_time (datetime): The start time of the interval fetch_interval (int): The interval in seconds @@ -145,17 +147,17 @@ def perform_long_running_loop(connection: EventConnection, fetch_interval: int): """ events = fetch_events(connection, fetch_interval) demisto.debug(f'{LOG_PREFIX} Adding {len(events)} Events to XSIAM') - - # Send the events to the XSIAM, with events from the context - # Need to add the option that if we have more then one failure of sending the events to xsiam then we stop fetching. consult with Meital and Dima # TODO + + # Send the events to the XSIAM. try: send_events_to_xsiam(events, vendor=VENDOR, product=PRODUCT) except DemistoException: demisto.error(f"Failed to send events to XSIAM. Error: {traceback.format_exc()}") - # save the events to the context so we can send them again in the next execution ''' COMMAND FUNCTIONS ''' + + def long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl): """ Performs the long running execution loop. @@ -185,9 +187,10 @@ def long_running_execution_command(url, token_id, fetch_interval, channel, verif time.sleep(FETCH_SLEEP) -def test_module(): # pragma: no cover +def test_module(): # pragma: no cover raise DemistoException( - "No test option is available due to API limitations. To verify the configuration, run the retarus-get-last-run-results command and ensure it returns no errors.") + "No test option is available due to API limitations.\ + To verify the configuration, run the retarus-get-last-run-results command and ensure it returns no errors.") def get_last_run_results_command(): @@ -195,9 +198,9 @@ def get_last_run_results_command(): if last_run_results: return CommandResults(readable_output=last_run_results) else: - return CommandResults(readable_output="No results from the last run yet. Ensure that a Retarus instance is configured and enabled. If it is, please wait one minute and try running the command again.") - - + return CommandResults(readable_output="No results from the last run yet. Ensure that a Retarus instance \ + is configured and enabled. If it is, please wait one minute and try running the command again.") + def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: int = 10) -> list[dict]: """ @@ -215,7 +218,7 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: event_ids = set() fetch_start_time = datetime.now().astimezone(timezone.utc) demisto.debug(f'{LOG_PREFIX} Starting to fetch events at {fetch_start_time}') - + while not is_interval_passed(fetch_start_time, fetch_interval): try: event = json.loads(connection.recv(timeout=recv_timeout)) @@ -223,38 +226,41 @@ def fetch_events(connection: EventConnection, fetch_interval: int, recv_timeout: continue except Exception as e: set_the_integration_context("last_run_results", - f"{str(e)} \n This error happened at {datetime.now().astimezone(timezone.utc)}") + f"{str(e)} \n This error happened at {datetime.now().astimezone(timezone.utc)}") raise DemistoException(str(e)) - + event_id = event.get("rmxId") event_ts = event.get("ts") if not event_ts: # if timestamp is not in the response, use the current time demisto.debug(f"{LOG_PREFIX} Event {event_id} does not have a timestamp, using current time") event_ts = datetime.now().isoformat() - + date = dateparser.parse(event_ts) if not date: demisto.debug(f"{LOG_PREFIX} Event {event_id} has an invalid timestamp, using current time") # if timestamp is not in correct format, use the current time date = datetime.now() - + event["_time"] = date.astimezone(timezone.utc).isoformat() event["SOURCE_LOG_TYPE"] = event.get("type") - + events.append(event) event_ids.add(event_id) - + num_events = len(events) demisto.debug(f"{LOG_PREFIX} Fetched {num_events} events") demisto.debug(f"{LOG_PREFIX} The fetched events ids are: " + ", ".join([str(event_id) for event_id in event_ids])) - + set_the_integration_context("last_run_results", - f"Got from connection {num_events} events starting at {str(fetch_start_time)} untill {datetime.now().astimezone(timezone.utc)}") + f"Got from connection {num_events} events starting\ + at {str(fetch_start_time)} untill {datetime.now().astimezone(timezone.utc)}") return events ''' MAIN FUNCTION ''' + + def main(): # pragma: no cover command = demisto.command() params = demisto.params() diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index aa4782f57ba7..9a69aa6de6b3 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -1,6 +1,9 @@ commonfields: id: Retarus Secure Email Gateway version: -1 +sectionOrder: +- Connect +- Collect name: Retarus Secure Email Gateway display: Retarus Secure Email Gateway category: Email diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py index ae51c085c388..19068025ab14 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py @@ -1,21 +1,15 @@ import json -import io -from unittest.mock import Mock, patch import uuid import pytest import demistomock as demisto from contextlib import contextmanager import RetarusSecureEmailGateway -import CommonServerPython from RetarusSecureEmailGateway import ( fetch_events, json, demisto, datetime, timedelta, - websocket_connection, - perform_long_running_loop, - DemistoException, Connection, EventConnection, long_running_execution_command, @@ -30,6 +24,7 @@ {"ts": "2023-08-12T13:24:11.147573+0000", "id": "3"} ] + def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: global CURRENT_TIME if not CURRENT_TIME: @@ -68,7 +63,7 @@ def recv(self, timeout): def pong(self): self.pongs += 1 - + def test_heartbeat(mocker, connection): """ @@ -132,7 +127,7 @@ def test_fetch_events(mocker, connection): assert len(events) == 2 assert events[0]["_time"] == "2023-08-16T12:24:12.147573+00:00" assert events[1]["_time"] == "2023-08-14T11:24:12.147573+00:00" - + debug_logs.assert_any_call("Retarus-logs Fetched 2 events") @@ -141,9 +136,9 @@ def test_get_last_run_results_command__with_results(mocker): mocker.patch.object(demisto, "getIntegrationContext", return_value=cnx) res = RetarusSecureEmailGateway.get_last_run_results_command() assert res.readable_output == "results" - - + + def test_get_last_run_results_command__no_results(mocker): mocker.patch.object(demisto, "getIntegrationContext", return_value={}) res = RetarusSecureEmailGateway.get_last_run_results_command() - assert res.readable_output == "No results from the last run yet. Ensure that a Retarus instance is configured and enabled. If it is, please wait one minute and try running the command again." \ No newline at end of file + assert "No results from the last run yet." in res.readable_output From f824e9966112ae5a693eee6b137c4c02cb914ea2 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 12:07:42 +0200 Subject: [PATCH 24/31] build improvements --- .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.py | 5 ++--- .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml | 1 + .../RetarusSecureEmailGateway_test.py | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 26de38c69cd9..9448bc6a4157 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -21,7 +21,6 @@ SERVER_IDLE_TIMEOUT = 60 DEFAULT_CHANNEL = "default" LOG_PREFIX = "Retarus-logs" -DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' # ISO8601 format with UTC, default in XSOAR ''' CLIENT CLASS ''' @@ -29,7 +28,7 @@ class EventConnection: def __init__(self, connection: Connection, fetch_interval: int = FETCH_INTERVAL_IN_SECONDS, - idle_timeout: int = SERVER_IDLE_TIMEOUT): + idle_timeout: int = SERVER_IDLE_TIMEOUT): # pragma: no cover self.connection = connection self.lock = threading.Lock() self.idle_timeout = idle_timeout @@ -123,7 +122,7 @@ def set_the_integration_context(key: str, val): # pragma: no cover demisto.setIntegrationContext(cnx) -def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: +def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: # pragma: no cover """Checks if the specified interval has passed since the given start time. This function is used within the fetch_events function to determine if the time to fetch events is over or not. diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 9a69aa6de6b3..958d7d44fc6d 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -21,6 +21,7 @@ configuration: required: true hiddenusername: true type: 9 + section: Connect - defaultvalue: default display: Channel name section: Collect diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py index 19068025ab14..2c35dbec854e 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_test.py @@ -1,4 +1,3 @@ -import json import uuid import pytest import demistomock as demisto @@ -7,7 +6,6 @@ from RetarusSecureEmailGateway import ( fetch_events, json, - demisto, datetime, timedelta, Connection, From cd7e7f47450c6f81bdd6ef69d8684df41e7a8abe Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 12:14:40 +0200 Subject: [PATCH 25/31] build improvements --- Packs/RetarusSecureEmailGateway/Author_image.png | 0 .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.py | 2 +- .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 Packs/RetarusSecureEmailGateway/Author_image.png diff --git a/Packs/RetarusSecureEmailGateway/Author_image.png b/Packs/RetarusSecureEmailGateway/Author_image.png deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 9448bc6a4157..eb409ceb793a 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -74,7 +74,7 @@ def push_events(events: list[dict]): # pragma: no cover @contextmanager -def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: str, verify_ssl: bool): +def websocket_connection(url: str, token_id: str, fetch_interval: int, channel: str, verify_ssl: bool): # pragma: no cover """ Create a connection to the api. diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 958d7d44fc6d..6041a0a281c3 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -44,6 +44,7 @@ configuration: name: insecure type: 8 required: false + section: Connect script: script: "" type: python From 9d535915cec2220e271666d23f7e0a4bef408ad1 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 22 Jan 2025 12:23:43 +0200 Subject: [PATCH 26/31] build --- .../Integrations/RetarusSecureEmailGateway/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index 8d4a8e09fe7f..1d985a1d6987 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -18,14 +18,10 @@ Integrate Retarus Secure Email Gateway to seamlessly fetch events and enhance em ### Troubleshooting -## Integration issues If you encounter any issues with this integration, follow these steps to troubleshoot: - Run the retarus-get-last-run-results command to obtain detailed information about the errors and problems you are facing. This command provides insights into the last execution of the integration and helps you understand the root cause of the issues. - If you receive an HTTP 400 or HTTP 401 status code when running the command, verify the token provided in the instance configuration. When opening a support case, include the results you obtained from running the retarus-get-last-run-results command. -## Only one instance can be configured on the same token and channel -Due to the Retarus API limitation, only one instance can be configured for each token and channel. It is important to note that while two instances with the same token but different channels are allowed, configuring two instances with the same token and channel may result in encountering errors and/or missing events. \ No newline at end of file +Note, due to the Retarus API limitation, only one instance can be configured for each token and channel. It is important to note that while two instances with the same token but different channels are allowed, configuring two instances with the same token and channel may result in encountering errors and/or missing events. \ No newline at end of file From 5b0cd71994b5d1cc7c7442ae859c8d3eafb07225 Mon Sep 17 00:00:00 2001 From: yshamai Date: Thu, 23 Jan 2025 20:19:37 +0200 Subject: [PATCH 27/31] docs improvements --- .../Integrations/RetarusSecureEmailGateway/README.md | 6 +++--- .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml | 4 ++-- .../RetarusSecureEmailGateway_description.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index 1d985a1d6987..c979dbfe6b07 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -3,7 +3,7 @@ Integrate Retarus Secure Email Gateway to seamlessly fetch events and enhance em ## Configure RetarusSecureEmailGateway Event Collector on Cortex XSIAM 1. Navigate to **Settings** > **Integrations** > **Servers & Services**. -2. Search for RetarusSecureEmailGateway. +2. Search for Retarus. 3. Click **Add instance** to create and configure a new integration instance. | **Parameter** | **Required** | **Description** | @@ -14,7 +14,7 @@ Integrate Retarus Secure Email Gateway to seamlessly fetch events and enhance em | Fetch interval in seconds | True | | | Trust any certificate (not secure) | False | | -5. No test button option available due to API limitation. Save the configured instance to test the connection. If you encounter an 'Authentication failed' error, check your configuration. +5. There is no test option available due to API limitations. To verify the configuration, run the retarus-get-last-run-results command and ensure it returns no errors." ### Troubleshooting @@ -24,4 +24,4 @@ If you receive an HTTP 400 or HTTP 401 status code when running the command, ver When opening a support case, include the results you obtained from running the retarus-get-last-run-results command. -Note, due to the Retarus API limitation, only one instance can be configured for each token and channel. It is important to note that while two instances with the same token but different channels are allowed, configuring two instances with the same token and channel may result in encountering errors and/or missing events. \ No newline at end of file +Note, due to the Retarus API limitation, only one instance can be configured for each token and channel. It is important to note that while two instances with the same token but different channels are allowed, configuring two instances with the same token and channel may result in errors and/or missing events. diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml index 6041a0a281c3..f895a2ea270d 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.yml @@ -7,7 +7,7 @@ sectionOrder: name: Retarus Secure Email Gateway display: Retarus Secure Email Gateway category: Email -description: Integrate RetarusSecureEmailGateway to seamlessly fetch events from SecureEmailGateway by Retarus and enhance email security. +description: Integrate Retarus Secure Email Gateway to seamlessly fetch events from Secure Email Gateway by Retarus and enhance email security. configuration: - defaultvalue: events.retarus.com display: Server URL @@ -16,7 +16,7 @@ configuration: type: 0 section: Connect - displaypassword: Token ID - additionalinfo: The Token ID to use for connection + additionalinfo: The Token ID to use for connection. name: credentials required: true hiddenusername: true diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md index aad7937a5a5e..a998ef51bcf7 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md @@ -1,4 +1,4 @@ ## Retarus Integration help -Provide the api token that Retarus has provided to you. +Provide the API token Retarus created for you. Provide the channel you created or use the default channel. \ No newline at end of file From 6907a5e950fb398ba5785321a905cbee3e5af34f Mon Sep 17 00:00:00 2001 From: yshamai Date: Tue, 28 Jan 2025 13:07:41 +0200 Subject: [PATCH 28/31] code review improvements --- .../Integrations/RetarusSecureEmailGateway/README.md | 4 +--- .../RetarusSecureEmailGateway.py | 9 +++++++-- .../RetarusSecureEmailGateway_description.md | 4 +++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md index c979dbfe6b07..bb775e4a7ca0 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/README.md @@ -10,12 +10,10 @@ Integrate Retarus Secure Email Gateway to seamlessly fetch events and enhance em | --- | --- | --- | | Server URL | True | | | Token ID | True | | - | Channel name | False | The channel to fetch events from | + | Channel name | False | The channel to fetch events from. In Retarus, a channel name represents a specific configuration or processing pipeline used to manage email traffic based on criteria like sender, recipient, domain, or metadata, enabling tailored routing, filtering, compliance, and logging rules.| | Fetch interval in seconds | True | | | Trust any certificate (not secure) | False | | -5. There is no test option available due to API limitations. To verify the configuration, run the retarus-get-last-run-results command and ensure it returns no errors." - ### Troubleshooting If you encounter any issues with this integration, follow these steps to troubleshoot: diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index eb409ceb793a..7b5d1a621ee1 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -133,8 +133,9 @@ def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: Returns: bool: True if the interval has passed, False otherwise """ - return fetch_start_time + timedelta(seconds=fetch_interval) < datetime.now().astimezone(timezone.utc) - + is_interval_passed = fetch_start_time + timedelta(seconds=fetch_interval) < datetime.now().astimezone(timezone.utc) + demisto.debug(f"returning {is_interval_passed=}") + return is_interval_passed def perform_long_running_loop(connection: EventConnection, fetch_interval: int): """ @@ -144,12 +145,14 @@ def perform_long_running_loop(connection: EventConnection, fetch_interval: int): connection (EventConnection): A connection object to fetch events from. fetch_interval (int): Fetch time for this fetching events cycle. """ + demisto.debug(f"{LOG_PREFIX} starting to fetch events") events = fetch_events(connection, fetch_interval) demisto.debug(f'{LOG_PREFIX} Adding {len(events)} Events to XSIAM') # Send the events to the XSIAM. try: send_events_to_xsiam(events, vendor=VENDOR, product=PRODUCT) + demisto.debug("Sended events to XSIAM successfully") except DemistoException: demisto.error(f"Failed to send events to XSIAM. Error: {traceback.format_exc()}") @@ -269,6 +272,8 @@ def main(): # pragma: no cover verify_ssl = argToBoolean(not params.get("insecure", False)) channel = params.get("channel", DEFAULT_CHANNEL) + demisto.debug(f"{LOG_PREFIX} command being called is {command}") + try: if command == "long-running-execution": return_results(long_running_execution_command(url, token_id, fetch_interval, channel, verify_ssl)) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md index a998ef51bcf7..4b8ab8f29e1c 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway_description.md @@ -1,4 +1,6 @@ ## Retarus Integration help Provide the API token Retarus created for you. -Provide the channel you created or use the default channel. \ No newline at end of file + +Provide the channel you created or use the default channel. +In Retarus, a channel name represents a specific configuration or processing pipeline used to manage email traffic based on criteria like sender, recipient, domain, or metadata, enabling tailored routing, filtering, compliance, and logging rules. \ No newline at end of file From e66df89b7496329a1a08cdba5efa6f783e510a92 Mon Sep 17 00:00:00 2001 From: yshamai Date: Tue, 28 Jan 2025 19:36:52 +0200 Subject: [PATCH 29/31] pre commit --- .../RetarusSecureEmailGateway/RetarusSecureEmailGateway.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py index 7b5d1a621ee1..067f9f5e83cf 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/RetarusSecureEmailGateway.py @@ -133,10 +133,11 @@ def is_interval_passed(fetch_start_time: datetime, fetch_interval: int) -> bool: Returns: bool: True if the interval has passed, False otherwise """ - is_interval_passed = fetch_start_time + timedelta(seconds=fetch_interval) < datetime.now().astimezone(timezone.utc) + is_interval_passed = fetch_start_time + timedelta(seconds=fetch_interval) < datetime.now().astimezone(timezone.utc) demisto.debug(f"returning {is_interval_passed=}") return is_interval_passed + def perform_long_running_loop(connection: EventConnection, fetch_interval: int): """ Long running loop iteration function. Fetches events from the connection and sends them to XSIAM. From 949134640c639b8de585790e9f792942a4742c1d Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 29 Jan 2025 09:16:29 +0200 Subject: [PATCH 30/31] add command example --- .../Integrations/RetarusSecureEmailGateway/command_examples | 1 + 1 file changed, 1 insertion(+) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples index e69de29bb2d1..06c7ddabe011 100644 --- a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples +++ b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples @@ -0,0 +1 @@ +!retarus-get-last-run-results \ No newline at end of file From 66f82f96927698c418b0e202c1573b197c0e2b13 Mon Sep 17 00:00:00 2001 From: yshamai Date: Wed, 29 Jan 2025 09:21:28 +0200 Subject: [PATCH 31/31] change command examples file name --- .../{command_examples => command_examples.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/{command_examples => command_examples.txt} (100%) diff --git a/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples b/Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples.txt similarity index 100% rename from Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples rename to Packs/RetarusSecureEmailGateway/Integrations/RetarusSecureEmailGateway/command_examples.txt