From cf4736109496b5f2b66e59ac7e752fab156d0560 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Fri, 1 Nov 2024 16:42:15 -0400 Subject: [PATCH 01/21] Add init support for SF log stream --- aws/logs_monitoring/customized_log_group.py | 5 +++++ aws/logs_monitoring/steps/enums.py | 1 - aws/logs_monitoring/steps/handlers/awslogs_handler.py | 8 +++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/aws/logs_monitoring/customized_log_group.py b/aws/logs_monitoring/customized_log_group.py index ad63a952..475d66e3 100644 --- a/aws/logs_monitoring/customized_log_group.py +++ b/aws/logs_monitoring/customized_log_group.py @@ -26,6 +26,11 @@ def is_lambda_customized_log_group(logstream_name): ) +# For both default and customzied Step Functions log groups, the log_stream starts with "states/" +def is_step_functions_log_group(logstream_name): + return logstream_name.startswith("states/") + + def get_lambda_function_name_from_logstream_name(logstream_name): try: # Not match the pattern for customized Lambda log group diff --git a/aws/logs_monitoring/steps/enums.py b/aws/logs_monitoring/steps/enums.py index 7a25a490..c81655bf 100644 --- a/aws/logs_monitoring/steps/enums.py +++ b/aws/logs_monitoring/steps/enums.py @@ -136,7 +136,6 @@ def __init__(self, string, event_source): RDS = ("/aws/rds", AwsEventSource.RDS) # e.g. sns/us-east-1/123456779121/SnsTopicX SNS = ("sns/", AwsEventSource.SNS) - STEPFUNCTION = ("/aws/vendedlogs/states", AwsEventSource.STEPFUNCTION) TRANSITGATEWAY = ("tgw-attach", AwsEventSource.TRANSITGATEWAY) def __str__(self): diff --git a/aws/logs_monitoring/steps/handlers/awslogs_handler.py b/aws/logs_monitoring/steps/handlers/awslogs_handler.py index a3f46aad..683210ab 100644 --- a/aws/logs_monitoring/steps/handlers/awslogs_handler.py +++ b/aws/logs_monitoring/steps/handlers/awslogs_handler.py @@ -13,6 +13,7 @@ ) from customized_log_group import ( is_lambda_customized_log_group, + is_step_functions_log_group, get_lambda_function_name_from_logstream_name, ) from steps.handlers.aws_attributes import AwsAttributes @@ -108,6 +109,10 @@ def set_source(self, event): if is_lambda_customized_log_group(log_stream): self.metadata[DD_SOURCE] = str(AwsEventSource.LAMBDA) + # Special handling for Step Functions log groups. Regardless of whether the log group is customized, the corresponding log stream starts with 'states/'." + if is_step_functions_log_group(log_stream): + self.metadata[DD_SOURCE] = str(AwsEventSource.STEPFUNCTION) + def add_cloudwatch_tags_from_cache(self): log_group_arn = self.aws_attributes.get_log_group_arn() formatted_tags = self.cache_layer.get_cloudwatch_log_group_tags_cache().get( @@ -159,9 +164,6 @@ def handle_rds_source(self): ) def handle_step_function_source(self): - if not self.aws_attributes.get_log_stream().startswith("states/"): - return - state_machine_arn = self.get_state_machine_arn() if not state_machine_arn: return From a28d570d820641eea6f38fcf2e4b39ff03ef2d10 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 10:57:26 -0500 Subject: [PATCH 02/21] Remove unnecessary unit tests --- aws/logs_monitoring/tests/test_parsing.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/aws/logs_monitoring/tests/test_parsing.py b/aws/logs_monitoring/tests/test_parsing.py index bb2ed83c..f5267dd8 100644 --- a/aws/logs_monitoring/tests/test_parsing.py +++ b/aws/logs_monitoring/tests/test_parsing.py @@ -295,14 +295,6 @@ def test_carbon_black_event(self): str(AwsEventSource.CARBONBLACK), ) - def test_step_function_event(self): - self.assertEqual( - parse_event_source( - {"awslogs": "logs"}, "/aws/vendedlogs/states/MyStateMachine-Logs" - ), - str(AwsEventSource.STEPFUNCTION), - ) - def test_cloudwatch_source_if_none_found(self): self.assertEqual( parse_event_source({"awslogs": "logs"}, ""), str(AwsEventSource.CLOUDWATCH) From 81f6c6ced049c6591d29541293b1ebf4e1d9636e Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 15:57:26 -0500 Subject: [PATCH 03/21] Test handle log for SF --- .../steps/handlers/awslogs_handler.py | 10 +++++----- aws/logs_monitoring/tests/test_awslogs_handler.py | 7 +++++++ .../tests/test_customized_log_group.py | 12 ++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/aws/logs_monitoring/steps/handlers/awslogs_handler.py b/aws/logs_monitoring/steps/handlers/awslogs_handler.py index 683210ab..8e4dd85d 100644 --- a/aws/logs_monitoring/steps/handlers/awslogs_handler.py +++ b/aws/logs_monitoring/steps/handlers/awslogs_handler.py @@ -103,13 +103,13 @@ def set_source(self, event): source = str(AwsEventSource.BEDROCK) self.metadata[DD_SOURCE] = parse_event_source(event, source) - # Special handling for customized log group of Lambda functions - # Multiple Lambda functions can share one single customized log group - # Need to parse logStream name to determine whether it is a Lambda function + # Special handling for customized log group of Lambda Functions and Step Functions + # Prefer to place the handling of customized log group at the bottom so that it can correct the source in some edge cases + # Multiple functions can share one single customized log group + # Need to parse logStream name to determine if is_lambda_customized_log_group(log_stream): self.metadata[DD_SOURCE] = str(AwsEventSource.LAMBDA) - - # Special handling for Step Functions log groups. Regardless of whether the log group is customized, the corresponding log stream starts with 'states/'." + # Regardless of whether the log group is customized, the corresponding log stream starts with 'states/'." if is_step_functions_log_group(log_stream): self.metadata[DD_SOURCE] = str(AwsEventSource.STEPFUNCTION) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 6c53670b..9edf3781 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -8,6 +8,8 @@ from approvaltests.approvals import verify_as_json from approvaltests.namer import NamerFactory +from aws.logs_monitoring.steps.enums import AwsEventSource + sys.modules["trace_forwarder.connection"] = MagicMock() sys.modules["datadog_lambda.wrapper"] = MagicMock() sys.modules["datadog_lambda.metric"] = MagicMock() @@ -22,6 +24,7 @@ }, ) env_patch.start() +from aws.logs_monitoring.settings import DD_SOURCE from steps.handlers.awslogs_handler import AwsLogsHandler from steps.handlers.aws_attributes import AwsAttributes from caching.cache_layer import CacheLayer @@ -117,6 +120,10 @@ def test_awslogs_handler_step_functions_tags_added_properly( awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) verify_as_json(list(awslogs_handler.handle(event))) verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) + self.assertEqual(awslogs_handler.metadata[DD_SOURCE], AwsEventSource.StepFunction); + self.assertEqual( + awslogs_handler.metadata[DD_SOURCE], AwsEventSource.StepFunction + ) def test_process_lambda_logs(self): # Non Lambda log diff --git a/aws/logs_monitoring/tests/test_customized_log_group.py b/aws/logs_monitoring/tests/test_customized_log_group.py index 9765ad29..2ab1a0f1 100644 --- a/aws/logs_monitoring/tests/test_customized_log_group.py +++ b/aws/logs_monitoring/tests/test_customized_log_group.py @@ -2,6 +2,7 @@ from customized_log_group import ( is_lambda_customized_log_group, get_lambda_function_name_from_logstream_name, + is_step_functions_log_group, ) @@ -58,3 +59,14 @@ def get_lambda_function_name_from_logstream_name(self): get_lambda_function_name_from_logstream_name(stepfunction_log_stream_name), None, ) + + def test_is_step_functions_log_group(self): + # Lambda logstream is false + lambda_log_stream_name = "2023/11/04/[$LATEST]4426346c2cdf4c54a74d3bd2b929fc44" + self.assertEqual(is_step_functions_log_group(lambda_log_stream_name), False) + + # SF logstream is true + step_functions_log_stream_name = "states/selfmonit-statemachine/2024-11-04-15-30/00000000" + self.assertEqual( + is_step_functions_log_group(step_functions_log_stream_name), True + ) From 4679d94d3b50aeed92be51afb3dc4363bd0e48e6 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:01:43 -0500 Subject: [PATCH 04/21] lint --- aws/logs_monitoring/tests/test_awslogs_handler.py | 4 +++- aws/logs_monitoring/tests/test_customized_log_group.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 9edf3781..499b556b 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -120,7 +120,9 @@ def test_awslogs_handler_step_functions_tags_added_properly( awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) verify_as_json(list(awslogs_handler.handle(event))) verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) - self.assertEqual(awslogs_handler.metadata[DD_SOURCE], AwsEventSource.StepFunction); + self.assertEqual( + awslogs_handler.metadata[DD_SOURCE], AwsEventSource.StepFunction + ) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.StepFunction ) diff --git a/aws/logs_monitoring/tests/test_customized_log_group.py b/aws/logs_monitoring/tests/test_customized_log_group.py index 2ab1a0f1..7074611b 100644 --- a/aws/logs_monitoring/tests/test_customized_log_group.py +++ b/aws/logs_monitoring/tests/test_customized_log_group.py @@ -66,7 +66,9 @@ def test_is_step_functions_log_group(self): self.assertEqual(is_step_functions_log_group(lambda_log_stream_name), False) # SF logstream is true - step_functions_log_stream_name = "states/selfmonit-statemachine/2024-11-04-15-30/00000000" + step_functions_log_stream_name = ( + "states/selfmonit-statemachine/2024-11-04-15-30/00000000" + ) self.assertEqual( is_step_functions_log_group(step_functions_log_stream_name), True ) From bfe8c96b965eedc1c2b5ed65f714805ffd111e2a Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:08:25 -0500 Subject: [PATCH 05/21] Fix error --- aws/logs_monitoring/tests/test_awslogs_handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 499b556b..38f40e2c 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -121,10 +121,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( verify_as_json(list(awslogs_handler.handle(event))) verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) self.assertEqual( - awslogs_handler.metadata[DD_SOURCE], AwsEventSource.StepFunction - ) - self.assertEqual( - awslogs_handler.metadata[DD_SOURCE], AwsEventSource.StepFunction + awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION ) def test_process_lambda_logs(self): From a3647c538cd6e2c55fca197390e88a446f729fd6 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:10:41 -0500 Subject: [PATCH 06/21] Assert python enum value --- aws/logs_monitoring/tests/test_awslogs_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 38f40e2c..be16ebd8 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -121,7 +121,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( verify_as_json(list(awslogs_handler.handle(event))) verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) self.assertEqual( - awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION + awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) def test_process_lambda_logs(self): From f95134853106e05df5d230f71ec3a87af6140dc7 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:19:33 -0500 Subject: [PATCH 07/21] Add more unit test --- .../tests/test_awslogs_handler.py | 71 ++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index be16ebd8..49f5d824 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -24,7 +24,7 @@ }, ) env_patch.start() -from aws.logs_monitoring.settings import DD_SOURCE +from aws.logs_monitoring.settings import DD_HOST, DD_SOURCE from steps.handlers.awslogs_handler import AwsLogsHandler from steps.handlers.aws_attributes import AwsAttributes from caching.cache_layer import CacheLayer @@ -124,6 +124,75 @@ def test_awslogs_handler_step_functions_tags_added_properly( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) + # SF customized log group + eventFromCustomizedLogGroup = { + "awslogs": { + "data": base64.b64encode( + gzip.compress( + bytes( + json.dumps( + { + "messageType": "DATA_MESSAGE", + "owner": "425362996713", + "logGroup": "test/logs", + "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", + "subscriptionFilters": ["testFilter"], + "logEvents": [ + { + "id": "37199773595581154154810589279545129148442535997644275712", + "timestamp": 1668095539607, + "message": '{"execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', + } + ], + } + ), + "utf-8", + ) + ) + ) + } + } + awslogs_handler.handle(event) + self.assertEqual( + awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value + ) + self.assertEqual( + awslogs_handler.metadata[DD_HOST], + "arn:aws:states:us-east-1:12345678910:statemachine:stepfunction1", + ) + + eventFromCustomizedLogGroup = { + "awslogs": { + "data": base64.b64encode( + gzip.compress( + bytes( + json.dumps( + { + "messageType": "DATA_MESSAGE", + "owner": "425362996713", + "logGroup": "test/logs", + "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", + "subscriptionFilters": ["testFilter"], + "logEvents": [ + { + "id": "37199773595581154154810589279545129148442535997644275712", + "timestamp": 1668095539607, + "message": '{"execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', + } + ], + } + ), + "utf-8", + ) + ) + ) + } + } + awslogs_handler.handle(eventFromCustomizedLogGroup) + self.assertNotEqual( + awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value + ) + def test_process_lambda_logs(self): # Non Lambda log stepfunction_loggroup = { From acb99c60ff61f41d5d9d66425cfaaafc04e897dc Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:23:52 -0500 Subject: [PATCH 08/21] fix unit test error --- aws/logs_monitoring/tests/test_awslogs_handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 49f5d824..65b0ab59 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -123,6 +123,10 @@ def test_awslogs_handler_step_functions_tags_added_properly( self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) + self.assertEqual( + awslogs_handler.metadata[DD_HOST], + "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", + ) # SF customized log group eventFromCustomizedLogGroup = { @@ -158,7 +162,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) self.assertEqual( awslogs_handler.metadata[DD_HOST], - "arn:aws:states:us-east-1:12345678910:statemachine:stepfunction1", + "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", ) eventFromCustomizedLogGroup = { From 158aeb662fcc78e57c521934b2d8420de1b2dda6 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:27:44 -0500 Subject: [PATCH 09/21] test error fix --- aws/logs_monitoring/tests/test_awslogs_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 65b0ab59..d8d49d1a 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -175,7 +175,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( "messageType": "DATA_MESSAGE", "owner": "425362996713", "logGroup": "test/logs", - "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", + "logStream": "state/states/2022-11-10-15-50/7851b2d9", "subscriptionFilters": ["testFilter"], "logEvents": [ { From ad3c71568aeca5814dfe07678b53f14bd812f194 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:32:01 -0500 Subject: [PATCH 10/21] Fix test error --- aws/logs_monitoring/tests/test_awslogs_handler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index d8d49d1a..24419441 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -145,7 +145,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( { "id": "37199773595581154154810589279545129148442535997644275712", "timestamp": 1668095539607, - "message": '{"execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', + "message": '{"execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction2:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', } ], } @@ -156,13 +156,14 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) } } + awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) awslogs_handler.handle(event) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) self.assertEqual( awslogs_handler.metadata[DD_HOST], - "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", + "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction2", ) eventFromCustomizedLogGroup = { @@ -181,7 +182,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( { "id": "37199773595581154154810589279545129148442535997644275712", "timestamp": 1668095539607, - "message": '{"execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', + "message": '{"execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction2:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', } ], } @@ -192,6 +193,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) } } + awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) awslogs_handler.handle(eventFromCustomizedLogGroup) self.assertNotEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value From a2f33c054a6fefd7de4879cd08032aed3c468dc6 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:34:29 -0500 Subject: [PATCH 11/21] . --- aws/logs_monitoring/tests/test_awslogs_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 24419441..ae30730b 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -157,7 +157,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( } } awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) - awslogs_handler.handle(event) + awslogs_handler.handle(eventFromCustomizedLogGroup) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) From 9a871cc12c770c76ff50f02cec729de05b867fb9 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:40:41 -0500 Subject: [PATCH 12/21] update metadata --- aws/logs_monitoring/tests/test_awslogs_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index ae30730b..e59a8a1d 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -156,7 +156,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) } } - awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) + awslogs_handler.metadata = {} awslogs_handler.handle(eventFromCustomizedLogGroup) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value @@ -193,7 +193,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) } } - awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) + awslogs_handler.metadata = {} awslogs_handler.handle(eventFromCustomizedLogGroup) self.assertNotEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value From 458b4bee9c4731ac0cd5a5baecea738dd6718335 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:45:12 -0500 Subject: [PATCH 13/21] fix test error --- aws/logs_monitoring/tests/test_awslogs_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index e59a8a1d..0c5a2c47 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -156,7 +156,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) } } - awslogs_handler.metadata = {} + awslogs_handler.metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} awslogs_handler.handle(eventFromCustomizedLogGroup) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value @@ -193,7 +193,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) } } - awslogs_handler.metadata = {} + awslogs_handler.metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} awslogs_handler.handle(eventFromCustomizedLogGroup) self.assertNotEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value From ebc2349367d75cb46ff18187c3637101d58e3627 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Mon, 4 Nov 2024 16:49:13 -0500 Subject: [PATCH 14/21] Create a new test --- .../tests/test_awslogs_handler.py | 53 +++++++------------ 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 0c5a2c47..79b80d25 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -128,6 +128,14 @@ def test_awslogs_handler_step_functions_tags_added_properly( "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", ) + @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.__init__") + @patch("caching.cloudwatch_log_group_cache.send_forwarder_internal_metrics") + @patch.dict("os.environ", {"DD_STEP_FUNCTIONS_TRACE_ENABLED": "true"}) + def test_awslogs_handler_step_functions_customized_log_group_properly( + self, + mock_forward_metrics, + mock_cache_init, + ): # SF customized log group eventFromCustomizedLogGroup = { "awslogs": { @@ -156,7 +164,17 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) } } - awslogs_handler.metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} + context = None + metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} + mock_forward_metrics.side_effect = MagicMock() + mock_cache_init.return_value = None + cache_layer = CacheLayer("") + cache_layer._step_functions_cache.get = MagicMock( + return_value=["test_tag_key:test_tag_value"] + ) + cache_layer._cloudwatch_log_group_cache.get = MagicMock() + + awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) awslogs_handler.handle(eventFromCustomizedLogGroup) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value @@ -166,39 +184,6 @@ def test_awslogs_handler_step_functions_tags_added_properly( "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction2", ) - eventFromCustomizedLogGroup = { - "awslogs": { - "data": base64.b64encode( - gzip.compress( - bytes( - json.dumps( - { - "messageType": "DATA_MESSAGE", - "owner": "425362996713", - "logGroup": "test/logs", - "logStream": "state/states/2022-11-10-15-50/7851b2d9", - "subscriptionFilters": ["testFilter"], - "logEvents": [ - { - "id": "37199773595581154154810589279545129148442535997644275712", - "timestamp": 1668095539607, - "message": '{"execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction2:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', - } - ], - } - ), - "utf-8", - ) - ) - ) - } - } - awslogs_handler.metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} - awslogs_handler.handle(eventFromCustomizedLogGroup) - self.assertNotEqual( - awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value - ) - def test_process_lambda_logs(self): # Non Lambda log stepfunction_loggroup = { From 42ddc381146c4c6b7e4a07a7a2261a8c78630a5a Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Tue, 5 Nov 2024 10:07:29 -0500 Subject: [PATCH 15/21] Tests failure --- aws/logs_monitoring/tests/test_awslogs_handler.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 79b80d25..39e43d4d 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -96,7 +96,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( { "id": "37199773595581154154810589279545129148442535997644275712", "timestamp": 1668095539607, - "message": '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', + "message": '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction3:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', } ], } @@ -125,13 +125,12 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) self.assertEqual( awslogs_handler.metadata[DD_HOST], - "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", + "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction3", ) @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.__init__") @patch("caching.cloudwatch_log_group_cache.send_forwarder_internal_metrics") - @patch.dict("os.environ", {"DD_STEP_FUNCTIONS_TRACE_ENABLED": "true"}) - def test_awslogs_handler_step_functions_customized_log_group_properly( + def test_awslogs_handler_step_functions_customized_log_group( self, mock_forward_metrics, mock_cache_init, @@ -165,7 +164,7 @@ def test_awslogs_handler_step_functions_customized_log_group_properly( } } context = None - metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} + metadata = {"ddtags": "env:dev"} mock_forward_metrics.side_effect = MagicMock() mock_cache_init.return_value = None cache_layer = CacheLayer("") @@ -175,7 +174,7 @@ def test_awslogs_handler_step_functions_customized_log_group_properly( cache_layer._cloudwatch_log_group_cache.get = MagicMock() awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) - awslogs_handler.handle(eventFromCustomizedLogGroup) + verify_as_json(list(awslogs_handler.handle(eventFromCustomizedLogGroup))) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) From cdeef41a5693d6d223944e9436de83176f037750 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Tue, 5 Nov 2024 10:21:32 -0500 Subject: [PATCH 16/21] test again --- aws/logs_monitoring/tests/test_awslogs_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 39e43d4d..44bcea8a 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -96,7 +96,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( { "id": "37199773595581154154810589279545129148442535997644275712", "timestamp": 1668095539607, - "message": '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction3:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', + "message": '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', } ], } @@ -125,7 +125,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( ) self.assertEqual( awslogs_handler.metadata[DD_HOST], - "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction3", + "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", ) @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.__init__") @@ -174,7 +174,7 @@ def test_awslogs_handler_step_functions_customized_log_group( cache_layer._cloudwatch_log_group_cache.get = MagicMock() awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) - verify_as_json(list(awslogs_handler.handle(eventFromCustomizedLogGroup))) + awslogs_handler.handle(eventFromCustomizedLogGroup) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) From 5f21538bffe95a7c3b253ed0bece22ae77829e35 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Tue, 5 Nov 2024 11:53:24 -0500 Subject: [PATCH 17/21] Test --- aws/logs_monitoring/tests/test_awslogs_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 44bcea8a..12d59917 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -96,7 +96,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( { "id": "37199773595581154154810589279545129148442535997644275712", "timestamp": 1668095539607, - "message": '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', + "message": '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction3:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', } ], } @@ -118,14 +118,14 @@ def test_awslogs_handler_step_functions_tags_added_properly( cache_layer._cloudwatch_log_group_cache.get = MagicMock() awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) - verify_as_json(list(awslogs_handler.handle(event))) - verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) + # verify_as_json(list(awslogs_handler.handle(event))) + # verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) self.assertEqual( awslogs_handler.metadata[DD_HOST], - "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", + "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction3", ) @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.__init__") From 65a9c500580545db88e34f441d60ee2b9c4e7c3c Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Tue, 5 Nov 2024 11:58:41 -0500 Subject: [PATCH 18/21] test --- aws/logs_monitoring/tests/test_awslogs_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 12d59917..c830d61b 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -118,6 +118,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( cache_layer._cloudwatch_log_group_cache.get = MagicMock() awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) + awslogs_handler.handle(event) # verify_as_json(list(awslogs_handler.handle(event))) # verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) self.assertEqual( From 7c7992fe06937e416a38b6a7c77b359be4f354a1 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Tue, 5 Nov 2024 12:03:35 -0500 Subject: [PATCH 19/21] test --- ...handler_step_functions_tags_added_properly.approved.json | 2 +- ...tep_functions_tags_added_properly.metadata.approved.json | 2 +- aws/logs_monitoring/tests/test_awslogs_handler.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json index 2dd05043..7e90adfc 100644 --- a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json @@ -8,7 +8,7 @@ } }, "id": "37199773595581154154810589279545129148442535997644275712", - "message": "{\"id\": \"1\",\"type\": \"ExecutionStarted\",\"details\": {\"input\": \"{}\",\"inputDetails\": {\"truncated\": \"false\"},\"roleArn\": \"arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt\"},\"previous_event_id\": \"0\",\"event_timestamp\": \"1716992192441\",\"execution_arn\": \"arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa\",\"redrive_count\": \"0\"}", + "message": "{\"id\": \"1\",\"type\": \"ExecutionStarted\",\"details\": {\"input\": \"{}\",\"inputDetails\": {\"truncated\": \"false\"},\"roleArn\": \"arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt\"},\"previous_event_id\": \"0\",\"event_timestamp\": \"1716992192441\",\"execution_arn\": \"arn:aws:states:us-east-1:12345678910:execution:StepFunction3:ccccccc-d1da-4c38-b32c-2b6b07d713fa\",\"redrive_count\": \"0\"}", "timestamp": 1668095539607 } ] diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json index 0310f084..cd9098d7 100644 --- a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json @@ -1,6 +1,6 @@ { "ddsource": "stepfunction", "ddtags": "env:dev,test_tag_key:test_tag_value,dd_step_functions_trace_enabled:true", - "host": "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", + "host": "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction3", "service": "stepfunction" } diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index c830d61b..3d5a5211 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -118,9 +118,9 @@ def test_awslogs_handler_step_functions_tags_added_properly( cache_layer._cloudwatch_log_group_cache.get = MagicMock() awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) - awslogs_handler.handle(event) - # verify_as_json(list(awslogs_handler.handle(event))) - # verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) + # awslogs_handler.handle(event) + verify_as_json(list(awslogs_handler.handle(event))) + verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) From 66c87082e5d3480eb406451973037896e85ffde3 Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Tue, 5 Nov 2024 12:31:53 -0500 Subject: [PATCH 20/21] Correct tests --- ...andler_step_functions_customized_log_group.json | 14 ++++++++++++++ ...ons_customized_log_group.metadata.approved.json | 6 ++++++ aws/logs_monitoring/tests/test_awslogs_handler.py | 5 +++-- 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.json create mode 100644 aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.metadata.approved.json diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.json new file mode 100644 index 00000000..3e49300f --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.json @@ -0,0 +1,14 @@ +[ + { + "aws": { + "awslogs": { + "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", + "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", + "owner": "425362996713" + } + }, + "id": "37199773595581154154810589279545129148442535997644275712", + "message": "{\"id\": \"1\",\"type\": \"ExecutionStarted\",\"details\": {\"input\": \"{}\",\"inputDetails\": {\"truncated\": \"false\"},\"roleArn\": \"arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt\"},\"previous_event_id\": \"0\",\"event_timestamp\": \"1716992192441\",\"execution_arn\": \"arn:aws:states:us-east-1:12345678910:execution:StepFunction2:ccccccc-d1da-4c38-b32c-2b6b07d713fa\",\"redrive_count\": \"0\"}", + "timestamp": 1668095539607 + } +] diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.metadata.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.metadata.approved.json new file mode 100644 index 00000000..3e65daa4 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.metadata.approved.json @@ -0,0 +1,6 @@ +{ + "ddsource": "stepfunction", + "ddtags": "env:dev,test_tag_key:test_tag_value,dd_step_functions_trace_enabled:true", + "host": "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction2", + "service": "stepfunction" +} diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 3d5a5211..8f7c2de3 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -153,7 +153,7 @@ def test_awslogs_handler_step_functions_customized_log_group( { "id": "37199773595581154154810589279545129148442535997644275712", "timestamp": 1668095539607, - "message": '{"execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction2:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', + "message": '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction2:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}', } ], } @@ -175,7 +175,8 @@ def test_awslogs_handler_step_functions_customized_log_group( cache_layer._cloudwatch_log_group_cache.get = MagicMock() awslogs_handler = AwsLogsHandler(context, metadata, cache_layer) - awslogs_handler.handle(eventFromCustomizedLogGroup) + verify_as_json(list(awslogs_handler.handle(eventFromCustomizedLogGroup))) + verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) self.assertEqual( awslogs_handler.metadata[DD_SOURCE], AwsEventSource.STEPFUNCTION.value ) From 49112f42625bb93e96e87d3297f4742199b90f7b Mon Sep 17 00:00:00 2001 From: "chenguang.xu@datadoghq.com" Date: Tue, 5 Nov 2024 12:38:24 -0500 Subject: [PATCH 21/21] Test issue --- ..._handler_step_functions_customized_log_group.approved.json} | 2 +- aws/logs_monitoring/tests/test_awslogs_handler.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) rename aws/logs_monitoring/tests/approved_files/{TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.json => TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.approved.json} (90%) diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.approved.json similarity index 90% rename from aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.json rename to aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.approved.json index 3e49300f..f99ff145 100644 --- a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.json +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.approved.json @@ -2,7 +2,7 @@ { "aws": { "awslogs": { - "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", + "logGroup": "test/logs", "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", "owner": "425362996713" } diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 8f7c2de3..88b9bf7d 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -131,6 +131,7 @@ def test_awslogs_handler_step_functions_tags_added_properly( @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.__init__") @patch("caching.cloudwatch_log_group_cache.send_forwarder_internal_metrics") + @patch.dict("os.environ", {"DD_STEP_FUNCTIONS_TRACE_ENABLED": "true"}) def test_awslogs_handler_step_functions_customized_log_group( self, mock_forward_metrics, @@ -165,7 +166,7 @@ def test_awslogs_handler_step_functions_customized_log_group( } } context = None - metadata = {"ddtags": "env:dev"} + metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} mock_forward_metrics.side_effect = MagicMock() mock_cache_init.return_value = None cache_layer = CacheLayer("")