From 9e481badab76005b52a1fd65e142c126784f4203 Mon Sep 17 00:00:00 2001 From: Katie McKew Date: Fri, 10 Nov 2023 17:08:41 -0500 Subject: [PATCH 1/5] [AWSMC-588] Support Bedrock log source --- aws/logs_monitoring/parsing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/parsing.py index ede21cd20..a4682c7f3 100644 --- a/aws/logs_monitoring/parsing.py +++ b/aws/logs_monitoring/parsing.py @@ -381,6 +381,7 @@ def find_s3_source(key): "network-firewall", "cloudfront", "verified-access", + "bedrock", ]: if source in key: return source.replace("amazon_", "") @@ -609,6 +610,10 @@ def awslogs_handler(event, context, metadata): metadata[DD_SOURCE] = "aws-iam-authenticator" # In case the conditions above don't match we maintain eks as the source + # Bedrock allows using any custom logGroup, but creates the logStream with this name + if logs["logStream"] == "aws/bedrock/modelinvocations": + metadata[DD_SOURCE] = "amazon_bedrock" + # Create and send structured logs to Datadog for log in logs["logEvents"]: yield merge_dicts(log, aws_attributes) From 637a6d7f882f57904256cf50ce252c5a7c08896e Mon Sep 17 00:00:00 2001 From: Katie McKew Date: Mon, 13 Nov 2023 15:00:49 -0500 Subject: [PATCH 2/5] Remove amazon_ prefix --- aws/logs_monitoring/parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/parsing.py index a4682c7f3..2a558e06e 100644 --- a/aws/logs_monitoring/parsing.py +++ b/aws/logs_monitoring/parsing.py @@ -612,7 +612,7 @@ def awslogs_handler(event, context, metadata): # Bedrock allows using any custom logGroup, but creates the logStream with this name if logs["logStream"] == "aws/bedrock/modelinvocations": - metadata[DD_SOURCE] = "amazon_bedrock" + metadata[DD_SOURCE] = "bedrock" # Create and send structured logs to Datadog for log in logs["logEvents"]: From bd346968764966344ed80f795d7031c30febe1c8 Mon Sep 17 00:00:00 2001 From: Katie McKew Date: Tue, 14 Nov 2023 14:38:22 -0500 Subject: [PATCH 3/5] Properly set source and service for cw logs --- aws/logs_monitoring/parsing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/parsing.py index 2a558e06e..5f3dfa73d 100644 --- a/aws/logs_monitoring/parsing.py +++ b/aws/logs_monitoring/parsing.py @@ -331,6 +331,7 @@ def find_cloudwatch_source(log_group): "elasticsearch", "transitgateway", "verified-access", + "bedrock" ]: if source in log_group: return source @@ -488,12 +489,14 @@ def awslogs_handler(event, context, metadata): # Set the source on the logs source = logs.get("logGroup", "cloudwatch") - # Use the logStream to identify if this is a CloudTrail event + # Use the logStream to identify if this is a CloudTrail, TransitGateway, or Bedrock event # i.e. 123456779121_CloudTrail_us-east-1 if "_CloudTrail_" in logs["logStream"]: source = "cloudtrail" if "tgw-attach" in logs["logStream"]: source = "transitgateway" + if logs["logStream"] == "aws/bedrock/modelinvocations": + source = "bedrock" metadata[DD_SOURCE] = parse_event_source(event, source) # Build aws attributes @@ -516,6 +519,7 @@ def awslogs_handler(event, context, metadata): ) # Set service from custom tags, which may include the tags set on the log group + # Returns DD_SOURCE by default metadata[DD_SERVICE] = get_service_from_tags(metadata) # Set host as log group where cloudwatch is source From ddcaa8f147e9ac5f8957578d53efbc1d573412ad Mon Sep 17 00:00:00 2001 From: Katie McKew Date: Tue, 14 Nov 2023 14:50:09 -0500 Subject: [PATCH 4/5] Lint --- aws/logs_monitoring/parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/parsing.py index 5f3dfa73d..172d562aa 100644 --- a/aws/logs_monitoring/parsing.py +++ b/aws/logs_monitoring/parsing.py @@ -331,7 +331,7 @@ def find_cloudwatch_source(log_group): "elasticsearch", "transitgateway", "verified-access", - "bedrock" + "bedrock", ]: if source in log_group: return source From cfff522fd59f4ff5ba4f76d55ecc343028a4c0f8 Mon Sep 17 00:00:00 2001 From: Katie McKew Date: Tue, 14 Nov 2023 16:16:23 -0500 Subject: [PATCH 5/5] Remove duplicate logStream check --- aws/logs_monitoring/parsing.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/aws/logs_monitoring/parsing.py b/aws/logs_monitoring/parsing.py index 172d562aa..8d0a32ebb 100644 --- a/aws/logs_monitoring/parsing.py +++ b/aws/logs_monitoring/parsing.py @@ -614,10 +614,6 @@ def awslogs_handler(event, context, metadata): metadata[DD_SOURCE] = "aws-iam-authenticator" # In case the conditions above don't match we maintain eks as the source - # Bedrock allows using any custom logGroup, but creates the logStream with this name - if logs["logStream"] == "aws/bedrock/modelinvocations": - metadata[DD_SOURCE] = "bedrock" - # Create and send structured logs to Datadog for log in logs["logEvents"]: yield merge_dicts(log, aws_attributes)