Enable container runtime metrics (#3625)

* Consolidate datadog env var defaults * Enable datadog runtime metrics collection in prod
usdigitalresponse · Oct 10, 2024 · 4a2f4b6 · 4a2f4b6
1 parent 77cdb1b
commit 4a2f4b6
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 27 deletions.
diff --git a/terraform/main.tf b/terraform/main.tf
@@ -185,7 +185,10 @@ module "api" {
   enable_saved_search_grants_digest  = var.api_enable_saved_search_grants_digest
   enable_grant_digest_scheduled_task = var.api_enable_grant_digest_scheduled_task
   unified_service_tags               = local.unified_service_tags
-  datadog_environment_variables      = var.api_datadog_environment_variables
+  datadog_environment_variables = merge(
+    var.default_datadog_environment_variables,
+    var.api_datadog_environment_variables,
+  )
   api_container_environment = merge(var.api_container_environment, {
     ARPA_AUDIT_REPORT_SQS_QUEUE_URL    = module.arpa_audit_report.sqs_queue_url
     ARPA_TREASURY_REPORT_SQS_QUEUE_URL = module.arpa_treasury_report.sqs_queue_url
@@ -226,10 +229,13 @@ module "consume_grants" {
   security_group_ids = [module.api_to_postgres_security_group.id]
 
   # Task configuration
-  ecs_cluster_name              = join("", aws_ecs_cluster.default[*].name)
-  docker_tag                    = var.api_container_image_tag
-  unified_service_tags          = local.unified_service_tags
-  datadog_environment_variables = var.consume_grants_datadog_environment_variables
+  ecs_cluster_name     = join("", aws_ecs_cluster.default[*].name)
+  docker_tag           = var.api_container_image_tag
+  unified_service_tags = local.unified_service_tags
+  datadog_environment_variables = merge(
+    var.default_datadog_environment_variables,
+    var.consume_grants_datadog_environment_variables,
+  )
 
   # Messaging
   grants_ingest_event_bus_name = var.consume_grants_source_event_bus_name
@@ -284,10 +290,7 @@ module "arpa_audit_report" {
     SES_CONFIGURATION_SET_DEFAULT = aws_sesv2_configuration_set.default.configuration_set_name
     WEBSITE_DOMAIN                = "https://${var.website_domain_name}"
   }
-  datadog_environment_variables = {
-    DD_LOGS_INJECTION    = "true"
-    DD_PROFILING_ENABLED = "true"
-  }
+  datadog_environment_variables = var.default_datadog_environment_variables
   consumer_task_efs_volume_mounts = [{
     name            = "data"
     container_path  = "/var/data"
@@ -374,10 +377,7 @@ module "arpa_treasury_report" {
     SES_CONFIGURATION_SET_DEFAULT = aws_sesv2_configuration_set.default.configuration_set_name
     WEBSITE_DOMAIN                = "https://${var.website_domain_name}"
   }
-  datadog_environment_variables = {
-    DD_LOGS_INJECTION    = "true"
-    DD_PROFILING_ENABLED = "true"
-  }
+  datadog_environment_variables = var.default_datadog_environment_variables
   consumer_task_efs_volume_mounts = [{
     name            = "data"
     container_path  = "/var/data"

diff --git a/terraform/prod.tfvars b/terraform/prod.tfvars
@@ -13,6 +13,11 @@ datadog_monitor_notification_handles = [
   "[email protected]",
 ]
 ses_datadog_events_enabled = true
+default_datadog_environment_variables = {
+  DD_LOGS_INJECTION          = true
+  DD_PROFILING_ENABLED       = true
+  DD_RUNTIME_METRICS_ENABLED = true
+}
 
 // Website
 website_enabled     = true
@@ -69,12 +74,9 @@ api_enable_new_team_terminology        = true
 api_enable_saved_search_grants_digest  = true
 api_enable_grant_digest_scheduled_task = true
 api_log_retention_in_days              = 30
-api_datadog_environment_variables = {
-  DD_PROFILING_ENABLED = true,
-}
 api_container_environment = {
-  NEW_GRANT_DETAILS_PAGE_ENABLED = true,
-  SHARE_TERMINOLOGY_ENABLED      = true,
+  NEW_GRANT_DETAILS_PAGE_ENABLED = true
+  SHARE_TERMINOLOGY_ENABLED      = true
 }
 
 // Postgres
@@ -86,9 +88,6 @@ postgres_ca_cert_identifier        = "rds-ca-rsa2048-g1"
 
 // Grant events consumer
 consume_grants_source_event_bus_name = "default"
-consume_grants_datadog_environment_variables = {
-  DD_PROFILING_ENABLED = true,
-}
 
 // Email
 email_enable_tracking = true
diff --git a/terraform/staging.tfvars b/terraform/staging.tfvars
@@ -12,6 +12,11 @@ datadog_monitor_notification_handles   = []
 ses_datadog_events_enabled             = true
 datadog_email_pipeline_enabled         = true
 datadog_email_pipeline_metrics_enabled = true
+default_datadog_environment_variables = {
+  DD_LOGS_INJECTION          = true
+  DD_PROFILING_ENABLED       = true
+  DD_RUNTIME_METRICS_ENABLED = true
+}
 
 // Website
 website_enabled     = true
@@ -68,9 +73,6 @@ api_enable_new_team_terminology        = true
 api_enable_saved_search_grants_digest  = true
 api_enable_grant_digest_scheduled_task = true
 api_log_retention_in_days              = 14
-api_datadog_environment_variables = {
-  DD_PROFILING_ENABLED = true,
-}
 api_container_environment = {
   NEW_GRANT_DETAILS_PAGE_ENABLED = true,
   SHARE_TERMINOLOGY_ENABLED      = true,
@@ -86,9 +88,6 @@ postgres_ca_cert_identifier        = "rds-ca-rsa2048-g1"
 
 // Grant events consumer
 consume_grants_source_event_bus_name = "default"
-consume_grants_datadog_environment_variables = {
-  DD_PROFILING_ENABLED = true,
-}
 
 // Email
 email_enable_tracking = true
diff --git a/terraform/variables.tf b/terraform/variables.tf
@@ -66,6 +66,12 @@ variable "datadog_email_pipeline_metrics_enabled" {
   default     = false
 }
 
+variable "default_datadog_environment_variables" {
+  description = "Default environment variables for Datadog configuration. May be overwritten by container-specific variables."
+  type        = map(string)
+  default     = {}
+}
+
 // Common
 variable "permissions_boundary_policy_name" {
   description = "Name of the permissions boundary for service roles"