fix: top-level ELB error metric and logs, downscale to min capacities (…

…#592)

* fix: upgrade grafonnet-lib

* feat: top-level metrics

* fix: disable invalid panel

* fix: downscale

* fix: central logging of server errors

* fix: lint

* chore: rename devloop

* chore: move up availability metric

* fix: use local

* fix: from grafonnet-lib upgrade

* fix: missing vars.namespace

* fix: use shared Redis panels

* chore: put Redis at the bottom

* fix: increase threshold for ELB errors

README.md

@@ -47,7 +47,7 @@ curl -X POST "http://localhost:3000/v1?chainId=eip155:1&projectId=someid" --data
 ## Testing
 
 ```bash
-just amigood
+just devloop
 ```
 
 ### Docker

justfile

@@ -62,7 +62,7 @@ docs target='all': (_check-string-in-set target "all,rust,tf")
   [[ '{{ target }}' == 'all' || '{{ target }}' == 'tf'    ]] && { just tf-docs;   }
 
 # Run linting and tests
-amigood: lint cargo-test-all
+devloop: lint cargo-test-all
 
 ################################################################################
 # Linting recipes

src/error.rs

@@ -7,7 +7,7 @@ use {
     axum::{response::IntoResponse, Json},
     cerberus::registry::RegistryError,
     hyper::StatusCode,
-    tracing::log::error,
+    tracing::{debug, log::error},
 };
 
 pub type RpcResult<T> = Result<T, RpcError>;
@@ -159,7 +159,7 @@ pub enum RpcError {
 
 impl IntoResponse for RpcError {
     fn into_response(self) -> axum::response::Response {
-        match self {
+        let response =  match &self {
             Self::AxumTungstenite(err) => (StatusCode::GONE, err.to_string()).into_response(),
             Self::UnsupportedChain(chain_id) => (
                 StatusCode::BAD_REQUEST,
@@ -357,15 +357,22 @@ impl IntoResponse for RpcError {
                 .into_response(),
 
             // Any other errors considering as 500
-            e => {
-                error!("Internal server error: {}", e);
-                (
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                    "Internal server error".to_string(),
-                )
-                    .into_response()
-            }
+            _ => (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "Internal server error".to_string(),
+            )
+                .into_response(),
+        };
+
+        if response.status().is_client_error() {
+            debug!("HTTP client error: {self:?}");
         }
+
+        if response.status().is_server_error() {
+            error!("HTTP server error: {self:?}");
+        }
+
+        response
     }
 }
 

terraform/ecs/cluster.tf

@@ -1,13 +1,18 @@
 locals {
   image = "${var.ecr_repository_url}:${var.image_version}"
 
+  desired_count = module.this.stage == "prod" ? var.autoscaling_desired_count : 1
+
+  task_cpu    = module.this.stage == "prod" ? var.task_cpu : 256
+  task_memory = module.this.stage == "prod" ? var.task_memory : 512
+
   otel_port   = var.port + 1
-  otel_cpu    = 128
-  otel_memory = 128
+  otel_cpu    = module.this.stage == "prod" ? 128 : 64
+  otel_memory = module.this.stage == "prod" ? 128 : 64
 
   prometheus_proxy_port   = var.port + 2
-  prometheus_proxy_cpu    = 128
-  prometheus_proxy_memory = 128
+  prometheus_proxy_cpu    = module.this.stage == "prod" ? 128 : 64
+  prometheus_proxy_memory = module.this.stage == "prod" ? 128 : 64
 
   file_descriptor_soft_limit = pow(2, 18)
   file_descriptor_hard_limit = local.file_descriptor_soft_limit * 2
@@ -16,8 +21,8 @@ locals {
 module "ecs_cpu_mem" {
   source  = "app.terraform.io/wallet-connect/ecs_cpu_mem/aws"
   version = "1.0.0"
-  cpu     = var.task_cpu + local.otel_cpu + local.prometheus_proxy_cpu
-  memory  = var.task_memory + local.otel_memory + local.prometheus_proxy_memory
+  cpu     = local.task_cpu
+  memory  = local.task_memory
 }
 
 #-------------------------------------------------------------------------------
@@ -65,8 +70,8 @@ resource "aws_ecs_task_definition" "app_task" {
     {
       name      = module.this.id,
       image     = local.image,
-      cpu       = var.task_cpu,
-      memory    = var.task_memory,
+      cpu       = local.task_cpu - local.otel_cpu - local.prometheus_proxy_cpu,
+      memory    = local.task_memory - local.otel_memory - local.prometheus_proxy_memory,
       essential = true,
 
       environment = [
@@ -206,7 +211,7 @@ resource "aws_ecs_service" "app_service" {
   cluster         = aws_ecs_cluster.app_cluster.id
   task_definition = aws_ecs_task_definition.app_task.arn
   launch_type     = "FARGATE"
-  desired_count   = var.autoscaling_desired_count
+  desired_count   = local.desired_count
   propagate_tags  = "TASK_DEFINITION"
 
   # Wait for the service deployment to succeed

terraform/ecs/cluster_autoscaling.tf

@@ -1,5 +1,9 @@
+locals {
+  autoscaling_min_capacity = module.this.stage == "prod" ? var.autoscaling_min_capacity : 1
+}
+
 resource "aws_appautoscaling_target" "ecs_target" {
-  min_capacity       = var.autoscaling_min_capacity
+  min_capacity       = local.autoscaling_min_capacity
   max_capacity       = var.autoscaling_max_capacity
   resource_id        = "service/${aws_ecs_cluster.app_cluster.name}/${aws_ecs_service.app_service.name}"
   scalable_dimension = "ecs:service:DesiredCount"

terraform/ecs/outputs.tf

@@ -32,3 +32,13 @@ output "load_balancer_arn_suffix" {
   description = "The ARN suffix of the load balancer"
   value       = aws_lb.load_balancer.arn_suffix
 }
+
+output "log_group_app_name" {
+  description = "The name of the log group for the app"
+  value       = aws_cloudwatch_log_group.cluster.name
+}
+
+output "log_group_app_arn" {
+  description = "The ARN of the log group for the app"
+  value       = aws_cloudwatch_log_group.cluster.arn
+}

terraform/monitoring/dashboard.jsonnet

@@ -17,13 +17,17 @@ local ds    = {
   },
 };
 local vars  = {
+  namespace:        'Notify',
   environment:      std.extVar('environment'),
   notifications:    std.parseJson(std.extVar('notifications')),
 
-  ecs_service_name: std.extVar('ecs_service_name'),
-  load_balancer:    std.extVar('load_balancer'),
-  target_group:     std.extVar('target_group'),
-  redis_cluster_id: std.extVar('redis_cluster_id'),
+  ecs_service_name:   std.extVar('ecs_service_name'),
+  load_balancer:      std.extVar('load_balancer'),
+  target_group:       std.extVar('target_group'),
+  redis_cluster_id:   std.extVar('redis_cluster_id'),
+  log_group_app_name: std.extVar('log_group_app_name'),
+  log_group_app_arn:  std.extVar('log_group_app_arn'),
+  aws_account_id:     std.extVar('aws_account_id'),
 };
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -52,8 +56,14 @@ dashboard.new(
 )
 
 .addPanels(layout.generate_grid([
-  row.new('ECS'),
+  row.new('Application'),
+    // panels.app.http_request_rate(ds, vars)          { gridPos: pos._4 },
+    // panels.app.http_request_latency(ds, vars)       { gridPos: pos._4 },
     panels.ecs.availability(ds, vars)                { gridPos: pos._3 },
+    panels.lb.error_5xx(ds, vars)                    { gridPos: pos._3 },
+    panels.lb.error_5xx_logs(ds, vars)               { gridPos: pos._3 },
+
+  row.new('ECS'),
     panels.ecs.memory(ds, vars)                      { gridPos: pos._3 },
     panels.ecs.cpu(ds, vars)                         { gridPos: pos._3 },
 
@@ -109,9 +119,6 @@ dashboard.new(
     panels.proxy.http_codes(ds, vars)                { gridPos: pos.two_thirds },
     panels.proxy.healthy_hosts(ds, vars)             { gridPos: pos._3 },
 
-  row.new('Database'),
-    panels.db.redis_cpu_memory(ds, vars)             { gridPos: pos._2 },
-
   row.new('History Metrics'),
     panels.history.requests(ds, vars)               { gridPos: pos_short._3 },
     panels.history.latency(ds, vars)                { gridPos: pos_short._3 },
@@ -123,4 +130,15 @@ dashboard.new(
     panels.identity.latency(ds, vars)                { gridPos: pos_short._2 },
     panels.identity.cache(ds, vars)                  { gridPos: pos_short._2 },
     panels.identity.usage(ds, vars)                  { gridPos: pos_short._2 },
+
+  row.new('Redis'),
+    panels.redis.cpu(ds, vars)                    { gridPos: pos._2 },
+    panels.redis.memory(ds, vars)                 { gridPos: pos._2 },
+
+  row.new('Load Balancer'),
+    panels.lb.active_connections(ds, vars)        { gridPos: pos._2 },
+    panels.lb.requests(ds, vars)                  { gridPos: pos._2 },
+
+    panels.lb.healthy_hosts(ds, vars)             { gridPos: pos._3 },
+    panels.lb.error_4xx(ds, vars)                 { gridPos: pos._3 },
 ]))

terraform/monitoring/main.tf

@@ -11,11 +11,14 @@ data "jsonnet_file" "dashboard" {
     environment   = module.this.stage
     notifications = jsonencode(var.notification_channels)
 
-    ecs_service_name = var.ecs_service_name
-    ecs_task_family  = var.ecs_task_family
-    load_balancer    = var.load_balancer_arn
-    target_group     = var.ecs_target_group_arn
-    redis_cluster_id = var.redis_cluster_id
+    ecs_service_name   = var.ecs_service_name
+    ecs_task_family    = var.ecs_task_family
+    load_balancer      = var.load_balancer_arn
+    target_group       = var.ecs_target_group_arn
+    redis_cluster_id   = var.redis_cluster_id
+    log_group_app_name = var.log_group_app_name
+    log_group_app_arn  = var.log_group_app_arn
+    aws_account_id     = var.aws_account_id
   }
 }
 

terraform/monitoring/panels/ecs/availability.libsonnet

@@ -44,7 +44,7 @@ local error_alert(vars) = alert.new(
           axisSoftMax = 100,
         )
     )
-    .setAlert(error_alert(vars))
+    .setAlert(vars.environment, error_alert(vars))
 
     .addTarget(targets.prometheus(
       datasource  = ds.prometheus,

terraform/monitoring/panels/ecs/cpu.libsonnet

@@ -14,7 +14,7 @@ local overrides       = defaults.overrides;
       datasource  = ds.prometheus,
     )
     .configure(overrides.cpu(defaults.configuration.timeseries_resource))
-    .setAlert(alert.new(
+    .setAlert(vars.environment, alert.new(
       namespace     = 'RPC Proxy',
       name          = "RPC %s - High CPU usage" % vars.environment,
       message       = "RPC %s - High CPU usage" % vars.environment,

terraform/monitoring/panels/ecs/memory.libsonnet

@@ -14,7 +14,7 @@ local overrides       = defaults.overrides;
       datasource  = ds.prometheus,
     )
     .configure(defaults.overrides.memory(defaults.configuration.timeseries_resource))
-    .setAlert(alert.new(
+    .setAlert(vars.environment, alert.new(
       namespace     = 'RPC Proxy',
       name          = "RPC %s - High Memory (RAM) usage" % vars.environment,
       message       = "RPC %s - High Memory (RAM) usage" % vars.environment,

terraform/monitoring/panels/lb/active_connections.libsonnet

@@ -0,0 +1,24 @@
+local grafana   = import '../../grafonnet-lib/grafana.libsonnet';
+local defaults  = import '../../grafonnet-lib/defaults.libsonnet';
+
+local panels    = grafana.panels;
+local targets   = grafana.targets;
+
+{
+  new(ds, vars)::
+    panels.timeseries(
+      title       = 'Active Connections',
+      datasource  = ds.cloudwatch,
+    )
+    .configure(defaults.configuration.timeseries)
+
+    .addTarget(targets.cloudwatch(
+      datasource      = ds.cloudwatch,
+      namespace     = 'AWS/ApplicationELB',
+      metricName    = 'ActiveConnectionCount',
+      dimensions    = {
+        LoadBalancer: vars.load_balancer
+      },
+      statistic     = 'Average',
+    ))
+}