Fix missing IAM permissions for Fluent Bit

The service account for Fluent Bit currently does not have permission to create the log streams with the new log stream template name. This means that non-system logs won't be sent to Cloudwatch. This updates the IAM policy to allow creating these log groups, set retention policies, and put log events within the Flightdeck namespace.
thoughtbot · Dec 18, 2023 · 7e103d6 · 7e103d6
1 parent 5b0bac9
commit 7e103d6
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 25 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -43,6 +43,7 @@ jobs:
       run: |
         CLUSTER=$(echo 'flightdeck-${{ github.ref_name }}' | cut -c1-20)
         CLUSTER="$CLUSTER-sandbox-v1"
+        echo "CLUSTER=$CLUSTER" >> "$GITHUB_ENV"
         aws \
           --region us-east-1 \
           eks \
@@ -64,4 +65,6 @@ jobs:
 
     - name: Run tests
       run: |
-        make tests ADDRESS=https://${{ github.ref_name }}.flightdeck-test.thoughtbot.com
+        make tests \
+          ADDRESS=https://${{ github.ref_name }}.flightdeck-test.thoughtbot.com \
+          CLUSTER="$CLUSTER"
diff --git a/aws/platform/main.tf b/aws/platform/main.tf
@@ -370,6 +370,7 @@ locals {
         [OUTPUT]
             Name cloudwatch_logs
             Match *
+            auto_create_group true
             region ${data.aws_region.current.name}
             log_group_name ${module.cloudwatch_logs.log_group_name}
             log_group_template ${var.logs_prefix}/$kubernetes['namespace_name']

diff --git a/aws/platform/modules/cloudwatch-logs/main.tf b/aws/platform/modules/cloudwatch-logs/main.tf
@@ -27,36 +27,17 @@ resource "aws_iam_role_policy_attachment" "this" {
 
 data "aws_iam_policy_document" "this" {
   statement {
-    sid = "AllowCreateLogEvents"
+    sid = "AllowWriteLogs"
     actions = [
-      "logs:DescribeLogStreams",
-      "logs:PutLogEvents"
-    ]
-    resources = [
-      "${aws_cloudwatch_log_group.this.arn}:log-stream:*"
-    ]
-  }
-
-  statement {
-    sid = "AllowCreateLogGroup"
-    actions = [
-      "logs:CreateLogGroup"
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+      "logs:PutRetentionPolicy",
     ]
     resources = [
       "${local.arn_prefix}:log-group:${var.log_group_prefix}/*"
     ]
   }
-
-  statement {
-    sid = "AllowCreateLogStream"
-    actions = [
-      "logs:CreateLogStream"
-    ]
-    resources = [
-      aws_cloudwatch_log_group.this.arn,
-      "${aws_cloudwatch_log_group.this.arn}:log-stream:*"
-    ]
-  }
 }
 
 data "aws_caller_identity" "current" {}

diff --git a/tests/fluentbit.bats b/tests/fluentbit.bats
@@ -29,3 +29,33 @@
     false
   fi
 }
+
+@test "creates log streams within groups for Kubernetes namespaces" {
+  expected="$RANDOM"
+  curl -v "$ADDRESS/echo?log=$expected"
+  pod=$(kubectl \
+    get pod \
+    --field-selector=status.phase=Running \
+    --selector=app=echoserver \
+    -n acceptance \
+    --output=name \
+    | cut -d'/' -f2)
+  logs=$(aws \
+    --region us-east-1 \
+    logs \
+    get-log-events \
+    --log-group-name "/flightdeck/acceptance" \
+    --log-stream-name "$pod.echoserver" \
+    --query 'events[*].[message]' \
+      --output text)
+
+  if ! echo "$logs" | grep -q "log=$expected"; then
+    echo "Failed to find log for test request." >&2
+    echo >&2
+    echo "Test request was: GET /echo?log=$expected" >&2
+    echo >&2
+    echo "Found log entries" >&2
+    echo "$logs" >&2
+    false
+  fi
+}