From c8c438ddccd9a8e97d80cf72ce6fa83f0c7e0074 Mon Sep 17 00:00:00 2001 From: Bar <51476543+bardabun@users.noreply.github.com> Date: Sun, 31 Mar 2024 12:21:26 +0300 Subject: [PATCH] Enhance Subscription Filter with Lambda Function Separation (#14) * Split Lambda function for EventBridge and CFN events * update mod.go name and remove comments * update workflow and readme --- .github/workflows/release.yaml | 98 ++++++-- .goreleaser.yml | 36 ++- README.md | 53 ++-- cfn-lambda/main.go | 400 +++++++++++++++++++++++++++++++ cloudformation/sam-template.yaml | 68 +++++- common/log_management.go | 126 ++++++++++ utils.go => common/utils.go | 23 +- eventbridge-lambda/main.go | 69 ++++++ go.mod | 2 +- go.sum | 2 - lambda_build.sh | 40 +++- main.go | 343 -------------------------- 12 files changed, 855 insertions(+), 405 deletions(-) create mode 100644 cfn-lambda/main.go create mode 100644 common/log_management.go rename utils.go => common/utils.go (76%) create mode 100644 eventbridge-lambda/main.go delete mode 100644 main.go diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ad90057..b7f5fb6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,34 +4,73 @@ on: types: [published] jobs: - build_function: + build_functions: name: Build function runs-on: ubuntu-latest steps: + # Check out the repository to the runner - name: Check out the repo uses: actions/checkout@v3 - - run: git fetch --force --tags + + # Fetch all history for all tags and branches + - run: git fetch --prune --unshallow + + # Setup Go environment - uses: actions/setup-go@v3 with: go-version: '1.19' + + # Import GPG key for GoReleaser - name: Import GPG key id: import_gpg uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} passphrase: ${{ secrets.PASSPHRASE }} + - name: Run GoReleaser uses: goreleaser/goreleaser-action@v4 with: version: latest - args: release --rm-dist + args: release --clean env: GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Build CloudFormation Lambda function + run: | + cd cfn-lambda + cp -R ../common ./common + GOOS=linux GOARCH=amd64 go build -o bootstrap . + zip -r ../cfn-lambda.zip . + cd .. + + - name: Build and Zip EventBridge Lambda function with common dependencies + run: | + cd eventbridge-lambda + cp -R ../common ./common + GOOS=linux GOARCH=amd64 go build -o bootstrap . + zip -r ../eventbridge-lambda.zip . + cd .. + + - name: Upload Lambdas ZIP as Artifact + uses: actions/upload-artifact@v2 + with: + name: lambdas + path: | + cfn-lambda.zip + eventbridge-lambda.zip + + - name: Cleanup common folders + run: | + rm -rf cfn-lambda/common + rm -rf eventbridge-lambda/common + + # Upload built artifacts to S3 upload_to_buckets: name: Upload to S3 buckets runs-on: ubuntu-latest - needs: build_function + needs: build_functions strategy: matrix: aws_region: @@ -40,38 +79,65 @@ jobs: - 'us-west-1' - 'us-west-2' - 'eu-central-1' + - 'eu-central-2' - 'eu-north-1' - 'eu-west-1' - 'eu-west-2' - 'eu-west-3' + - 'eu-south-1' + - 'eu-south-2' - 'sa-east-1' - 'ap-northeast-1' - 'ap-northeast-2' - 'ap-northeast-3' - 'ap-south-1' + - 'ap-south-2' - 'ap-southeast-1' - 'ap-southeast-2' + - 'ap-southeast-3' + - 'ap-southeast-4' + - 'ap-east-1' - 'ca-central-1' + - 'ca-west-1' + - 'af-south-1' + - 'me-south-1' + - 'me-central-1' + - 'il-central-1' steps: - name: Check out the repo uses: actions/checkout@v3 - - name: download zip - run: wget -c https://github.com/logzio/firehose-logs/releases/download/${{ github.event.release.tag_name }}/firehose-logs_${{ github.event.release.tag_name }}_linux_amd64.zip -O function.zip - - name: create new version + + - name: Download Artifact for Lambdas + uses: actions/download-artifact@v2 + with: + name: lambdas + path: . + + - name: Configure AWS CLI + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} + aws-region: ${{ matrix.aws_region }} + + - name: Upload CloudFormation Lambda ZIP to S3 + run: | + aws s3 cp ./cfn-lambda.zip s3://logzio-aws-integrations-${{ matrix.aws_region }}/firehose-logs/${{ github.event.release.tag_name }}/cfn-lambda.zip --acl public-read + + - name: Upload EventBridge Lambda ZIP to S3 + run: | + aws s3 cp ./eventbridge-lambda.zip s3://logzio-aws-integrations-${{ matrix.aws_region }}/firehose-logs/${{ github.event.release.tag_name }}/eventbridge-lambda.zip --acl public-read + + - name: Prepare SAM Template run: | cp ./cloudformation/sam-template.yaml ./sam-template-${{ matrix.aws_region }}.yaml sed -i "s/<>/${{ github.event.release.tag_name }}/" "./sam-template-${{ matrix.aws_region }}.yaml" sed -i "s/<>/${{ matrix.aws_region }}/" "./sam-template-${{ matrix.aws_region }}.yaml" - - name: Upload to aws + + - name: Upload SAM Template to S3 run: | - sudo apt-get update - sudo apt-get install awscli - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_KEY }} - aws configure set region ${{ matrix.aws_region }} aws s3 cp ./sam-template-${{ matrix.aws_region }}.yaml s3://logzio-aws-integrations-${{ matrix.aws_region }}/firehose-logs/${{ github.event.release.tag_name }}/sam-template.yaml --acl public-read - aws s3 cp ./function.zip s3://logzio-aws-integrations-${{ matrix.aws_region }}/firehose-logs/${{ github.event.release.tag_name }}/function.zip --acl public-read + - name: Clean run: | - rm ./sam-template-${{ matrix.aws_region }}.yaml - rm ./function.zip \ No newline at end of file + rm ./sam-template-${{ matrix.aws_region }}.yaml \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index cf3c2e8..6e5b8de 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,7 +1,27 @@ before: hooks: + - go mod tidy builds: - - env: + - id: "cfn-lambda" + dir: "cfn-lambda/" + main: "." + binary: "bootstrap" + env: + - CGO_ENABLED=0 + mod_timestamp: '{{ .CommitTimestamp }}' + flags: + - -trimpath + ldflags: + - '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}}' + goos: + - linux + goarch: + - amd64 + - id: "eventbridge-lambda" + dir: "eventbridge-lambda/" + main: "." + binary: "bootstrap" + env: - CGO_ENABLED=0 mod_timestamp: '{{ .CommitTimestamp }}' flags: @@ -12,10 +32,18 @@ builds: - linux goarch: - amd64 - binary: 'bootstrap' archives: - - format: zip - name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}' + - id: "cfn-lambda" + builds: + - "cfn-lambda" + format: zip + name_template: 'cfn-lambda_{{ .Version }}_{{ .Os }}_{{ .Arch }}' + - id: "eventbridge-lambda" + builds: + - "eventbridge-lambda" + format: zip + name_template: 'eventbridge-lambda_{{ .Version }}_{{ .Os }}_{{ .Arch }}' release: + draft: true changelog: skip: true \ No newline at end of file diff --git a/README.md b/README.md index 3e863bf..2762cae 100644 --- a/README.md +++ b/README.md @@ -13,25 +13,37 @@ This project will uses a Cloudformation template to create a Stack that deploys: To deploy this project, click the button that matches the region you wish to deploy your Stack to: -| Region | Deployment | -|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `us-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-east-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `us-east-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-east-2.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `us-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-west-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `us-west-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-west-2.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `eu-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-central-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `eu-north-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-north-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-north-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `eu-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `eu-west-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-2.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `eu-west-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-3.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `sa-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=sa-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-sa-east-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `ap-northeast-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `ap-northeast-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-2.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `ap-northeast-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-3.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `ap-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-south-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `ap-southeast-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `ap-southeast-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-2.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | -| `ca-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ca-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ca-central-1.s3.amazonaws.com/firehose-logs/0.0.2/sam-template.yaml&stackName=logzio-firehose) | +| Region | Deployment | +|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `us-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-east-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `us-east-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-east-2.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `us-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-west-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `us-west-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-west-2.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-central-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-central-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-central-2.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-north-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-north-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-north-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-west-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-2.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-west-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-3.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-south-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-south-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-south-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-south-2.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `sa-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=sa-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-sa-east-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-northeast-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-northeast-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-2.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-northeast-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-3.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-south-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-south-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-south-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-south-2.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-southeast-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-southeast-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-2.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-southeast-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-3.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-southeast-4` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-4#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-4.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-east-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ca-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ca-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ca-central-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `ca-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ca-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ca-west-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `af-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=af-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-af-south-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `me-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=me-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-me-south-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `me-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=me-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-me-central-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | +| `il-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=il-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-il-central-1.s3.amazonaws.com/firehose-logs/0.1.0/sam-template.yaml&stackName=logzio-firehose) | ### 1. Specify stack details @@ -66,6 +78,7 @@ Once new logs are added to your chosen log group, they will be sent to your Logz If you've used the `services` field, you'll have to **wait 6 minutes** before creating new log groups for your chosen services. This is due to cold start and custom resource invocation, that can cause the Lambda to behave unexpectedly. ### Changelog: - +- **0.1.0**: + Introduced the ability to directly update service and custom log parameters within the stack. - **0.0.2**: Fix for RDS service - look for prefix `/aws/rds/` - **0.0.1**: Initial release. diff --git a/cfn-lambda/main.go b/cfn-lambda/main.go new file mode 100644 index 0000000..824e80b --- /dev/null +++ b/cfn-lambda/main.go @@ -0,0 +1,400 @@ +package main + +import ( + "context" + "fmt" + "github.com/aws/aws-lambda-go/cfn" + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/logzio/firehose-logs/common" + lp "github.com/logzio/firehose-logs/logger" + "go.uber.org/zap" + "os" +) + +var sugLog *zap.SugaredLogger + +func main() { + lambda.Start(cfn.LambdaWrap(HandleRequest)) +} + +func HandleRequest(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { + logger := lp.GetLogger() + defer logger.Sync() + sugLog = logger.Sugar() + sugLog.Info("Starting handling event...") + sugLog.Debug("Handling event: ", event) + err = common.ValidateRequired() + if err != nil { + sugLog.Debug("Lambda finished with error") + return "", nil, err + } + + // CloudFormation custom resource handling logic + switch event.RequestType { + case "Create": + sugLog.Debug("Detected CloudFormation Create event") + return customResourceRun(ctx, event) + case "Update": + sugLog.Debug("Detected CloudFormation Update event") + return customResourceRunUpdate(ctx, event) + case "Delete": + sugLog.Debug("Detected CloudFormation Delete event") + return customResourceRunDelete(ctx, event) + default: + sugLog.Debug("Detected unsupported request type") + return customResourceRunDoNothing(ctx, event) + } +} + +func generatePhysicalResourceId(event cfn.Event) string { + // Concatenate StackId and LogicalResourceId to form a unique PhysicalResourceId + physicalResourceId := fmt.Sprintf("%s-%s", event.StackID, event.LogicalResourceID) + return physicalResourceId +} + +func customResourceRunUpdate(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { + oldConfig := event.OldResourceProperties + newConfig := event.ResourceProperties + + physicalResourceID = generatePhysicalResourceId(event) + + err = updateConfiguration(ctx, oldConfig, newConfig) + if err != nil { + sugLog.Error("Error during update: ", err) + return physicalResourceID, nil, err + } + + // Populate your data map as needed for the update + data = make(map[string]interface{}) + // Populate data as needed + + return physicalResourceID, data, nil +} + +func updateConfiguration(ctx context.Context, oldConfig, newConfig map[string]interface{}) error { + sess, err := common.GetSession() + if err != nil { + sugLog.Error("Error while creating session: ", err.Error()) + return err + } + + sugLog.Info("Extracting configuration strings...") + + // Helper function to extract and validate configuration strings + extractConfigString := func(config map[string]interface{}, key string) (string, error) { + value, exists := config[key] + if !exists { + return "", nil + } + strValue, ok := value.(string) + if !ok { + sugLog.Errorf("Invalid type for %s; expected string", key) + return "", fmt.Errorf("invalid configuration type for %s", key) + } + return strValue, nil + } + + // Extract and validate services and custom log group strings from the configurations + oldServicesStr, err := extractConfigString(oldConfig, "Services") + newServicesStr, err := extractConfigString(newConfig, "Services") + oldCustomGroupsStr, err := extractConfigString(oldConfig, "CustomLogGroups") + newCustomGroupsStr, err := extractConfigString(newConfig, "CustomLogGroups") + if err != nil { + return err + } + + // Parse services and custom log groups + oldServices := common.ParseServices(oldServicesStr) + newServices := common.ParseServices(newServicesStr) + oldCustomGroups := common.ParseServices(oldCustomGroupsStr) + newCustomGroups := common.ParseServices(newCustomGroupsStr) + + // Find differences in services and custom log groups + servicesToAdd, servicesToRemove := findDifferences(oldServices, newServices) + customGroupsToAdd, customGroupsToRemove := findDifferences(oldCustomGroups, newCustomGroups) + + // Update subscription filters + if err := updateSubscriptionFilters(sess, servicesToAdd, servicesToRemove, customGroupsToAdd, customGroupsToRemove); err != nil { + sugLog.Errorf("Error updating subscription filters: %v", err) + return err + } + + return nil +} + +func updateSubscriptionFilters(sess *session.Session, servicesToAdd, servicesToRemove, customGroupsToAdd, customGroupsToRemove []string) error { + // Add subscription filters for new services + if len(servicesToAdd) > 0 { + addedServices, err := addServices(sess, servicesToAdd) + if err != nil { + sugLog.Errorf("Error adding subscriptions for services: %v", err) + return err + } + sugLog.Infof("Added subscriptions for services: %v", addedServices) + } + + // Add subscription filters for new custom log groups + if len(customGroupsToAdd) > 0 { + addedCustomGroups, err := addCustom(sess, customGroupsToAdd, nil) // Assuming the third parameter is handled within the function + if err != nil { + sugLog.Errorf("Error adding subscriptions for custom log groups: %v", err) + return err + } + sugLog.Infof("Added subscriptions for custom log groups: %v", addedCustomGroups) + } + + // Remove subscription filters from services no longer needed + if len(servicesToRemove) > 0 { + removedServices, err := deleteServices(sess, servicesToRemove) + if err != nil { + sugLog.Errorf("Error removing subscriptions for services: %v", err) + return err + } + sugLog.Infof("Removed subscriptions for services: %v", removedServices) + } + + // Remove subscription filters from custom log groups no longer needed + if len(customGroupsToRemove) > 0 { + removedCustomGroups, err := deleteCustom(sess, customGroupsToRemove) + if err != nil { + sugLog.Errorf("Error removing subscriptions for custom log groups: %v", err) + return err + } + sugLog.Infof("Removed subscriptions for custom log groups: %v", removedCustomGroups) + } + + return nil +} + +// findDifferences finds elements in 'new' that are not in 'old', and vice versa. +func findDifferences(old, new []string) (toAdd, toRemove []string) { + oldSet := make(map[string]struct{}) + newSet := make(map[string]struct{}) + + // Populate 'oldSet' with elements from the 'old' slice. + for _, item := range old { + oldSet[item] = struct{}{} + } + + for _, item := range new { + newSet[item] = struct{}{} + } + + // Find elements in 'new' that are not in 'old' and add them to 'toAdd'. + for item := range newSet { + _, exists := oldSet[item] // Check if 'item' exists in 'oldSet' + if !exists { + toAdd = append(toAdd, item) + } + } + + for item := range oldSet { + _, exists := newSet[item] + if !exists { + toRemove = append(toRemove, item) + } + } + + return toAdd, toRemove +} + +// Wrapper for first invocation from cloud formation custom resource +func customResourceRun(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { + physicalResourceID = generatePhysicalResourceId(event) + + err = handleFirstInvocation() + if err != nil { + sugLog.Error("Error while handling first invocation: ", err.Error()) + return physicalResourceID, nil, err + } + + // Populate your data map as needed for the update + data = make(map[string]interface{}) + // Populate data as needed + + return physicalResourceID, data, nil +} + +// Wrapper for invocation from cloudformation custom resource - for read, update +func customResourceRunDoNothing(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { + return +} + +// Wrapper for invocation from cloudformation custom resource - delete +func customResourceRunDelete(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { + sess, err := common.GetSession() + if err != nil { + sugLog.Error("Error while creating session: ", err.Error()) + } + + deleted := make([]string, 0) + servicesToDelete := common.GetServices() + if servicesToDelete != nil { + newDeleted, err := deleteServices(sess, servicesToDelete) + deleted = append(deleted, newDeleted...) + if err != nil { + sugLog.Error(err.Error()) + } + } + + pathsToDelete := common.GetCustomPaths() + if pathsToDelete != nil { + newDeleted, err := deleteCustom(sess, pathsToDelete) + deleted = append(deleted, newDeleted...) + if err != nil { + sugLog.Error(err.Error()) + } + } + + sugLog.Info("Deleted subscription filters for the following log groups: ", deleted) + + physicalResourceID = generatePhysicalResourceId(event) + // Populate your data map as needed for the update + data = make(map[string]interface{}) + // Populate data as needed + + return physicalResourceID, data, nil +} + +func handleFirstInvocation() error { + sess, err := common.GetSession() + if err != nil { + return err + } + + added := make([]string, 0) + servicesToAdd := common.GetServices() + if servicesToAdd != nil { + newAdded, err := addServices(sess, servicesToAdd) + added = append(added, newAdded...) + if err != nil { + sugLog.Error(err.Error()) + } + } + + pathsToAdd := common.GetCustomPaths() + if pathsToAdd != nil { + newAdded, err := addCustom(sess, pathsToAdd, added) + added = append(added, newAdded...) + if err != nil { + sugLog.Error(err.Error()) + } + } + + sugLog.Info("Following these log groups: ", added) + + return nil +} + +func addCustom(sess *session.Session, customGroup, added []string) ([]string, error) { + logsClient := cloudwatchlogs.New(sess) + toAdd := make([]string, 0) + lambdaNameTrigger := common.LambdaPrefix + os.Getenv(common.EnvFunctionName) + for _, customLogGroup := range customGroup { + if !common.ListContains(customLogGroup, added) { + // Prevent a situation where we put subscription filter on the trigger function + if customLogGroup != lambdaNameTrigger { + toAdd = append(toAdd, customLogGroup) + } + } + } + + newAdded := common.PutSubscriptionFilter(toAdd, logsClient) + + return newAdded, nil +} + +func addServices(sess *session.Session, servicesToAdd []string) ([]string, error) { + logsClient := cloudwatchlogs.New(sess) + logGroups := getLogGroups(servicesToAdd, logsClient) + if len(logGroups) > 0 { + sugLog.Debug("Detected the following services: ", logGroups) + newAdded := common.PutSubscriptionFilter(logGroups, logsClient) + return newAdded, nil + } else { + return nil, fmt.Errorf("Could not retrieve any log groups") + } +} + +func getLogGroups(services []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { + logGroupsToAdd := make([]string, 0) + serviceToPrefix := common.GetServicesMap() + for _, service := range services { + if prefix, ok := serviceToPrefix[service]; ok { + sugLog.Debug("Working on prefix: ", prefix) + newLogGroups, err := logGroupsPagination(prefix, logsClient) + if err != nil { + sugLog.Errorf("Error while searching for log groups of %s: %s", service, err.Error()) + } + + logGroupsToAdd = append(logGroupsToAdd, newLogGroups...) + } else { + sugLog.Errorf("Service %s is not supported. Skipping.", service) + } + } + + return logGroupsToAdd +} + +func logGroupsPagination(prefix string, logsClient *cloudwatchlogs.CloudWatchLogs) ([]string, error) { + var nextToken *string + logGroups := make([]string, 0) + for { + describeOutput, err := logsClient.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{ + LogGroupNamePrefix: &prefix, + NextToken: nextToken, + }) + + if err != nil { + return nil, err + } + if describeOutput != nil { + nextToken = describeOutput.NextToken + for _, logGroup := range describeOutput.LogGroups { + // Prevent a situation where we put subscription filter on the trigger and shipper function + if *logGroup.LogGroupName != common.LambdaPrefix+os.Getenv(common.EnvFunctionName) { + logGroups = append(logGroups, *logGroup.LogGroupName) + } + } + } + + if nextToken == nil { + break + } + } + + return logGroups, nil +} + +func deleteServices(sess *session.Session, servicesToDelete []string) ([]string, error) { + logsClient := cloudwatchlogs.New(sess) + logGroups := getLogGroups(servicesToDelete, logsClient) + + sugLog.Infow("Attempting to delete subscription filters", + "servicesToDelete", servicesToDelete, + "logGroups", logGroups) + + if len(logGroups) > 0 { + newDeleted := common.DeleteSubscriptionFilter(logGroups, logsClient) + sugLog.Infow("Deleted subscription filters", + "deletedLogGroups", newDeleted) + return newDeleted, nil + } else { + sugLog.Info("No log groups found for deletion") + return nil, fmt.Errorf("Could not delete any log groups") + } +} + +func deleteCustom(sess *session.Session, customGroup []string) ([]string, error) { + logsClient := cloudwatchlogs.New(sess) + + newDeleted := common.DeleteSubscriptionFilter(customGroup, logsClient) + + // Log the outcome of the deletion attempts + sugLog.Infow("Deleted custom subscription filters", + "deletedCustomLogGroups", newDeleted) + + return newDeleted, nil +} diff --git a/cloudformation/sam-template.yaml b/cloudformation/sam-template.yaml index c32d0c9..7051b06 100644 --- a/cloudformation/sam-template.yaml +++ b/cloudformation/sam-template.yaml @@ -67,17 +67,17 @@ Conditions: - '' Resources: - logzioFirehoseSubscriptionFiltersFunction: + CfnLambdaFunction: Type: 'AWS::Lambda::Function' DependsOn: logzioFirehose Properties: Code: S3Bucket: logzio-aws-integrations-<> - S3Key: firehose-logs/<>/function.zip - FunctionName: !Join ['-', [!Ref AWS::StackName, 'sf-func']] + S3Key: firehose-logs/<>/cfn-lambda.zip + FunctionName: !Join [ '-', [ !Ref AWS::StackName, 'cfn-lambda' ] ] Handler: bootstrap Runtime: provided.al2 - Role: !GetAtt firehoseSubscriptionFilterLambdaRole.Arn + Role: !GetAtt lambdaExecutionRole.Arn Timeout: !Ref triggerLambdaTimeout MemorySize: !Ref triggerLambdaMemory ReservedConcurrentExecutions: 1 @@ -90,7 +90,53 @@ Resources: FIREHOSE_ARN: !GetAtt logzioFirehose.Arn LOG_LEVEL: !Ref triggerLambdaLogLevel PUT_SF_ROLE: !GetAtt firehosePutSubscriptionFilterRole.Arn - firehoseSubscriptionFilterLambdaRole: + + EventBridgeLambdaFunction: + Type: 'AWS::Lambda::Function' + DependsOn: logzioFirehose + Properties: + Code: + S3Bucket: logzio-aws-integrations-<> + S3Key: firehose-logs/<>/eventbridge-lambda.zip + FunctionName: !Join [ '-', [ !Ref AWS::StackName, 'eventbridge-lambda' ] ] + Handler: bootstrap + Runtime: provided.al2 + Role: !GetAtt lambdaExecutionRole.Arn + Timeout: !Ref triggerLambdaTimeout + MemorySize: !Ref triggerLambdaMemory + ReservedConcurrentExecutions: 1 + Environment: + Variables: + SERVICES: !Ref services + CUSTOM_GROUPS: !Ref customLogGroups + ACCOUNT_ID: !Ref AWS::AccountId + AWS_PARTITION: !Ref AWS::Partition + FIREHOSE_ARN: !GetAtt logzioFirehose.Arn + LOG_LEVEL: !Ref triggerLambdaLogLevel + PUT_SF_ROLE: !GetAtt firehosePutSubscriptionFilterRole.Arn + LogGroupCreationEventRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - 'aws.logs' + detail-type: + - 'AWS API Call via CloudTrail' + detail: + eventName: + - 'CreateLogGroup' + State: ENABLED + Targets: + - Arn: !GetAtt EventBridgeLambdaFunction.Arn + Id: 'EventBridgeLambdaFunctionTarget' + PermissionForEventBridgeToInvokeLambda: + Type: 'AWS::Lambda::Permission' + Properties: + Action: 'lambda:InvokeFunction' + FunctionName: !GetAtt EventBridgeLambdaFunction.Arn + Principal: 'events.amazonaws.com' + SourceArn: !GetAtt LogGroupCreationEventRule.Arn + lambdaExecutionRole: Type: 'AWS::IAM::Role' Properties: RoleName: !Join [ '-', [ 'logzioRole', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] @@ -129,10 +175,12 @@ Resources: Resource: !GetAtt firehosePutSubscriptionFilterRole.Arn triggerPrimerInvoke: Type: AWS::CloudFormation::CustomResource - DependsOn: logzioFirehoseSubscriptionFiltersFunction + DependsOn: CfnLambdaFunction Version: '1.0' Properties: - ServiceToken: !GetAtt logzioFirehoseSubscriptionFiltersFunction.Arn + ServiceToken: !GetAtt CfnLambdaFunction.Arn + Services: !Ref services + CustomLogGroups: !Ref customLogGroups logGroupCreationEvent: Condition: createEventbridgeTrigger Type: AWS::Events::Rule @@ -151,13 +199,13 @@ Resources: Name: !Join [ '-', [ 'logGroupCreated', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] State: ENABLED Targets: - - Arn: !GetAtt logzioFirehoseSubscriptionFiltersFunction.Arn - Id: !Join [ '-', [ 'logzioSubscriptionFilter', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] + - Arn: !GetAtt EventBridgeLambdaFunction.Arn + Id: 'EventBridgeLambdaFunctionTarget' permissionForEventsToInvokeLambda: Condition: createEventbridgeTrigger Type: AWS::Lambda::Permission Properties: - FunctionName: !Ref logzioFirehoseSubscriptionFiltersFunction + FunctionName: !Ref EventBridgeLambdaFunction Action: 'lambda:InvokeFunction' Principal: 'events.amazonaws.com' SourceArn: !GetAtt logGroupCreationEvent.Arn diff --git a/common/log_management.go b/common/log_management.go new file mode 100644 index 0000000..7b6a7b3 --- /dev/null +++ b/common/log_management.go @@ -0,0 +1,126 @@ +package common + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "go.uber.org/zap" + "os" +) + +var sugLog *zap.SugaredLogger + +func PutSubscriptionFilter(logGroups []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { + // Early return if logsClient is nil to avoid panic + if logsClient == nil { + fmt.Println("CloudWatch Logs client is nil") + return nil + } + + // Initialize logger if it's nil + if sugLog == nil { + initLogger() + } + + destinationArn := os.Getenv(envFirehoseArn) + roleArn := os.Getenv(envPutSubscriptionFilterRole) + filterPattern := "" + filterName := subscriptionFilterName + added := make([]string, 0) + for _, logGroup := range logGroups { + _, err := logsClient.PutSubscriptionFilter(&cloudwatchlogs.PutSubscriptionFilterInput{ + DestinationArn: &destinationArn, + FilterName: &filterName, + LogGroupName: &logGroup, + FilterPattern: &filterPattern, + RoleArn: &roleArn, + }) + + if err != nil { + sugLog.Error("Error while trying to add subscription filter for ", logGroup, ": ", err.Error()) + continue + } + + added = append(added, logGroup) + } + + return added +} + +// Ensure sugLog is safely initialized before use +func initLogger() { + // Basic logger initialization, replace with your actual logger configuration + logger, err := zap.NewProduction() + if err != nil { + fmt.Printf("Failed to initialize logger: %v\n", err) + os.Exit(1) // Or handle the error according to your application's requirements + } + sugLog = logger.Sugar() +} + +func DeleteSubscriptionFilter(logGroups []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { + // Early return if logsClient is nil to avoid panic + if logsClient == nil { + fmt.Println("CloudWatch Logs client is nil") + return nil + } + + // Initialize logger if it's nil + if sugLog == nil { + initLogger() + } + + filterName := subscriptionFilterName + deleted := make([]string, 0) + for _, logGroup := range logGroups { + _, err := logsClient.DeleteSubscriptionFilter(&cloudwatchlogs.DeleteSubscriptionFilterInput{ + FilterName: &filterName, + LogGroupName: &logGroup, + }) + + if err != nil { + sugLog.Error("Error while trying to delete subscription filter for ", logGroup, ": ", err.Error()) + continue + } + + deleted = append(deleted, logGroup) + sugLog.Info("Detected the following services for deletion2: ", deleted) + + } + + return deleted +} + +func GetSession() (*session.Session, error) { + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Region: aws.String(os.Getenv(envAwsRegion)), + }, + }) + + if err != nil { + return nil, fmt.Errorf("error occurred while trying to create a connection to aws: %s. Aborting", err.Error()) + } + + return sess, nil +} + +func ValidateRequired() error { + destinationArn := os.Getenv(envFirehoseArn) + if destinationArn == emptyString { + return fmt.Errorf("destination ARN must be set") + } + + accountId := os.Getenv(envAccountId) + if accountId == emptyString { + return fmt.Errorf("account id must be set") + } + + awsPartition := os.Getenv(envAwsPartition) + if awsPartition == emptyString { + return fmt.Errorf("aws partition must be set") + } + + return nil +} diff --git a/utils.go b/common/utils.go similarity index 76% rename from utils.go rename to common/utils.go index d051c6d..b8a9442 100644 --- a/utils.go +++ b/common/utils.go @@ -1,4 +1,4 @@ -package main +package common import ( "os" @@ -8,7 +8,7 @@ import ( const ( envServices = "SERVICES" envAwsRegion = "AWS_REGION" // reserved env - envFunctionName = "AWS_LAMBDA_FUNCTION_NAME" // reserved env + EnvFunctionName = "AWS_LAMBDA_FUNCTION_NAME" // reserved env envFirehoseArn = "FIREHOSE_ARN" envAccountId = "ACCOUNT_ID" envCustomGroups = "CUSTOM_GROUPS" @@ -17,11 +17,11 @@ const ( valuesSeparator = "," emptyString = "" - lambdaPrefix = "/aws/lambda/" + LambdaPrefix = "/aws/lambda/" subscriptionFilterName = "logzio_firehose" ) -func getServices() []string { +func GetServices() []string { servicesStr := os.Getenv(envServices) if servicesStr == emptyString { return nil @@ -31,7 +31,7 @@ func getServices() []string { return strings.Split(servicesStr, valuesSeparator) } -func getServicesMap() map[string]string { +func GetServicesMap() map[string]string { return map[string]string{ "apigateway": "/aws/apigateway/", "rds": "/aws/rds/", @@ -50,7 +50,7 @@ func getServicesMap() map[string]string { } } -func getCustomPaths() []string { +func GetCustomPaths() []string { pathsStr := os.Getenv(envCustomGroups) if pathsStr == emptyString { return nil @@ -60,7 +60,16 @@ func getCustomPaths() []string { return strings.Split(pathsStr, valuesSeparator) } -func listContains(s string, l []string) bool { +func ParseServices(servicesStr string) []string { + if servicesStr == emptyString { + return nil + } + + servicesStr = strings.ReplaceAll(servicesStr, " ", "") + return strings.Split(servicesStr, valuesSeparator) +} + +func ListContains(s string, l []string) bool { for _, item := range l { if s == item { return true diff --git a/eventbridge-lambda/main.go b/eventbridge-lambda/main.go new file mode 100644 index 0000000..3a66e23 --- /dev/null +++ b/eventbridge-lambda/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/logzio/firehose-logs/common" + lp "github.com/logzio/firehose-logs/logger" + "go.uber.org/zap" + "os" + "strings" +) + +var sugLog *zap.SugaredLogger + +func main() { + lambda.Start(HandleEventBridgeRequest) +} + +func HandleEventBridgeRequest(ctx context.Context, event map[string]interface{}) (string, error) { + logger := lp.GetLogger() + defer logger.Sync() + sugLog = logger.Sugar() + sugLog.Info("Starting handling EventBridge event...") + sugLog.Debug("Handling event: ", event) + err := common.ValidateRequired() + if err != nil { + return "Lambda finished with error", err + } + + if _, ok := event["detail"]; ok { + // Extracted EventBridge event handling logic + newLogGroupCreated(event["detail"].(map[string]interface{})["requestParameters"].(map[string]interface{})["logGroupName"].(string)) + } + + return "EventBridge event processed", nil +} + +func newLogGroupCreated(logGroup string) { + // Prevent a situation where we put subscription filter on the trigger function + if logGroup == common.LambdaPrefix+os.Getenv(common.EnvFunctionName) { + return + } + + servicesToAdd := common.GetServices() + var added []string + if servicesToAdd != nil { + serviceToPrefix := common.GetServicesMap() + sess, err := common.GetSession() + if err != nil { + sugLog.Error("Could not create aws session: ", err.Error()) + return + } + logsClient := cloudwatchlogs.New(sess) + for _, service := range servicesToAdd { + if prefix, ok := serviceToPrefix[service]; ok { + if strings.Contains(logGroup, prefix) { + added = common.PutSubscriptionFilter([]string{logGroup}, logsClient) + if len(added) > 0 { + sugLog.Info("Added log group: ", logGroup) + return + } + } + } + } + } + + sugLog.Info("Log group ", logGroup, " does not match any of the selected services: ", servicesToAdd) +} diff --git a/go.mod b/go.mod index 93eadf7..fd50592 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module main +module github.com/logzio/firehose-logs go 1.19 diff --git a/go.sum b/go.sum index 29f9014..a050a62 100644 --- a/go.sum +++ b/go.sum @@ -31,7 +31,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -47,7 +46,6 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/lambda_build.sh b/lambda_build.sh index a188ec8..75e9695 100644 --- a/lambda_build.sh +++ b/lambda_build.sh @@ -1,3 +1,39 @@ #!/bin/bash -GOOS=linux GOARCH=amd64 go build -o bootstrap main && chmod 644 $(find . -type f) && chmod 755 $(find . -type d) && zip function.zip bootstrap -rm bootstrap \ No newline at end of file + +# Ensure the script exits on first error +set -e + +# Define the function directories +FUNCTION_DIRS=("cfn-lambda" "eventbridge-lambda") + +# Loop through each function directory to build and zip +for dir in "${FUNCTION_DIRS[@]}"; do + echo "Building and zipping $dir..." + + # Copy the common directory to the current function directory + cp -R common "$dir/" + + # Move into the function directory + cd "$dir" + + # Build the Go binary named bootstrap + GOOS=linux GOARCH=amd64 go build -o bootstrap main.go + + # Set the file permissions as needed + chmod 644 $(find . -type f) + chmod 755 $(find . -type d) bootstrap + + # Zip the binary into a package named after the directory + zip "../${dir}.zip" bootstrap + + # Clean up: Remove the binary and copied common directory + rm bootstrap + rm -rf common + + # Move back to the project root directory + cd .. + + echo "$dir build and packaging complete." +done + +echo "All Lambda functions have been built and packaged." diff --git a/main.go b/main.go deleted file mode 100644 index 2aa5dc2..0000000 --- a/main.go +++ /dev/null @@ -1,343 +0,0 @@ -package main - -import ( - "context" - "fmt" - "github.com/aws/aws-lambda-go/cfn" - "github.com/aws/aws-lambda-go/lambda" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "go.uber.org/zap" - lp "main/logger" - "os" - "strings" -) - -var sugLog *zap.SugaredLogger - -func main() { - lambda.Start(HandleRequest) -} - -func HandleRequest(ctx context.Context, event map[string]interface{}) (string, error) { - logger := lp.GetLogger() - defer logger.Sync() - sugLog = logger.Sugar() - sugLog.Info("Starting handling event...") - sugLog.Debug("Handling event: ", event) - err := validateRequired() - if err != nil { - return "Lambda finished with error", err - } - - if _, ok := event["detail"]; ok { - // Create log group invocation - sugLog.Debug("Detected Eventbridge event") - newLogGroupCreated(event["detail"].(map[string]interface{})["requestParameters"].(map[string]interface{})["logGroupName"].(string)) - } else { - // First invocation - if event["RequestType"].(string) == "Create" { - sugLog.Debug("Detected Cloudformation Create event") - lambda.Start(cfn.LambdaWrap(customResourceRun)) - } else if event["RequestType"].(string) == "Update" { - sugLog.Debug("Detected Cloudformation Update event") - // TODO - implement update - lambda.Start(cfn.LambdaWrap(customResourceRunDoNothing)) - } else if event["RequestType"].(string) == "Delete" { - sugLog.Debug("Detected Cloudformation delete event") - lambda.Start(cfn.LambdaWrap(customResourceRunDelete)) - } else { - lambda.Start(cfn.LambdaWrap(customResourceRunDoNothing)) - } - } - - return "Lambda finished", nil -} - -func validateRequired() error { - destinationArn := os.Getenv(envFirehoseArn) - if destinationArn == emptyString { - return fmt.Errorf("destination ARN must be set") - } - - accountId := os.Getenv(envAccountId) - if accountId == emptyString { - return fmt.Errorf("account id must be set") - } - - awsPartition := os.Getenv(envAwsPartition) - if awsPartition == emptyString { - return fmt.Errorf("aws partition must be set") - } - - return nil -} - -// Wrapper for first invocation from cloud formation custom resource -func customResourceRun(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { - err = handleFirstInvocation() - if err != nil { - sugLog.Error("Error while handling first invocation: ", err.Error()) - return - } - - return -} - -// Wrapper for invocation from cloudformation custom resource - for read, update -func customResourceRunDoNothing(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { - return -} - -// Wrapper for invocation from cloudformation custom resource - delete -func customResourceRunDelete(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { - sess, err := getSession() - if err != nil { - sugLog.Error("Error while creating session: ", err.Error()) - } - - deleted := make([]string, 0) - servicesToDelete := getServices() - if servicesToDelete != nil { - newDeleted, err := deleteServices(sess, servicesToDelete) - deleted = append(deleted, newDeleted...) - if err != nil { - sugLog.Error(err.Error()) - } - } - - pathsToDelete := getCustomPaths() - if pathsToDelete != nil { - newDeleted, err := deleteCustom(sess, pathsToDelete) - deleted = append(deleted, newDeleted...) - if err != nil { - sugLog.Error(err.Error()) - } - } - - sugLog.Info("Deleted subscription filters for the following log groups: ", deleted) - - return -} - -func newLogGroupCreated(logGroup string) { - // Prevent a situation where we put subscription filter on the trigger function - if logGroup == lambdaPrefix+os.Getenv(envFunctionName) { - return - } - - servicesToAdd := getServices() - var added []string - if servicesToAdd != nil { - serviceToPrefix := getServicesMap() - sess, err := getSession() - if err != nil { - sugLog.Error("Could not create aws session: ", err.Error()) - return - } - logsClient := cloudwatchlogs.New(sess) - for _, service := range servicesToAdd { - if prefix, ok := serviceToPrefix[service]; ok { - if strings.Contains(logGroup, prefix) { - added = putSubscriptionFilter([]string{logGroup}, logsClient) - if len(added) > 0 { - sugLog.Info("Added log group: ", logGroup) - return - } - } - } - } - } - - sugLog.Info("Log group ", logGroup, " does not match any of the selected services: ", servicesToAdd) -} - -func handleFirstInvocation() error { - sess, err := getSession() - if err != nil { - return err - } - - added := make([]string, 0) - servicesToAdd := getServices() - if servicesToAdd != nil { - newAdded, err := addServices(sess, servicesToAdd) - added = append(added, newAdded...) - if err != nil { - sugLog.Error(err.Error()) - } - } - - pathsToAdd := getCustomPaths() - if pathsToAdd != nil { - newAdded, err := addCustom(sess, pathsToAdd, added) - added = append(added, newAdded...) - if err != nil { - sugLog.Error(err.Error()) - } - } - - sugLog.Info("Following these log groups: ", added) - - return nil -} - -func addCustom(sess *session.Session, customGroup, added []string) ([]string, error) { - logsClient := cloudwatchlogs.New(sess) - toAdd := make([]string, 0) - lambdaNameTrigger := lambdaPrefix + os.Getenv(envFunctionName) - for _, customLogGroup := range customGroup { - if !listContains(customLogGroup, added) { - // Prevent a situation where we put subscription filter on the trigger function - if customLogGroup != lambdaNameTrigger { - toAdd = append(toAdd, customLogGroup) - } - } - } - - newAdded := putSubscriptionFilter(toAdd, logsClient) - - return newAdded, nil -} - -func addServices(sess *session.Session, servicesToAdd []string) ([]string, error) { - logsClient := cloudwatchlogs.New(sess) - logGroups := getLogGroups(servicesToAdd, logsClient) - if len(logGroups) > 0 { - sugLog.Debug("Detected the following services: ", logGroups) - newAdded := putSubscriptionFilter(logGroups, logsClient) - return newAdded, nil - } else { - return nil, fmt.Errorf("Could not retrieve any log groups") - } -} - -func putSubscriptionFilter(logGroups []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { - destinationArn := os.Getenv(envFirehoseArn) - roleArn := os.Getenv(envPutSubscriptionFilterRole) - filterPattern := "" - filterName := subscriptionFilterName - added := make([]string, 0) - for _, logGroup := range logGroups { - _, err := logsClient.PutSubscriptionFilter(&cloudwatchlogs.PutSubscriptionFilterInput{ - DestinationArn: &destinationArn, - FilterName: &filterName, - LogGroupName: &logGroup, - FilterPattern: &filterPattern, - RoleArn: &roleArn, - }) - - if err != nil { - sugLog.Error("Error while trying to add subscription filter for ", logGroup, ": ", err.Error()) - continue - } - - added = append(added, logGroup) - } - - return added -} - -func getLogGroups(services []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { - logGroupsToAdd := make([]string, 0) - serviceToPrefix := getServicesMap() - for _, service := range services { - if prefix, ok := serviceToPrefix[service]; ok { - sugLog.Debug("Working on prefix: ", prefix) - newLogGroups, err := logGroupsPagination(prefix, logsClient) - if err != nil { - sugLog.Errorf("Error while searching for log groups of %s: %s", service, err.Error()) - } - - logGroupsToAdd = append(logGroupsToAdd, newLogGroups...) - } else { - sugLog.Errorf("Service %s is not supported. Skipping.", service) - } - } - - return logGroupsToAdd -} - -func logGroupsPagination(prefix string, logsClient *cloudwatchlogs.CloudWatchLogs) ([]string, error) { - var nextToken *string - logGroups := make([]string, 0) - for { - describeOutput, err := logsClient.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{ - LogGroupNamePrefix: &prefix, - NextToken: nextToken, - }) - - if err != nil { - return nil, err - } - if describeOutput != nil { - nextToken = describeOutput.NextToken - for _, logGroup := range describeOutput.LogGroups { - // Prevent a situation where we put subscription filter on the trigger and shipper function - if *logGroup.LogGroupName != lambdaPrefix+os.Getenv(envFunctionName) { - logGroups = append(logGroups, *logGroup.LogGroupName) - } - } - } - - if nextToken == nil { - break - } - } - - return logGroups, nil -} - -func getSession() (*session.Session, error) { - sess, err := session.NewSessionWithOptions(session.Options{ - Config: aws.Config{ - Region: aws.String(os.Getenv(envAwsRegion)), - }, - }) - - if err != nil { - return nil, fmt.Errorf("error occurred while trying to create a connection to aws: %s. Aborting", err.Error()) - } - - return sess, nil -} - -func deleteServices(sess *session.Session, servicesToDelete []string) ([]string, error) { - logsClient := cloudwatchlogs.New(sess) - logGroups := getLogGroups(servicesToDelete, logsClient) - if len(logGroups) > 0 { - sugLog.Debug("Detected the following services for deletion: ", logGroups) - newDeleted := deleteSubscriptionFilter(logGroups, logsClient) - return newDeleted, nil - } else { - return nil, fmt.Errorf("Could not delete any log groups") - } -} - -func deleteSubscriptionFilter(logGroups []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { - filterName := subscriptionFilterName - deleted := make([]string, 0) - for _, logGroup := range logGroups { - _, err := logsClient.DeleteSubscriptionFilter(&cloudwatchlogs.DeleteSubscriptionFilterInput{ - FilterName: &filterName, - LogGroupName: &logGroup, - }) - - if err != nil { - sugLog.Error("Error while trying to delete subscription filter for ", logGroup, ": ", err.Error()) - continue - } - - deleted = append(deleted, logGroup) - } - - return deleted -} - -func deleteCustom(sess *session.Session, customGroup []string) ([]string, error) { - logsClient := cloudwatchlogs.New(sess) - newDeleted := deleteSubscriptionFilter(customGroup, logsClient) - - return newDeleted, nil -}