From 24f04b9f71d88b294e80aa4f851f26ddd749c7d2 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 29 Aug 2022 13:36:01 +0200 Subject: [PATCH 01/12] feat: add smoke testing Add a terraform script and an example js function. The function is pushed to aws with the following layers: - apm js agent - locally built lambda extension Log level and aws region can be configured using variables. --- testdata/function/index.js | 8 ++++++++ tf/main.tf | 42 ++++++++++++++++++++++++++++++++++++++ tf/providers.tf | 8 ++++++++ tf/variables.tf | 12 +++++++++++ 4 files changed, 70 insertions(+) create mode 100644 testdata/function/index.js create mode 100644 tf/main.tf create mode 100644 tf/providers.tf create mode 100644 tf/variables.tf diff --git a/testdata/function/index.js b/testdata/function/index.js new file mode 100644 index 00000000..c6cb24e0 --- /dev/null +++ b/testdata/function/index.js @@ -0,0 +1,8 @@ +exports.handler = async (event) => { + const response = { + statusCode: 200, + body: JSON.stringify('Hello from Lambda!'), + }; + return response; +}; + diff --git a/tf/main.tf b/tf/main.tf new file mode 100644 index 00000000..4a93300f --- /dev/null +++ b/tf/main.tf @@ -0,0 +1,42 @@ +provider "aws" { + region = var.aws_region +} + +module "lambda_function" { + source = "terraform-aws-modules/lambda/aws" + + function_name = "smoke-testing-test" + description = "Example Lambda function for smoke testing" + handler = "index.handler" + runtime = "nodejs16.x" + + source_path = "../testdata/function/" + + layers = [ + module.lambda_layer_local.lambda_layer_arn, + "arn:aws:lambda:eu-central-1:267093732750:layer:elastic-apm-node-ver-3-38-0:1", + ] + + environment_variables = { + NODE_OPTIONS = "-r elastic-apm-node/start" + ELASTIC_APM_LOG_LEVEL = var.log_level + ELASTIC_APM_LAMBDA_APM_SERVER = "CHANGEME" + ELASTIC_APM_SECRET_TOKEN = "CHANGEME" + } + + tags = { + Name = "my-lambda" + } +} + +module "lambda_layer_local" { + source = "terraform-aws-modules/lambda/aws" + + create_layer = true + + layer_name = "apm-lambda-extension-smoke-testing" + description = "AWS Lambda Extension Layer for Elastic APM - smoke testing" + compatible_runtimes = ["nodejs16.x"] + + source_path = "../bin/" +} diff --git a/tf/providers.tf b/tf/providers.tf new file mode 100644 index 00000000..c01034cc --- /dev/null +++ b/tf/providers.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">=4.28.0" + } + } +} diff --git a/tf/variables.tf b/tf/variables.tf new file mode 100644 index 00000000..d5059a1b --- /dev/null +++ b/tf/variables.tf @@ -0,0 +1,12 @@ +variable "aws_region" { + type = string + description = "aws region" + default = "eu-central-1" +} + +variable "log_level" { + type = string + description = "lambda extension log level" + default = "trace" +} + From 2d8ffa6e738842c052220cac74fbac12a45a0cf4 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 5 Sep 2022 16:19:20 +0200 Subject: [PATCH 02/12] feat: add support for creating elastic cloud deployments create an elastic cloud deployment and auto set the apm server url and secret token --- tf/main.tf | 51 +++++++++++++++++++++++++++++++++++++++++++++++-- tf/providers.tf | 4 ++++ tf/variables.tf | 11 +++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/tf/main.tf b/tf/main.tf index 4a93300f..8b73b0fc 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -2,6 +2,53 @@ provider "aws" { region = var.aws_region } +provider "ec" {} + +data "ec_stack" "latest" { + version_regex = "latest" + region = var.ec_region +} + +resource "ec_deployment" "ec_aws_lambda_minimal" { + name = "aws-lambda-smoke-testing-deployment" + region = var.ec_region + version = data.ec_stack.latest.version + deployment_template_id = var.ec_deployment_template + + elasticsearch { + autoscale = "false" + + topology { + id = "hot_content" + size = "1g" + zone_count = 1 + } + } +} + +resource "ec_deployment" "basic_datasource" { + name = "aws-lambda-smoke-testing-deployment-data" + region = var.ec_region + version = data.ec_stack.latest.version + deployment_template_id = var.ec_deployment_template + + elasticsearch { + topology { + id = "hot_content" + size = "1g" + zone_count = 1 + } + } + + kibana {} + + apm {} + + observability { + deployment_id = ec_deployment.ec_aws_lambda_minimal.id + } +} + module "lambda_function" { source = "terraform-aws-modules/lambda/aws" @@ -20,8 +67,8 @@ module "lambda_function" { environment_variables = { NODE_OPTIONS = "-r elastic-apm-node/start" ELASTIC_APM_LOG_LEVEL = var.log_level - ELASTIC_APM_LAMBDA_APM_SERVER = "CHANGEME" - ELASTIC_APM_SECRET_TOKEN = "CHANGEME" + ELASTIC_APM_LAMBDA_APM_SERVER = ec_deployment.basic_datasource.apm[0].https_endpoint + ELASTIC_APM_SECRET_TOKEN = ec_deployment.basic_datasource.apm_secret_token } tags = { diff --git a/tf/providers.tf b/tf/providers.tf index c01034cc..506fb456 100644 --- a/tf/providers.tf +++ b/tf/providers.tf @@ -4,5 +4,9 @@ terraform { source = "hashicorp/aws" version = ">=4.28.0" } + ec = { + source = "elastic/ec" + version = ">=0.4.1" + } } } diff --git a/tf/variables.tf b/tf/variables.tf index d5059a1b..56288fab 100644 --- a/tf/variables.tf +++ b/tf/variables.tf @@ -10,3 +10,14 @@ variable "log_level" { default = "trace" } +variable "ec_region" { + type = string + description = "ec region" + default = "aws-eu-central-1" +} + +variable "ec_deployment_template" { + type = string + description = "ec deployment template" + default = "aws-storage-optimized-v2" +} From 59ecb411d5cdcdb2d43a893de725159dd95a36da Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 6 Sep 2022 21:33:29 +0200 Subject: [PATCH 03/12] fix: use provided aws region when retrieving lambda layers --- tf/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/main.tf b/tf/main.tf index 8b73b0fc..af93b55a 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -61,7 +61,7 @@ module "lambda_function" { layers = [ module.lambda_layer_local.lambda_layer_arn, - "arn:aws:lambda:eu-central-1:267093732750:layer:elastic-apm-node-ver-3-38-0:1", + "arn:aws:lambda:${var.aws_region}:267093732750:layer:elastic-apm-node-ver-3-38-0:1", ] environment_variables = { From 59bb7e39c8b711e7ce753f892ffbc082a90b6d70 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:55:09 +0200 Subject: [PATCH 04/12] feat: move to custom modules in the apm-server repository --- tf/main.tf | 57 ++++++++++-------------------------------------------- 1 file changed, 10 insertions(+), 47 deletions(-) diff --git a/tf/main.tf b/tf/main.tf index af93b55a..23363b61 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -2,51 +2,14 @@ provider "aws" { region = var.aws_region } -provider "ec" {} - -data "ec_stack" "latest" { - version_regex = "latest" - region = var.ec_region -} - -resource "ec_deployment" "ec_aws_lambda_minimal" { - name = "aws-lambda-smoke-testing-deployment" - region = var.ec_region - version = data.ec_stack.latest.version - deployment_template_id = var.ec_deployment_template - - elasticsearch { - autoscale = "false" - - topology { - id = "hot_content" - size = "1g" - zone_count = 1 - } - } -} - -resource "ec_deployment" "basic_datasource" { - name = "aws-lambda-smoke-testing-deployment-data" - region = var.ec_region - version = data.ec_stack.latest.version - deployment_template_id = var.ec_deployment_template - - elasticsearch { - topology { - id = "hot_content" - size = "1g" - zone_count = 1 - } - } - - kibana {} - - apm {} - - observability { - deployment_id = ec_deployment.ec_aws_lambda_minimal.id - } +module "ec_deployment" { + source = "github.com/elastic/apm-server//testing/infra/terraform/modules/ec_deployment?depth=1" + deployment_name_prefix = "apm-aws-lambda-smoke-testing" + integrations_server = true + apm_server_expvar = false + apm_server_pprof = false + region = var.ec_region + deployment_template = var.ec_deployment_template } module "lambda_function" { @@ -67,8 +30,8 @@ module "lambda_function" { environment_variables = { NODE_OPTIONS = "-r elastic-apm-node/start" ELASTIC_APM_LOG_LEVEL = var.log_level - ELASTIC_APM_LAMBDA_APM_SERVER = ec_deployment.basic_datasource.apm[0].https_endpoint - ELASTIC_APM_SECRET_TOKEN = ec_deployment.basic_datasource.apm_secret_token + ELASTIC_APM_LAMBDA_APM_SERVER = module.ec_deployment.apm_url + ELASTIC_APM_SECRET_TOKEN = module.ec_deployment.apm_secret_token } tags = { From 780c6973929f6a1f82c22122d28a4bdd3ffc1199 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Sep 2022 01:58:52 +0200 Subject: [PATCH 05/12] build: add terraform gitignore --- tf/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tf/.gitignore diff --git a/tf/.gitignore b/tf/.gitignore new file mode 100644 index 00000000..09b1e21a --- /dev/null +++ b/tf/.gitignore @@ -0,0 +1,4 @@ +.terraform/ + +*.tfstate +*.tfstate.* From 9a098938a47c1b9ecf5e4438d1674f667e7e609a Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Sep 2022 02:01:44 +0200 Subject: [PATCH 06/12] refactor: rename ec_region to ess_region --- tf/main.tf | 2 +- tf/variables.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tf/main.tf b/tf/main.tf index 23363b61..b2483bb5 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -8,7 +8,7 @@ module "ec_deployment" { integrations_server = true apm_server_expvar = false apm_server_pprof = false - region = var.ec_region + region = var.ess_region deployment_template = var.ec_deployment_template } diff --git a/tf/variables.tf b/tf/variables.tf index 56288fab..4035d34a 100644 --- a/tf/variables.tf +++ b/tf/variables.tf @@ -10,9 +10,9 @@ variable "log_level" { default = "trace" } -variable "ec_region" { +variable "ess_region" { type = string - description = "ec region" + description = "ess region" default = "aws-eu-central-1" } From 866790f3ee14a779fd9460b2f7b60ebf4ba83b0f Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Sep 2022 19:45:30 +0200 Subject: [PATCH 07/12] docs: update template description Co-authored-by: Andrew Wilkins --- tf/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/variables.tf b/tf/variables.tf index 4035d34a..f0ddd421 100644 --- a/tf/variables.tf +++ b/tf/variables.tf @@ -18,6 +18,6 @@ variable "ess_region" { variable "ec_deployment_template" { type = string - description = "ec deployment template" + description = "Elastic Cloud deployment template" default = "aws-storage-optimized-v2" } From 60974ba2740b0668c2e2595e4bba8714a2f15b34 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Wed, 14 Sep 2022 23:24:09 +0200 Subject: [PATCH 08/12] fix: move to gcp-us-west1 / CFT region --- tf/variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/variables.tf b/tf/variables.tf index f0ddd421..aaa56564 100644 --- a/tf/variables.tf +++ b/tf/variables.tf @@ -13,11 +13,11 @@ variable "log_level" { variable "ess_region" { type = string description = "ess region" - default = "aws-eu-central-1" + default = "gcp-us-west2" } variable "ec_deployment_template" { type = string description = "Elastic Cloud deployment template" - default = "aws-storage-optimized-v2" + default = "gcp-compute-optimized" } From 01626d6ec4f7d80541a04f082768073f7abf751f Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Wed, 14 Sep 2022 23:25:54 +0200 Subject: [PATCH 09/12] refactor: rename ec deployment template to ess deployment template --- tf/main.tf | 2 +- tf/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/main.tf b/tf/main.tf index b2483bb5..6324c00e 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -9,7 +9,7 @@ module "ec_deployment" { apm_server_expvar = false apm_server_pprof = false region = var.ess_region - deployment_template = var.ec_deployment_template + deployment_template = var.ess_deployment_template } module "lambda_function" { diff --git a/tf/variables.tf b/tf/variables.tf index aaa56564..c25268e5 100644 --- a/tf/variables.tf +++ b/tf/variables.tf @@ -16,7 +16,7 @@ variable "ess_region" { default = "gcp-us-west2" } -variable "ec_deployment_template" { +variable "ess_deployment_template" { type = string description = "Elastic Cloud deployment template" default = "gcp-compute-optimized" From eadce13075ced38f7c6cdff055b45e451a1ef1ad Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Thu, 15 Sep 2022 01:25:08 +0200 Subject: [PATCH 10/12] feat: add bash script to validate smoke testing output --- tf/output.tf | 16 ++++++++++++++++ tf/test.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 tf/output.tf create mode 100755 tf/test.sh diff --git a/tf/output.tf b/tf/output.tf new file mode 100644 index 00000000..ef0576ae --- /dev/null +++ b/tf/output.tf @@ -0,0 +1,16 @@ +output "elasticsearch_url" { + value = module.ec_deployment.elasticsearch_url + description = "The secure Elasticsearch URL" +} + +output "elasticsearch_username" { + value = module.ec_deployment.elasticsearch_username + description = "The Elasticsearch username" + sensitive = true +} + +output "elasticsearch_password" { + value = module.ec_deployment.elasticsearch_password + description = "The Elasticsearch password" + sensitive = true +} diff --git a/tf/test.sh b/tf/test.sh new file mode 100755 index 00000000..ea470965 --- /dev/null +++ b/tf/test.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e + +export TF_IN_AUTOMATION=1 +export TF_CLI_ARGS=-no-color + +cleanup() { + terraform destroy -auto-approve >> tf.log +} + +trap "cleanup" EXIT + +echo "-> Creating the underlying infrastructure..." +terraform init | tee tf.log +terraform apply -auto-approve | tee -a tf.log + +echo "-> Calling the lambda function..." +aws lambda invoke --function-name smoke-testing-test response.json +aws lambda invoke --function-name smoke-testing-test response.json + +echo "-> Waiting for the agent documents to be indexed in Elasticsearch..." + +ES_HOST=$(terraform output -raw elasticsearch_url) +ES_USER=$(terraform output -raw elasticsearch_username) +ES_PASS=$(terraform output -raw elasticsearch_password) + +hits=0 +attempts=0 +while [ ${hits} -eq 0 ]; do + # Check that ES has transaction documents on the GET endpoint. + resp=$(curl -s -H 'Content-Type: application/json' -u ${ES_USER}:${ES_PASS} "${ES_HOST}/traces-apm-*/_search" -d '{ + "query": { + "match": { + "agent.name": "nodejs" + } + } + }') + hits=$(echo ${resp} | jq '.hits.total.value') + if [[ ${attempts} -ge 30 ]]; then + echo "-> Didn't find any traces for the NodeJS agents after ${attempts} attempts" + exit 1 + fi + let "attempts+=1" + sleep 1 +done + +echo "-> Smoke tests passed!" From cff7ae7d919451349e6cf8c6222d0449996b1e0a Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Thu, 15 Sep 2022 01:25:24 +0200 Subject: [PATCH 11/12] lint: runt terraform fmt --- tf/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tf/main.tf b/tf/main.tf index 6324c00e..d37b33b1 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -3,13 +3,13 @@ provider "aws" { } module "ec_deployment" { - source = "github.com/elastic/apm-server//testing/infra/terraform/modules/ec_deployment?depth=1" + source = "github.com/elastic/apm-server//testing/infra/terraform/modules/ec_deployment?depth=1" deployment_name_prefix = "apm-aws-lambda-smoke-testing" - integrations_server = true - apm_server_expvar = false - apm_server_pprof = false - region = var.ess_region - deployment_template = var.ess_deployment_template + integrations_server = true + apm_server_expvar = false + apm_server_pprof = false + region = var.ess_region + deployment_template = var.ess_deployment_template } module "lambda_function" { From bf8a16820fe52f2a0ed2f184b9b32608efa3477d Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Thu, 15 Sep 2022 02:51:36 +0200 Subject: [PATCH 12/12] fix: explicitly pass the region to the aws cli Co-authored-by: Andrew Wilkins --- tf/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/test.sh b/tf/test.sh index ea470965..3280828e 100755 --- a/tf/test.sh +++ b/tf/test.sh @@ -16,8 +16,8 @@ terraform init | tee tf.log terraform apply -auto-approve | tee -a tf.log echo "-> Calling the lambda function..." -aws lambda invoke --function-name smoke-testing-test response.json -aws lambda invoke --function-name smoke-testing-test response.json +aws lambda invoke --region=eu-central-1 --function-name smoke-testing-test response.json +aws lambda invoke --region=eu-central-1 --function-name smoke-testing-test response.json echo "-> Waiting for the agent documents to be indexed in Elasticsearch..."