Merge pull request #116 from zotoio/feature/encrypted-env

Feature/encrypted env

.envExample

@@ -1,36 +1,29 @@
 GTM_AWS_REGION=ap-southeast-2
+GTM_AWS_STAGE=dev
 GTM_SQS_PENDING_QUEUE=gtmPendingQueue
 GTM_SQS_RESULTS_QUEUE=gtmResultsQueue
 GTM_SNS_RESULTS_TOPIC=gtmResultsSNSTopic
-GTM_GITHUB_WEBHOOK_SECRET=<redacted>
-GTM_GITHUB_TOKEN=<redacted>
-GTM_GITHUB_TOKEN_FUNCTIONAL_TESTS=<redacted>
 GTM_GITHUB_HOST=api.github.com
 GTM_GITHUB_DEBUG=true
 GTM_GITHUB_TIMEOUT=5000
 GTM_GITHUB_PATH_PREFIX=
 GTM_GITHUB_PROXY=
 GTM_TASK_CONFIG_FILENAME=.githubTaskManager.json
 GTM_AGENT_PORT=9091
-GTM_AGENT_AWS_ACCESS_KEY_ID=<redacted>
-GTM_AGENT_AWS_SECRET_ACCESS_KEY=<redacted>
 GTM_JENKINS_USER=<redacted>
 GTM_JENKINS_URL=<redacted>
 GTM_JENKINS_CSRF=<true|false>
 GTM_TEAMCITY_USER=<redacted>
-GTM_TEAMCITY_PASSCODE=<redacted>
 GTM_TEAMCITY_URL=<redacted>
 GTM_DOCKER_IMAGE_WHITELIST=alpine:*,zotoio/*
 GTM_DOCKER_IMAGE_WHITELIST_FILE=.dockerImageWhitelistExample
 GTM_DOCKER_COMMANDS_ALLOWED=true
 GTM_DOCKER_ALLOW_PULL=true
 GTM_DOCKER_DEFAULT_WORKER_IMAGE=zotoio/gtm-worker:latest
 IAM_ENABLED=<true|undefined>
-LAUNCHDARKLY_API_TOKEN=<redacted>
 GTM_LOGSTASH_HOST=
 GTM_LOGSTASH_PORT=
 GTM_SONAR_HOST_URL=http://localhost:9000
-GTM_SONAR_GITHUB_OAUTH=<redacted>
 GTM_SONAR_GITHUB_ENDPOINT=<http://enterprise.github/api/v3/>
 GTM_SONAR_LOGIN=<redacted>
 GTM_SONAR_PROJECTNAME_PREFIX=github::
@@ -46,4 +39,27 @@ GTM_AWS_VPC_ID=<redacted>
 GTM_BASE_URL=http://localhost:9091
 GTM_S3_DEPENDENCY_BUCKET=gtmstorage
 GTM_WELCOME_MESSAGE_ENABLED=true
-GTM_REPO_BLACKLIST=.*ignore-repo.*,.*another-repo.*
+GTM_REPO_BLACKLIST=.*ignore-repo.*,.*another-repo.*
+GTM_AWS_KMS_KEY_ID=<redacted>
+GTM_SLS_EXECUTOR_AWS_STAGE=<stage for lambdas deployed by ExecutorDockerServerless>
+GTM_SLS_EXECUTOR_AWS_REGION=<region for lambdas>
+GTM_SLS_EXECUTOR_AWS_EXECUTION_ROLE=<iam role for lambda execution>
+
+GTM_CRYPT_GITHUB_TOKEN=<redacted>
+GTM_CRYPT_GITHUB_WEBHOOK_SECRET=<redacted>
+GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID=<redacted>
+GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY=<redacted>
+GTM_CRYPT_LAUNCHDARKLY_API_TOKEN=<redacted>
+GTM_CRYPT_SONAR_GITHUB_OAUTH=<redacted>
+GTM_CRYPT_SONAR_LOGIN=<redacted>
+GTM_CRYPT_JENKINS_TOKEN=<redacted>
+
+# use the following to encrypt values, collect from kms-secrets-* file created, and add above
+#npm run sls-encrypt GTM_CRYPT_GITHUB_TOKEN <redacted>
+#npm run sls-encrypt GTM_CRYPT_GITHUB_WEBHOOK_SECRET <redacted>
+#npm run sls-encrypt GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID <redacted>
+#npm run sls-encrypt GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY <redacted>
+#npm run sls-encrypt GTM_CRYPT_LAUNCHDARKLY_API_TOKEN <redacted>
+#npm run sls-encrypt GTM_CRYPT_SONAR_GITHUB_OAUTH <redacted>
+#npm run sls-encrypt GTM_CRYPT_SONAR_LOGIN <redacted>
+#npm run sls-encrypt GTM_CRYPT_JENKINS_TOKEN <redacted>

README.md

@@ -40,33 +40,37 @@ Create an asynchronous CI agnostic mechanism for running custom test stage gates
 - npm install
 - setup serverless aws creds per https://github.com/serverless/serverless/blob/master/docs/providers/aws/guide/credentials.md
 - setup a .env file in the repo root (copy from .envExample and modify)
+- create and AWS KMS key, and capture the id for var `GTM_AWS_KMS_KEY_ID`
 
 | Environment variable | description |
 | -------------------- | ----------- |
+|GTM_AWS_KMS_KEY_ID | aws kms key id |
+|GTM_CRYPT_GITHUB_TOKEN | encrypted access token for accessing github |
+|GTM_CRYPT_GITHUB_WEBHOOK_SECRET | encrypted shared secret from github webook config |
+|GTM_CRYPT_AWS_ACCESS_KEY_ID | encrypted aws key id - for agent only |
+|GTM_CRYPT_AWS_SECRET_ACCESS_KEY | encrypted aws secret - for agent only |
+|GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY|secret key for agent|
+|GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID|access key for agent|
+|GTM_CRYPT_JENKINS_TOKEN| encrypted token |
+|GTM_CRYPT_TEAMCITY_PASSCODE| encrypted teamcity executor passcode|
+|GTM_CRYPT_SONAR_LOGIN| encrypted sonar access token |
+|GTM_CRYPT_SONAR_GITHUB_OAUTH| encrypted github token for sonar to post comments and status |
 |GTM_AWS_REGION | awsregion to create resources in |
 |GTM_SQS_PENDING_QUEUE | name of SQS queue for new event |
 |GTM_SQS_RESULTS_QUEUE | name of SQS queue for results |
 |GTM_SNS_RESULTS_TOPIC | name of SNS topic for result ping |
-|GTM_GITHUB_WEBHOOK_SECRET | shared secret from github webook config |
-|GTM_GITHUB_TOKEN | access token for accessing github |
-|GTM_GITHUB_TOKEN_FUNCTIONAL_TESTS | access token for individual test type.  each task type can have a different token |
 |GTM_GITHUB_HOST | api hostname can be updated for github enterprise |
 |GTM_GITHUB_DEBUG | debug mode for api calls |
 |GTM_GITHUB_TIMEOUT | github api timeout |
 |GTM_GITHUB_PATH_PREFIX | path prefix for github enterprise |
 |GTM_GITHUB_PROXY | github api client proxy |
 |GTM_TASK_CONFIG_FILENAME | filename in repo to look for for task config - default is .githubTaskManager |
-|GTM_AWS_ACCESS_KEY_ID | aws key id - for agent only |
-|GTM_AWS_SECRET_ACCESS_KEY | aws secret - for agent only |
 |AWS_PROXY|URL of proxy to use for network requests. Optional|
 |GTM_AGENT_PORT| defaults to 9091 |
-|GTM_AGENT_AWS_ACCESS_KEY_ID|access key for agent|
-|GTM_AGENT_AWS_SECRET_ACCESS_KEY|secret key for agent|
 |GTM_JENKINS_USER|login for jenkins executor|
 |GTM_JENKINS_URL|url executor uses to talk to jenkins|
 |GTM_JENKINS_CSRF| is csrf enabled? true or false|
 |GTM_TEAMCITY_USER|teamcity executor user|
-|GTM_TEAMCITY_PASSCODE|teamcity executor passcode|
 |GTM_TEAMCITY_URL|teamcity api url|
 |GTM_DOCKER_IMAGE_WHITELIST| comma separated list of regex of allows docker images eg. alpine:*,bash:latest|
 |GTM_DOCKER_IMAGE_WHITELIST_FILE|use an optional docker whitelist file .dockerImageWhitelistExample|
@@ -78,10 +82,8 @@ Create an asynchronous CI agnostic mechanism for running custom test stage gates
 |GTM_LOGSTASH_HOST|optional logstash host for elasticsearch analysis|
 |GTM_LOGSTASH_PORT|optional logstash port|
 |GTM_SONAR_HOST_URL| sonar host url to connect to |
-|GTM_SONAR_LOGIN| sonar access token |
 |GTM_SONAR_PROJECTNAME_PREFIX| prefix if reporting to sonarqube |
 |GTM_SONAR_ANALYSIS_MODE| mode for sonar runner, default preview for PRs |
-|GTM_SONAR_GITHUB_OAUTH| github token for sonar to post comments and status |
 |GTM_SONAR_SOURCES| default source dir is `src`|
 |GTM_SONAR_JAVA_BINARIES| default is `target`|
 |GTM_SONAR_MODULES| comma separated modules|
@@ -96,6 +98,11 @@ Create an asynchronous CI agnostic mechanism for running custom test stage gates
 |GTM_S3_DEPENDENCY_BUCKET| aws s3 storage of build dependencies|
 |GTM_AWS_S3_PROXY| https_proxy for aws s3 |
 |GTM_REPO_BLACKLIST| comma separated list of regex to blackist repo names from triggering events |
+|GTM_SLS_EXECUTOR_AWS_STAGE| stage override from default calculation of dev/test|
+|GTM_SLS_EXECUTOR_AWS_REGION| aws region for lambdas default ap-southeast-2|
+|GTM_SLS_EXECUTOR_AWS_EXECUTION_ROLE| docker serverless lambda execution role |
+
+> important: values of env vars prefixed with `GTM_CRYPT_*` must be created via `npm run sls-encrypt [name] [value]`
 
 ## Configure and deploy
 - run: `npm run sls-deploy` - note that this will create aws re$ources..

docker-compose.yml

@@ -15,16 +15,16 @@ services:
       - DOCKER_HOST=tcp://docker-in-docker:2375
       - NODE_ENV=development
       - GTM_AGENT_PORT=${GTM_AGENT_PORT}
-      - GTM_AGENT_AWS_ACCESS_KEY_ID=${GTM_AGENT_AWS_ACCESS_KEY_ID}
-      - GTM_AGENT_AWS_SECRET_ACCESS_KEY=${GTM_AGENT_AWS_SECRET_ACCESS_KEY}
+      - GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID=${GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID}
+      - GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY=${GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY}
       - GTM_AWS_REGION=${GTM_AWS_REGION}
       - GTM_SQS_PENDING_QUEUE=${GTM_SQS_PENDING_QUEUE}
       - GTM_SQS_RESULTS_QUEUE=${GTM_SQS_RESULTS_QUEUE}
       - GTM_SNS_RESULTS_TOPIC=${GTM_SNS_RESULTS_TOPIC}
       - GTM_TASK_CONFIG_FILENAME=${GTM_TASK_CONFIG_FILENAME}
       - GTM_AGENT_GROUP=${GTM_AGENT_GROUP}
-      - GTM_GITHUB_TOKEN=${GTM_GITHUB_TOKEN}
-      - GTM_GITHUB_WEBHOOK_SECRET=${GTM_GITHUB_WEBHOOK_SECRET}
+      - GTM_CRYPT_GITHUB_TOKEN=${GTM_CRYPT_GITHUB_TOKEN}
+      - GTM_CRYPT_GITHUB_WEBHOOK_SECRET=${GTM_CRYPT_GITHUB_WEBHOOK_SECRET}
       - GTM_GITHUB_HOST=${GTM_GITHUB_HOST}
       - GTM_GITHUB_DEBUG=${GTM_GITHUB_DEBUG}
       - GTM_GITHUB_TIMEOUT=${GTM_GITHUB_TIMEOUT}

dotenv.js

@@ -0,0 +1,8 @@
+const dotenv = require('dotenv');
+const config = dotenv.config();
+
+if (config.error) {
+    console.error(config.error);
+}
+
+//console.log(json.plain(_.extend(sharedConfig.parsed, localConfig.parsed)));

encrypt.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+display_usage() {
+    echo "Please supply env var name and value to encrypt"
+    echo "eg. ./encrypt.sh GTM_CRYPT_MY_VARIABLE supers3cret"
+}
+
+if [ $# -eq 0 ]; then
+    display_usage
+    exit 1
+fi
+
+export $(cat .env | grep -v ^# | xargs)
+node --require ./dotenv.js ./node_modules/serverless/bin/serverless encrypt -k $GTM_AWS_KMS_KEY_ID -n $1 -v $2
+
+# todo add encrypted values to .env from kms secrets

k8s/k8s-gtm-agent.yml

@@ -9,12 +9,12 @@ data:
   GTM_SQS_PENDING_QUEUE: gtmPendingQueue
   GTM_SQS_RESULTS_QUEUE: gtmResultsQueue
   GTM_SNS_RESULTS_TOPIC: gtmResultsSNSTopic
-  GTM_GITHUB_WEBHOOK_SECRET: ${GTM_GITHUB_WEBHOOK_SECRET}
+  GTM_CRYPT_GITHUB_WEBHOOK_SECRET: ${GTM_CRYPT_GITHUB_WEBHOOK_SECRET}
   GTM_AGENT_PORT: "9091"
   GTM_AGENT_GROUP: ${GTM_AGENT_GROUP}
   GTM_AGENT_CLOUDWATCH_LOGS_GROUP: ${GTM_AGENT_CLOUDWATCH_LOGS_GROUP}
-  GTM_AGENT_AWS_ACCESS_KEY_ID: ${GTM_AGENT_AWS_ACCESS_KEY_ID}
-  GTM_AGENT_AWS_SECRET_ACCESS_KEY: ${GTM_AGENT_AWS_SECRET_ACCESS_KEY}
+  GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID: ${GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID}
+  GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY: ${GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY}
 
 ---
 apiVersion: apps/v1beta2

kms-secrets.dev.ap-southeast-2.yml

@@ -0,0 +1,10 @@
+secrets:
+    GTM_CRYPT_GITHUB_TOKEN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEJcnivPXbcJwnZzTQTxu31AAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFP2pE25RzVb1ZrnlAIBEIBDfziIOzj0CgNt/OuDDoWtIpDXtHRvyVKxMtUOHalsNs8RDVQFPTMGX0qr/BkMeEIO0yG1UF4g2i5p7ZRUJQ68IEgRAQ==
+    GTM_CRYPT_GITHUB_WEBHOOK_SECRET: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEEkJghXzLnVn+mkZOvU24nAAAAbjBsBgkqhkiG9w0BBwagXzBdAgEAMFgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM8JJgdprhT1xeMJ2iAgEQgCtCYXg7hhBX4A1PvVqN5adTeGBGVvoQ5o2Mv9y8ryfoybUqSSVclmJ4XMu0
+    GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQGVqoF0YYPkVr2XywzcnsPtAAAAcjBwBgkqhkiG9w0BBwagYzBhAgEAMFwGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMs0VIMb7Ver78YYylAgEQgC+bNI7V97dCWeE4n9D7CKFUbD8i4t3ZbbgniTsaoThOczAlYhBxmLtVcAQ78b6mnw==
+    GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQH3RvbGVR/b3CF8NkbkIjcYAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDC+dhWEBgen13NUSJwIBEIBDaG2tl4rs0hmHn7XIAVMU9VFYAq8LcQ2Tu0WvXYwO4GVi9nXQfVq5qrxTc62ajNHjp9wOKCeakOC7pbpsWktb3dc9nQ==
+    GTM_CRYPT_LAUNCHDARKLY_API_TOKEN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQHMMlqueqRkis1P7Ahnf6UbAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOagJHWpec2No95GBwIBEIBD3vl3qwULarfWZjMn3mJzQNDdWJmfyvuMCYDAW0f0dWvVWDzOwKo2crAnJF66Wb94xMSoGKa3Drsx8VxRvX1W1F91uA==
+    GTM_CRYPT_SONAR_GITHUB_OAUTH: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEth3/8M3N831hAAbUkHNvIAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDC2qYfZIYSSq16VjzwIBEIBD3YHX3Hwh26MolZK37h1ZlKU2ClfHPO4l166g86jTo3n2dBbvyH6tIWHgpoRfVnlHlUzwa34GVUqUhFh0s9lVFEyoMA==
+    GTM_CRYPT_SONAR_LOGIN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQGHbKJhON3qOYWjAW7D3AwbAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDCFV2CfHNMu2kNrZnwIBEIBDhnUFDqiS24L6GfJn2bEbZqVSJTEOsMcYtuDiAB5d53a+fYkP1a3pk3JSRtTmDOhbIeXQbIVnssw1WxItlXvyZlJa0Q==
+    GTM_CRYPT_JENKINS_TOKEN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEmEA1yZBLobp7pyJTncAPiAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMofG5CUTNcaNv5quYAgEQgDu5LVDrZZ6uvquX+9xNjzyVV1muTU1WKIq7w01TuHY5ffPc4BCaReP8CoqzLjhD+nMQHVL2WGWEfQSa0A==
+keyArn: 'arn:aws:kms:ap-southeast-2:347186442473:key/f7caff09-20ff-4698-b6a5-5ebe3ccf8556'