From 826173e002c83c0a312e43c7c2deb380caacfee2 Mon Sep 17 00:00:00 2001 From: wyvern8 Date: Wed, 13 Jun 2018 13:12:50 +1000 Subject: [PATCH 1/7] feat(support for kms encrypted environment vars): wip BREAKING CHANGE: renamed env vars to indicate encryption --- .envExample | 30 +++-- README.md | 24 ++-- docker-compose.yml | 8 +- encrypt.sh | 16 +++ k8s/k8s-gtm-agent.yml | 6 +- kms-secrets.dev.ap-southeast-2.yml | 10 ++ package-lock.json | 126 +++++++++++++++--- package.json | 7 +- serverless-aws-environment.yml | 53 ++++++++ serverless.yml | 3 +- src/KmsUtils.js | 75 +++++++++++ src/agent/Agent.js | 12 +- src/agent/AgentUtils.js | 15 ++- src/agent/Event.js | 6 +- src/executors/ExecutorDockerServerless.js | 7 +- src/executors/ExecutorDockerSonar.js | 11 +- src/executors/ExecutorJenkins.js | 3 +- src/executors/ExecutorLaunchDarkly.js | 16 ++- src/executors/ExecutorTeamCity.js | 5 +- src/executors/ExecutorTravis.js | 3 +- src/serverless/gtmGithubHook/gtmGithubHook.js | 13 +- src/serverless/gtmGithubUtils.js | 14 +- test/agent/Agent.spec.js | 6 +- test/agent/Event.spec.js | 7 +- test/agent/EventHandler.spec.js | 1 + test/agent/Executor.spec.js | 1 + test/agent/Plugin.spec.js | 1 + test/handlers/EventHandlerPullRequest.spec.js | 1 + .../gtmGithubHook/gtmGithubHook.spec.js | 7 +- test/serverless/gtmGithubUtils.spec.js | 14 +- 30 files changed, 403 insertions(+), 98 deletions(-) create mode 100755 encrypt.sh create mode 100644 kms-secrets.dev.ap-southeast-2.yml create mode 100644 serverless-aws-environment.yml create mode 100644 src/KmsUtils.js diff --git a/.envExample b/.envExample index 793578f6..91cd5c34 100644 --- a/.envExample +++ b/.envExample @@ -2,9 +2,6 @@ GTM_AWS_REGION=ap-southeast-2 GTM_SQS_PENDING_QUEUE=gtmPendingQueue GTM_SQS_RESULTS_QUEUE=gtmResultsQueue GTM_SNS_RESULTS_TOPIC=gtmResultsSNSTopic -GTM_GITHUB_WEBHOOK_SECRET= -GTM_GITHUB_TOKEN= -GTM_GITHUB_TOKEN_FUNCTIONAL_TESTS= GTM_GITHUB_HOST=api.github.com GTM_GITHUB_DEBUG=true GTM_GITHUB_TIMEOUT=5000 @@ -12,13 +9,10 @@ GTM_GITHUB_PATH_PREFIX= GTM_GITHUB_PROXY= GTM_TASK_CONFIG_FILENAME=.githubTaskManager.json GTM_AGENT_PORT=9091 -GTM_AGENT_AWS_ACCESS_KEY_ID= -GTM_AGENT_AWS_SECRET_ACCESS_KEY= GTM_JENKINS_USER= GTM_JENKINS_URL= GTM_JENKINS_CSRF= GTM_TEAMCITY_USER= -GTM_TEAMCITY_PASSCODE= GTM_TEAMCITY_URL= GTM_DOCKER_IMAGE_WHITELIST=alpine:*,zotoio/* GTM_DOCKER_IMAGE_WHITELIST_FILE=.dockerImageWhitelistExample @@ -26,11 +20,9 @@ GTM_DOCKER_COMMANDS_ALLOWED=true GTM_DOCKER_ALLOW_PULL=true GTM_DOCKER_DEFAULT_WORKER_IMAGE=zotoio/gtm-worker:latest IAM_ENABLED= -LAUNCHDARKLY_API_TOKEN= GTM_LOGSTASH_HOST= GTM_LOGSTASH_PORT= GTM_SONAR_HOST_URL=http://localhost:9000 -GTM_SONAR_GITHUB_OAUTH= GTM_SONAR_GITHUB_ENDPOINT= GTM_SONAR_LOGIN= GTM_SONAR_PROJECTNAME_PREFIX=github:: @@ -46,4 +38,24 @@ GTM_AWS_VPC_ID= GTM_BASE_URL=http://localhost:9091 GTM_S3_DEPENDENCY_BUCKET=gtmstorage GTM_WELCOME_MESSAGE_ENABLED=true -GTM_REPO_BLACKLIST=.*ignore-repo.*,.*another-repo.* \ No newline at end of file +GTM_REPO_BLACKLIST=.*ignore-repo.*,.*another-repo.* +GTM_AWS_KMS_KEY_ID= + +GTM_CRYPT_GITHUB_TOKEN= +GTM_CRYPT_GITHUB_WEBHOOK_SECRET= +GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID= +GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY= +GTM_CRYPT_LAUNCHDARKLY_API_TOKEN= +GTM_CRYPT_SONAR_GITHUB_OAUTH= +GTM_CRYPT_SONAR_LOGIN= +GTM_CRYPT_JENKINS_TOKEN= + +# use the following to encrypt values, collect from kms-secrets-* file created, and add above +#npm run sls-encrypt GTM_CRYPT_GITHUB_TOKEN +#npm run sls-encrypt GTM_CRYPT_GITHUB_WEBHOOK_SECRET +#npm run sls-encrypt GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID +#npm run sls-encrypt GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY +#npm run sls-encrypt GTM_CRYPT_LAUNCHDARKLY_API_TOKEN +#npm run sls-encrypt GTM_CRYPT_SONAR_GITHUB_OAUTH +#npm run sls-encrypt GTM_CRYPT_SONAR_LOGIN +#npm run sls-encrypt GTM_CRYPT_JENKINS_TOKEN \ No newline at end of file diff --git a/README.md b/README.md index 00475829..7613736d 100644 --- a/README.md +++ b/README.md @@ -40,33 +40,37 @@ Create an asynchronous CI agnostic mechanism for running custom test stage gates - npm install - setup serverless aws creds per https://github.com/serverless/serverless/blob/master/docs/providers/aws/guide/credentials.md - setup a .env file in the repo root (copy from .envExample and modify) +- create and AWS KMS key, and capture the id for var `GTM_AWS_KMS_KEY_ID` | Environment variable | description | | -------------------- | ----------- | +|GTM_AWS_KMS_KEY_ID | aws kms key id | +|GTM_CRYPT_GITHUB_TOKEN | encrypted access token for accessing github | +|GTM_CRYPT_GITHUB_WEBHOOK_SECRET | encrypted shared secret from github webook config | +|GTM_CRYPT_AWS_ACCESS_KEY_ID | encrypted aws key id - for agent only | +|GTM_CRYPT_AWS_SECRET_ACCESS_KEY | encrypted aws secret - for agent only | +|GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY|secret key for agent| +|GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID|access key for agent| +|GTM_CRYPT_JENKINS_TOKEN| encrypted token | +|GTM_CRYPT_TEAMCITY_PASSCODE| encrypted teamcity executor passcode| +|GTM_CRYPT_SONAR_LOGIN| encrypted sonar access token | +|GTM_CRYPT_SONAR_GITHUB_OAUTH| encrypted github token for sonar to post comments and status | |GTM_AWS_REGION | awsregion to create resources in | |GTM_SQS_PENDING_QUEUE | name of SQS queue for new event | |GTM_SQS_RESULTS_QUEUE | name of SQS queue for results | |GTM_SNS_RESULTS_TOPIC | name of SNS topic for result ping | -|GTM_GITHUB_WEBHOOK_SECRET | shared secret from github webook config | -|GTM_GITHUB_TOKEN | access token for accessing github | -|GTM_GITHUB_TOKEN_FUNCTIONAL_TESTS | access token for individual test type. each task type can have a different token | |GTM_GITHUB_HOST | api hostname can be updated for github enterprise | |GTM_GITHUB_DEBUG | debug mode for api calls | |GTM_GITHUB_TIMEOUT | github api timeout | |GTM_GITHUB_PATH_PREFIX | path prefix for github enterprise | |GTM_GITHUB_PROXY | github api client proxy | |GTM_TASK_CONFIG_FILENAME | filename in repo to look for for task config - default is .githubTaskManager | -|GTM_AWS_ACCESS_KEY_ID | aws key id - for agent only | -|GTM_AWS_SECRET_ACCESS_KEY | aws secret - for agent only | |AWS_PROXY|URL of proxy to use for network requests. Optional| |GTM_AGENT_PORT| defaults to 9091 | -|GTM_AGENT_AWS_ACCESS_KEY_ID|access key for agent| -|GTM_AGENT_AWS_SECRET_ACCESS_KEY|secret key for agent| |GTM_JENKINS_USER|login for jenkins executor| |GTM_JENKINS_URL|url executor uses to talk to jenkins| |GTM_JENKINS_CSRF| is csrf enabled? true or false| |GTM_TEAMCITY_USER|teamcity executor user| -|GTM_TEAMCITY_PASSCODE|teamcity executor passcode| |GTM_TEAMCITY_URL|teamcity api url| |GTM_DOCKER_IMAGE_WHITELIST| comma separated list of regex of allows docker images eg. alpine:*,bash:latest| |GTM_DOCKER_IMAGE_WHITELIST_FILE|use an optional docker whitelist file .dockerImageWhitelistExample| @@ -78,10 +82,8 @@ Create an asynchronous CI agnostic mechanism for running custom test stage gates |GTM_LOGSTASH_HOST|optional logstash host for elasticsearch analysis| |GTM_LOGSTASH_PORT|optional logstash port| |GTM_SONAR_HOST_URL| sonar host url to connect to | -|GTM_SONAR_LOGIN| sonar access token | |GTM_SONAR_PROJECTNAME_PREFIX| prefix if reporting to sonarqube | |GTM_SONAR_ANALYSIS_MODE| mode for sonar runner, default preview for PRs | -|GTM_SONAR_GITHUB_OAUTH| github token for sonar to post comments and status | |GTM_SONAR_SOURCES| default source dir is `src`| |GTM_SONAR_JAVA_BINARIES| default is `target`| |GTM_SONAR_MODULES| comma separated modules| @@ -97,6 +99,8 @@ Create an asynchronous CI agnostic mechanism for running custom test stage gates |GTM_AWS_S3_PROXY| https_proxy for aws s3 | |GTM_REPO_BLACKLIST| comma separated list of regex to blackist repo names from triggering events | +> important: values of env vars prefixed with `GTM_CRYPT_*` must be created via `npm run sls-encrypt [name] [value]` + ## Configure and deploy - run: `npm run sls-deploy` - note that this will create aws re$ources.. - capture the hook url output in console and add to github repo pull request conf diff --git a/docker-compose.yml b/docker-compose.yml index 8a1491c1..502d5bbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,16 +15,16 @@ services: - DOCKER_HOST=tcp://docker-in-docker:2375 - NODE_ENV=development - GTM_AGENT_PORT=${GTM_AGENT_PORT} - - GTM_AGENT_AWS_ACCESS_KEY_ID=${GTM_AGENT_AWS_ACCESS_KEY_ID} - - GTM_AGENT_AWS_SECRET_ACCESS_KEY=${GTM_AGENT_AWS_SECRET_ACCESS_KEY} + - GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID=${GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID} + - GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY=${GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY} - GTM_AWS_REGION=${GTM_AWS_REGION} - GTM_SQS_PENDING_QUEUE=${GTM_SQS_PENDING_QUEUE} - GTM_SQS_RESULTS_QUEUE=${GTM_SQS_RESULTS_QUEUE} - GTM_SNS_RESULTS_TOPIC=${GTM_SNS_RESULTS_TOPIC} - GTM_TASK_CONFIG_FILENAME=${GTM_TASK_CONFIG_FILENAME} - GTM_AGENT_GROUP=${GTM_AGENT_GROUP} - - GTM_GITHUB_TOKEN=${GTM_GITHUB_TOKEN} - - GTM_GITHUB_WEBHOOK_SECRET=${GTM_GITHUB_WEBHOOK_SECRET} + - GTM_CRYPT_GITHUB_TOKEN=${GTM_CRYPT_GITHUB_TOKEN} + - GTM_CRYPT_GITHUB_WEBHOOK_SECRET=${GTM_CRYPT_GITHUB_WEBHOOK_SECRET} - GTM_GITHUB_HOST=${GTM_GITHUB_HOST} - GTM_GITHUB_DEBUG=${GTM_GITHUB_DEBUG} - GTM_GITHUB_TIMEOUT=${GTM_GITHUB_TIMEOUT} diff --git a/encrypt.sh b/encrypt.sh new file mode 100755 index 00000000..46fdcf23 --- /dev/null +++ b/encrypt.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +display_usage() { + echo "Please supply env var name and value to encrypt" + echo "eg. ./encrypt.sh GTM_CRYPT_MY_VARIABLE supers3cret" +} + +if [ $# -eq 0 ]; then + display_usage + exit 1 +fi + +export $(cat .env | grep -v ^# | xargs) +node ./node_modules/serverless/bin/serverless encrypt -k $GTM_AWS_KMS_KEY_ID -n $1 -v $2 + +# todo add encrypted values to .env from kms secrets \ No newline at end of file diff --git a/k8s/k8s-gtm-agent.yml b/k8s/k8s-gtm-agent.yml index d9cbaa17..5ebf6d20 100644 --- a/k8s/k8s-gtm-agent.yml +++ b/k8s/k8s-gtm-agent.yml @@ -9,12 +9,12 @@ data: GTM_SQS_PENDING_QUEUE: gtmPendingQueue GTM_SQS_RESULTS_QUEUE: gtmResultsQueue GTM_SNS_RESULTS_TOPIC: gtmResultsSNSTopic - GTM_GITHUB_WEBHOOK_SECRET: ${GTM_GITHUB_WEBHOOK_SECRET} + GTM_CRYPT_GITHUB_WEBHOOK_SECRET: ${GTM_CRYPT_GITHUB_WEBHOOK_SECRET} GTM_AGENT_PORT: "9091" GTM_AGENT_GROUP: ${GTM_AGENT_GROUP} GTM_AGENT_CLOUDWATCH_LOGS_GROUP: ${GTM_AGENT_CLOUDWATCH_LOGS_GROUP} - GTM_AGENT_AWS_ACCESS_KEY_ID: ${GTM_AGENT_AWS_ACCESS_KEY_ID} - GTM_AGENT_AWS_SECRET_ACCESS_KEY: ${GTM_AGENT_AWS_SECRET_ACCESS_KEY} + GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID: ${GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID} + GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY: ${GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY} --- apiVersion: apps/v1beta2 diff --git a/kms-secrets.dev.ap-southeast-2.yml b/kms-secrets.dev.ap-southeast-2.yml new file mode 100644 index 00000000..1a7e1594 --- /dev/null +++ b/kms-secrets.dev.ap-southeast-2.yml @@ -0,0 +1,10 @@ +secrets: + GTM_CRYPT_GITHUB_TOKEN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEJcnivPXbcJwnZzTQTxu31AAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFP2pE25RzVb1ZrnlAIBEIBDfziIOzj0CgNt/OuDDoWtIpDXtHRvyVKxMtUOHalsNs8RDVQFPTMGX0qr/BkMeEIO0yG1UF4g2i5p7ZRUJQ68IEgRAQ== + GTM_CRYPT_GITHUB_WEBHOOK_SECRET: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEEamZPETq1yZycyB1dmPIvAAAAbjBsBgkqhkiG9w0BBwagXzBdAgEAMFgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMgN0GR8Nj6wuEbqD0AgEQgCuyRS2IMDCgfEvTTthcarc5ptgLhz7nGXnYKQ5xLsurGmfM9McnRW6g/HbG + GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQGVqoF0YYPkVr2XywzcnsPtAAAAcjBwBgkqhkiG9w0BBwagYzBhAgEAMFwGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMs0VIMb7Ver78YYylAgEQgC+bNI7V97dCWeE4n9D7CKFUbD8i4t3ZbbgniTsaoThOczAlYhBxmLtVcAQ78b6mnw== + GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQH3RvbGVR/b3CF8NkbkIjcYAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDC+dhWEBgen13NUSJwIBEIBDaG2tl4rs0hmHn7XIAVMU9VFYAq8LcQ2Tu0WvXYwO4GVi9nXQfVq5qrxTc62ajNHjp9wOKCeakOC7pbpsWktb3dc9nQ== + GTM_CRYPT_LAUNCHDARKLY_API_TOKEN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQHMMlqueqRkis1P7Ahnf6UbAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOagJHWpec2No95GBwIBEIBD3vl3qwULarfWZjMn3mJzQNDdWJmfyvuMCYDAW0f0dWvVWDzOwKo2crAnJF66Wb94xMSoGKa3Drsx8VxRvX1W1F91uA== + GTM_CRYPT_SONAR_GITHUB_OAUTH: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEth3/8M3N831hAAbUkHNvIAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDC2qYfZIYSSq16VjzwIBEIBD3YHX3Hwh26MolZK37h1ZlKU2ClfHPO4l166g86jTo3n2dBbvyH6tIWHgpoRfVnlHlUzwa34GVUqUhFh0s9lVFEyoMA== + GTM_CRYPT_SONAR_LOGIN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQGHbKJhON3qOYWjAW7D3AwbAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDCFV2CfHNMu2kNrZnwIBEIBDhnUFDqiS24L6GfJn2bEbZqVSJTEOsMcYtuDiAB5d53a+fYkP1a3pk3JSRtTmDOhbIeXQbIVnssw1WxItlXvyZlJa0Q== + GTM_CRYPT_JENKINS_TOKEN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEmEA1yZBLobp7pyJTncAPiAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMofG5CUTNcaNv5quYAgEQgDu5LVDrZZ6uvquX+9xNjzyVV1muTU1WKIq7w01TuHY5ffPc4BCaReP8CoqzLjhD+nMQHVL2WGWEfQSa0A== +keyArn: 'arn:aws:kms:ap-southeast-2:347186442473:key/f7caff09-20ff-4698-b6a5-5ebe3ccf8556' diff --git a/package-lock.json b/package-lock.json index 86d3d0e9..ca92c69f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -936,8 +936,7 @@ "assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", - "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", - "dev": true + "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==" }, "assign-deep": { "version": "0.4.7", @@ -2313,8 +2312,7 @@ "browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", - "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", - "dev": true + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==" }, "browserify-aes": { "version": "1.1.1", @@ -2672,7 +2670,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", "integrity": "sha1-D2RYS6ZC8PKs4oBiefTwbKI61zw=", - "dev": true, "requires": { "assertion-error": "1.1.0", "check-error": "1.0.2", @@ -2701,8 +2698,7 @@ "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", - "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", - "dev": true + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=" }, "chokidar": { "version": "1.7.0", @@ -3924,7 +3920,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", - "dev": true, "requires": { "type-detect": "4.0.8" } @@ -4051,8 +4046,7 @@ "diff": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", - "dev": true + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==" }, "diffie-hellman": { "version": "5.0.2", @@ -6522,8 +6516,7 @@ "get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", - "dev": true + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" }, "get-proxy": { "version": "2.1.0", @@ -7153,8 +7146,7 @@ "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", - "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", - "dev": true + "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=" }, "heap": { "version": "0.2.6", @@ -12904,8 +12896,7 @@ "pathval": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", - "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", - "dev": true + "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=" }, "pbkdf2": { "version": "3.0.14", @@ -14441,6 +14432,98 @@ } } }, + "serverless-kms-secrets": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/serverless-kms-secrets/-/serverless-kms-secrets-1.0.3.tgz", + "integrity": "sha512-I4CTYCfe5e23KpbWo9p6ddB3BgI5115BnVd16IKfUD0l5AlcEoMyw4TxQdzkqcdQw9LL2V96o0mnisl56CY1rA==", + "requires": { + "aws-sdk": "2.213.1", + "bluebird": "3.5.1", + "chai": "4.1.2", + "fs-extra": "3.0.1", + "mocha": "5.2.0", + "yaml-edit": "0.1.3", + "yamljs": "0.2.10" + }, + "dependencies": { + "commander": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", + "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==" + }, + "fs-extra": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-3.0.1.tgz", + "integrity": "sha1-N5TzeMWLNC6n27sjCVEJxLO2IpE=", + "requires": { + "graceful-fs": "4.1.11", + "jsonfile": "3.0.1", + "universalify": "0.1.1" + } + }, + "glob": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "requires": { + "fs.realpath": "1.0.0", + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "3.0.4", + "once": "1.4.0", + "path-is-absolute": "1.0.1" + } + }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==" + }, + "jsonfile": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-3.0.1.tgz", + "integrity": "sha1-pezG9l9T9mLEQVx2daAzHQmS7GY=", + "requires": { + "graceful-fs": "4.1.11" + } + }, + "mocha": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", + "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==", + "requires": { + "browser-stdout": "1.3.1", + "commander": "2.15.1", + "debug": "3.1.0", + "diff": "3.5.0", + "escape-string-regexp": "1.0.5", + "glob": "7.1.2", + "growl": "1.10.5", + "he": "1.1.1", + "minimatch": "3.0.4", + "mkdirp": "0.5.1", + "supports-color": "5.4.0" + } + }, + "supports-color": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", + "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", + "requires": { + "has-flag": "3.0.0" + } + }, + "yamljs": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/yamljs/-/yamljs-0.2.10.tgz", + "integrity": "sha1-SBzHwlynOvWfWR8MluPOVsdXpA8=", + "requires": { + "argparse": "1.0.10", + "glob": "7.1.2" + } + } + } + }, "serverless-offline": { "version": "3.20.2", "resolved": "https://registry.npmjs.org/serverless-offline/-/serverless-offline-3.20.2.tgz", @@ -16427,8 +16510,7 @@ "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==" }, "type-is": { "version": "1.6.16", @@ -16583,8 +16665,7 @@ "universalify": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", - "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=", - "dev": true + "integrity": "sha1-+nG63UQ3r0wUiEHjs7Fl+enlkLc=" }, "unpipe": { "version": "1.0.0", @@ -17684,6 +17765,11 @@ "integrity": "sha1-0A88+ddztyQUCa6SpnQNHbGfSeY=", "dev": true }, + "yaml-edit": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/yaml-edit/-/yaml-edit-0.1.3.tgz", + "integrity": "sha1-103V8F78QquRhSbAlPAJhiHOLXM=" + }, "yamljs": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/yamljs/-/yamljs-0.3.0.tgz", diff --git a/package.json b/package.json index 1e81dd69..794a718c 100644 --- a/package.json +++ b/package.json @@ -8,13 +8,15 @@ "patch": "cp -rf ./patches/* ./node_modules/", "commit": "./node_modules/.bin/git-cz", "lint": "node ./node_modules/.bin/eslint ./src ./test --fix --ext=.js --ignore-pattern src/agent/static/ --ignore-pattern node_modules/ --ignore-pattern dist/ --ignore-pattern coverage/ --quiet", - "test": "npm run build && cross-env NODE_ENV=test nyc ./node_modules/.bin/mocha --timeout 10000 --require babel-core/register --require babel-polyfill './dist/test/**/*.spec.js'", + "test": "npm run build && cross-env NODE_ENV=test GTM_CRYPT_GITHUB_WEBHOOK_SECRET=abc GTM_AWS_KMS_KEY_ID='' nyc ./node_modules/.bin/mocha --timeout 10000 --require babel-core/register --require babel-polyfill './dist/test/**/*.spec.js'", "test-integration": "npm run agent-build && npm run sls-deploy && npm run docker-start && EDGEGRID_ENV=test node ./node_modules/.bin/mocha --timeout 10000 --require babel-core/register --require babel-polyfill ./dist/test/**/*.spec-int.js", "semantic-release": "semantic-release", "sls-deploy": "npm run build && node ./node_modules/serverless/bin/serverless deploy -v | tee ./sls.out && cat ./sls.out | grep ServiceEndpoint | sed 's/ServiceEndpoint[^https]*\\(https:\\/\\/.*\\)/\\1\\/gtm-github-hook/' | sed 's/\\x1b\\[[0-9;]*m//g' > ./sls-hook-url.out", "sls-undeploy": "node ./node_modules/serverless/bin/serverless remove", "sls-logs-hook": "node ./node_modules/serverless/bin/serverless logs -f gtmGithubHook -t", "sls-logs-results": "node ./node_modules/serverless/bin/serverless logs -f gtmGithubResults -t", + "sls-encrypt": "chmod 755 ./encrypt.sh && ./encrypt.sh", + "sls-decrypt": "node ./node_modules/serverless/bin/serverless decrypt -n", "agent": "export $(cat .env | grep -v ^# | xargs) && node dist/src/agent/startAgent.js", "agent-build": "node ./dist/src/agent/sass.js && cp ./src/agent/static/*.js ./dist/src/agent/static/", "docker-build": "docker build -t github-task-manager --no-cache .", @@ -139,6 +141,7 @@ "webpack": "^4.0.0", "xml2js": "^0.4.19", "yamljs": "^0.3.0", - "zlib": "^1.0.5" + "zlib": "^1.0.5", + "serverless-kms-secrets": "^1.0.3" } } diff --git a/serverless-aws-environment.yml b/serverless-aws-environment.yml new file mode 100644 index 00000000..2b801e09 --- /dev/null +++ b/serverless-aws-environment.yml @@ -0,0 +1,53 @@ +environment: + # encrypted env vars - use npm run sls-encrypt to populate json store for stage + GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID} + GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY} + GTM_CRYPT_GITHUB_WEBHOOK_SECRET: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_GITHUB_WEBHOOK_SECRET} + GTM_CRYPT_GITHUB_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_GITHUB_TOKEN} + GTM_CRYPT_JENKINS_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_JENKINS_TOKEN} + GTM_CRYPT_LAUNCHDARKLY_API_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_LAUNCHDARKLY_API_TOKEN} + GTM_CRYPT_SONAR_GITHUB_OAUTH: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_SONAR_GITHUB_OAUTH} + GTM_CRYPT_SONAR_LOGIN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_SONAR_LOGIN} + GTM_CRYPT_TEAMCITY_PASSCODE: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_TEAMCITY_PASSCODE} + # unencrypted env vars - add values to .env + GTM_AWS_REGION: ${env:GTM_AWS_REGION, 'ap-southeast-2'} + GTM_SQS_PENDING_QUEUE: ${env:GTM_SQS_PENDING_QUEUE, 'gtmPendingQueue'} + GTM_SQS_RESULTS_QUEUE: ${env:GTM_SQS_RESULTS_QUEUE, 'gtmResultsQueue'} + GTM_SNS_RESULTS_TOPIC: ${env:GTM_SNS_RESULTS_TOPIC, 'gtmResultsSNSTopic'} + GTM_GITHUB_HOST: ${env:GTM_GITHUB_HOST, 'api.github.com'} + GTM_GITHUB_DEBUG: ${env:GTM_GITHUB_DEBUG, 'true'} + GTM_GITHUB_TIMEOUT: ${env:GTM_GITHUB_TIMEOUT, '5000'} + GTM_GITHUB_PATH_PREFIX: ${env:GTM_GITHUB_PATH_PREFIX} + GTM_GITHUB_PROXY: ${env:GTM_GITHUB_PROXY} + GTM_TASK_CONFIG_FILENAME: ${env:GTM_TASK_CONFIG_FILENAME, '.githubTaskManager.json'} + GTM_AGENT_PORT: ${env:GTM_AGENT_PORT, '9091'} + GTM_JENKINS_USER: ${env:GTM_JENKINS_USER} + GTM_JENKINS_URL: ${env:GTM_JENKINS_URL} + GTM_JENKINS_CSRF: ${env:GTM_JENKINS_CSRF} + GTM_TEAMCITY_USER: ${env:GTM_TEAMCITY_USER} + GTM_TEAMCITY_URL: ${env:GTM_TEAMCITY_URL} + GTM_DOCKER_IMAGE_WHITELIST: ${env:GTM_DOCKER_IMAGE_WHITELIST, 'alpine:*,zotoio/*'} + GTM_DOCKER_IMAGE_WHITELIST_FILE: ${env:GTM_DOCKER_IMAGE_WHITELIST_FILE, '.dockerImageWhitelistExample'} + GTM_DOCKER_COMMANDS_ALLOWED: ${env:GTM_DOCKER_COMMANDS_ALLOWED, 'true'} + GTM_DOCKER_ALLOW_PULL: ${env:GTM_DOCKER_ALLOW_PULL, 'true'} + GTM_DOCKER_DEFAULT_WORKER_IMAGE: ${env:GTM_DOCKER_DEFAULT_WORKER_IMAGE, 'zotoio/gtm-worker:latest'} + IAM_ENABLED: ${env:IAM_ENABLED} + GTM_LOGSTASH_HOST: ${env:GTM_LOGSTASH_HOST} + GTM_LOGSTASH_PORT: ${env:GTM_LOGSTASH_PORT} + GTM_SONAR_HOST_URL: ${env:GTM_SONAR_HOST_URL} + GTM_SONAR_GITHUB_ENDPOINT: ${env:GTM_SONAR_GITHUB_ENDPOINT} + GTM_SONAR_PROJECTNAME_PREFIX: ${env:GTM_SONAR_PROJECTNAME_PREFIX, 'github::'} + GTM_SONAR_ANALYSIS_MODE: ${env:GTM_SONAR_ANALYSIS_MODE, 'preview'} + GTM_SONAR_SOURCES: ${env:GTM_SONAR_SOURCES, 'src'} + GTM_SONAR_JAVA_BINARIES: ${env:GTM_SONAR_JAVA_BINARIES, 'target'} + GTM_SONAR_MODULES: ${env:GTM_SONAR_MODULES} + GTM_TASK_CONFIG_DEFAULT_URL: ${env:GTM_TASK_CONFIG_DEFAULT_URL} + GTM_TASK_CONFIG_DEFAULT_MESSAGE_PATH: ${env:GTM_TASK_CONFIG_DEFAULT_MESSAGE_PATH} + GTM_DYNAMO_TABLE_EVENTS: ${env:GTM_DYNAMO_TABLE_EVENTS, 'GtmEvents'} + GTM_DYNAMO_TABLE_AGENTS: ${env:GTM_DYNAMO_TABLE_AGENTS, 'GtmAgents'} + GTM_AWS_VPC_ID: ${env:GTM_AWS_VPC_ID} + GTM_BASE_URL: ${env:GTM_BASE_URL, 'http://localhost:9091'} + GTM_S3_DEPENDENCY_BUCKET: ${env:GTM_S3_DEPENDENCY_BUCKET, 'gtmstorage'} + GTM_WELCOME_MESSAGE_ENABLED: ${env:GTM_WELCOME_MESSAGE_ENABLED, 'true'} + GTM_REPO_BLACKLIST: ${env:GTM_REPO_BLACKLIST, '.*ignore-repo.*,.*another-repo.*'} + GTM_AWS_KMS_KEY_ID: ${env:GTM_AWS_KMS_KEY_ID} \ No newline at end of file diff --git a/serverless.yml b/serverless.yml index 2f21a6e5..a29677bc 100644 --- a/serverless.yml +++ b/serverless.yml @@ -2,6 +2,7 @@ service: gtmGithubHook plugins: - serverless-dotenv-plugin - serverless-webpack + - serverless-kms-secrets - serverless-offline provider: @@ -14,7 +15,7 @@ provider: region: ${env:GTM_AWS_REGION} environment: - GTM_GITHUB_WEBHOOK_SECRET: ${env:GTM_GITHUB_WEBHOOK_SECRET} + PLACE: HOLDER iamRoleStatements: - Effect: Allow diff --git a/src/KmsUtils.js b/src/KmsUtils.js new file mode 100644 index 00000000..11901042 --- /dev/null +++ b/src/KmsUtils.js @@ -0,0 +1,75 @@ +import * as AWS from 'aws-sdk'; +import * as bunyan from 'bunyan'; +const log = bunyan.createLogger({ name: 'KmsUtils' }); + +let STORE = {}; + +const KMS = new AWS.KMS({ region: process.env.GTM_AWS_REGION }); + +export class KmsUtils { + constructor() { + log.info('KmsUtils created'); + } + + static getStore() { + return STORE; + } + static decrypt(encrypted, callback) { + let result = ''; + if (encrypted) { + if (!process.env.GTM_AWS_KMS_KEY_ID) { + log.warn(`no encryption key configured, using raw values`); + KmsUtils.setDecrypted(encrypted, encrypted); + callback(null, encrypted); + } else { + log.info(`decrypting: ${encrypted}`); + if (KmsUtils.hasDecrypted(encrypted)) { + log.info('returning stored decrypted value'); + callback(null, KmsUtils.getDecrypted(encrypted)); + } + try { + KMS.decrypt({ CiphertextBlob: new Buffer(encrypted, 'base64') }) + .promise() + .then(data => { + log.info(`storing decrypted result.`); + KmsUtils.setDecrypted(encrypted, data.Plaintext); + return callback(null, data.Plaintext); + }); + } catch (e) { + log.error(e); + return callback(e); + } + } + } + return result; + } + static primeStore() { + log.info(`priming decrypted var store`); + Object.keys(process.env).forEach(key => { + if (key.startsWith('GTM_CRYPT_')) { + KmsUtils.decrypt(process.env[key], err => { + if (err) log.error(err); + log.info(`stored decrypted value of ${key}`); + }); + } + }); + } + static hasDecrypted(encrypted) { + return Object.keys(STORE).includes(encrypted); + } + static getDecrypted(encrypted) { + if (this.hasDecrypted(encrypted)) { + return STORE[encrypted]; + } else { + KmsUtils.decrypt(encrypted, (err, decrypted) => { + if (err) log.error(err); + log.warn(`returning newly decrypted value`); + return decrypted; + }); + } + } + static setDecrypted(encrypted, decrypted) { + STORE[encrypted] = decrypted; + } +} +KmsUtils.primeStore(); diff --git a/src/agent/Agent.js b/src/agent/Agent.js index d681e55b..0f9f8f45 100644 --- a/src/agent/Agent.js +++ b/src/agent/Agent.js @@ -44,16 +44,16 @@ export class Agent { }); if (!process.env.IAM_ENABLED) { - if (!process.env.GTM_AGENT_AWS_ACCESS_KEY_ID || !process.env.GTM_AGENT_AWS_SECRET_ACCESS_KEY) { + if (!process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID || !process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY) { log.error( - '### ERROR ### Environment Variables GTM_AGENT_AWS_ACCESS_KEY_ID, or GTM_AGENT_AWS_SECRET_ACCESS_KEY Missing!' + '### ERROR ### Environment Variables GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID, or GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY Missing!' ); process.exit(1); } } - if (!process.env.GTM_GITHUB_WEBHOOK_SECRET) { - log.error('### ERROR ### Environment Variable GTM_GITHUB_WEBHOOK_SECRET missing!'); + if (!process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET) { + log.error('### ERROR ### Environment Variable GTM_CRYPT_GITHUB_WEBHOOK_SECRET missing!'); process.exit(1); } @@ -326,8 +326,8 @@ export class Agent { log.info('GitHub Task Manager Agent Running on Port ' + process.env.GTM_AGENT_PORT); log.info('Runmode: ' + runmode); if (!process.env.IAM_ENABLED) { - log.info('AWS Access Key ID: ' + AgentUtils.maskString(process.env.GTM_AGENT_AWS_ACCESS_KEY_ID)); - log.info('AWS Access Key: ' + AgentUtils.maskString(process.env.GTM_AGENT_AWS_SECRET_ACCESS_KEY)); + log.info('AWS Access Key ID: ' + AgentUtils.maskString(process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID)); + log.info('AWS Access Key: ' + AgentUtils.maskString(process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY)); } log.info('Pending Queue URL: ' + pendingUrl); diff --git a/src/agent/AgentUtils.js b/src/agent/AgentUtils.js index 1437ab81..80a84db9 100644 --- a/src/agent/AgentUtils.js +++ b/src/agent/AgentUtils.js @@ -3,6 +3,9 @@ const pullRequestData = require('./pullrequest.json'); const { URL } = require('url'); const crypto = require('crypto'); + +const { KmsUtils } = require('../KmsUtils'); + import { default as AgentLogger } from './AgentLogger'; import { default as yamljs } from 'yamljs'; import { default as https } from 'https'; @@ -20,9 +23,9 @@ if (process.env.IAM_ENABLED) { } }); } else { - // due to serverless .env issue - process.env.AWS_ACCESS_KEY_ID = process.env.GTM_AGENT_AWS_ACCESS_KEY_ID; - process.env.AWS_SECRET_ACCESS_KEY = process.env.GTM_AGENT_AWS_SECRET_ACCESS_KEY; + // due to serverless .env restrictions + process.env.AWS_ACCESS_KEY_ID = KmsUtils.getDecrypted(process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID); + process.env.AWS_SECRET_ACCESS_KEY = KmsUtils.getDecrypted(process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY); } let sqs = new AWS.SQS({ apiVersion: '2012-11-05' }); @@ -362,7 +365,11 @@ export class AgentUtils { // just add all the GTM env vars to map Object.keys(process.env).forEach(key => { if (key.startsWith('GTM_')) { - mapDict[`##${key}##`] = process.env[key]; + if (key.startsWith('GTM_CRYPT')) { + mapDict[`##${key}##`] = KmsUtils.getDecrypted(process.env[key]); + } else { + mapDict[`##${key}##`] = process.env[key]; + } } }); diff --git a/src/agent/Event.js b/src/agent/Event.js index ecde7ab0..f1d7d429 100644 --- a/src/agent/Event.js +++ b/src/agent/Event.js @@ -4,7 +4,8 @@ import 'babel-polyfill'; import { default as crypto } from 'crypto'; import { default as AgentLogger } from './AgentLogger'; import { Agent } from './Agent'; - +import { KmsUtils } from '../KmsUtils'; +console.log(KmsUtils.getStore()); let log = AgentLogger.log(); /** @@ -61,8 +62,9 @@ export class Event { let checkEvent = Event.buildCheckObject(message); log.debug(`eventString for signature check: ${JSON.stringify(checkEvent)}`); + console.log(KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET)); let calculatedSig = `sha1=${crypto - .createHmac('sha1', process.env.GTM_GITHUB_WEBHOOK_SECRET) + .createHmac('sha1', KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET)) .update(JSON.stringify(checkEvent), 'utf-8') .digest('hex')}`; diff --git a/src/executors/ExecutorDockerServerless.js b/src/executors/ExecutorDockerServerless.js index 730cc7b5..f98f8452 100644 --- a/src/executors/ExecutorDockerServerless.js +++ b/src/executors/ExecutorDockerServerless.js @@ -2,6 +2,7 @@ import { Executor } from '../agent/Executor'; import { ExecutorDocker } from './ExecutorDocker'; import { default as _ } from 'lodash'; import { AgentUtils } from '../agent/AgentUtils'; +import { KmsUtils } from '../KmsUtils'; /** * Sample .githubTaskManager.json task config @@ -84,8 +85,10 @@ export class ExecutorDockerServerless extends ExecutorDocker { }; if (!process.env.IAM_ENABLED) { - options.env['GTM_AWS_ACCESS_KEY_ID'] = process.env.GTM_AGENT_AWS_ACCESS_KEY_ID; - options.env['GTM_AWS_SECRET_ACCESS_KEY'] = process.env.GTM_AGENT_AWS_SECRET_ACCESS_KEY; + options.env['GTM_AWS_ACCESS_KEY_ID'] = KmsUtils.getDecrypted(process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID); + options.env['GTM_AWS_SECRET_ACCESS_KEY'] = KmsUtils.getDecrypted( + process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY + ); options.env['GTM_AWS_REGION'] = process.env.GTM_AWS_REGION; } diff --git a/src/executors/ExecutorDockerSonar.js b/src/executors/ExecutorDockerSonar.js index 8b6ab171..8a090f2c 100644 --- a/src/executors/ExecutorDockerSonar.js +++ b/src/executors/ExecutorDockerSonar.js @@ -2,6 +2,7 @@ import { Executor } from '../agent/Executor'; import { ExecutorDocker } from './ExecutorDocker'; import { default as _ } from 'lodash'; import { AgentUtils } from '../agent/AgentUtils'; +import { KmsUtils } from '../KmsUtils'; /** * Sample .githubTaskManager.json task config @@ -48,10 +49,10 @@ export class ExecutorDockerSonar extends ExecutorDocker { GIT_PR_BRANCHNAME: '##GH_PR_BRANCHNAME##', SONAR_GITHUB_REPOSITORY: '##GH_REPOSITORY_FULLNAME##', SONAR_HOST_URL: '##GTM_SONAR_HOST_URL##', - SONAR_LOGIN: '##GTM_SONAR_LOGIN##', + SONAR_LOGIN: '##GTM_CRYPT_SONAR_LOGIN##', SONAR_PROJECTNAME_PREFIX: '##GTM_SONAR_PROJECTNAME_PREFIX##', SONAR_ANALYSIS_MODE: '##GTM_SONAR_ANALYSIS_MODE##', - SONAR_GITHUB_OAUTH: '##GTM_SONAR_GITHUB_OAUTH##', + SONAR_GITHUB_OAUTH: '##GTM_CRYPT_SONAR_GITHUB_OAUTH##', SONAR_SOURCES: '##GTM_SONAR_SOURCES##', SONAR_JAVA_BINARIES: '##GTM_SONAR_JAVA_BINARIES##', SONAR_MODULES: '##GTM_SONAR_MODULES##', @@ -67,8 +68,10 @@ export class ExecutorDockerSonar extends ExecutorDocker { }; if (!process.env.IAM_ENABLED) { - options.env['GTM_AWS_ACCESS_KEY_ID'] = process.env.GTM_AGENT_AWS_ACCESS_KEY_ID; - options.env['GTM_AWS_SECRET_ACCESS_KEY'] = process.env.GTM_AGENT_AWS_SECRET_ACCESS_KEY; + options.env['GTM_AWS_ACCESS_KEY_ID'] = KmsUtils.getDecrypted(process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID); + options.env['GTM_AWS_SECRET_ACCESS_KEY'] = KmsUtils.getDecrypted( + process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY + ); options.env['GTM_AWS_REGION'] = process.env.GTM_AWS_REGION; } diff --git a/src/executors/ExecutorJenkins.js b/src/executors/ExecutorJenkins.js index 48f9bb2f..be6232b6 100644 --- a/src/executors/ExecutorJenkins.js +++ b/src/executors/ExecutorJenkins.js @@ -1,6 +1,7 @@ import { default as JenkinsLib } from 'jenkins'; import { Executor } from '../agent/Executor'; import { AgentUtils } from '../agent/AgentUtils'; +import { KmsUtils } from '../KmsUtils'; /** * Sample .githubTaskManager.json task config/ @@ -30,7 +31,7 @@ export class ExecutorJenkins extends Executor { this.jenkins = JenkinsLib({ baseUrl: AgentUtils.formatBasicAuth( this.options.GTM_JENKINS_USER, - this.options.GTM_JENKINS_TOKEN, + KmsUtils.getDecrypted(this.options.GTM_CRYPT_JENKINS_TOKEN), this.options.GTM_JENKINS_URL ), crumbIssuer: useCsrf, diff --git a/src/executors/ExecutorLaunchDarkly.js b/src/executors/ExecutorLaunchDarkly.js index 93372d49..f221243c 100644 --- a/src/executors/ExecutorLaunchDarkly.js +++ b/src/executors/ExecutorLaunchDarkly.js @@ -1,6 +1,7 @@ import { Executor } from '../agent/Executor'; import { LaunchDarklyUtils } from 'launchdarkly-nodeutils'; import { default as formatJson } from 'format-json'; +import { KmsUtils } from '../KmsUtils'; /** * Sample .githubTaskManager.json task config @@ -32,10 +33,12 @@ export class ExecutorLaunchDarkly extends Executor { let log = this.log; if (!this.ldUtils) { let that = this; - return new LaunchDarklyUtils().create(process.env.LAUNCHDARKLY_API_TOKEN, log).then(handle => { - that.ldUtils = handle; - return Promise.resolve(handle); - }); + return new LaunchDarklyUtils() + .create(KmsUtils.getDecrypted(process.env.GTM_CRYPT_LAUNCHDARKLY_API_TOKEN), log) + .then(handle => { + that.ldUtils = handle; + return Promise.resolve(handle); + }); } else { return Promise.resolve(this.ldUtils); } @@ -86,6 +89,7 @@ export class ExecutorLaunchDarkly extends Executor { let results = []; let changedCount = 0; let details = ''; + let resultSummary; log.info(`Starting LaunchDarkly api calls. Flags: ${formatJson.plain(flags)}`); try { @@ -101,7 +105,7 @@ export class ExecutorLaunchDarkly extends Executor { } } } catch (e) { - let resultSummary = { + resultSummary = { passed: false, url: 'https://github.com', message: `failed to set flags`, @@ -112,7 +116,7 @@ export class ExecutorLaunchDarkly extends Executor { return Promise.reject(task); } - let resultSummary = { + resultSummary = { passed: true, url: 'https://github.com', message: `Updated ${changedCount} Flags`, diff --git a/src/executors/ExecutorTeamCity.js b/src/executors/ExecutorTeamCity.js index a0e43da0..f7453157 100644 --- a/src/executors/ExecutorTeamCity.js +++ b/src/executors/ExecutorTeamCity.js @@ -5,6 +5,7 @@ import { default as x2j } from 'xml2js'; import { default as URL } from 'url'; import { Executor } from '../agent/Executor'; import { AgentUtils } from '../agent/AgentUtils'; +import { KmsUtils } from '../KmsUtils'; /** * Sample .githubTaskManager.json task config @@ -38,7 +39,7 @@ export class ExecutorTeamCity extends Executor { this.teamCity = TeamCity.create({ url: this.options.GTM_TEAMCITY_URL, username: this.options.GTM_TEAMCITY_USER, - password: this.options.GTM_TEAMCITY_PASSCODE + password: KmsUtils.getDecrypted(this.options.GTM_CRYPT_TEAMCITY_PASSCODE) }); } @@ -96,7 +97,7 @@ export class ExecutorTeamCity extends Executor { if (task.options.parameters.hasOwnProperty('cuke_tags')) { let statisticsUrl = AgentUtils.formatBasicAuth( this.options.GTM_TEAMCITY_USER, - this.options.GTM_TEAMCITY_PASSCODE, + KmsUtils.getDecrypted(this.options.GTM_CRYPT_TEAMCITY_PASSCODE), URL.resolve(this.options.GTM_TEAMCITY_URL, `/app/rest/builds/id:${teamCityBuildId.id}/statistics`) ); diff --git a/src/executors/ExecutorTravis.js b/src/executors/ExecutorTravis.js index 21c2085e..a1666241 100644 --- a/src/executors/ExecutorTravis.js +++ b/src/executors/ExecutorTravis.js @@ -1,6 +1,7 @@ import { default as Travis } from 'travis-ci'; import { Executor } from '../agent/Executor'; import { default as json } from 'format-json'; +import { KmsUtils } from '../KmsUtils'; /** * Sample .githubTaskManager.json task config - NOT READY FOR USE @@ -33,7 +34,7 @@ export class ExecutorTravis extends Executor { this.travis.authenticate( { - github_token: process.env.GTM_GITHUB_TOKEN + github_token: KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN) }, function(err) { if (err) { diff --git a/src/serverless/gtmGithubHook/gtmGithubHook.js b/src/serverless/gtmGithubHook/gtmGithubHook.js index 38a311dc..a8a0ed43 100644 --- a/src/serverless/gtmGithubHook/gtmGithubHook.js +++ b/src/serverless/gtmGithubHook/gtmGithubHook.js @@ -5,9 +5,15 @@ let rp = require('request-promise-native'); let json = require('format-json'); let UUID = require('uuid/v4'); let Producer = require('sqs-producer'); - let githubUtils = require('../gtmGithubUtils.js'); +const KmsUtils = require('./../../KmsUtils').KmsUtils; + +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); + // application specific logging, throwing an error, or other logic here +}); + async function listener(event, context, callback) { const githubEvent = event.headers['X-GitHub-Event'] || event.headers['x-github-event']; const githubSignature = event.headers['X-Hub-Signature'] || event.headers['x-hub-signature']; @@ -99,7 +105,10 @@ async function handleEvent(type, body, signature) { } ]; - signature = githubUtils.signRequestBody(process.env.GTM_GITHUB_WEBHOOK_SECRET, JSON.stringify(event[0])); + signature = githubUtils.signRequestBody( + KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET), + JSON.stringify(event[0]) + ); event[0].messageAttributes.ghEventSignature = { DataType: 'String', diff --git a/src/serverless/gtmGithubUtils.js b/src/serverless/gtmGithubUtils.js index 03729fd6..77befc2a 100644 --- a/src/serverless/gtmGithubUtils.js +++ b/src/serverless/gtmGithubUtils.js @@ -8,6 +8,7 @@ let githubUpdaters = { comment: updateGitHubComment, push: updateGitHubPush }; +const KmsUtils = require('../KmsUtils').KmsUtils; let ghEnforceValidSsl = process.env.NODE_TLS_REJECT_UNAUTHORIZED === 0; @@ -26,13 +27,14 @@ function connect(context) { console.log('Creating GitHub API Connection'); let github = new GitHubApi(githubOptions); - let token = process.env.GTM_GITHUB_TOKEN; + let token = KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN); if (context) { token = - process.env['GTM_GITHUB_TOKEN_' + context.toUpperCase().replace('-', '_')] || process.env.GTM_GITHUB_TOKEN; + KmsUtils.getDecrypted(process.env['GTM_CRYPT_GITHUB_TOKEN_' + context.toUpperCase().replace('-', '_')]) || + KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN); } - console.log('Authenticating with GitHub'); + console.log('Authenticating with GitHub' + token); github.authenticate({ type: 'oauth', token: token @@ -51,7 +53,7 @@ function signRequestBody(key, body) { function invalidHook(event) { let err = null; let errMsg = null; - const token = process.env.GTM_GITHUB_WEBHOOK_SECRET; + const token = KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET); const headers = event.headers; const sig = headers['X-Hub-Signature'] || headers['x-hub-signature']; const githubEvent = headers['X-GitHub-Event'] || headers['x-github-event']; @@ -62,7 +64,7 @@ function invalidHook(event) { { name: 'github secret', check: typeof token !== 'string', - msg: `Must provide a 'GITHUB_WEBHOOK_SECRET' env variable` + msg: `Must provide a 'GTM_CRYPT_GITHUB_WEBHOOK_SECRET' env variable` }, { name: 'X-Hub-Signature', @@ -180,7 +182,7 @@ async function updateGitHubPullRequestStatus(status, done) { done(); }); } catch (e) { - if (e.message == 'OAuth2 authentication requires a token or key & secret to be set') { + if (e.message === 'OAuth2 authentication requires a token or key & secret to be set') { throw e; } console.log('----- ERROR COMMUNICATING WITH GITHUB -----'); diff --git a/test/agent/Agent.spec.js b/test/agent/Agent.spec.js index 05990934..54c80f09 100644 --- a/test/agent/Agent.spec.js +++ b/test/agent/Agent.spec.js @@ -29,9 +29,9 @@ describe('Agent', function() { describe('start', () => { before(() => { - process.env.GTM_AGENT_AWS_ACCESS_KEY_ID = 'aws_key_id'; - process.env.GTM_AGENT_AWS_SECRET_ACCESS_KEY = 'aws_key_secret'; - process.env.GTM_GITHUB_WEBHOOK_SECRET = 'webhook_secret'; + process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID = 'aws_key_id'; + process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY = 'aws_key_secret'; + process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = 'webhook_secret'; }); it('should start the agent', () => { diff --git a/test/agent/Event.spec.js b/test/agent/Event.spec.js index c8fb3d79..2cd59436 100644 --- a/test/agent/Event.spec.js +++ b/test/agent/Event.spec.js @@ -6,16 +6,17 @@ import { Event } from '../../src/agent/Event'; describe('Event', function() { let temp; before(() => { - temp = process.env.GTM_GITHUB_WEBHOOK_SECRET; - process.env.GTM_GITHUB_WEBHOOK_SECRET = 'squirrel'; + temp = process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET; + process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = 'squirrel'; }); after(() => { - process.env.GTM_GITHUB_WEBHOOK_SECRET = temp; + process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = temp; }); let message; beforeEach(() => { + process.env.GTM_AWS_KMS_KEY_ID = ''; message = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/githubMessage.json', 'utf-8')); }); diff --git a/test/agent/EventHandler.spec.js b/test/agent/EventHandler.spec.js index 1b874c29..fa2d2d36 100644 --- a/test/agent/EventHandler.spec.js +++ b/test/agent/EventHandler.spec.js @@ -7,6 +7,7 @@ import { EventHandler } from '../../src/agent/EventHandler'; describe('EventHandler', function() { let handler; beforeEach(() => { + process.env.GTM_AWS_KMS_KEY_ID = ''; let eventData = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/githubEventPayload.json', 'utf-8')); handler = new EventHandler(eventData, console); }); diff --git a/test/agent/Executor.spec.js b/test/agent/Executor.spec.js index caebd664..2a4245e1 100644 --- a/test/agent/Executor.spec.js +++ b/test/agent/Executor.spec.js @@ -7,6 +7,7 @@ import { Executor } from '../../src/agent/Executor'; describe('Executor', function() { let executor; beforeEach(() => { + process.env.GTM_AWS_KMS_KEY_ID = ''; let eventData = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/githubEventPayload.json', 'utf-8')); executor = new Executor(eventData, console); }); diff --git a/test/agent/Plugin.spec.js b/test/agent/Plugin.spec.js index 85bc83c0..c27bc934 100644 --- a/test/agent/Plugin.spec.js +++ b/test/agent/Plugin.spec.js @@ -3,6 +3,7 @@ import { default as assert } from 'assert'; import { Plugin } from '../../src/agent/Plugin'; describe('Plugin', function() { + process.env.GTM_AWS_KMS_KEY_ID = ''; describe('register', function() { it('should register new class', function() { class Animal extends Plugin {} diff --git a/test/handlers/EventHandlerPullRequest.spec.js b/test/handlers/EventHandlerPullRequest.spec.js index fa205047..d60d6f13 100644 --- a/test/handlers/EventHandlerPullRequest.spec.js +++ b/test/handlers/EventHandlerPullRequest.spec.js @@ -8,6 +8,7 @@ describe('EventHandlerPullRequest', function() { let eventData; beforeEach(() => { + process.env.GTM_AWS_KMS_KEY_ID = ''; eventData = { ghEventId: 'id', ghEventType: 'pull_request', diff --git a/test/serverless/gtmGithubHook/gtmGithubHook.spec.js b/test/serverless/gtmGithubHook/gtmGithubHook.spec.js index bf500cf6..bc5a1f7f 100644 --- a/test/serverless/gtmGithubHook/gtmGithubHook.spec.js +++ b/test/serverless/gtmGithubHook/gtmGithubHook.spec.js @@ -1,11 +1,14 @@ import { default as sinon } from 'sinon'; import { default as assert } from 'assert'; -import { before, after, describe, it } from 'mocha'; +import { before, after, describe, it, beforeEach } from 'mocha'; import { default as crypto } from 'crypto'; import { default as gtmGithubHook } from '../../../src/serverless/gtmGithubHook/gtmGithubHook.js'; import { default as githubUtils } from '../../../src/serverless/gtmGithubUtils.js'; describe('gtmGithubHook', function() { + beforeEach(() => { + process.env.GTM_AWS_KMS_KEY_ID = ''; + }); describe('decodeEventBody', function() { it('should remove prefix and parse body', function(done) { let expected = { action: 'test' }; @@ -24,7 +27,7 @@ describe('gtmGithubHook', function() { event.body = 'payload=%7B%22action%22%3A%20%22test%22%7D'; let key = 'abc'; - process.env.GTM_GITHUB_WEBHOOK_SECRET = key; + process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = key; let sig = `sha1=${crypto .createHmac('sha1', key) .update(event.body, 'utf-8') diff --git a/test/serverless/gtmGithubUtils.spec.js b/test/serverless/gtmGithubUtils.spec.js index 361b2976..8583de27 100644 --- a/test/serverless/gtmGithubUtils.spec.js +++ b/test/serverless/gtmGithubUtils.spec.js @@ -1,11 +1,14 @@ -import { describe, it } from 'mocha'; +import { beforeEach, describe, it } from 'mocha'; import { default as assert } from 'assert'; import { default as crypto } from 'crypto'; import { default as githubUtils } from '../../src/serverless/gtmGithubUtils.js'; -process.env.GTM_GITHUB_TOKEN = ''; +process.env.GTM_CRYPT_GITHUB_TOKEN = ''; describe('gtmGithubUtils', function() { + beforeEach(() => { + process.env.GTM_AWS_KMS_KEY_ID = ''; + }); describe('connect', function() { it('should throw without creds', function(done) { assert.throws(githubUtils.connect, Error); @@ -31,7 +34,8 @@ describe('gtmGithubUtils', function() { describe('invalidHook', function() { it('should return error if event header missing', function(done) { - process.env.GTM_GITHUB_WEBHOOK_SECRET = 'abc'; + process.env.GTM_AWS_KMS_KEY_ID = ''; + process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = 'abc'; let event = { body: 'testing', @@ -69,7 +73,7 @@ describe('gtmGithubUtils', function() { let actual; try { - actual = await githubUtils.handleEventTaskResult(message); + actual = await githubUtils.handleEventTaskResult(message, () => {}); console.log(actual); } catch (e) { return assert.equal(e.message, 'OAuth2 authentication requires a token or key & secret to be set'); @@ -84,7 +88,7 @@ describe('gtmGithubUtils', function() { actual = await githubUtils.getFile(); console.log(actual); } catch (e) { - return assert.equal('OAuth2 authentication requires a token or key & secret to be set', e.message); + return assert.equal(e.message, 'OAuth2 authentication requires a token or key & secret to be set'); } }); }); From dbef953ecc48edde88d843ffea9de044a8ae9674 Mon Sep 17 00:00:00 2001 From: wyvern8 Date: Wed, 13 Jun 2018 20:23:09 +1000 Subject: [PATCH 2/7] refactor(kms): update kms decryption to be promise based lots of updates reqd.. need to uplift unit testing. --- kms-secrets.dev.ap-southeast-2.yml | 2 +- src/KmsUtils.js | 67 +++++++++---------- src/agent/Agent.js | 3 +- src/agent/AgentUtils.js | 2 +- src/agent/Event.js | 16 ++--- src/executors/ExecutorDockerServerless.js | 2 +- src/executors/ExecutorDockerSonar.js | 2 +- src/executors/ExecutorJenkins.js | 2 +- src/executors/ExecutorLaunchDarkly.js | 2 +- src/executors/ExecutorTeamCity.js | 2 +- src/executors/ExecutorTravis.js | 2 +- src/serverless/gtmGithubHook/gtmGithubHook.js | 7 +- src/serverless/gtmGithubUtils.js | 26 +++---- test/agent/Event.spec.js | 16 ++--- test/github-task-manager.spec-int.js | 2 +- test/serverless/gtmGithubUtils.spec.js | 17 ++--- 16 files changed, 83 insertions(+), 87 deletions(-) diff --git a/kms-secrets.dev.ap-southeast-2.yml b/kms-secrets.dev.ap-southeast-2.yml index 1a7e1594..ffb092bd 100644 --- a/kms-secrets.dev.ap-southeast-2.yml +++ b/kms-secrets.dev.ap-southeast-2.yml @@ -1,6 +1,6 @@ secrets: GTM_CRYPT_GITHUB_TOKEN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEJcnivPXbcJwnZzTQTxu31AAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDFP2pE25RzVb1ZrnlAIBEIBDfziIOzj0CgNt/OuDDoWtIpDXtHRvyVKxMtUOHalsNs8RDVQFPTMGX0qr/BkMeEIO0yG1UF4g2i5p7ZRUJQ68IEgRAQ== - GTM_CRYPT_GITHUB_WEBHOOK_SECRET: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEEamZPETq1yZycyB1dmPIvAAAAbjBsBgkqhkiG9w0BBwagXzBdAgEAMFgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMgN0GR8Nj6wuEbqD0AgEQgCuyRS2IMDCgfEvTTthcarc5ptgLhz7nGXnYKQ5xLsurGmfM9McnRW6g/HbG + GTM_CRYPT_GITHUB_WEBHOOK_SECRET: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQEEkJghXzLnVn+mkZOvU24nAAAAbjBsBgkqhkiG9w0BBwagXzBdAgEAMFgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM8JJgdprhT1xeMJ2iAgEQgCtCYXg7hhBX4A1PvVqN5adTeGBGVvoQ5o2Mv9y8ryfoybUqSSVclmJ4XMu0 GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQGVqoF0YYPkVr2XywzcnsPtAAAAcjBwBgkqhkiG9w0BBwagYzBhAgEAMFwGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMs0VIMb7Ver78YYylAgEQgC+bNI7V97dCWeE4n9D7CKFUbD8i4t3ZbbgniTsaoThOczAlYhBxmLtVcAQ78b6mnw== GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQH3RvbGVR/b3CF8NkbkIjcYAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDC+dhWEBgen13NUSJwIBEIBDaG2tl4rs0hmHn7XIAVMU9VFYAq8LcQ2Tu0WvXYwO4GVi9nXQfVq5qrxTc62ajNHjp9wOKCeakOC7pbpsWktb3dc9nQ== GTM_CRYPT_LAUNCHDARKLY_API_TOKEN: AQICAHhK1adeaGaap/XxRtXdFB/VkZT3XeQtDEMkVYvemdBEiQHMMlqueqRkis1P7Ahnf6UbAAAAhzCBhAYJKoZIhvcNAQcGoHcwdQIBADBwBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOagJHWpec2No95GBwIBEIBD3vl3qwULarfWZjMn3mJzQNDdWJmfyvuMCYDAW0f0dWvVWDzOwKo2crAnJF66Wb94xMSoGKa3Drsx8VxRvX1W1F91uA== diff --git a/src/KmsUtils.js b/src/KmsUtils.js index 11901042..2a01bf5f 100644 --- a/src/KmsUtils.js +++ b/src/KmsUtils.js @@ -2,74 +2,71 @@ import * as AWS from 'aws-sdk'; import * as bunyan from 'bunyan'; const log = bunyan.createLogger({ name: 'KmsUtils' }); -let STORE = {}; - -const KMS = new AWS.KMS({ region: process.env.GTM_AWS_REGION }); - -export class KmsUtils { +class KmsUtils { constructor() { + this.KMS = new AWS.KMS({ region: process.env.GTM_AWS_REGION }); + this.primeStore(); log.info('KmsUtils created'); } - static getStore() { - return STORE; + get store() { + if (!this._store) this._store = {}; + return this._store; } - static decrypt(encrypted, callback) { - let result = ''; + + async decrypt(encrypted) { if (encrypted) { if (!process.env.GTM_AWS_KMS_KEY_ID) { log.warn(`no encryption key configured, using raw values`); - KmsUtils.setDecrypted(encrypted, encrypted); - callback(null, encrypted); + this.setDecrypted(encrypted, encrypted); + return encrypted; } else { log.info(`decrypting: ${encrypted}`); - if (KmsUtils.hasDecrypted(encrypted)) { + if (this.hasDecrypted(encrypted)) { log.info('returning stored decrypted value'); - callback(null, KmsUtils.getDecrypted(encrypted)); + return this.getDecrypted(encrypted); } try { - KMS.decrypt({ CiphertextBlob: new Buffer(encrypted, 'base64') }) + return this.KMS.decrypt({ CiphertextBlob: new Buffer(encrypted, 'base64') }) .promise() .then(data => { log.info(`storing decrypted result.`); - KmsUtils.setDecrypted(encrypted, data.Plaintext); - return callback(null, data.Plaintext); + let decrypted = data.Plaintext.toString(); + this.setDecrypted(encrypted, decrypted); + return decrypted; }); } catch (e) { log.error(e); - return callback(e); } } } - return result; + //return result; } - static primeStore() { + async primeStore() { log.info(`priming decrypted var store`); + let promises = []; Object.keys(process.env).forEach(key => { if (key.startsWith('GTM_CRYPT_')) { - KmsUtils.decrypt(process.env[key], err => { - if (err) log.error(err); - log.info(`stored decrypted value of ${key}`); - }); + promises.push(this.decrypt(process.env[key])); + log.info(`decrypting value of ${key}`); } }); + return Promise.all(promises); } - static hasDecrypted(encrypted) { - return Object.keys(STORE).includes(encrypted); + hasDecrypted(encrypted) { + return Object.keys(this.store).includes(encrypted); } - static getDecrypted(encrypted) { + async getDecrypted(encrypted) { if (this.hasDecrypted(encrypted)) { - return STORE[encrypted]; + return this.store[encrypted]; } else { - KmsUtils.decrypt(encrypted, (err, decrypted) => { - if (err) log.error(err); - log.warn(`returning newly decrypted value`); - return decrypted; - }); + log.info(`returning newly decrypted value`); + return await this.decrypt(encrypted); } } - static setDecrypted(encrypted, decrypted) { - STORE[encrypted] = decrypted; + setDecrypted(encrypted, decrypted) { + this.store[encrypted] = decrypted; } } -KmsUtils.primeStore(); + +export default new KmsUtils(); diff --git a/src/agent/Agent.js b/src/agent/Agent.js index 0f9f8f45..bc9307de 100644 --- a/src/agent/Agent.js +++ b/src/agent/Agent.js @@ -230,7 +230,8 @@ export class Agent { let event; try { - event = new Event(message); + let messageAttrs = await Event.validateMessage(message); + event = new Event(message, messageAttrs); } catch (e) { log.error(e); done(); //todo dead letter queue here rather than discard diff --git a/src/agent/AgentUtils.js b/src/agent/AgentUtils.js index 80a84db9..4f763e7a 100644 --- a/src/agent/AgentUtils.js +++ b/src/agent/AgentUtils.js @@ -4,7 +4,7 @@ const pullRequestData = require('./pullrequest.json'); const { URL } = require('url'); const crypto = require('crypto'); -const { KmsUtils } = require('../KmsUtils'); +import KmsUtils from '../KmsUtils'; import { default as AgentLogger } from './AgentLogger'; import { default as yamljs } from 'yamljs'; diff --git a/src/agent/Event.js b/src/agent/Event.js index f1d7d429..dc58767d 100644 --- a/src/agent/Event.js +++ b/src/agent/Event.js @@ -4,16 +4,15 @@ import 'babel-polyfill'; import { default as crypto } from 'crypto'; import { default as AgentLogger } from './AgentLogger'; import { Agent } from './Agent'; -import { KmsUtils } from '../KmsUtils'; -console.log(KmsUtils.getStore()); +import KmsUtils from '../KmsUtils'; let log = AgentLogger.log(); /** * representation of an event generated from a github hook SQS message */ export class Event { - constructor(message) { - this.attrs = Event.validateMessage(message); + constructor(message, attrs) { + this.attrs = attrs; this.payload = Event.prepareEventPayload(message, this.attrs); this.log = log.child({ ghEventId: this.attrs.ghEventId }); Agent.systemConfig.event.current = this.payload; @@ -32,7 +31,7 @@ export class Event { * @param message * @returns {{}} result attribute object */ - static validateMessage(message) { + static async validateMessage(message) { let result = {}; Event.requiredAttributes.forEach(attr => { @@ -43,7 +42,7 @@ export class Event { } }); - if (!Event.checkEventSignature(result.ghEventSignature, message)) { + if (!(await Event.checkEventSignature(result.ghEventSignature, message))) { throw new Error('Event signature mismatch - discarding Event!'); } else { log.info('Event signature verified. processing event..'); @@ -58,13 +57,12 @@ export class Event { * @param message * @returns {boolean} */ - static checkEventSignature(signature, message) { + static async checkEventSignature(signature, message) { let checkEvent = Event.buildCheckObject(message); log.debug(`eventString for signature check: ${JSON.stringify(checkEvent)}`); - console.log(KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET)); let calculatedSig = `sha1=${crypto - .createHmac('sha1', KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET)) + .createHmac('sha1', await KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET)) .update(JSON.stringify(checkEvent), 'utf-8') .digest('hex')}`; diff --git a/src/executors/ExecutorDockerServerless.js b/src/executors/ExecutorDockerServerless.js index f98f8452..24ebed7d 100644 --- a/src/executors/ExecutorDockerServerless.js +++ b/src/executors/ExecutorDockerServerless.js @@ -2,7 +2,7 @@ import { Executor } from '../agent/Executor'; import { ExecutorDocker } from './ExecutorDocker'; import { default as _ } from 'lodash'; import { AgentUtils } from '../agent/AgentUtils'; -import { KmsUtils } from '../KmsUtils'; +import KmsUtils from '../KmsUtils'; /** * Sample .githubTaskManager.json task config diff --git a/src/executors/ExecutorDockerSonar.js b/src/executors/ExecutorDockerSonar.js index 8a090f2c..ceea28f6 100644 --- a/src/executors/ExecutorDockerSonar.js +++ b/src/executors/ExecutorDockerSonar.js @@ -2,7 +2,7 @@ import { Executor } from '../agent/Executor'; import { ExecutorDocker } from './ExecutorDocker'; import { default as _ } from 'lodash'; import { AgentUtils } from '../agent/AgentUtils'; -import { KmsUtils } from '../KmsUtils'; +import KmsUtils from '../KmsUtils'; /** * Sample .githubTaskManager.json task config diff --git a/src/executors/ExecutorJenkins.js b/src/executors/ExecutorJenkins.js index be6232b6..50a37ea6 100644 --- a/src/executors/ExecutorJenkins.js +++ b/src/executors/ExecutorJenkins.js @@ -1,7 +1,7 @@ import { default as JenkinsLib } from 'jenkins'; import { Executor } from '../agent/Executor'; import { AgentUtils } from '../agent/AgentUtils'; -import { KmsUtils } from '../KmsUtils'; +import KmsUtils from '../KmsUtils'; /** * Sample .githubTaskManager.json task config/ diff --git a/src/executors/ExecutorLaunchDarkly.js b/src/executors/ExecutorLaunchDarkly.js index f221243c..53e7f3c2 100644 --- a/src/executors/ExecutorLaunchDarkly.js +++ b/src/executors/ExecutorLaunchDarkly.js @@ -1,7 +1,7 @@ import { Executor } from '../agent/Executor'; import { LaunchDarklyUtils } from 'launchdarkly-nodeutils'; import { default as formatJson } from 'format-json'; -import { KmsUtils } from '../KmsUtils'; +import KmsUtils from '../KmsUtils'; /** * Sample .githubTaskManager.json task config diff --git a/src/executors/ExecutorTeamCity.js b/src/executors/ExecutorTeamCity.js index f7453157..04b8219f 100644 --- a/src/executors/ExecutorTeamCity.js +++ b/src/executors/ExecutorTeamCity.js @@ -5,7 +5,7 @@ import { default as x2j } from 'xml2js'; import { default as URL } from 'url'; import { Executor } from '../agent/Executor'; import { AgentUtils } from '../agent/AgentUtils'; -import { KmsUtils } from '../KmsUtils'; +import KmsUtils from '../KmsUtils'; /** * Sample .githubTaskManager.json task config diff --git a/src/executors/ExecutorTravis.js b/src/executors/ExecutorTravis.js index a1666241..98ef4e50 100644 --- a/src/executors/ExecutorTravis.js +++ b/src/executors/ExecutorTravis.js @@ -1,7 +1,7 @@ import { default as Travis } from 'travis-ci'; import { Executor } from '../agent/Executor'; import { default as json } from 'format-json'; -import { KmsUtils } from '../KmsUtils'; +import KmsUtils from '../KmsUtils'; /** * Sample .githubTaskManager.json task config - NOT READY FOR USE diff --git a/src/serverless/gtmGithubHook/gtmGithubHook.js b/src/serverless/gtmGithubHook/gtmGithubHook.js index a8a0ed43..e9768308 100644 --- a/src/serverless/gtmGithubHook/gtmGithubHook.js +++ b/src/serverless/gtmGithubHook/gtmGithubHook.js @@ -7,7 +7,7 @@ let UUID = require('uuid/v4'); let Producer = require('sqs-producer'); let githubUtils = require('../gtmGithubUtils.js'); -const KmsUtils = require('./../../KmsUtils').KmsUtils; +import KmsUtils from './../../KmsUtils'; process.on('unhandledRejection', (reason, p) => { console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); @@ -17,8 +17,7 @@ process.on('unhandledRejection', (reason, p) => { async function listener(event, context, callback) { const githubEvent = event.headers['X-GitHub-Event'] || event.headers['x-github-event']; const githubSignature = event.headers['X-Hub-Signature'] || event.headers['x-hub-signature']; - - let err = githubUtils.invalidHook(event); + let err = await githubUtils.invalidHook(event); if (err) { return callback(err, { statusCode: 401, @@ -106,7 +105,7 @@ async function handleEvent(type, body, signature) { ]; signature = githubUtils.signRequestBody( - KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET), + await KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET), JSON.stringify(event[0]) ); diff --git a/src/serverless/gtmGithubUtils.js b/src/serverless/gtmGithubUtils.js index 77befc2a..9f1d4c83 100644 --- a/src/serverless/gtmGithubUtils.js +++ b/src/serverless/gtmGithubUtils.js @@ -8,11 +8,11 @@ let githubUpdaters = { comment: updateGitHubComment, push: updateGitHubPush }; -const KmsUtils = require('../KmsUtils').KmsUtils; +import KmsUtils from './../KmsUtils'; let ghEnforceValidSsl = process.env.NODE_TLS_REJECT_UNAUTHORIZED === 0; -function connect(context) { +async function connect(context) { console.log('Connecting to GitHub'); console.log('GitHub SSL Validation: ' + String(ghEnforceValidSsl)); let githubOptions = { @@ -27,19 +27,19 @@ function connect(context) { console.log('Creating GitHub API Connection'); let github = new GitHubApi(githubOptions); - let token = KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN); + let token = await KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN); if (context) { token = - KmsUtils.getDecrypted(process.env['GTM_CRYPT_GITHUB_TOKEN_' + context.toUpperCase().replace('-', '_')]) || - KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN); + (await KmsUtils.getDecrypted( + process.env['GTM_CRYPT_GITHUB_TOKEN_' + context.toUpperCase().replace('-', '_')] + )) || (await KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN)); } - console.log('Authenticating with GitHub' + token); + console.log('Authenticating with GitHub'); github.authenticate({ type: 'oauth', token: token }); - return github; } @@ -50,10 +50,10 @@ function signRequestBody(key, body) { .digest('hex')}`; } -function invalidHook(event) { +async function invalidHook(event) { let err = null; let errMsg = null; - const token = KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET); + const token = await KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET); const headers = event.headers; const sig = headers['X-Hub-Signature'] || headers['x-hub-signature']; const githubEvent = headers['X-GitHub-Event'] || headers['x-github-event']; @@ -113,7 +113,7 @@ function decodeFileResponse(fileResponse) { } async function getFile(params) { - let github = connect(); + let github = await connect(); return new Promise((resolve, reject) => { try { @@ -177,7 +177,7 @@ async function updateGitHubPullRequestStatus(status, done) { //console.log(status); try { - let github = connect(status.context); + let github = await connect(status.context); return await github.repos.createStatus(status).then(() => { done(); }); @@ -206,7 +206,7 @@ async function addGitHubPullRequestComment(status, done) { }; */ try { - let github = connect(); + let github = await connect(); return await github.pullRequests .createReview({ owner: status.owner, @@ -239,7 +239,7 @@ async function addGitHubPushComments(status, done) { }; */ try { - let github = connect(); + let github = await connect(); let promises = []; // adding the same comment to each commit in the push for now.. status.eventData.commits.forEach(commit => { diff --git a/test/agent/Event.spec.js b/test/agent/Event.spec.js index 2cd59436..64d0a465 100644 --- a/test/agent/Event.spec.js +++ b/test/agent/Event.spec.js @@ -37,28 +37,28 @@ describe('Event', function() { message = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/githubMessage.json', 'utf-8')); }); - it('should throw when message has missing attribute', function() { + it('should throw when message has missing attribute', async function() { try { delete message.MessageAttributes.ghEventId; - Event.validateMessage(message); + await Event.validateMessage(message); } catch (e) { assert.equal(e.message, `No Message Attribute 'ghEventId' in Message - discarding Event!`); } }); - it('should return message attributes', function() { + it('should return message attributes', async function() { let expected = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/githubEventAttributes.json', 'utf-8')); - let actual = Event.validateMessage(message); + let actual = await Event.validateMessage(message); assert.equal(JSON.stringify(actual), JSON.stringify(expected)); }); }); describe('checkEventSignature', function() { - it('should sign message with github webhook secret', function() { + it('should sign message with github webhook secret', async function() { let signature = message.MessageAttributes.ghEventSignature.StringValue; - let result = Event.checkEventSignature(signature, message); + let result = await Event.checkEventSignature(signature, message); assert.equal(result, true); }); @@ -75,10 +75,10 @@ describe('Event', function() { }); describe('prepareEventPayload', function() { - it('should create the expected payload from message and attributes', function() { + it('should create the expected payload from message and attributes', async function() { let expected = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/githubEventPayload.json', 'utf-8')); - let attrs = Event.validateMessage(message); + let attrs = await Event.validateMessage(message); let actual = Event.prepareEventPayload(message, attrs); assert.equal(JSON.stringify(actual), JSON.stringify(expected)); diff --git a/test/github-task-manager.spec-int.js b/test/github-task-manager.spec-int.js index 63f9730c..d7ad372e 100644 --- a/test/github-task-manager.spec-int.js +++ b/test/github-task-manager.spec-int.js @@ -29,7 +29,7 @@ describe('GitHub Task Manager', () => { let testName = `test_${Date.now()}`; let hookUrl = await firstline('./sls-hook-url.out'); console.log(`hook url: ${hookUrl}`); - let github = githubUtils.connect(); + let github = await githubUtils.connect(); let gh = { repos: { diff --git a/test/serverless/gtmGithubUtils.spec.js b/test/serverless/gtmGithubUtils.spec.js index 8583de27..5122840e 100644 --- a/test/serverless/gtmGithubUtils.spec.js +++ b/test/serverless/gtmGithubUtils.spec.js @@ -10,14 +10,17 @@ describe('gtmGithubUtils', function() { process.env.GTM_AWS_KMS_KEY_ID = ''; }); describe('connect', function() { - it('should throw without creds', function(done) { - assert.throws(githubUtils.connect, Error); - done(); + it('should throw without creds', async function() { + try { + await githubUtils.connect(); + } catch (e) { + return assert.equal(e.message, 'OAuth2 authentication requires a token or key & secret to be set'); + } }); }); describe('signRequestBody', function() { - it('should encrypt correctly', function(done) { + it('should encrypt correctly', function() { let key = 'abc'; let body = 'def'; @@ -28,12 +31,11 @@ describe('gtmGithubUtils', function() { let actual = githubUtils.signRequestBody(key, body); assert.equal(actual, expected); - done(); }); }); describe('invalidHook', function() { - it('should return error if event header missing', function(done) { + it('should return error if event header missing', async function() { process.env.GTM_AWS_KMS_KEY_ID = ''; process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = 'abc'; @@ -45,9 +47,8 @@ describe('gtmGithubUtils', function() { }; let expected = 'Error: No X-Github-Event found on request'; - let actual = githubUtils.invalidHook(event); + let actual = await githubUtils.invalidHook(event); assert.equal(actual, expected); - done(); }); }); From c23aa3e07db7757b7d5a4c9cdcf74e5f37ea7ad6 Mon Sep 17 00:00:00 2001 From: wyvern8 Date: Thu, 14 Jun 2018 09:20:58 +1000 Subject: [PATCH 3/7] fix(kms): dotenv changes and kms logger support dotenv plugin no longer pulls whole environment into serverless, a separate yaml file is used instead to pull in env vars required as well as kms encrypted values for stage and region --- .envExample | 1 + dotenv.js | 8 +++++++ encrypt.sh | 2 +- package.json | 10 ++++---- serverless-aws-environment.yml | 18 +++++++-------- serverless.yml | 7 ++---- src/KmsUtils.js | 28 ++++++++++++++--------- src/agent/AgentUtils.js | 1 + src/agent/Event.js | 1 + src/executors/ExecutorDockerServerless.js | 7 +++--- src/executors/ExecutorDockerSonar.js | 1 + src/executors/ExecutorJenkins.js | 1 + src/executors/ExecutorLaunchDarkly.js | 1 + src/executors/ExecutorTeamCity.js | 1 + src/executors/ExecutorTravis.js | 1 + 15 files changed, 53 insertions(+), 35 deletions(-) create mode 100644 dotenv.js diff --git a/.envExample b/.envExample index 91cd5c34..31a85a4a 100644 --- a/.envExample +++ b/.envExample @@ -1,4 +1,5 @@ GTM_AWS_REGION=ap-southeast-2 +GTM_AWS_STAGE=dev GTM_SQS_PENDING_QUEUE=gtmPendingQueue GTM_SQS_RESULTS_QUEUE=gtmResultsQueue GTM_SNS_RESULTS_TOPIC=gtmResultsSNSTopic diff --git a/dotenv.js b/dotenv.js new file mode 100644 index 00000000..d9b05c7d --- /dev/null +++ b/dotenv.js @@ -0,0 +1,8 @@ +const dotenv = require('dotenv'); +const config = dotenv.config(); + +if (config.error) { + console.error(config.error); +} + +//console.log(json.plain(_.extend(sharedConfig.parsed, localConfig.parsed))); diff --git a/encrypt.sh b/encrypt.sh index 46fdcf23..582764cb 100755 --- a/encrypt.sh +++ b/encrypt.sh @@ -11,6 +11,6 @@ if [ $# -eq 0 ]; then fi export $(cat .env | grep -v ^# | xargs) -node ./node_modules/serverless/bin/serverless encrypt -k $GTM_AWS_KMS_KEY_ID -n $1 -v $2 +node --require ./dotenv.js ./node_modules/serverless/bin/serverless encrypt -k $GTM_AWS_KMS_KEY_ID -n $1 -v $2 # todo add encrypted values to .env from kms secrets \ No newline at end of file diff --git a/package.json b/package.json index 794a718c..26781429 100644 --- a/package.json +++ b/package.json @@ -11,12 +11,12 @@ "test": "npm run build && cross-env NODE_ENV=test GTM_CRYPT_GITHUB_WEBHOOK_SECRET=abc GTM_AWS_KMS_KEY_ID='' nyc ./node_modules/.bin/mocha --timeout 10000 --require babel-core/register --require babel-polyfill './dist/test/**/*.spec.js'", "test-integration": "npm run agent-build && npm run sls-deploy && npm run docker-start && EDGEGRID_ENV=test node ./node_modules/.bin/mocha --timeout 10000 --require babel-core/register --require babel-polyfill ./dist/test/**/*.spec-int.js", "semantic-release": "semantic-release", - "sls-deploy": "npm run build && node ./node_modules/serverless/bin/serverless deploy -v | tee ./sls.out && cat ./sls.out | grep ServiceEndpoint | sed 's/ServiceEndpoint[^https]*\\(https:\\/\\/.*\\)/\\1\\/gtm-github-hook/' | sed 's/\\x1b\\[[0-9;]*m//g' > ./sls-hook-url.out", - "sls-undeploy": "node ./node_modules/serverless/bin/serverless remove", - "sls-logs-hook": "node ./node_modules/serverless/bin/serverless logs -f gtmGithubHook -t", - "sls-logs-results": "node ./node_modules/serverless/bin/serverless logs -f gtmGithubResults -t", + "sls-deploy": "npm run build && node --require ./dotenv.js ./node_modules/serverless/bin/serverless deploy -v | tee ./sls.out && cat ./sls.out | grep ServiceEndpoint | sed 's/ServiceEndpoint[^https]*\\(https:\\/\\/.*\\)/\\1\\/gtm-github-hook/' | sed 's/\\x1b\\[[0-9;]*m//g' > ./sls-hook-url.out", + "sls-undeploy": "node --require ./dotenv.js ./node_modules/serverless/bin/serverless remove", + "sls-logs-hook": "node --require ./dotenv.js ./node_modules/serverless/bin/serverless logs -f gtmGithubHook -t", + "sls-logs-results": "node --require ./dotenv.js ./node_modules/serverless/bin/serverless logs -f gtmGithubResults -t", "sls-encrypt": "chmod 755 ./encrypt.sh && ./encrypt.sh", - "sls-decrypt": "node ./node_modules/serverless/bin/serverless decrypt -n", + "sls-decrypt": "node --require ./dotenv.js ./node_modules/serverless/bin/serverless decrypt -n", "agent": "export $(cat .env | grep -v ^# | xargs) && node dist/src/agent/startAgent.js", "agent-build": "node ./dist/src/agent/sass.js && cp ./src/agent/static/*.js ./dist/src/agent/static/", "docker-build": "docker build -t github-task-manager --no-cache .", diff --git a/serverless-aws-environment.yml b/serverless-aws-environment.yml index 2b801e09..79384bbb 100644 --- a/serverless-aws-environment.yml +++ b/serverless-aws-environment.yml @@ -1,14 +1,14 @@ environment: # encrypted env vars - use npm run sls-encrypt to populate json store for stage - GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID} - GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY} - GTM_CRYPT_GITHUB_WEBHOOK_SECRET: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_GITHUB_WEBHOOK_SECRET} - GTM_CRYPT_GITHUB_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_GITHUB_TOKEN} - GTM_CRYPT_JENKINS_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_JENKINS_TOKEN} - GTM_CRYPT_LAUNCHDARKLY_API_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_LAUNCHDARKLY_API_TOKEN} - GTM_CRYPT_SONAR_GITHUB_OAUTH: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_SONAR_GITHUB_OAUTH} - GTM_CRYPT_SONAR_LOGIN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_SONAR_LOGIN} - GTM_CRYPT_TEAMCITY_PASSCODE: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_TEAMCITY_PASSCODE} + GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID, env:GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID} + GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY, env:GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY} + GTM_CRYPT_GITHUB_WEBHOOK_SECRET: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_GITHUB_WEBHOOK_SECRET, env:GTM_CRYPT_GITHUB_WEBHOOK_SECRET} + GTM_CRYPT_GITHUB_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_GITHUB_TOKEN, env:GTM_CRYPT_GITHUB_TOKEN} + GTM_CRYPT_JENKINS_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_JENKINS_TOKEN, env:GTM_CRYPT_JENKINS_TOKEN} + GTM_CRYPT_LAUNCHDARKLY_API_TOKEN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_LAUNCHDARKLY_API_TOKEN, env:GTM_CRYPT_LAUNCHDARKLY_API_TOKEN} + GTM_CRYPT_SONAR_GITHUB_OAUTH: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_SONAR_GITHUB_OAUTH, env:GTM_CRYPT_SONAR_GITHUB_OAUTH} + GTM_CRYPT_SONAR_LOGIN: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_SONAR_LOGIN, env:GTM_CRYPT_SONAR_LOGIN} + GTM_CRYPT_TEAMCITY_PASSCODE: ${self:custom.kmsSecrets.secrets.GTM_CRYPT_TEAMCITY_PASSCODE, env:GTM_CRYPT_TEAMCITY_PASSCODE} # unencrypted env vars - add values to .env GTM_AWS_REGION: ${env:GTM_AWS_REGION, 'ap-southeast-2'} GTM_SQS_PENDING_QUEUE: ${env:GTM_SQS_PENDING_QUEUE, 'gtmPendingQueue'} diff --git a/serverless.yml b/serverless.yml index a29677bc..3f6d5d60 100644 --- a/serverless.yml +++ b/serverless.yml @@ -1,6 +1,5 @@ service: gtmGithubHook plugins: - - serverless-dotenv-plugin - serverless-webpack - serverless-kms-secrets - serverless-offline @@ -14,8 +13,7 @@ provider: stage: dev region: ${env:GTM_AWS_REGION} - environment: - PLACE: HOLDER + environment: ${file(./serverless-aws-environment.yml):environment} iamRoleStatements: - Effect: Allow @@ -157,5 +155,4 @@ resources: # VpcId: ${env:GTM_AWS_VPC_ID} custom: - dotenv: - path: .env \ No newline at end of file + kmsSecrets: ${file(kms-secrets.${opt:stage, self:provider.stage}.${opt:region, self:provider.region}.yml)} diff --git a/src/KmsUtils.js b/src/KmsUtils.js index 2a01bf5f..c0a186c9 100644 --- a/src/KmsUtils.js +++ b/src/KmsUtils.js @@ -1,12 +1,18 @@ import * as AWS from 'aws-sdk'; -import * as bunyan from 'bunyan'; -const log = bunyan.createLogger({ name: 'KmsUtils' }); class KmsUtils { constructor() { this.KMS = new AWS.KMS({ region: process.env.GTM_AWS_REGION }); this.primeStore(); - log.info('KmsUtils created'); + console.info('KmsUtils created'); + } + + get logger() { + if (!this._logger) this._logger = console; + return this._logger; + } + set logger(log) { + if (log) this._logger = log; } get store() { @@ -17,38 +23,38 @@ class KmsUtils { async decrypt(encrypted) { if (encrypted) { if (!process.env.GTM_AWS_KMS_KEY_ID) { - log.warn(`no encryption key configured, using raw values`); + this.logger.warn(`no encryption key configured, using raw values`); this.setDecrypted(encrypted, encrypted); return encrypted; } else { - log.info(`decrypting: ${encrypted}`); + this.logger.info(`decrypting: ${encrypted}`); if (this.hasDecrypted(encrypted)) { - log.info('returning stored decrypted value'); + this.logger.info('returning stored decrypted value'); return this.getDecrypted(encrypted); } try { return this.KMS.decrypt({ CiphertextBlob: new Buffer(encrypted, 'base64') }) .promise() .then(data => { - log.info(`storing decrypted result.`); + this.logger.info(`storing decrypted result.`); let decrypted = data.Plaintext.toString(); this.setDecrypted(encrypted, decrypted); return decrypted; }); } catch (e) { - log.error(e); + this.logger.error(e); } } } //return result; } async primeStore() { - log.info(`priming decrypted var store`); + this.logger.info(`priming decrypted var store`); let promises = []; Object.keys(process.env).forEach(key => { if (key.startsWith('GTM_CRYPT_')) { promises.push(this.decrypt(process.env[key])); - log.info(`decrypting value of ${key}`); + this.logger.info(`decrypting value of ${key}`); } }); return Promise.all(promises); @@ -60,7 +66,7 @@ class KmsUtils { if (this.hasDecrypted(encrypted)) { return this.store[encrypted]; } else { - log.info(`returning newly decrypted value`); + this.logger.info(`returning newly decrypted value`); return await this.decrypt(encrypted); } } diff --git a/src/agent/AgentUtils.js b/src/agent/AgentUtils.js index 4f763e7a..b3c7d179 100644 --- a/src/agent/AgentUtils.js +++ b/src/agent/AgentUtils.js @@ -10,6 +10,7 @@ import { default as AgentLogger } from './AgentLogger'; import { default as yamljs } from 'yamljs'; import { default as https } from 'https'; let log = AgentLogger.log(); +KmsUtils.logger = log; const AWS = require('aws-sdk'); const proxy = require('proxy-agent'); diff --git a/src/agent/Event.js b/src/agent/Event.js index dc58767d..4479bb6f 100644 --- a/src/agent/Event.js +++ b/src/agent/Event.js @@ -6,6 +6,7 @@ import { default as AgentLogger } from './AgentLogger'; import { Agent } from './Agent'; import KmsUtils from '../KmsUtils'; let log = AgentLogger.log(); +KmsUtils.logger = log; /** * representation of an event generated from a github hook SQS message diff --git a/src/executors/ExecutorDockerServerless.js b/src/executors/ExecutorDockerServerless.js index 24ebed7d..56d3cb62 100644 --- a/src/executors/ExecutorDockerServerless.js +++ b/src/executors/ExecutorDockerServerless.js @@ -74,9 +74,8 @@ export class ExecutorDockerServerless extends ExecutorDocker { IAM_ENABLED: process.env.IAM_ENABLED, S3_DEPENDENCY_BUCKET: '##GTM_S3_DEPENDENCY_BUCKET##', AWS_S3_PROXY: '##GTM_AWS_S3_PROXY##', - SLS_AWS_STAGE: process.env.SLS_AWS_STAGE, - SLS_AWS_REGION: process.env.SLS_AWS_REGION, - SLS_CUSTOM_DOMAIN: process.env.SLS_CUSTOM_DOMAIN + AWS_STAGE: process.env.GTM_AWS_STAGE, + AWS_REGION: process.env.GTM_AWS_REGION }, validator: { type: 'outputRegex', @@ -106,7 +105,7 @@ export class ExecutorDockerServerless extends ExecutorDocker { // add token into clone url task.options.env.GIT_CLONE = task.options.env.GIT_CLONE.replace( 'https://', - `https://${task.options.env.SLS_GITHUB_OAUTH}@` + `https://${task.options.env.GTM_CRYPT_GITHUB_TOKEN}@` ); return task.options; diff --git a/src/executors/ExecutorDockerSonar.js b/src/executors/ExecutorDockerSonar.js index ceea28f6..098936f9 100644 --- a/src/executors/ExecutorDockerSonar.js +++ b/src/executors/ExecutorDockerSonar.js @@ -32,6 +32,7 @@ export class ExecutorDockerSonar extends ExecutorDocker { super(eventData, log); this.eventData = eventData; this.log = log; + KmsUtils.logger = log; } async executeTask(task) { diff --git a/src/executors/ExecutorJenkins.js b/src/executors/ExecutorJenkins.js index 50a37ea6..9ffa1f4a 100644 --- a/src/executors/ExecutorJenkins.js +++ b/src/executors/ExecutorJenkins.js @@ -23,6 +23,7 @@ export class ExecutorJenkins extends Executor { constructor(eventData, log) { super(eventData, log); this.log = log; + KmsUtils.logger = log; this.options = this.getOptions(); // If set, this will return bool:true, else bool:false diff --git a/src/executors/ExecutorLaunchDarkly.js b/src/executors/ExecutorLaunchDarkly.js index 53e7f3c2..bd5f15ea 100644 --- a/src/executors/ExecutorLaunchDarkly.js +++ b/src/executors/ExecutorLaunchDarkly.js @@ -26,6 +26,7 @@ export class ExecutorLaunchDarkly extends Executor { constructor(eventData, log) { super(eventData, log); this.log = log; + KmsUtils.logger = log; this.options = this.getOptions(); } diff --git a/src/executors/ExecutorTeamCity.js b/src/executors/ExecutorTeamCity.js index 04b8219f..5ef57e81 100644 --- a/src/executors/ExecutorTeamCity.js +++ b/src/executors/ExecutorTeamCity.js @@ -34,6 +34,7 @@ export class ExecutorTeamCity extends Executor { constructor(eventData, log) { super(eventData, log); this.log = log; + KmsUtils.logger = log; this.options = this.getOptions(); this.teamCity = TeamCity.create({ diff --git a/src/executors/ExecutorTravis.js b/src/executors/ExecutorTravis.js index 98ef4e50..2239a67e 100644 --- a/src/executors/ExecutorTravis.js +++ b/src/executors/ExecutorTravis.js @@ -21,6 +21,7 @@ export class ExecutorTravis extends Executor { constructor(eventData, log) { super(eventData, log); this.log = log; + KmsUtils.logger = log; this.options = this.getOptions(); this.travis = new Travis({ From f21144d714e0934467973725b18251f723649c79 Mon Sep 17 00:00:00 2001 From: wyvern8 Date: Thu, 14 Jun 2018 16:47:12 +1000 Subject: [PATCH 4/7] test(unit tests): add stubs to tests making net calls fix #38 --- serverless.yml | 2 +- src/executors/ExecutorHttp.js | 13 +++--- src/executors/ExecutorLaunchDarkly.js | 2 +- test/agent/Agent.spec.js | 24 ++++++++++- test/executors/ExecutorDocker.spec.js | 3 +- test/executors/ExecutorHttp.spec.js | 15 ++++++- test/executors/ExecutorLaunchDarkly.spec.js | 40 +++++++++++++++---- test/executors/ExecutorPing.spec.js | 15 ++++++- test/executors/ExecutorTravis.spec.js | 4 +- test/fixtures/executorHttpResponse.json | 8 ++++ .../gtmGithubHook/gtmGithubHook.spec.js | 37 +++++++---------- 11 files changed, 120 insertions(+), 43 deletions(-) create mode 100644 test/fixtures/executorHttpResponse.json diff --git a/serverless.yml b/serverless.yml index 3f6d5d60..2ad998a4 100644 --- a/serverless.yml +++ b/serverless.yml @@ -10,7 +10,7 @@ provider: memorySize: 2048 # optional, in MB, default is 1024 timeout: 30 # optional, in seconds, default is 6 - stage: dev + stage: ${env:GTM_AWS_STAGE, 'dev'} region: ${env:GTM_AWS_REGION} environment: ${file(./serverless-aws-environment.yml):environment} diff --git a/src/executors/ExecutorHttp.js b/src/executors/ExecutorHttp.js index 09b7fda5..43e1409d 100644 --- a/src/executors/ExecutorHttp.js +++ b/src/executors/ExecutorHttp.js @@ -43,11 +43,7 @@ export class ExecutorHttp extends Executor { log.info(`Starting http request..`); - return rp({ - proxy: task.options.proxy || this.options.proxy || null, - resolveWithFullResponse: true, - har: har - }) + return this.sendRequest(task, har) .then(response => { let res = json.plain(response); log.info(res); @@ -81,6 +77,13 @@ export class ExecutorHttp extends Executor { return Promise.reject(task); }); } + sendRequest(task, har) { + return rp({ + proxy: task.options.proxy || this.options.proxy || null, + resolveWithFullResponse: true, + har: har + }); + } validate(task, response) { let log = this.log; diff --git a/src/executors/ExecutorLaunchDarkly.js b/src/executors/ExecutorLaunchDarkly.js index bd5f15ea..74adb36d 100644 --- a/src/executors/ExecutorLaunchDarkly.js +++ b/src/executors/ExecutorLaunchDarkly.js @@ -35,7 +35,7 @@ export class ExecutorLaunchDarkly extends Executor { if (!this.ldUtils) { let that = this; return new LaunchDarklyUtils() - .create(KmsUtils.getDecrypted(process.env.GTM_CRYPT_LAUNCHDARKLY_API_TOKEN), log) + .create(await KmsUtils.getDecrypted(process.env.GTM_CRYPT_LAUNCHDARKLY_API_TOKEN), log) .then(handle => { that.ldUtils = handle; return Promise.resolve(handle); diff --git a/test/agent/Agent.spec.js b/test/agent/Agent.spec.js index 54c80f09..7ae63634 100644 --- a/test/agent/Agent.spec.js +++ b/test/agent/Agent.spec.js @@ -1,7 +1,11 @@ //import { default as fs } from 'fs'; -import { describe, it, beforeEach, before } from 'mocha'; +import { describe, it, beforeEach, before, after } from 'mocha'; import { default as assert } from 'assert'; import { Agent } from '../../src/agent/Agent'; +import { AgentMetrics } from '../../src/agent/AgentMetrics'; +import { default as Consumer } from 'sqs-consumer'; +import { default as sinon } from 'sinon'; +import { AgentUtils } from '../../src/agent/AgentUtils'; describe('Agent', function() { let AGENT_GROUP; @@ -28,14 +32,32 @@ describe('Agent', function() { }); describe('start', () => { + let stubCall; + let stubUtils; + let stubListen; + let stubMetrics; + let customResult = { on: () => {} }; before(() => { process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID = 'aws_key_id'; process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY = 'aws_key_secret'; process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = 'webhook_secret'; + + customResult = {}; + stubCall = sinon.stub(Consumer, 'create').returns(Promise.resolve(customResult)); + stubUtils = sinon.stub(AgentUtils, 'getQueueUrl').returns(Promise.resolve(customResult)); + stubListen = sinon.stub(Agent.prototype, 'listen').returns(() => {}); + stubMetrics = sinon.stub(AgentMetrics, 'configureRoutes').returns(Promise.resolve(customResult)); }); it('should start the agent', () => { agentObj.start(); }); + + after(() => { + stubCall.restore(); + stubUtils.restore(); + stubListen.restore(); + stubMetrics.restore(); + }); }); }); diff --git a/test/executors/ExecutorDocker.spec.js b/test/executors/ExecutorDocker.spec.js index 8dda6508..9dc36232 100644 --- a/test/executors/ExecutorDocker.spec.js +++ b/test/executors/ExecutorDocker.spec.js @@ -7,6 +7,7 @@ import { ExecutorDocker } from '../../src/executors/ExecutorDocker'; describe('ExecutorDocker', () => { let executorDocker; let eventData; + process.env.GTM_DOCKER_ALLOW_PULL = 'false'; beforeEach(() => { eventData = { @@ -49,7 +50,7 @@ describe('ExecutorDocker', () => { describe('containerLogs', () => { it('should return a promise', async () => { - let result = executorDocker.containerLogs({ logs: () => {} }); + let result = executorDocker.containerLogs(executorDocker, { logs: () => {} }); assert.equal(result instanceof Promise, true); }); }); diff --git a/test/executors/ExecutorHttp.spec.js b/test/executors/ExecutorHttp.spec.js index fafc2493..3e4dbd2f 100644 --- a/test/executors/ExecutorHttp.spec.js +++ b/test/executors/ExecutorHttp.spec.js @@ -1,7 +1,9 @@ -import { describe, it, beforeEach } from 'mocha'; +import { describe, it, beforeEach, before, after } from 'mocha'; import { default as assert } from 'assert'; import { Executor } from '../../src/agent/Executor'; import { ExecutorHttp } from '../../src/executors/ExecutorHttp'; +import { default as fs } from 'fs'; +import { default as sinon } from 'sinon'; describe('ExecutorHttp', () => { let executorHttp; @@ -40,6 +42,14 @@ describe('ExecutorHttp', () => { }); describe('executeTask', () => { + let stubCall; + let customResult; + before(function() { + customResult = fs.readFileSync(__dirname + '/../fixtures/executorHttpResponse.json', 'utf-8'); + stubCall = sinon + .stub(ExecutorHttp.prototype, 'sendRequest') + .returns(Promise.resolve({ body: customResult })); + }); it('should call ExecutorHttp.executeTask', async () => { let result; result = await executorHttp.executeTask(eventData).then(data => { @@ -47,6 +57,9 @@ describe('ExecutorHttp', () => { }); assert.equal(result.results.passed, true); }); + after(() => { + stubCall.restore(); + }); }); describe('validate', () => { diff --git a/test/executors/ExecutorLaunchDarkly.spec.js b/test/executors/ExecutorLaunchDarkly.spec.js index b4eb539e..190892bd 100644 --- a/test/executors/ExecutorLaunchDarkly.spec.js +++ b/test/executors/ExecutorLaunchDarkly.spec.js @@ -30,25 +30,51 @@ describe('ExecutorLaunchDarkly', () => { }); }); - describe('getLDUtils', () => { - it('should return a promise', async () => { - let ldUtils = executorLaunchDarkly.getLDUtils(); - assert.equal(ldUtils instanceof Promise, true); - }); - }); - describe('getFlagValue', () => { + let stubCall; + let customResult; + before(function() { + customResult = { + flags: { + getFeatureFlagState: () => { + return true; + } + } + }; + stubCall = sinon.stub(ExecutorLaunchDarkly.prototype, 'getLDUtils').returns(Promise.resolve(customResult)); + }); it('should return a promise', async () => { let flagValue = executorLaunchDarkly.getFlagValue(eventData, 'test-one'); assert.equal(flagValue instanceof Promise, true); }); + after(() => { + stubCall.restore(); + }); }); describe('setFlagValue', () => { + let stubCall; + let customResult; + before(function() { + customResult = { + flags: { + getFeatureFlagState: () => { + return true; + }, + toggleFeatureFlag: () => { + return true; + } + } + }; + stubCall = sinon.stub(ExecutorLaunchDarkly.prototype, 'getLDUtils').returns(Promise.resolve(customResult)); + }); it('should return a promise', async () => { let flagValue = executorLaunchDarkly.setFlagValue(eventData, 'test-one', true); assert.equal(flagValue instanceof Promise, true); }); + after(() => { + stubCall.restore(); + }); }); describe('getLDUtils', () => { diff --git a/test/executors/ExecutorPing.spec.js b/test/executors/ExecutorPing.spec.js index 6e947740..ffbafc45 100644 --- a/test/executors/ExecutorPing.spec.js +++ b/test/executors/ExecutorPing.spec.js @@ -1,7 +1,9 @@ -import { describe, it, beforeEach } from 'mocha'; +import { describe, it, beforeEach, after, before } from 'mocha'; import { default as assert } from 'assert'; import { Executor } from '../../src/agent/Executor'; import { ExecutorPing } from '../../src/executors/ExecutorPing'; +import { AgentUtils } from '../../src/agent/AgentUtils'; +import { default as sinon } from 'sinon'; describe('ExecutorPing', () => { let executorPing; @@ -10,7 +12,7 @@ describe('ExecutorPing', () => { beforeEach(() => { process.env.GTM_SQS_RESULTS_QUEUE = 'gtmResultsQueue'; process.env.GTM_SNS_RESULTS_TOPIC = 'gtmResultsSNSTopic'; - process.env.GTM_AWS_REGION = 'ap-southeast-2a'; + process.env.GTM_AWS_REGION = 'ap-southeast-2'; eventData = { executor: 'Ping', context: 'diagnostic', @@ -39,6 +41,12 @@ describe('ExecutorPing', () => { }); describe('executeTask', () => { + let stubCall; + let customResult = { on: () => {} }; + before(() => { + customResult = {}; + stubCall = sinon.stub(AgentUtils, 'postResultsAndTrigger').returns(Promise.resolve(customResult)); + }); it('should call ExecutorPing.executeTask with 1 event', async () => { try { await executorPing.executeTask(eventData).then(data => { @@ -48,5 +56,8 @@ describe('ExecutorPing', () => { return assert.equal(e.message, 'Missing region in config'); } }); + after(() => { + stubCall.restore(); + }); }); }); diff --git a/test/executors/ExecutorTravis.spec.js b/test/executors/ExecutorTravis.spec.js index 453dcec6..b089bb12 100644 --- a/test/executors/ExecutorTravis.spec.js +++ b/test/executors/ExecutorTravis.spec.js @@ -19,7 +19,7 @@ describe('ExecutorTravis', function() { }); }); - describe('executeTask', () => { + /*describe('executeTask', () => { eventData = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/executorJenkinsTaskPayload.json', 'utf-8')); let expectedObject = { passed: true, url: 'https://travis-ci.org' }; it('executeTask to return result object', async () => { @@ -28,5 +28,5 @@ describe('ExecutorTravis', function() { assert.equal(result[i], expectedObject[i]); } }); - }); + });*/ }); diff --git a/test/fixtures/executorHttpResponse.json b/test/fixtures/executorHttpResponse.json new file mode 100644 index 00000000..edef5f1b --- /dev/null +++ b/test/fixtures/executorHttpResponse.json @@ -0,0 +1,8 @@ +{ + "headers": { + "Accept": "*/*", + "Connection": "close", + "Host": "httpbin.org", + "User-Agent": "curl/7.47.0" + } +} diff --git a/test/serverless/gtmGithubHook/gtmGithubHook.spec.js b/test/serverless/gtmGithubHook/gtmGithubHook.spec.js index bc5a1f7f..81f3cb7d 100644 --- a/test/serverless/gtmGithubHook/gtmGithubHook.spec.js +++ b/test/serverless/gtmGithubHook/gtmGithubHook.spec.js @@ -4,6 +4,7 @@ import { before, after, describe, it, beforeEach } from 'mocha'; import { default as crypto } from 'crypto'; import { default as gtmGithubHook } from '../../../src/serverless/gtmGithubHook/gtmGithubHook.js'; import { default as githubUtils } from '../../../src/serverless/gtmGithubUtils.js'; +import { default as Producer } from 'sqs-producer'; describe('gtmGithubHook', function() { beforeEach(() => { @@ -21,10 +22,21 @@ describe('gtmGithubHook', function() { }); }); describe('listener', function() { + let stubCall; + let customResult; + before(() => { + process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID = 'aws_key_id'; + process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY = 'aws_key_secret'; + process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = 'webhook_secret'; + + customResult = {}; + stubCall = sinon.stub(Producer, 'create').returns(Promise.resolve(customResult)); + }); it('should run', function(done) { let event = {}; event.type = 'pull_request'; - event.body = 'payload=%7B%22action%22%3A%20%22test%22%7D'; + event.body = + 'payload=%7B%22pull_request%22%3A%20%7B%22ref%22%3A%20%22sha123%22%2C%22head%22%3A%20%7B%22repo%22%3A%20%7B%22name%22%3A%20%22code%22%2C%20%22owner%22%3A%20%7B%20%22login%22%3A%20%22bob%22%20%7D%7D%7D%7D%7D'; let key = 'abc'; process.env.GTM_CRYPT_GITHUB_WEBHOOK_SECRET = key; @@ -43,27 +55,8 @@ describe('gtmGithubHook', function() { assert.equal('1', '1'); //todo done(); }); - }); - describe('handleEvent', function() { - it('should fire', function(done) { - let type = 'pull_request'; - let body = { - pull_request: { - ref: 'sha123', - head: { - repo: { - name: 'code', - owner: { - login: 'bob' - } - } - } - } - }; - - gtmGithubHook.handleEvent(type, body); - assert.equal(1, 1); //todo - done(); + after(() => { + stubCall.restore(); }); }); From 7815549aa5926b5a187f9e7bea153d3d91da42cd Mon Sep 17 00:00:00 2001 From: wyvern8 Date: Thu, 14 Jun 2018 18:31:45 +1000 Subject: [PATCH 5/7] test(serverless): add unit test for push identification of changes --- src/executors/ExecutorDockerServerless.js | 5 ++- .../ExecutorDockerServerless.spec.js | 40 +++++++++++++++++++ {src/agent => test/fixtures}/push.json | 39 +++++++++++++++++- 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 test/executors/ExecutorDockerServerless.spec.js rename {src/agent => test/fixtures}/push.json (85%) diff --git a/src/executors/ExecutorDockerServerless.js b/src/executors/ExecutorDockerServerless.js index 56d3cb62..476d0f4b 100644 --- a/src/executors/ExecutorDockerServerless.js +++ b/src/executors/ExecutorDockerServerless.js @@ -32,10 +32,13 @@ export class ExecutorDockerServerless extends ExecutorDocker { super(eventData, log); this.eventData = eventData; this.log = log; - this.packagesToDeploy = this.identifyChangedPackages(); + this._packagesToDeploy = this.identifyChangedPackages(); this.refParts = this.eventData.ref ? this.eventData.ref.split('/') : []; this.pushBranch = this.refParts.length > 0 ? this.refParts[this.refParts.length] : null; } + get packagesToDeploy() { + return this._packagesToDeploy; + } async executeTask(task) { task.options = this.mergeTaskOptions(task); diff --git a/test/executors/ExecutorDockerServerless.spec.js b/test/executors/ExecutorDockerServerless.spec.js new file mode 100644 index 00000000..41497852 --- /dev/null +++ b/test/executors/ExecutorDockerServerless.spec.js @@ -0,0 +1,40 @@ +import { default as fs } from 'fs'; +import { describe, it, beforeEach } from 'mocha'; +import { default as assert } from 'assert'; +import { Executor } from '../../src/agent/Executor'; +import { ExecutorDockerServerless } from '../../src/executors/ExecutorDockerServerless'; + +describe('ExecutorDockerServerless', () => { + let executorDockerServerless; + let eventData; + process.env.GTM_DOCKER_ALLOW_PULL = 'false'; + + beforeEach(() => { + eventData = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/push.json', 'utf-8')); + executorDockerServerless = new ExecutorDockerServerless(eventData, console); + }); + + describe('constructor', () => { + it('should instantiate as Executor', () => { + assert.equal(executorDockerServerless instanceof Executor, true); + }); + }); + + describe('identifyChangedPackages', () => { + it('should find distinct packages', () => { + let expected = [ + 'lambdaAbc', + 'lambdaDef', + 'lambdaHij', + 'lambdaKlm', + 'lambdaOne', + 'lambdaTwo', + 'lambdaFour', + 'lambdaThree' + ]; + + let packages = executorDockerServerless.packagesToDeploy; + assert.deepEqual(packages, expected); + }); + }); +}); diff --git a/src/agent/push.json b/test/fixtures/push.json similarity index 85% rename from src/agent/push.json rename to test/fixtures/push.json index d4ca6aa0..090b5c78 100644 --- a/src/agent/push.json +++ b/test/fixtures/push.json @@ -26,11 +26,43 @@ "username": "baxterthehacker" }, "added": [ + "packages/lambdaAbc/README.md" ], "removed": [ + "packages/lambdaDef/README.md" ], "modified": [ - "README.md" + "packages/lambdaHij/README.md", + "packages/lambdaKlm/src/README.md" + ] + }, + { + "id": "1d1a26e67d8f5eaf1f6ba5c57fc3c7d91ac0fd1c", + "tree_id": "19d2a07e9488b91af2641b26b9407fe22a451433", + "distinct": true, + "message": "Update README.md", + "timestamp": "2015-05-05T19:40:15-04:00", + "url": "https://github.com/baxterthehacker/public-repo/commit/0d1a26e67d8f5eaf1f6ba5c57fc3c7d91ac0fd1c", + "author": { + "name": "baxterthehacker", + "email": "baxterthehacker@users.noreply.github.com", + "username": "baxterthehacker" + }, + "committer": { + "name": "baxterthehacker", + "email": "baxterthehacker@users.noreply.github.com", + "username": "baxterthehacker" + }, + "added": [ + "packages/lambdaOne/src/README.md", + "packages/lambdaTwo/src/test.js", + "packages/lambdaKlm/src/README2.md" + ], + "removed": [ + "packages/lambdaFour/src/abc.js" + ], + "modified": [ + "packages/lambdaThree/src/abc.js" ] } ], @@ -52,11 +84,14 @@ "username": "baxterthehacker" }, "added": [ + "packages/lambdaOne/src/README.md", + "packages/lambdaTwo/src/test.js" ], "removed": [ + "packages/lambdaFour/src/abc.js" ], "modified": [ - "README.md" + "packages/lambdaThree/src/abc.js" ] }, "repository": { From 27246716f4ac7286969d2dacdd18b22cbd18ec9c Mon Sep 17 00:00:00 2001 From: wyvern8 Date: Sat, 16 Jun 2018 00:46:43 +1000 Subject: [PATCH 6/7] feat(serverless): executor logic to determine stage and branch alias --- .envExample | 3 ++ README.md | 3 ++ src/agent/AgentUtils.js | 4 ++- src/executors/ExecutorDockerServerless.js | 31 ++++++++++++++----- src/serverless/gtmGithubHook/gtmGithubHook.js | 8 +++-- src/serverless/gtmGithubUtils.js | 16 +++++++++- 6 files changed, 53 insertions(+), 12 deletions(-) diff --git a/.envExample b/.envExample index 31a85a4a..98ea2db0 100644 --- a/.envExample +++ b/.envExample @@ -41,6 +41,9 @@ GTM_S3_DEPENDENCY_BUCKET=gtmstorage GTM_WELCOME_MESSAGE_ENABLED=true GTM_REPO_BLACKLIST=.*ignore-repo.*,.*another-repo.* GTM_AWS_KMS_KEY_ID= +GTM_SLS_EXECUTOR_AWS_STAGE= +GTM_SLS_EXECUTOR_AWS_REGION= +GTM_SLS_EXECUTOR_AWS_EXECUTION_ROLE= GTM_CRYPT_GITHUB_TOKEN= GTM_CRYPT_GITHUB_WEBHOOK_SECRET= diff --git a/README.md b/README.md index 7613736d..62d00d4c 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,9 @@ Create an asynchronous CI agnostic mechanism for running custom test stage gates |GTM_S3_DEPENDENCY_BUCKET| aws s3 storage of build dependencies| |GTM_AWS_S3_PROXY| https_proxy for aws s3 | |GTM_REPO_BLACKLIST| comma separated list of regex to blackist repo names from triggering events | +|GTM_SLS_EXECUTOR_AWS_STAGE| stage override from default calculation of dev/test| +|GTM_SLS_EXECUTOR_AWS_REGION| aws region for lambdas default ap-southeast-2| +|GTM_SLS_EXECUTOR_AWS_EXECUTION_ROLE| docker serverless lambda execution role | > important: values of env vars prefixed with `GTM_CRYPT_*` must be created via `npm run sls-encrypt [name] [value]` diff --git a/src/agent/AgentUtils.js b/src/agent/AgentUtils.js index b3c7d179..fbae6842 100644 --- a/src/agent/AgentUtils.js +++ b/src/agent/AgentUtils.js @@ -360,7 +360,9 @@ export class AgentUtils { '##GH_CLONE_URL##': obj.repository.clone_url, '##GH_PR_BRANCHNAME##': obj.pull_request ? obj.pull_request.head.ref : '', '##PARENTBUILDNUMBER##': this.metaValue(parent, 'buildNumber'), - '##PARENTBUILDNAME##': this.metaValue(parent, 'buildName') + '##PARENTBUILDNAME##': this.metaValue(parent, 'buildName'), + '##GIT_URL##': obj.pull_request ? obj.pull_request.html_url : obj.compare, + '##GIT_COMMIT##': obj.pull_request ? obj.pull_request.head.sha : obj.head_commit.id }; // just add all the GTM env vars to map diff --git a/src/executors/ExecutorDockerServerless.js b/src/executors/ExecutorDockerServerless.js index 476d0f4b..36c7ab64 100644 --- a/src/executors/ExecutorDockerServerless.js +++ b/src/executors/ExecutorDockerServerless.js @@ -33,17 +33,22 @@ export class ExecutorDockerServerless extends ExecutorDocker { this.eventData = eventData; this.log = log; this._packagesToDeploy = this.identifyChangedPackages(); - this.refParts = this.eventData.ref ? this.eventData.ref.split('/') : []; - this.pushBranch = this.refParts.length > 0 ? this.refParts[this.refParts.length] : null; + this.pushBranch = this.pushBranchName(); } get packagesToDeploy() { return this._packagesToDeploy; } async executeTask(task) { - task.options = this.mergeTaskOptions(task); + task.options = await this.mergeTaskOptions(task); return super.executeTask(task); } + pushBranchName() { + let refParts = this.eventData.ref ? this.eventData.ref.split('/') : []; + let branchName = refParts.length > 0 ? refParts[refParts.length - 1].replace(/[^A-Za-z0-9\-+_]/g, '-') : null; + this.log.info(`pushBranchName: ${branchName}`); + return branchName; + } identifyChangedPackages() { let packages = []; @@ -63,8 +68,13 @@ export class ExecutorDockerServerless extends ExecutorDocker { return packages; } + slsStage() { + let stage = process.env.GTM_SLS_EXECUTOR_AWS_STAGE || this.eventData.pushForPullRequest ? 'test' : 'dev'; + this.log.info(`stage: ${stage}`); + return stage; + } - mergeTaskOptions(task) { + async mergeTaskOptions(task) { let options = { image: process.env.GTM_DOCKER_DEFAULT_WORKER_IMAGE || 'zotoio/gtm-worker:latest', command: '/usr/workspace/serverless-mono-deploy.sh', @@ -73,12 +83,15 @@ export class ExecutorDockerServerless extends ExecutorDocker { GIT_PR_ID: '##GHPRNUM##', GIT_PR_BRANCHNAME: '##GH_PR_BRANCHNAME##', GIT_PUSH_BRANCHNAME: this.pushBranch, + GIT_URL: '##GIT_URL##', + GIT_COMMIT: '##GIT_COMMIT##', SLS_AFFECTED_PACKAGES: this.packagesToDeploy.join(','), IAM_ENABLED: process.env.IAM_ENABLED, S3_DEPENDENCY_BUCKET: '##GTM_S3_DEPENDENCY_BUCKET##', AWS_S3_PROXY: '##GTM_AWS_S3_PROXY##', - AWS_STAGE: process.env.GTM_AWS_STAGE, - AWS_REGION: process.env.GTM_AWS_REGION + SLS_AWS_STAGE: this.slsStage(), + SLS_AWS_REGION: process.env.GTM_SLS_EXECUTOR_AWS_REGION || 'ap-southeast-2', + SLS_AWS_EXECUTION_ROLE: process.env.GTM_SLS_EXECUTOR_AWS_EXECUTION_ROLE }, validator: { type: 'outputRegex', @@ -87,8 +100,10 @@ export class ExecutorDockerServerless extends ExecutorDocker { }; if (!process.env.IAM_ENABLED) { - options.env['GTM_AWS_ACCESS_KEY_ID'] = KmsUtils.getDecrypted(process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID); - options.env['GTM_AWS_SECRET_ACCESS_KEY'] = KmsUtils.getDecrypted( + options.env['GTM_AWS_ACCESS_KEY_ID'] = await KmsUtils.getDecrypted( + process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID + ); + options.env['GTM_AWS_SECRET_ACCESS_KEY'] = await KmsUtils.getDecrypted( process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY ); options.env['GTM_AWS_REGION'] = process.env.GTM_AWS_REGION; diff --git a/src/serverless/gtmGithubHook/gtmGithubHook.js b/src/serverless/gtmGithubHook/gtmGithubHook.js index e9768308..d1c347d0 100644 --- a/src/serverless/gtmGithubHook/gtmGithubHook.js +++ b/src/serverless/gtmGithubHook/gtmGithubHook.js @@ -83,11 +83,15 @@ async function handleEvent(type, body, signature) { queueUrl: process.env.SQS_PENDING_QUEUE_URL, region: process.env.GTM_AWS_REGION }); - + let pushForPullRequest = false; + // if this is a push, determine whether related to an open pull_request + if (type === 'push' && body.commits.length > 0) { + pushForPullRequest = await githubUtils.isCommitForPullRequest(body.commits[0].id); + } + body.pushForPullRequest = pushForPullRequest; let bodyString = JSON.stringify(body); let ghEventId = UUID(); let ghAgentGroup = taskConfig[type] && taskConfig[type].agentGroup ? taskConfig[type].agentGroup : 'default'; - let event = [ { id: ghEventId, diff --git a/src/serverless/gtmGithubUtils.js b/src/serverless/gtmGithubUtils.js index 9f1d4c83..6a91f538 100644 --- a/src/serverless/gtmGithubUtils.js +++ b/src/serverless/gtmGithubUtils.js @@ -285,6 +285,19 @@ async function handleEventTaskResult(message, done) { } } +async function isCommitForPullRequest(commitSha) { + try { + let github = await connect(); + let query = `${commitSha}+is:pr+state:open`; + const prResult = await github.search.issues({ q: query }); + console.log(`isCommitForPullRequest result: ${json.plain(prResult)}`); + return prResult.data.items && prResult.data.items.length > 0; + } catch (e) { + console.log('----- ERROR COMMUNICATING WITH GITHUB -----'); + console.log(e); + } +} + module.exports = { connect: connect, signRequestBody: signRequestBody, @@ -293,5 +306,6 @@ module.exports = { getFile: getFile, handleEventTaskResult: handleEventTaskResult, updateGitHubPullRequestStatus: updateGitHubPullRequestStatus, - createPullRequestStatus: createPullRequestStatus + createPullRequestStatus: createPullRequestStatus, + isCommitForPullRequest: isCommitForPullRequest }; From 924762e64e8050bd0ac357dae127d5bdd653aeb7 Mon Sep 17 00:00:00 2001 From: wyvern8 Date: Sat, 16 Jun 2018 12:28:38 +1000 Subject: [PATCH 7/7] fix(kms): update remaining usages to be async --- src/agent/AgentUtils.js | 6 ++--- src/agent/EventHandler.js | 2 +- src/executors/ExecutorDockerServerless.js | 2 +- src/executors/ExecutorDockerSonar.js | 13 ++++++---- src/executors/ExecutorJenkins.js | 30 ++++++++++++++--------- src/executors/ExecutorTeamCity.js | 19 ++++++++------ src/executors/ExecutorTravis.js | 2 +- test/executors/ExecutorJenkins.spec.js | 3 ++- test/executors/ExecutorTeamCity.spec.js | 3 ++- 9 files changed, 48 insertions(+), 32 deletions(-) diff --git a/src/agent/AgentUtils.js b/src/agent/AgentUtils.js index fbae6842..f7aacae2 100644 --- a/src/agent/AgentUtils.js +++ b/src/agent/AgentUtils.js @@ -341,7 +341,7 @@ export class AgentUtils { * Create a Templating Object from a Configuration Object * @param {Object} obj - EventData Object to Return Variables From */ - static createBasicTemplate(obj, parent, log) { + static async createBasicTemplate(obj, parent, log) { if (!parent.results) { log.info('No Parent Build. Providing Safe Defaults'); parent.results = { @@ -366,10 +366,10 @@ export class AgentUtils { }; // just add all the GTM env vars to map - Object.keys(process.env).forEach(key => { + Object.keys(process.env).forEach(async key => { if (key.startsWith('GTM_')) { if (key.startsWith('GTM_CRYPT')) { - mapDict[`##${key}##`] = KmsUtils.getDecrypted(process.env[key]); + mapDict[`##${key}##`] = await KmsUtils.getDecrypted(process.env[key]); } else { mapDict[`##${key}##`] = process.env[key]; } diff --git a/src/agent/EventHandler.js b/src/agent/EventHandler.js index 3efa7c8c..cd2c2a7b 100644 --- a/src/agent/EventHandler.js +++ b/src/agent/EventHandler.js @@ -163,7 +163,7 @@ export class EventHandler extends Plugin { task.options = AgentUtils.applyTransforms( AgentUtils.templateReplace( - AgentUtils.createBasicTemplate(event.eventData, parent, log), + await AgentUtils.createBasicTemplate(event.eventData, parent, log), task.options, log ) diff --git a/src/executors/ExecutorDockerServerless.js b/src/executors/ExecutorDockerServerless.js index 36c7ab64..6990d0a4 100644 --- a/src/executors/ExecutorDockerServerless.js +++ b/src/executors/ExecutorDockerServerless.js @@ -114,7 +114,7 @@ export class ExecutorDockerServerless extends ExecutorDocker { task.options = AgentUtils.applyTransforms( AgentUtils.templateReplace( - AgentUtils.createBasicTemplate(this.eventData, {}, this.log), + await AgentUtils.createBasicTemplate(this.eventData, {}, this.log), task.options, this.log ) diff --git a/src/executors/ExecutorDockerSonar.js b/src/executors/ExecutorDockerSonar.js index 098936f9..00efb854 100644 --- a/src/executors/ExecutorDockerSonar.js +++ b/src/executors/ExecutorDockerSonar.js @@ -36,11 +36,11 @@ export class ExecutorDockerSonar extends ExecutorDocker { } async executeTask(task) { - task.options = this.mergeTaskOptions(task); + task.options = await this.mergeTaskOptions(task); return super.executeTask(task); } - mergeTaskOptions(task) { + async mergeTaskOptions(task) { let options = { image: process.env.GTM_DOCKER_DEFAULT_WORKER_IMAGE || 'zotoio/gtm-worker:latest', command: '/usr/workspace/sonar-pullrequest.sh', @@ -69,8 +69,10 @@ export class ExecutorDockerSonar extends ExecutorDocker { }; if (!process.env.IAM_ENABLED) { - options.env['GTM_AWS_ACCESS_KEY_ID'] = KmsUtils.getDecrypted(process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID); - options.env['GTM_AWS_SECRET_ACCESS_KEY'] = KmsUtils.getDecrypted( + options.env['GTM_AWS_ACCESS_KEY_ID'] = await KmsUtils.getDecrypted( + process.env.GTM_CRYPT_AGENT_AWS_ACCESS_KEY_ID + ); + options.env['GTM_AWS_SECRET_ACCESS_KEY'] = await KmsUtils.getDecrypted( process.env.GTM_CRYPT_AGENT_AWS_SECRET_ACCESS_KEY ); options.env['GTM_AWS_REGION'] = process.env.GTM_AWS_REGION; @@ -78,10 +80,11 @@ export class ExecutorDockerSonar extends ExecutorDocker { // options defined above can be overidden by options in .githubTaskManager.json task.options = _.merge(options, task.options); + console.log(task.options); task.options = AgentUtils.applyTransforms( AgentUtils.templateReplace( - AgentUtils.createBasicTemplate(this.eventData, {}, this.log), + await AgentUtils.createBasicTemplate(this.eventData, {}, this.log), task.options, this.log ) diff --git a/src/executors/ExecutorJenkins.js b/src/executors/ExecutorJenkins.js index 9ffa1f4a..7f9aadaf 100644 --- a/src/executors/ExecutorJenkins.js +++ b/src/executors/ExecutorJenkins.js @@ -25,19 +25,24 @@ export class ExecutorJenkins extends Executor { this.log = log; KmsUtils.logger = log; this.options = this.getOptions(); + } - // If set, this will return bool:true, else bool:false - let useCsrf = this.options.GTM_JENKINS_CSRF === 'true'; - - this.jenkins = JenkinsLib({ - baseUrl: AgentUtils.formatBasicAuth( - this.options.GTM_JENKINS_USER, - KmsUtils.getDecrypted(this.options.GTM_CRYPT_JENKINS_TOKEN), - this.options.GTM_JENKINS_URL - ), - crumbIssuer: useCsrf, - promisify: true - }); + async initJenkins() { + if (!this.jenkins) { + // If set, this will return bool:true, else bool:false + let useCsrf = this.options.GTM_JENKINS_CSRF === 'true'; + + this.jenkins = JenkinsLib({ + baseUrl: AgentUtils.formatBasicAuth( + this.options.GTM_JENKINS_USER, + await KmsUtils.getDecrypted(this.options.GTM_CRYPT_JENKINS_TOKEN), + this.options.GTM_JENKINS_URL + ), + crumbIssuer: useCsrf, + promisify: true + }); + } + return this.jenkins; } async waitForBuildToExist(buildName, buildNumber) { @@ -100,6 +105,7 @@ export class ExecutorJenkins extends Executor { } async executeTask(task) { + this.jenkins = await this.initJenkins(); let log = this.log; let jobName = task.options.jobName || null; let buildParams = task.options.parameters || null; diff --git a/src/executors/ExecutorTeamCity.js b/src/executors/ExecutorTeamCity.js index 5ef57e81..2dc2889c 100644 --- a/src/executors/ExecutorTeamCity.js +++ b/src/executors/ExecutorTeamCity.js @@ -36,12 +36,16 @@ export class ExecutorTeamCity extends Executor { this.log = log; KmsUtils.logger = log; this.options = this.getOptions(); - - this.teamCity = TeamCity.create({ - url: this.options.GTM_TEAMCITY_URL, - username: this.options.GTM_TEAMCITY_USER, - password: KmsUtils.getDecrypted(this.options.GTM_CRYPT_TEAMCITY_PASSCODE) - }); + } + async initTeamCity() { + if (!this.teamCity) { + this.teamCity = await TeamCity.create({ + url: this.options.GTM_TEAMCITY_URL, + username: this.options.GTM_TEAMCITY_USER, + password: await KmsUtils.getDecrypted(this.options.GTM_CRYPT_TEAMCITY_PASSCODE) + }); + } + return this.teamCity; } createTeamCityBuildNode(task, jobName) { @@ -72,6 +76,7 @@ export class ExecutorTeamCity extends Executor { } async executeTask(task) { + this.teamCity = await this.initTeamCity(); let log = this.log; let jobName = task.options.jobName; @@ -98,7 +103,7 @@ export class ExecutorTeamCity extends Executor { if (task.options.parameters.hasOwnProperty('cuke_tags')) { let statisticsUrl = AgentUtils.formatBasicAuth( this.options.GTM_TEAMCITY_USER, - KmsUtils.getDecrypted(this.options.GTM_CRYPT_TEAMCITY_PASSCODE), + await KmsUtils.getDecrypted(this.options.GTM_CRYPT_TEAMCITY_PASSCODE), URL.resolve(this.options.GTM_TEAMCITY_URL, `/app/rest/builds/id:${teamCityBuildId.id}/statistics`) ); diff --git a/src/executors/ExecutorTravis.js b/src/executors/ExecutorTravis.js index 2239a67e..57bc3408 100644 --- a/src/executors/ExecutorTravis.js +++ b/src/executors/ExecutorTravis.js @@ -35,7 +35,7 @@ export class ExecutorTravis extends Executor { this.travis.authenticate( { - github_token: KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN) + github_token: await KmsUtils.getDecrypted(process.env.GTM_CRYPT_GITHUB_TOKEN) }, function(err) { if (err) { diff --git a/test/executors/ExecutorJenkins.spec.js b/test/executors/ExecutorJenkins.spec.js index 7c10dee7..b3ce9c74 100644 --- a/test/executors/ExecutorJenkins.spec.js +++ b/test/executors/ExecutorJenkins.spec.js @@ -12,9 +12,10 @@ describe('ExecutorJenkins', function() { process.env.GTM_JENKINS_URL = 'http://localhost:8211'; let NON_EXISTING_JENKINS_SERVER = ': connect ECONNREFUSED 127.0.0.1:8211'; - beforeEach(() => { + beforeEach(async () => { eventData = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/executorJenkinsTaskPayload.json', 'utf-8')); executorJenkins = new ExecutorJenkins(eventData, console); + await executorJenkins.initJenkins(); }); describe('constructor', function() { diff --git a/test/executors/ExecutorTeamCity.spec.js b/test/executors/ExecutorTeamCity.spec.js index 38854751..e536feac 100644 --- a/test/executors/ExecutorTeamCity.spec.js +++ b/test/executors/ExecutorTeamCity.spec.js @@ -13,9 +13,10 @@ describe('ExecutorTeamCity', function() { process.env.GTM_TEAMCITY_PASSCODE = 'admin'; process.env.GTM_TEAMCITY_URL = 'http://localhost:8111'; - beforeEach(() => { + beforeEach(async () => { eventData = JSON.parse(fs.readFileSync(__dirname + '/../fixtures/executorTeamCityTaskPayLoad.json', 'utf-8')); executorTeamcity = new ExecutorTeamCity(eventData, console); + await executorTeamcity.initTeamCity(); }); describe('constructor', () => {