DRIVERS-2585 Use AWS Secrets Manager for AWS-Related Test Secrets

.evergreen/auth_aws/setup_secrets.py

@@ -0,0 +1,68 @@
+import json
+import os
+import sys
+import yaml
+
+import boto3
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+aws_lib = os.path.join(os.path.dirname(HERE), 'auth_aws', 'lib')
+sys.path.insert(0, aws_lib)
+
+DEFAULT_CLIENT = "0oadp0hpl7q3UIehP297"
+
+
+def get_secrets(profile, *vaults):
+    """Get the driver secret values."""
+    # Handle local credentials.
+    if len(profile) != 0:
+        session = boto3.Session(profile_name=profile)
+        client = session.client(service_name='secretsmanager', region_name='us-west-2')
+    else:
+        if "AWS_SESSION_TOKEN" not in os.environ:
+            if "AWS_ROLE_ARN" in os.environ:
+                session = boto3.Session(aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'])
+                client = session.client(service_name='sts', region_name='us-west-2')
+                creds = client.assume_role(RoleArn=os.environ['AWS_ROLE_ARN'], RoleSessionName='test')['Credentials']
+                os.environ['AWS_ACCESS_KEY_ID'] = creds['AccessKeyId']
+                os.environ['AWS_SECRET_ACCESS_KEY'] = creds['SecretAccessKey']
+                os.environ['AWS_SESSION_TOKEN'] = creds['SessionToken']
+            else:
+                raise ValueError('Missing AWS credentials')
+
+        # Create a session using the given creds
+        session = boto3.Session(aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], aws_session_token=os.environ['AWS_SESSION_TOKEN'])
+        client = session.client(service_name='secretsmanager', region_name='us-west-2')
+
+    secrets = []
+    try:
+        for vault in vaults:
+            secrets.append(client.get_secret_value(
+                SecretId=vault
+            )['SecretString'])
+    except Exception as e:
+        # For a list of exceptions thrown, see
+        # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
+        raise e
+
+    # Decrypts secret using the associated KMS key.
+    return [json.loads(s) for s in secrets]
+
+
+def write_secrets(profile, *vaults):
+    pairs = {}
+    secrets = get_secrets(profile, *vaults)
+    for secret in secrets:
+        for key, val in secret.items():
+            pairs[key.upper()] = val
+
+    with open("secrets-expansion.yml", "w") as yaml_out:
+        yaml.dump(pairs, yaml_out, default_flow_style=False, allow_unicode=True, default_style='"')
+
+    with open("secrets-export.sh", "w") as out:
+        out.write("#!/usr/bin/env bash" + "\n\n")
+        for key, val in pairs.items():
+            out.write("export " + key + "=" + "\"" + val + "\"\n")
+
+
+write_secrets(sys.argv[1], *sys.argv[2:])

.evergreen/auth_oidc/activate-authoidcvenv.sh

@@ -25,13 +25,13 @@ fi
 # Automatically invoked by activate-authoidcvenv.sh.
 activate_authoidcvenv() {
   # shellcheck source=.evergreen/venv-utils.sh
-  . ../venv-utils.sh || return
+  . drivers-evergreen-tools/.evergreen/venv-utils.sh || return
 
   if [[ -d authoidcvenv ]]; then
     venvactivate authoidcvenv || return
   else
     # shellcheck source=.evergreen/find-python3.sh
-    . ../find-python3.sh || return
+    . drivers-evergreen-tools/.evergreen/find-python3.sh || return
 
     venvcreate "$(find_python3)" authoidcvenv || return
 

.evergreen/auth_oidc/oidc_get_tokens.py

@@ -9,29 +9,28 @@
 def main():
     token_dir = os.environ['OIDC_TOKEN_DIR']
     os.makedirs(token_dir, exist_ok=True)
-    secrets = get_secrets()
     config = {
-        "issuer": secrets['oidc_issuer_1_uri'],
-        "jwks_uri": secrets['oidc_jwks_uri'],
-        'rsa_key': secrets['oidc_rsa_key'],
+        "issuer": os.getenv('oidc_issuer_1_uri'.upper()),
+        "jwks_uri": os.getenv('oidc_jwks_uri'.upper()),
+        'rsa_key': os.getenv('oidc_rsa_key'.upper()),
         'audience': DEFAULT_CLIENT,
         'client_id': DEFAULT_CLIENT,
-        'client_secret':secrets['oidc_client_secret'],
+        'client_secret': os.getenv('oidc_client_secret'),
         'username': 'test_user1',
         'token_file': os.path.join(token_dir, 'test_user1')
     }
     get_id_token(config)
     for i in range(2):
         config['token_file'] = os.path.join(token_dir, f'test_user1_{i+1}')
         get_id_token(config)
-    config['issuer'] = secrets['oidc_issuer_2_uri']
+    config['issuer'] = os.getenv('oidc_issuer_2_uri'.upper())
     config['username'] = 'test_user2'
     config['token_file'] = os.path.join(token_dir, 'test_user2')
     get_id_token(config)
     for i in range(2):
         config['token_file'] = os.path.join(token_dir, f'test_user2_{i+1}')
         get_id_token(config)
-    config['issuer'] = secrets['oidc_issuer_1_uri']
+    config['issuer'] = os.getenv('oidc_issuer_1_uri'.upper())
     config['username'] = 'test_user1'
     config['token_file'] = os.path.join(token_dir, 'test_user1_expires')
     get_id_token(config, expires=60)

.evergreen/auth_oidc/utils.py

@@ -7,7 +7,6 @@
 HERE = os.path.abspath(os.path.dirname(__file__))
 aws_lib = os.path.join(os.path.dirname(HERE), 'auth_aws', 'lib')
 sys.path.insert(0, aws_lib)
-from aws_handle_oidc_creds import get_id_token, MOCK_ENDPOINT
 
 DEFAULT_CLIENT = "0oadp0hpl7q3UIehP297"
 

.gitignore

@@ -106,3 +106,5 @@ lb-expansion.yml
 .DS_Store
 *.log
 orchestration.config
+secrets-expansion.yml
+secrets-export.sh