Skip to content

Commit

Permalink
CLOUDP-153003: Encryption at Rest Tests for GCP & Azure (#1035)
Browse files Browse the repository at this point in the history
  • Loading branch information
roothorp authored and cveticm committed Aug 10, 2023
1 parent c8b4266 commit ffe7705
Show file tree
Hide file tree
Showing 11 changed files with 352 additions and 407 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/test-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -223,11 +223,11 @@ jobs:
AWS_ACCOUNT_ARN_LIST: ${{ secrets.AWS_ACCOUNT_ARN_LIST }}
AWS_KMS_KEY_ID: ${{ secrets.AWS_KMS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID_NEW_TEST }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET_NEW_TEST }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
GCP_SA_CRED: ${{ secrets.GCP_SA_CRED }}
GCP_SA_CRED: ${{ secrets.GCP_SA_CRED_NEW_TEST }}
DATADOG_KEY: ${{ secrets.DATADOG_KEY }}
run: |
helm version
Expand Down
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ module github.com/mongodb/mongodb-atlas-kubernetes
go 1.20

require (
cloud.google.com/go/kms v1.12.1
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
github.com/Azure/go-autorest/autorest v0.11.29
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
Expand Down Expand Up @@ -36,9 +38,11 @@ require (
)

require (
cloud.google.com/go/iam v1.1.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
github.com/google/s2a-go v0.1.4 // indirect
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94=
cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U=
cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
Expand All @@ -45,6 +49,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9Orh
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0 h1:8d4U82r7ItT1Es91x3eUcAQweih36KWvUha8AZ9X0Rs=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0/go.mod h1:/1bkGperHinQbAHMWivoec/Ucu6//iXo6jn5mhmqCVU=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
Expand Down Expand Up @@ -90,6 +96,7 @@ github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzAR
github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down
1 change: 1 addition & 0 deletions pkg/api/v1/encryption_at_rest.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,5 +94,6 @@ func (az AzureKeyVault) ToAtlas() mongodbatlas.AzureKeyVault {
KeyVaultName: az.KeyVaultName,
KeyIdentifier: az.KeyIdentifier,
TenantID: az.TenantID,
Secret: az.Secret,
}
}
4 changes: 3 additions & 1 deletion pkg/controller/atlasproject/encryption_at_rest.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,14 +103,16 @@ func readAndFillGoogleSecret(kubeClient client.Client, parentNs string, gkms *md
}

func readAndFillAzureSecret(kubeClient client.Client, parentNs string, azureVault *mdbv1.AzureKeyVault) (*watch.WatchedObject, error) {
fieldData, watchObj, err := readSecretData(kubeClient, azureVault.SecretRef, parentNs, "ClientID", "AzureEnvironment", "SubscriptionID", "ResourceGroupName", "KeyVaultName", "KeyIdentifier")
fieldData, watchObj, err := readSecretData(kubeClient, azureVault.SecretRef, parentNs, "ClientID", "Secret", "AzureEnvironment", "SubscriptionID", "ResourceGroupName", "KeyVaultName", "KeyIdentifier", "TenantID")
if err != nil {
return watchObj, err
}

azureVault.ClientID = fieldData["ClientID"]
azureVault.Secret = fieldData["Secret"]
azureVault.AzureEnvironment = fieldData["AzureEnvironment"]
azureVault.SubscriptionID = fieldData["SubscriptionID"]
azureVault.TenantID = fieldData["TenantID"]
azureVault.ResourceGroupName = fieldData["ResourceGroupName"]
azureVault.KeyVaultName = fieldData["KeyVaultName"]
azureVault.KeyIdentifier = fieldData["KeyIdentifier"]
Expand Down
8 changes: 6 additions & 2 deletions pkg/controller/atlasproject/encryption_at_rest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,8 +211,10 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
t.Run("Azure with correct secret data", func(t *testing.T) {
secretData := map[string][]byte{
"ClientID": []byte("testClientID"),
"Secret": []byte("testClientSecret"),
"AzureEnvironment": []byte("testAzureEnvironment"),
"SubscriptionID": []byte("testSubscriptionID"),
"TenantID": []byte("testTenantID"),
"ResourceGroupName": []byte("testResourceGroupName"),
"KeyVaultName": []byte("testKeyVaultName"),
"KeyIdentifier": []byte("testKeyIdentifier"),
Expand All @@ -226,7 +228,7 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "gcp-secret",
Name: "azure-secret",
Namespace: "test",
},
},
Expand All @@ -238,7 +240,7 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
AzureKeyVault: mdbv1.AzureKeyVault{
Enabled: toptr.MakePtr(true),
SecretRef: common.ResourceRefNamespaced{
Name: "gcp-secret",
Name: "azure-secret",
},
},
}
Expand All @@ -247,8 +249,10 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
assert.Nil(t, err)

assert.Equal(t, string(secretData["ClientID"]), encRest.AzureKeyVault.ClientID)
assert.Equal(t, string(secretData["Secret"]), encRest.AzureKeyVault.Secret)
assert.Equal(t, string(secretData["AzureEnvironment"]), encRest.AzureKeyVault.AzureEnvironment)
assert.Equal(t, string(secretData["SubscriptionID"]), encRest.AzureKeyVault.SubscriptionID)
assert.Equal(t, string(secretData["TenantID"]), encRest.AzureKeyVault.TenantID)
assert.Equal(t, string(secretData["ResourceGroupName"]), encRest.AzureKeyVault.ResourceGroupName)
assert.Equal(t, string(secretData["KeyVaultName"]), encRest.AzureKeyVault.KeyVaultName)
assert.Equal(t, string(secretData["KeyIdentifier"]), encRest.AzureKeyVault.KeyIdentifier)
Expand Down
141 changes: 135 additions & 6 deletions test/e2e/actions/cloud/aws.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
package cloud

import (
"encoding/json"
"errors"
"fmt"
"os"
"strings"

"github.com/aws/aws-sdk-go/aws"
aws_sdk "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/kms"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/onsi/ginkgo/v2/dsl/core"

"github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/toptr"
"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/aws"
"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config"
)

type AwsAction struct {
Expand All @@ -25,13 +31,136 @@ type awsNetwork struct {
Subnets []*string
}

func NewAwsAction() *AwsAction {
return new(AwsAction)
type principal struct {
AWS []string `json:"AWS,omitempty"`
}

func (awsAction *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) {
session := aws.SessionAWS(region)
return session.GetCustomerMasterKeyID(atlasAccountArn, assumedRoleArn)
type kmsPolicy struct {
Version string `json:"Version"`
Statement []statement `json:"Statement"`
}

type statement struct {
Sid string `json:"Sid"`
Effect string `json:"Effect"`
Principal principal `json:"Principal"`
Action []string `json:"Action"`
Resource string `json:"Resource"`
}

func (a *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) {
a.t.Helper()

kmsClient := kms.New(a.session, aws.NewConfig().WithRegion(config.AWSRegionUS))

keyId, adminARNs, err := getKeyIDAndAdminARNs()
if err != nil {
return "", err
}

policyString, err := rolePolicyString(atlasAccountArn, assumedRoleArn, adminARNs)
if err != nil {
return "", err
}

policyInput := &kms.PutKeyPolicyInput{
KeyId: &keyId,
PolicyName: aws_sdk.String("default"),
Policy: aws_sdk.String(policyString),
}

_, err = kmsClient.PutKeyPolicy(policyInput)
if err != nil {
return "", err
}

return keyId, nil
}

func getKeyIDAndAdminARNs() (keyID string, adminARNs []string, err error) {
keyID = os.Getenv("AWS_KMS_KEY_ID")
if keyID == "" {
err = errors.New("AWS_KMS_KEY_ID secret is empty")
return
}
adminArnString := os.Getenv("AWS_ACCOUNT_ARN_LIST")
if adminArnString == "" {
err = errors.New("AWS_ACCOUNT_ARN_LIST secret is empty")
return
}

adminARNs = strings.Split(adminArnString, ",")
if len(adminARNs) == 0 {
err = errors.New("AWS_ACCOUNT_ARN_LIST wasn't parsed properly, please separate accounts via a comma")
return
}

return keyID, adminARNs, nil
}

func rolePolicyString(atlasAccountARN, assumedRoleARN string, adminARNs []string) (string, error) {
policy := defaultKMSPolicy(atlasAccountARN, assumedRoleARN, adminARNs)
byteStr, err := json.Marshal(policy)
if err != nil {
return "", err
}
return string(byteStr), nil
}

func defaultKMSPolicy(atlasAccountArn, assumedRoleArn string, adminARNs []string) kmsPolicy {
return kmsPolicy{
Version: "2012-10-17",
Statement: []statement{
{
Sid: "Enable IAM User Permissions",
Effect: "Allow",
Principal: principal{
AWS: []string{atlasAccountArn},
},
Action: []string{"kms:*"},
Resource: "*",
},
{
Sid: "Allow access for Key Administrators",
Effect: "Allow",
Principal: principal{
AWS: adminARNs,
},
Action: []string{
"kms:Create*",
"kms:Describe*",
"kms:Enable*",
"kms:List*",
"kms:Put*",
"kms:Update*",
"kms:Revoke*",
"kms:Disable*",
"kms:Get*",
"kms:Delete*",
"kms:TagResource",
"kms:UntagResource",
"kms:ScheduleKeyDeletion",
"kms:CancelKeyDeletion",
},
Resource: "*",
},
{
Sid: "Allow use of the key",
Effect: "Allow",
Principal: principal{
AWS: []string{assumedRoleArn},
},
Action: []string{
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:DescribeKey",
},
Resource: "*",
},
},
}
}

func (a *AwsAction) GetAccountID() (string, error) {
Expand Down
Loading

0 comments on commit ffe7705

Please sign in to comment.