Fix EaT flakiness

.github/workflows/cleanup-all.yml

@@ -55,7 +55,6 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_ACCOUNT_ARN_LIST: ${{ secrets.AWS_ACCOUNT_ARN_LIST }}
-          AWS_KMS_KEY_ID: ${{ secrets.AWS_KMS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}

.github/workflows/test-e2e.yml

@@ -222,7 +222,6 @@ jobs:
           OPENSHIFT_PASS: ${{ secrets.OPENSHIFT_PASS }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_ACCOUNT_ARN_LIST: ${{ secrets.AWS_ACCOUNT_ARN_LIST }}
-          AWS_KMS_KEY_ID: ${{ secrets.AWS_KMS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID_NEW_TEST }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}

pkg/controller/atlasproject/encryption_at_rest.go

@@ -297,6 +297,8 @@ func balanceAsymmetricalFields(atlas *mongodbatlas.EncryptionAtRest, spec *mongo
 	if isNotNilAndFalse(atlas.GoogleCloudKms.Enabled) {
 		spec.GoogleCloudKms.Enabled = toptr.MakePtr(false)
 	}
+
+	spec.Valid = atlas.Valid
 }
 
 func IsEncryptionSpecEmpty(spec *mdbv1.EncryptionAtRest) bool {

test/e2e/actions/cloud/aws.go

@@ -16,7 +16,6 @@ import (
 	"github.com/onsi/ginkgo/v2/dsl/core"
 
 	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/toptr"
-	"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config"
 )
 
 type AwsAction struct {
@@ -48,12 +47,12 @@ type statement struct {
 	Resource  string    `json:"Resource"`
 }
 
-func (a *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) {
+func (a *AwsAction) CreateKMS(alias, region, atlasAccountArn, assumedRoleArn string) (string, error) {
 	a.t.Helper()
 
-	kmsClient := kms.New(a.session, aws.NewConfig().WithRegion(config.AWSRegionUS))
+	kmsClient := kms.New(a.session, aws.NewConfig().WithRegion(region))
 
-	keyId, adminARNs, err := getKeyIDAndAdminARNs()
+	adminARNs, err := getAdminARNs()
 	if err != nil {
 		return "", err
 	}
@@ -63,39 +62,53 @@ func (a *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (k
 		return "", err
 	}
 
-	policyInput := &kms.PutKeyPolicyInput{
-		KeyId:      &keyId,
-		PolicyName: aws_sdk.String("default"),
-		Policy:     aws_sdk.String(policyString),
-	}
+	key, err := kmsClient.CreateKey(&kms.CreateKeyInput{
+		Description: aws.String("Key for E2E test"),
+		KeySpec:     aws.String("SYMMETRIC_DEFAULT"),
+		KeyUsage:    aws.String("ENCRYPT_DECRYPT"),
+		MultiRegion: aws.Bool(false),
+		Origin:      aws.String("AWS_KMS"),
+		Policy:      aws.String(policyString),
+	})
 
-	_, err = kmsClient.PutKeyPolicy(policyInput)
 	if err != nil {
 		return "", err
 	}
 
-	return keyId, nil
-}
+	_, err = kmsClient.CreateAlias(&kms.CreateAliasInput{
+		AliasName:   aws.String("alias/" + strings.ToLower(strings.ReplaceAll(alias, " ", "-"))),
+		TargetKeyId: key.KeyMetadata.KeyId,
+	})
 
-func getKeyIDAndAdminARNs() (keyID string, adminARNs []string, err error) {
-	keyID = os.Getenv("AWS_KMS_KEY_ID")
-	if keyID == "" {
-		err = errors.New("AWS_KMS_KEY_ID secret is empty")
-		return
+	if err != nil {
+		a.t.Log(fmt.Sprintf("failed to create alias to key %s(%s): %s", alias, *key.KeyMetadata.KeyId, err))
 	}
+
+	a.t.Cleanup(func() {
+		_, err = kmsClient.ScheduleKeyDeletion(&kms.ScheduleKeyDeletionInput{
+			KeyId:               key.KeyMetadata.KeyId,
+			PendingWindowInDays: aws.Int64(7), // this is the minimum possible and can be up to 24h longer than value set
+		})
+		if err != nil {
+			a.t.Error(err)
+		}
+	})
+
+	return *key.KeyMetadata.KeyId, nil
+}
+
+func getAdminARNs() ([]string, error) {
 	adminArnString := os.Getenv("AWS_ACCOUNT_ARN_LIST")
 	if adminArnString == "" {
-		err = errors.New("AWS_ACCOUNT_ARN_LIST secret is empty")
-		return
+		return nil, errors.New("AWS_ACCOUNT_ARN_LIST secret is empty")
 	}
 
-	adminARNs = strings.Split(adminArnString, ",")
+	adminARNs := strings.Split(adminArnString, ",")
 	if len(adminARNs) == 0 {
-		err = errors.New("AWS_ACCOUNT_ARN_LIST wasn't parsed properly, please separate accounts via a comma")
-		return
+		return nil, errors.New("AWS_ACCOUNT_ARN_LIST wasn't parsed properly, please separate accounts via a comma")
 	}
 
-	return keyID, adminARNs, nil
+	return adminARNs, nil
 }
 
 func rolePolicyString(atlasAccountARN, assumedRoleARN string, adminARNs []string) (string, error) {

test/e2e/e2e_suite_test.go

@@ -65,7 +65,6 @@ func checkUpAWSEnvironment() {
 	Expect(os.Getenv("AWS_ACCESS_KEY_ID")).ShouldNot(BeEmpty(), "Please, setup AWS_ACCESS_KEY_ID environment variable for test with AWS")
 	Expect(os.Getenv("AWS_SECRET_ACCESS_KEY")).ShouldNot(BeEmpty(), "Please, setup AWS_SECRET_ACCESS_KEY environment variable for test with AWS")
 	Expect(os.Getenv("AWS_ACCOUNT_ARN_LIST")).ShouldNot(BeEmpty(), "Please, setup AWS_ACCOUNT_ARN_LIST environment variable for test with AWS")
-	Expect(os.Getenv("AWS_KMS_KEY_ID")).ShouldNot(BeEmpty(), "Please, setup AWS_KMS_KEY_ID environment variable for test with AWS")
 }
 
 func checkUpAzureEnvironment() {

test/e2e/encryption_at_rest_test.go

@@ -153,7 +153,7 @@ func encryptionAtRestFlow(userData *model.TestDataProvider, encAtRest v1.Encrypt
 			aRole = userData.Project.Status.CloudProviderAccessRoles[0]
 		}
 
-		fillKMSforAWS(&encAtRest, aRole.AtlasAWSAccountArn, aRole.IamAssumedRoleArn)
+		fillKMSforAWS(fmt.Sprintf("%s-kms", userData.Project.Spec.Name), &encAtRest, aRole.AtlasAWSAccountArn, aRole.IamAssumedRoleArn)
 		fillVaultforAzure(&encAtRest)
 		fillKMSforGCP(&encAtRest)
 
@@ -188,15 +188,15 @@ func encryptionAtRestFlow(userData *model.TestDataProvider, encAtRest v1.Encrypt
 	})
 }
 
-func fillKMSforAWS(encAtRest *v1.EncryptionAtRest, atlasAccountArn, assumedRoleArn string) {
+func fillKMSforAWS(alias string, encAtRest *v1.EncryptionAtRest, atlasAccountArn, assumedRoleArn string) {
 	if (encAtRest.AwsKms == v1.AwsKms{}) {
 		return
 	}
 
 	Expect(encAtRest.AwsKms.Region).NotTo(Equal(""))
 	awsAction, err := cloud.NewAWSAction(GinkgoT())
 	Expect(err).ToNot(HaveOccurred())
-	CustomerMasterKeyID, err := awsAction.CreateKMS(config.AWSRegionUS, atlasAccountArn, assumedRoleArn)
+	CustomerMasterKeyID, err := awsAction.CreateKMS(alias, config.AWSRegionUS, atlasAccountArn, assumedRoleArn)
 	Expect(err).ToNot(HaveOccurred())
 	Expect(CustomerMasterKeyID).NotTo(Equal(""))
 
@@ -345,7 +345,7 @@ var _ = Describe("Encryption at rest AWS", Label("encryption-at-rest"), Ordered,
 			Expect(len(userData.Project.Status.CloudProviderAccessRoles)).NotTo(Equal(0))
 			aRole := userData.Project.Status.CloudProviderAccessRoles[0]
 
-			fillKMSforAWS(&encAtRest, aRole.AtlasAWSAccountArn, aRole.IamAssumedRoleArn)
+			fillKMSforAWS(fmt.Sprintf("%s-kms", userData.Project.Spec.Name), &encAtRest, aRole.AtlasAWSAccountArn, aRole.IamAssumedRoleArn)
 			fillVaultforAzure(&encAtRest)
 			fillKMSforGCP(&encAtRest)
 
@@ -447,7 +447,7 @@ var _ = Describe("Encryption at rest AWS", Label("encryption-at-rest"), Ordered,
 
 			encAtRest.AwsKms.Region = string(secret.Data["Region"])
 
-			fillKMSforAWS(&encAtRest, aRole.AtlasAWSAccountArn, aRole.IamAssumedRoleArn)
+			fillKMSforAWS(fmt.Sprintf("%s-kms", userData.Project.Spec.Name), &encAtRest, aRole.AtlasAWSAccountArn, aRole.IamAssumedRoleArn)
 
 			Expect(userData.K8SClient.Get(userData.Context, types.NamespacedName{Name: userData.Project.Name,
 				Namespace: userData.Resources.Namespace}, userData.Project)).Should(Succeed())

test/e2e/project_deletion_protection_test.go

@@ -232,15 +232,15 @@ var _ = Describe("Project Deletion Protection", Label("project", "deletion-prote
 		By("Adding AWS Encryption At Rest to the project", func() {
 			awsAction, err := cloud.NewAWSAction(GinkgoT())
 			Expect(err).ToNot(HaveOccurred())
-			customerMasterKeyID, err = awsAction.CreateKMS(config.AWSRegionUS, atlasAccountARN, awsRoleARN)
+			customerMasterKeyID, err = awsAction.CreateKMS(fmt.Sprintf("%s-kms", projectName), "eu-west-2", atlasAccountARN, awsRoleARN)
 			Expect(err).ToNot(HaveOccurred())
 
 			_, _, err = atlasClient.Client.EncryptionsAtRest.Create(ctx, &mongodbatlas.EncryptionAtRest{
 				GroupID: projectID,
 				AwsKms: mongodbatlas.AwsKms{
 					Enabled:             toptr.MakePtr(true),
 					CustomerMasterKeyID: customerMasterKeyID,
-					Region:              "US_EAST_1",
+					Region:              "EU_WEST_2",
 					RoleID:              atlasRoleID,
 				},
 			})
@@ -764,7 +764,7 @@ var _ = Describe("Project Deletion Protection", Label("project", "deletion-prote
 
 		By("Encryption At Rest is ready after configured properly", func() {
 			Expect(testData.K8SClient.Get(context.TODO(), client.ObjectKeyFromObject(testData.Project), testData.Project)).To(Succeed())
-			testData.Project.Spec.EncryptionAtRest.AwsKms.Region = "US_EAST_1"
+			testData.Project.Spec.EncryptionAtRest.AwsKms.Region = "EU_WEST_2"
 			Expect(testData.K8SClient.Update(context.TODO(), testData.Project)).To(Succeed())
 
 			Eventually(func(g Gomega) {