From c679bfd8ecbdb9dee1620b9348a1e20c4bfcc1b3 Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Thu, 19 Dec 2024 16:09:17 +0000 Subject: [PATCH 1/4] init --- pkg/networking/backend_sg_provider.go | 25 +- pkg/networking/backend_sg_provider_test.go | 442 +++++++++++++-------- 2 files changed, 298 insertions(+), 169 deletions(-) diff --git a/pkg/networking/backend_sg_provider.go b/pkg/networking/backend_sg_provider.go index 8b3900e52..1bb141105 100644 --- a/pkg/networking/backend_sg_provider.go +++ b/pkg/networking/backend_sg_provider.go @@ -5,14 +5,15 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/aws/smithy-go" "regexp" "sort" "strings" "sync" "time" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/smithy-go" + awssdk "github.com/aws/aws-sdk-go-v2/aws" ec2sdk "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/go-logr/logr" @@ -234,6 +235,12 @@ func (p *defaultBackendSGProvider) allocateBackendSG(ctx context.Context, resour if len(sgID) > 1 { p.logger.V(1).Info("Existing SG found", "id", sgID) p.autoGeneratedSG = sgID + + if err = p.syncBackendSGTags(ctx, sgID); err != nil { + p.logger.Error(err, "failed to synchronize tags of existing securityGroup", sgID) + return err + } + p.logger.Info("added resource tags", "resourceID", sgID) return nil } @@ -253,6 +260,20 @@ func (p *defaultBackendSGProvider) allocateBackendSG(ctx context.Context, resour return nil } +func (p *defaultBackendSGProvider) syncBackendSGTags(ctx context.Context, sgID string) error { + tagSpecifications := p.buildBackendSGTags(ctx) + + createReq := &ec2sdk.CreateTagsInput{ + Resources: []string{sgID}, + Tags: tagSpecifications[0].Tags, + } + + if _, err := p.ec2Client.CreateTagsWithContext(ctx, createReq); err != nil { + return err + } + return nil +} + func (p *defaultBackendSGProvider) buildBackendSGTags(_ context.Context) []ec2types.TagSpecification { var defaultTags []ec2types.Tag for key, val := range p.defaultTags { diff --git a/pkg/networking/backend_sg_provider_test.go b/pkg/networking/backend_sg_provider_test.go index 4850d7d30..3dd3219b8 100644 --- a/pkg/networking/backend_sg_provider_test.go +++ b/pkg/networking/backend_sg_provider_test.go @@ -2,12 +2,13 @@ package networking import ( "context" + "reflect" + "testing" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/smithy-go" "k8s.io/apimachinery/pkg/types" - "reflect" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" - "testing" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -41,13 +42,19 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { resp *ec2sdk.CreateSecurityGroupOutput err error } + type createTagsWithContextCall struct { + req *ec2sdk.CreateTagsInput + resp *ec2sdk.CreateTagsOutput + err error + } type fields struct { - backendSG string - ingResources []*networking.Ingress - svcResource *corev1.Service - defaultTags map[string]string - describeSGCalls []describeSecurityGroupsAsListCall - createSGCalls []createSecurityGroupWithContexCall + backendSG string + ingResources []*networking.Ingress + svcResource *corev1.Service + defaultTags map[string]string + describeSGCalls []describeSecurityGroupsAsListCall + createSGCalls []createSecurityGroupWithContexCall + createSGTagsCalls []createTagsWithContextCall } defaultEC2Filters := []ec2types.Filter{ { @@ -75,28 +82,47 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { Name: "name", }, } - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "awesome-ns", - Name: "awesome-svc", - }, - } + // svc := &corev1.Service{ + // ObjectMeta: metav1.ObjectMeta{ + // Namespace: "awesome-ns", + // Name: "awesome-svc", + // }, + // } tests := []struct { name string want string fields fields wantErr error }{ + // { + // name: "backend sg enabled", + // fields: fields{ + // backendSG: "sg-xxx", + // ingResources: []*networking.Ingress{ing}, + // }, + // want: "sg-xxx", + // }, + // { + // name: "backend sg enabled, auto-gen, SG exists", + // fields: fields{ + // describeSGCalls: []describeSecurityGroupsAsListCall{ + // { + // req: &ec2sdk.DescribeSecurityGroupsInput{ + // Filters: defaultEC2Filters, + // }, + // resp: []ec2types.SecurityGroup{ + // { + // GroupId: awssdk.String("sg-autogen"), + // }, + // }, + // }, + // }, + // ingResources: []*networking.Ingress{ing, ing1}, + // }, + // want: "sg-autogen", + // }, { - name: "backend sg enabled", - fields: fields{ - backendSG: "sg-xxx", - ingResources: []*networking.Ingress{ing}, - }, - want: "sg-xxx", - }, - { - name: "backend sg enabled, auto-gen, SG exists", + name: "backend sg enabled, auto-gen, SG exists, try to sync tags", fields: fields{ describeSGCalls: []describeSecurityGroupsAsListCall{ { @@ -110,99 +136,32 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { }, }, }, - ingResources: []*networking.Ingress{ing, ing1}, - }, - want: "sg-autogen", - }, - { - name: "backend sg enabled, auto-gen new SG", - fields: fields{ - describeSGCalls: []describeSecurityGroupsAsListCall{ - { - req: &ec2sdk.DescribeSecurityGroupsInput{ - Filters: defaultEC2Filters, - }, - err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, - }, - }, - createSGCalls: []createSecurityGroupWithContexCall{ + createSGTagsCalls: []createTagsWithContextCall{ { - req: &ec2sdk.CreateSecurityGroupInput{ - Description: awssdk.String(sgDescription), - GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), - TagSpecifications: []ec2types.TagSpecification{ + req: &ec2sdk.CreateTagsInput{ + Resources: []string{"sg-autogen"}, + Tags: []ec2types.Tag{ { - ResourceType: ec2types.ResourceType("security-group"), - Tags: []ec2types.Tag{ - { - Key: awssdk.String("elbv2.k8s.aws/cluster"), - Value: awssdk.String(defaultClusterName), - }, - { - Key: awssdk.String("elbv2.k8s.aws/resource"), - Value: awssdk.String("backend-sg"), - }, - }, + Key: awssdk.String("KubernetesCluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("defaultTag"), + Value: awssdk.String("specified"), }, - }, - VpcId: awssdk.String(defaultVPCID), - }, - resp: &ec2sdk.CreateSecurityGroupOutput{ - GroupId: awssdk.String("sg-newauto"), - }, - }, - }, - ingResources: []*networking.Ingress{ing, ing1}, - }, - want: "sg-newauto", - }, - { - name: "backend sg enabled, auto-gen new SG with additional defaultTags", - fields: fields{ - describeSGCalls: []describeSecurityGroupsAsListCall{ - { - req: &ec2sdk.DescribeSecurityGroupsInput{ - Filters: defaultEC2Filters, - }, - err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, - }, - }, - createSGCalls: []createSecurityGroupWithContexCall{ - { - req: &ec2sdk.CreateSecurityGroupInput{ - Description: awssdk.String(sgDescription), - GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), - TagSpecifications: []ec2types.TagSpecification{ { - ResourceType: ec2types.ResourceType("security-group"), - Tags: []ec2types.Tag{ - { - Key: awssdk.String("KubernetesCluster"), - Value: awssdk.String(defaultClusterName), - }, - { - Key: awssdk.String("defaultTag"), - Value: awssdk.String("specified"), - }, - { - Key: awssdk.String("zzzKey"), - Value: awssdk.String("value"), - }, - { - Key: awssdk.String("elbv2.k8s.aws/cluster"), - Value: awssdk.String(defaultClusterName), - }, - { - Key: awssdk.String("elbv2.k8s.aws/resource"), - Value: awssdk.String("backend-sg"), - }, - }, + Key: awssdk.String("zzzKey"), + Value: awssdk.String("value"), + }, + { + Key: awssdk.String("elbv2.k8s.aws/cluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("elbv2.k8s.aws/resource"), + Value: awssdk.String("sg-autogen"), }, }, - VpcId: awssdk.String(defaultVPCID), - }, - resp: &ec2sdk.CreateSecurityGroupOutput{ - GroupId: awssdk.String("sg-newauto"), }, }, }, @@ -211,65 +170,211 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { "KubernetesCluster": defaultClusterName, "defaultTag": "specified", }, - svcResource: svc, - }, - want: "sg-newauto", - }, - { - name: "describe SG call returns error", - fields: fields{ - describeSGCalls: []describeSecurityGroupsAsListCall{ - { - req: &ec2sdk.DescribeSecurityGroupsInput{ - Filters: defaultEC2Filters, - }, - err: &smithy.GenericAPIError{Code: "Some.Other.Error", Message: "describe security group as list error"}, - }, - }, - ingResources: []*networking.Ingress{ing}, - }, - wantErr: errors.New("api error Some.Other.Error: describe security group as list error"), - }, - { - name: "create SG call returns error", - fields: fields{ - describeSGCalls: []describeSecurityGroupsAsListCall{ - { - req: &ec2sdk.DescribeSecurityGroupsInput{ - Filters: defaultEC2Filters, - }, - err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, - }, - }, - createSGCalls: []createSecurityGroupWithContexCall{ - { - req: &ec2sdk.CreateSecurityGroupInput{ - Description: awssdk.String(sgDescription), - GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceType("security-group"), - Tags: []ec2types.Tag{ - { - Key: awssdk.String("elbv2.k8s.aws/cluster"), - Value: awssdk.String(defaultClusterName), - }, - { - Key: awssdk.String("elbv2.k8s.aws/resource"), - Value: awssdk.String("backend-sg"), - }, - }, - }, - }, - VpcId: awssdk.String(defaultVPCID), - }, - err: &smithy.GenericAPIError{Code: "Create.Error", Message: "unable to create security group"}, - }, - }, - ingResources: []*networking.Ingress{ing1}, + ingResources: []*networking.Ingress{ing, ing1}, }, - wantErr: errors.New("api error Create.Error: unable to create security group"), + want: "sg-autogen", }, + // { + // name: "backend sg enabled, auto-gen, SG exists, tags sync error", + // fields: fields{ + // describeSGCalls: []describeSecurityGroupsAsListCall{ + // { + // req: &ec2sdk.DescribeSecurityGroupsInput{ + // Filters: defaultEC2Filters, + // }, + // resp: []ec2types.SecurityGroup{ + // { + // GroupId: awssdk.String("sg-autogen"), + // }, + // }, + // }, + // }, + // createSGTagsCalls: []createTagsWithContextCall{ + // { + // req: &ec2sdk.CreateTagsInput{ + // Resources: []string{"sg-autogen"}, + // Tags: []ec2types.Tag{ + // { + // Key: awssdk.String("KubernetesCluster"), + // Value: awssdk.String(defaultClusterName), + // }, + // { + // Key: awssdk.String("defaultTag"), + // Value: awssdk.String("specified"), + // }, + // { + // Key: awssdk.String("zzzKey"), + // Value: awssdk.String("value"), + // }, + // }, + // }, + // err: awserr.New("Some.Error", "create tags error", nil), + // }, + // }, + // defaultTags: map[string]string{ + // "zzzKey": "value", + // "KubernetesCluster": defaultClusterName, + // "defaultTag": "specified", + // }, + // }, + // wantErr: errors.New("api error Some.Other.Error: describe security group as list error"), + // }, + // { + // name: "backend sg enabled, auto-gen new SG", + // fields: fields{ + // describeSGCalls: []describeSecurityGroupsAsListCall{ + // { + // req: &ec2sdk.DescribeSecurityGroupsInput{ + // Filters: defaultEC2Filters, + // }, + // err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, + // }, + // }, + // createSGCalls: []createSecurityGroupWithContexCall{ + // { + // req: &ec2sdk.CreateSecurityGroupInput{ + // Description: awssdk.String(sgDescription), + // GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), + // TagSpecifications: []ec2types.TagSpecification{ + // { + // ResourceType: ec2types.ResourceType("security-group"), + // Tags: []ec2types.Tag{ + // { + // Key: awssdk.String("elbv2.k8s.aws/cluster"), + // Value: awssdk.String(defaultClusterName), + // }, + // { + // Key: awssdk.String("elbv2.k8s.aws/resource"), + // Value: awssdk.String("backend-sg"), + // }, + // }, + // }, + // }, + // VpcId: awssdk.String(defaultVPCID), + // }, + // resp: &ec2sdk.CreateSecurityGroupOutput{ + // GroupId: awssdk.String("sg-newauto"), + // }, + // }, + // }, + // ingResources: []*networking.Ingress{ing, ing1}, + // }, + // want: "sg-newauto", + // }, + // { + // name: "backend sg enabled, auto-gen new SG with additional defaultTags", + // fields: fields{ + // describeSGCalls: []describeSecurityGroupsAsListCall{ + // { + // req: &ec2sdk.DescribeSecurityGroupsInput{ + // Filters: defaultEC2Filters, + // }, + // err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, + // }, + // }, + // createSGCalls: []createSecurityGroupWithContexCall{ + // { + // req: &ec2sdk.CreateSecurityGroupInput{ + // Description: awssdk.String(sgDescription), + // GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), + // TagSpecifications: []ec2types.TagSpecification{ + // { + // ResourceType: ec2types.ResourceType("security-group"), + // Tags: []ec2types.Tag{ + // { + // Key: awssdk.String("KubernetesCluster"), + // Value: awssdk.String(defaultClusterName), + // }, + // { + // Key: awssdk.String("defaultTag"), + // Value: awssdk.String("specified"), + // }, + // { + // Key: awssdk.String("zzzKey"), + // Value: awssdk.String("value"), + // }, + // { + // Key: awssdk.String("elbv2.k8s.aws/cluster"), + // Value: awssdk.String(defaultClusterName), + // }, + // { + // Key: awssdk.String("elbv2.k8s.aws/resource"), + // Value: awssdk.String("backend-sg"), + // }, + // }, + // }, + // }, + // VpcId: awssdk.String(defaultVPCID), + // }, + // resp: &ec2sdk.CreateSecurityGroupOutput{ + // GroupId: awssdk.String("sg-newauto"), + // }, + // }, + // }, + // defaultTags: map[string]string{ + // "zzzKey": "value", + // "KubernetesCluster": defaultClusterName, + // "defaultTag": "specified", + // }, + // svcResource: svc, + // }, + // want: "sg-newauto", + // }, + // { + // name: "describe SG call returns error", + // fields: fields{ + // describeSGCalls: []describeSecurityGroupsAsListCall{ + // { + // req: &ec2sdk.DescribeSecurityGroupsInput{ + // Filters: defaultEC2Filters, + // }, + // err: &smithy.GenericAPIError{Code: "Some.Other.Error", Message: "describe security group as list error"}, + // }, + // }, + // ingResources: []*networking.Ingress{ing}, + // }, + // wantErr: errors.New("api error Some.Other.Error: describe security group as list error"), + // }, + // { + // name: "create SG call returns error", + // fields: fields{ + // describeSGCalls: []describeSecurityGroupsAsListCall{ + // { + // req: &ec2sdk.DescribeSecurityGroupsInput{ + // Filters: defaultEC2Filters, + // }, + // err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, + // }, + // }, + // createSGCalls: []createSecurityGroupWithContexCall{ + // { + // req: &ec2sdk.CreateSecurityGroupInput{ + // Description: awssdk.String(sgDescription), + // GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), + // TagSpecifications: []ec2types.TagSpecification{ + // { + // ResourceType: ec2types.ResourceType("security-group"), + // Tags: []ec2types.Tag{ + // { + // Key: awssdk.String("elbv2.k8s.aws/cluster"), + // Value: awssdk.String(defaultClusterName), + // }, + // { + // Key: awssdk.String("elbv2.k8s.aws/resource"), + // Value: awssdk.String("backend-sg"), + // }, + // }, + // }, + // }, + // VpcId: awssdk.String(defaultVPCID), + // }, + // err: &smithy.GenericAPIError{Code: "Create.Error", Message: "unable to create security group"}, + // }, + // }, + // ingResources: []*networking.Ingress{ing1}, + // }, + // wantErr: errors.New("api error Create.Error: unable to create security group"), + // }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -283,6 +388,9 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { for _, call := range tt.fields.createSGCalls { ec2Client.EXPECT().CreateSecurityGroupWithContext(context.Background(), call.req).Return(call.resp, call.err) } + for _, call := range tt.fields.createSGTagsCalls { + ec2Client.EXPECT().CreateTagsWithContext(context.Background(), call.req).Return(call.resp, call.err) + } k8sClient := mock_client.NewMockClient(ctrl) sgProvider := NewBackendSGProvider(defaultClusterName, tt.fields.backendSG, defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, logr.New(&log.NullLogSink{})) From 3f312a02f81ce1997d109d695a925884afcf6d8e Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Fri, 20 Dec 2024 04:13:01 +0000 Subject: [PATCH 2/4] unit test --- pkg/networking/backend_sg_provider_test.go | 425 +++++++++++---------- 1 file changed, 217 insertions(+), 208 deletions(-) diff --git a/pkg/networking/backend_sg_provider_test.go b/pkg/networking/backend_sg_provider_test.go index 3dd3219b8..81b79bd5b 100644 --- a/pkg/networking/backend_sg_provider_test.go +++ b/pkg/networking/backend_sg_provider_test.go @@ -82,12 +82,12 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { Name: "name", }, } - // svc := &corev1.Service{ - // ObjectMeta: metav1.ObjectMeta{ - // Namespace: "awesome-ns", - // Name: "awesome-svc", - // }, - // } + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "awesome-svc", + }, + } tests := []struct { name string want string @@ -159,7 +159,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { }, { Key: awssdk.String("elbv2.k8s.aws/resource"), - Value: awssdk.String("sg-autogen"), + Value: awssdk.String("backend-sg"), }, }, }, @@ -174,207 +174,216 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { }, want: "sg-autogen", }, - // { - // name: "backend sg enabled, auto-gen, SG exists, tags sync error", - // fields: fields{ - // describeSGCalls: []describeSecurityGroupsAsListCall{ - // { - // req: &ec2sdk.DescribeSecurityGroupsInput{ - // Filters: defaultEC2Filters, - // }, - // resp: []ec2types.SecurityGroup{ - // { - // GroupId: awssdk.String("sg-autogen"), - // }, - // }, - // }, - // }, - // createSGTagsCalls: []createTagsWithContextCall{ - // { - // req: &ec2sdk.CreateTagsInput{ - // Resources: []string{"sg-autogen"}, - // Tags: []ec2types.Tag{ - // { - // Key: awssdk.String("KubernetesCluster"), - // Value: awssdk.String(defaultClusterName), - // }, - // { - // Key: awssdk.String("defaultTag"), - // Value: awssdk.String("specified"), - // }, - // { - // Key: awssdk.String("zzzKey"), - // Value: awssdk.String("value"), - // }, - // }, - // }, - // err: awserr.New("Some.Error", "create tags error", nil), - // }, - // }, - // defaultTags: map[string]string{ - // "zzzKey": "value", - // "KubernetesCluster": defaultClusterName, - // "defaultTag": "specified", - // }, - // }, - // wantErr: errors.New("api error Some.Other.Error: describe security group as list error"), - // }, - // { - // name: "backend sg enabled, auto-gen new SG", - // fields: fields{ - // describeSGCalls: []describeSecurityGroupsAsListCall{ - // { - // req: &ec2sdk.DescribeSecurityGroupsInput{ - // Filters: defaultEC2Filters, - // }, - // err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, - // }, - // }, - // createSGCalls: []createSecurityGroupWithContexCall{ - // { - // req: &ec2sdk.CreateSecurityGroupInput{ - // Description: awssdk.String(sgDescription), - // GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), - // TagSpecifications: []ec2types.TagSpecification{ - // { - // ResourceType: ec2types.ResourceType("security-group"), - // Tags: []ec2types.Tag{ - // { - // Key: awssdk.String("elbv2.k8s.aws/cluster"), - // Value: awssdk.String(defaultClusterName), - // }, - // { - // Key: awssdk.String("elbv2.k8s.aws/resource"), - // Value: awssdk.String("backend-sg"), - // }, - // }, - // }, - // }, - // VpcId: awssdk.String(defaultVPCID), - // }, - // resp: &ec2sdk.CreateSecurityGroupOutput{ - // GroupId: awssdk.String("sg-newauto"), - // }, - // }, - // }, - // ingResources: []*networking.Ingress{ing, ing1}, - // }, - // want: "sg-newauto", - // }, - // { - // name: "backend sg enabled, auto-gen new SG with additional defaultTags", - // fields: fields{ - // describeSGCalls: []describeSecurityGroupsAsListCall{ - // { - // req: &ec2sdk.DescribeSecurityGroupsInput{ - // Filters: defaultEC2Filters, - // }, - // err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, - // }, - // }, - // createSGCalls: []createSecurityGroupWithContexCall{ - // { - // req: &ec2sdk.CreateSecurityGroupInput{ - // Description: awssdk.String(sgDescription), - // GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), - // TagSpecifications: []ec2types.TagSpecification{ - // { - // ResourceType: ec2types.ResourceType("security-group"), - // Tags: []ec2types.Tag{ - // { - // Key: awssdk.String("KubernetesCluster"), - // Value: awssdk.String(defaultClusterName), - // }, - // { - // Key: awssdk.String("defaultTag"), - // Value: awssdk.String("specified"), - // }, - // { - // Key: awssdk.String("zzzKey"), - // Value: awssdk.String("value"), - // }, - // { - // Key: awssdk.String("elbv2.k8s.aws/cluster"), - // Value: awssdk.String(defaultClusterName), - // }, - // { - // Key: awssdk.String("elbv2.k8s.aws/resource"), - // Value: awssdk.String("backend-sg"), - // }, - // }, - // }, - // }, - // VpcId: awssdk.String(defaultVPCID), - // }, - // resp: &ec2sdk.CreateSecurityGroupOutput{ - // GroupId: awssdk.String("sg-newauto"), - // }, - // }, - // }, - // defaultTags: map[string]string{ - // "zzzKey": "value", - // "KubernetesCluster": defaultClusterName, - // "defaultTag": "specified", - // }, - // svcResource: svc, - // }, - // want: "sg-newauto", - // }, - // { - // name: "describe SG call returns error", - // fields: fields{ - // describeSGCalls: []describeSecurityGroupsAsListCall{ - // { - // req: &ec2sdk.DescribeSecurityGroupsInput{ - // Filters: defaultEC2Filters, - // }, - // err: &smithy.GenericAPIError{Code: "Some.Other.Error", Message: "describe security group as list error"}, - // }, - // }, - // ingResources: []*networking.Ingress{ing}, - // }, - // wantErr: errors.New("api error Some.Other.Error: describe security group as list error"), - // }, - // { - // name: "create SG call returns error", - // fields: fields{ - // describeSGCalls: []describeSecurityGroupsAsListCall{ - // { - // req: &ec2sdk.DescribeSecurityGroupsInput{ - // Filters: defaultEC2Filters, - // }, - // err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, - // }, - // }, - // createSGCalls: []createSecurityGroupWithContexCall{ - // { - // req: &ec2sdk.CreateSecurityGroupInput{ - // Description: awssdk.String(sgDescription), - // GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), - // TagSpecifications: []ec2types.TagSpecification{ - // { - // ResourceType: ec2types.ResourceType("security-group"), - // Tags: []ec2types.Tag{ - // { - // Key: awssdk.String("elbv2.k8s.aws/cluster"), - // Value: awssdk.String(defaultClusterName), - // }, - // { - // Key: awssdk.String("elbv2.k8s.aws/resource"), - // Value: awssdk.String("backend-sg"), - // }, - // }, - // }, - // }, - // VpcId: awssdk.String(defaultVPCID), - // }, - // err: &smithy.GenericAPIError{Code: "Create.Error", Message: "unable to create security group"}, - // }, - // }, - // ingResources: []*networking.Ingress{ing1}, - // }, - // wantErr: errors.New("api error Create.Error: unable to create security group"), - // }, + { + name: "backend sg enabled, auto-gen, SG exists, tags sync error", + fields: fields{ + describeSGCalls: []describeSecurityGroupsAsListCall{ + { + req: &ec2sdk.DescribeSecurityGroupsInput{ + Filters: defaultEC2Filters, + }, + resp: []ec2types.SecurityGroup{ + { + GroupId: awssdk.String("sg-autogen"), + }, + }, + }, + }, + createSGTagsCalls: []createTagsWithContextCall{ + { + req: &ec2sdk.CreateTagsInput{ + Resources: []string{"sg-autogen"}, + Tags: []ec2types.Tag{ + { + Key: awssdk.String("KubernetesCluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("defaultTag"), + Value: awssdk.String("specified"), + }, + { + Key: awssdk.String("zzzKey"), + Value: awssdk.String("value"), + }, + { + Key: awssdk.String("elbv2.k8s.aws/cluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("elbv2.k8s.aws/resource"), + Value: awssdk.String("backend-sg"), + }, + }, + }, + err: &smithy.GenericAPIError{Code: "Some.Other.Error", Message: "unable to tag security group"}, + }, + }, + defaultTags: map[string]string{ + "zzzKey": "value", + "KubernetesCluster": defaultClusterName, + "defaultTag": "specified", + }, + svcResource: svc, + }, + wantErr: errors.New("api error Some.Other.Error: unable to tag security group"), + }, + { + name: "backend sg enabled, auto-gen new SG", + fields: fields{ + describeSGCalls: []describeSecurityGroupsAsListCall{ + { + req: &ec2sdk.DescribeSecurityGroupsInput{ + Filters: defaultEC2Filters, + }, + err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, + }, + }, + createSGCalls: []createSecurityGroupWithContexCall{ + { + req: &ec2sdk.CreateSecurityGroupInput{ + Description: awssdk.String(sgDescription), + GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceType("security-group"), + Tags: []ec2types.Tag{ + { + Key: awssdk.String("elbv2.k8s.aws/cluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("elbv2.k8s.aws/resource"), + Value: awssdk.String("backend-sg"), + }, + }, + }, + }, + VpcId: awssdk.String(defaultVPCID), + }, + resp: &ec2sdk.CreateSecurityGroupOutput{ + GroupId: awssdk.String("sg-newauto"), + }, + }, + }, + ingResources: []*networking.Ingress{ing, ing1}, + }, + want: "sg-newauto", + }, + { + name: "backend sg enabled, auto-gen new SG with additional defaultTags", + fields: fields{ + describeSGCalls: []describeSecurityGroupsAsListCall{ + { + req: &ec2sdk.DescribeSecurityGroupsInput{ + Filters: defaultEC2Filters, + }, + err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, + }, + }, + createSGCalls: []createSecurityGroupWithContexCall{ + { + req: &ec2sdk.CreateSecurityGroupInput{ + Description: awssdk.String(sgDescription), + GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceType("security-group"), + Tags: []ec2types.Tag{ + { + Key: awssdk.String("KubernetesCluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("defaultTag"), + Value: awssdk.String("specified"), + }, + { + Key: awssdk.String("zzzKey"), + Value: awssdk.String("value"), + }, + { + Key: awssdk.String("elbv2.k8s.aws/cluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("elbv2.k8s.aws/resource"), + Value: awssdk.String("backend-sg"), + }, + }, + }, + }, + VpcId: awssdk.String(defaultVPCID), + }, + resp: &ec2sdk.CreateSecurityGroupOutput{ + GroupId: awssdk.String("sg-newauto"), + }, + }, + }, + defaultTags: map[string]string{ + "zzzKey": "value", + "KubernetesCluster": defaultClusterName, + "defaultTag": "specified", + }, + svcResource: svc, + }, + want: "sg-newauto", + }, + { + name: "describe SG call returns error", + fields: fields{ + describeSGCalls: []describeSecurityGroupsAsListCall{ + { + req: &ec2sdk.DescribeSecurityGroupsInput{ + Filters: defaultEC2Filters, + }, + err: &smithy.GenericAPIError{Code: "Some.Other.Error", Message: "describe security group as list error"}, + }, + }, + ingResources: []*networking.Ingress{ing}, + }, + wantErr: errors.New("api error Some.Other.Error: describe security group as list error"), + }, + { + name: "create SG call returns error", + fields: fields{ + describeSGCalls: []describeSecurityGroupsAsListCall{ + { + req: &ec2sdk.DescribeSecurityGroupsInput{ + Filters: defaultEC2Filters, + }, + err: &smithy.GenericAPIError{Code: "InvalidGroup.NotFound", Message: ""}, + }, + }, + createSGCalls: []createSecurityGroupWithContexCall{ + { + req: &ec2sdk.CreateSecurityGroupInput{ + Description: awssdk.String(sgDescription), + GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"), + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceType("security-group"), + Tags: []ec2types.Tag{ + { + Key: awssdk.String("elbv2.k8s.aws/cluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("elbv2.k8s.aws/resource"), + Value: awssdk.String("backend-sg"), + }, + }, + }, + }, + VpcId: awssdk.String(defaultVPCID), + }, + err: &smithy.GenericAPIError{Code: "Create.Error", Message: "unable to create security group"}, + }, + }, + ingResources: []*networking.Ingress{ing1}, + }, + wantErr: errors.New("api error Create.Error: unable to create security group"), + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 597eb701ba7608f1c7f52e42168eb089d592ca6e Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Fri, 20 Dec 2024 04:29:49 +0000 Subject: [PATCH 3/4] cleanup --- pkg/networking/backend_sg_provider.go | 5 +- pkg/networking/backend_sg_provider_test.go | 96 +++++++++++++--------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/pkg/networking/backend_sg_provider.go b/pkg/networking/backend_sg_provider.go index 1bb141105..6f8f1974b 100644 --- a/pkg/networking/backend_sg_provider.go +++ b/pkg/networking/backend_sg_provider.go @@ -5,15 +5,14 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/smithy-go" "regexp" "sort" "strings" "sync" "time" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/aws/smithy-go" - awssdk "github.com/aws/aws-sdk-go-v2/aws" ec2sdk "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/go-logr/logr" diff --git a/pkg/networking/backend_sg_provider_test.go b/pkg/networking/backend_sg_provider_test.go index 81b79bd5b..bcd551e96 100644 --- a/pkg/networking/backend_sg_provider_test.go +++ b/pkg/networking/backend_sg_provider_test.go @@ -2,13 +2,12 @@ package networking import ( "context" - "reflect" - "testing" - ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/smithy-go" "k8s.io/apimachinery/pkg/types" + "reflect" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" + "testing" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -48,13 +47,13 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { err error } type fields struct { - backendSG string - ingResources []*networking.Ingress - svcResource *corev1.Service - defaultTags map[string]string - describeSGCalls []describeSecurityGroupsAsListCall - createSGCalls []createSecurityGroupWithContexCall - createSGTagsCalls []createTagsWithContextCall + backendSG string + ingResources []*networking.Ingress + svcResource *corev1.Service + defaultTags map[string]string + describeSGCalls []describeSecurityGroupsAsListCall + createSGCalls []createSecurityGroupWithContexCall + createTagsWithContextCalls []createTagsWithContextCall } defaultEC2Filters := []ec2types.Filter{ { @@ -94,33 +93,50 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { fields fields wantErr error }{ - // { - // name: "backend sg enabled", - // fields: fields{ - // backendSG: "sg-xxx", - // ingResources: []*networking.Ingress{ing}, - // }, - // want: "sg-xxx", - // }, - // { - // name: "backend sg enabled, auto-gen, SG exists", - // fields: fields{ - // describeSGCalls: []describeSecurityGroupsAsListCall{ - // { - // req: &ec2sdk.DescribeSecurityGroupsInput{ - // Filters: defaultEC2Filters, - // }, - // resp: []ec2types.SecurityGroup{ - // { - // GroupId: awssdk.String("sg-autogen"), - // }, - // }, - // }, - // }, - // ingResources: []*networking.Ingress{ing, ing1}, - // }, - // want: "sg-autogen", - // }, + { + name: "backend sg enabled", + fields: fields{ + backendSG: "sg-xxx", + ingResources: []*networking.Ingress{ing}, + }, + want: "sg-xxx", + }, + { + name: "backend sg enabled, auto-gen, SG exists", + fields: fields{ + describeSGCalls: []describeSecurityGroupsAsListCall{ + { + req: &ec2sdk.DescribeSecurityGroupsInput{ + Filters: defaultEC2Filters, + }, + resp: []ec2types.SecurityGroup{ + { + GroupId: awssdk.String("sg-autogen"), + }, + }, + }, + }, + createTagsWithContextCalls: []createTagsWithContextCall{ + { + req: &ec2sdk.CreateTagsInput{ + Resources: []string{"sg-autogen"}, + Tags: []ec2types.Tag{ + { + Key: awssdk.String("elbv2.k8s.aws/cluster"), + Value: awssdk.String(defaultClusterName), + }, + { + Key: awssdk.String("elbv2.k8s.aws/resource"), + Value: awssdk.String("backend-sg"), + }, + }, + }, + }, + }, + ingResources: []*networking.Ingress{ing, ing1}, + }, + want: "sg-autogen", + }, { name: "backend sg enabled, auto-gen, SG exists, try to sync tags", fields: fields{ @@ -136,7 +152,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { }, }, }, - createSGTagsCalls: []createTagsWithContextCall{ + createTagsWithContextCalls: []createTagsWithContextCall{ { req: &ec2sdk.CreateTagsInput{ Resources: []string{"sg-autogen"}, @@ -189,7 +205,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { }, }, }, - createSGTagsCalls: []createTagsWithContextCall{ + createTagsWithContextCalls: []createTagsWithContextCall{ { req: &ec2sdk.CreateTagsInput{ Resources: []string{"sg-autogen"}, @@ -397,7 +413,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { for _, call := range tt.fields.createSGCalls { ec2Client.EXPECT().CreateSecurityGroupWithContext(context.Background(), call.req).Return(call.resp, call.err) } - for _, call := range tt.fields.createSGTagsCalls { + for _, call := range tt.fields.createTagsWithContextCalls { ec2Client.EXPECT().CreateTagsWithContext(context.Background(), call.req).Return(call.resp, call.err) } k8sClient := mock_client.NewMockClient(ctrl) From e175472a7e9a0e6451d2e980615954e97c1ae925 Mon Sep 17 00:00:00 2001 From: phuhung273 Date: Tue, 14 Jan 2025 12:09:01 +0000 Subject: [PATCH 4/4] improve reconcile tag logic --- pkg/networking/backend_sg_provider.go | 86 ++++++++++++++++++---- pkg/networking/backend_sg_provider_test.go | 44 +++++++++-- 2 files changed, 108 insertions(+), 22 deletions(-) diff --git a/pkg/networking/backend_sg_provider.go b/pkg/networking/backend_sg_provider.go index 6f8f1974b..7cbd35d40 100644 --- a/pkg/networking/backend_sg_provider.go +++ b/pkg/networking/backend_sg_provider.go @@ -20,6 +20,8 @@ import ( corev1 "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" "sigs.k8s.io/aws-load-balancer-controller/pkg/k8s" "sigs.k8s.io/aws-load-balancer-controller/pkg/runtime" @@ -227,15 +229,16 @@ func (p *defaultBackendSGProvider) allocateBackendSG(ctx context.Context, resour } sgName := p.getBackendSGName() - sgID, err := p.getBackendSGFromEC2(ctx, sgName, p.vpcID) + sg, err := p.getBackendSGFromEC2(ctx, sgName, p.vpcID) if err != nil { return err } - if len(sgID) > 1 { + if sg != nil { + sgID := awssdk.ToString(sg.GroupId) p.logger.V(1).Info("Existing SG found", "id", sgID) p.autoGeneratedSG = sgID - if err = p.syncBackendSGTags(ctx, sgID); err != nil { + if err = p.reconcileTags(ctx, *sg); err != nil { p.logger.Error(err, "failed to synchronize tags of existing securityGroup", sgID) return err } @@ -259,20 +262,59 @@ func (p *defaultBackendSGProvider) allocateBackendSG(ctx context.Context, resour return nil } -func (p *defaultBackendSGProvider) syncBackendSGTags(ctx context.Context, sgID string) error { - tagSpecifications := p.buildBackendSGTags(ctx) +func (p *defaultBackendSGProvider) reconcileTags(ctx context.Context, sg ec2types.SecurityGroup) error { + desiredTags := p.buildBackendSGTagsMap() + currentTags := make(map[string]string) + for _, tag := range sg.Tags { + currentTags[awssdk.ToString(tag.Key)] = awssdk.ToString(tag.Value) + } + + tagsToUpdate, tagsToRemove := algorithm.DiffStringMap(desiredTags, currentTags) + + if len(tagsToUpdate) > 0 { + req := &ec2sdk.CreateTagsInput{ + Resources: []string{*sg.GroupId}, + Tags: convertTagsToSDKTags(tagsToUpdate), + } - createReq := &ec2sdk.CreateTagsInput{ - Resources: []string{sgID}, - Tags: tagSpecifications[0].Tags, + p.logger.Info("adding resource tags", + "resourceID", sg.GroupId, + "change", tagsToUpdate) + if _, err := p.ec2Client.CreateTagsWithContext(ctx, req); err != nil { + return err + } + p.logger.Info("added resource tags", + "resourceID", sg.GroupId) } - if _, err := p.ec2Client.CreateTagsWithContext(ctx, createReq); err != nil { - return err + if len(tagsToRemove) > 0 { + req := &ec2sdk.DeleteTagsInput{ + Resources: []string{*sg.GroupId}, + Tags: convertTagsToSDKTags(tagsToRemove), + } + + p.logger.Info("removing resource tags", + "resourceID", sg.GroupId, + "change", tagsToRemove) + if _, err := p.ec2Client.DeleteTagsWithContext(ctx, req); err != nil { + return err + } + p.logger.Info("removed resource tags", + "resourceID", sg.GroupId) } return nil } +func (p *defaultBackendSGProvider) buildBackendSGTagsMap() map[string]string { + defaultTags := make(map[string]string) + for key, val := range p.defaultTags { + defaultTags[key] = val + } + defaultTags[tagKeyK8sCluster] = p.clusterName + defaultTags[tagKeyResource] = tagValueBackend + return defaultTags +} + func (p *defaultBackendSGProvider) buildBackendSGTags(_ context.Context) []ec2types.TagSpecification { var defaultTags []ec2types.Tag for key, val := range p.defaultTags { @@ -301,7 +343,7 @@ func (p *defaultBackendSGProvider) buildBackendSGTags(_ context.Context) []ec2ty } } -func (p *defaultBackendSGProvider) getBackendSGFromEC2(ctx context.Context, sgName string, vpcID string) (string, error) { +func (p *defaultBackendSGProvider) getBackendSGFromEC2(ctx context.Context, sgName string, vpcID string) (*ec2types.SecurityGroup, error) { req := &ec2sdk.DescribeSecurityGroupsInput{ Filters: []ec2types.Filter{ { @@ -321,12 +363,12 @@ func (p *defaultBackendSGProvider) getBackendSGFromEC2(ctx context.Context, sgNa p.logger.V(1).Info("Queriying existing SG", "vpc-id", vpcID, "name", sgName) sgs, err := p.ec2Client.DescribeSecurityGroupsAsList(ctx, req) if err != nil && !isEC2SecurityGroupNotFoundError(err) { - return "", err + return nil, err } if len(sgs) > 0 { - return awssdk.ToString(sgs[0].GroupId), nil + return &sgs[0], nil } - return "", nil + return nil, nil } func (p *defaultBackendSGProvider) releaseSG(ctx context.Context) error { @@ -384,3 +426,19 @@ func isEC2SecurityGroupNotFoundError(err error) bool { func getObjectKey(resourceType ResourceType, resource types.NamespacedName) string { return string(resourceType) + "/" + resource.String() } + +// convert tags into AWS SDK tag presentation. +func convertTagsToSDKTags(tags map[string]string) []ec2types.Tag { + if len(tags) == 0 { + return nil + } + sdkTags := make([]ec2types.Tag, 0, len(tags)) + + for _, key := range sets.StringKeySet(tags).List() { + sdkTags = append(sdkTags, ec2types.Tag{ + Key: awssdk.String(key), + Value: awssdk.String(tags[key]), + }) + } + return sdkTags +} diff --git a/pkg/networking/backend_sg_provider_test.go b/pkg/networking/backend_sg_provider_test.go index bcd551e96..0af9ed6cf 100644 --- a/pkg/networking/backend_sg_provider_test.go +++ b/pkg/networking/backend_sg_provider_test.go @@ -46,6 +46,11 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { resp *ec2sdk.CreateTagsOutput err error } + type deleteTagsWithContextCall struct { + req *ec2sdk.DeleteTagsInput + resp *ec2sdk.DeleteTagsOutput + err error + } type fields struct { backendSG string ingResources []*networking.Ingress @@ -54,6 +59,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { describeSGCalls []describeSecurityGroupsAsListCall createSGCalls []createSecurityGroupWithContexCall createTagsWithContextCalls []createTagsWithContextCall + deleteTagsWithContextCalls []deleteTagsWithContextCall } defaultEC2Filters := []ec2types.Filter{ { @@ -148,6 +154,12 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { resp: []ec2types.SecurityGroup{ { GroupId: awssdk.String("sg-autogen"), + Tags: []ec2types.Tag{ + { + Key: awssdk.String("tag-to-be-deleted"), + Value: awssdk.String("delete-me"), + }, + }, }, }, }, @@ -165,10 +177,6 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { Key: awssdk.String("defaultTag"), Value: awssdk.String("specified"), }, - { - Key: awssdk.String("zzzKey"), - Value: awssdk.String("value"), - }, { Key: awssdk.String("elbv2.k8s.aws/cluster"), Value: awssdk.String(defaultClusterName), @@ -177,6 +185,23 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { Key: awssdk.String("elbv2.k8s.aws/resource"), Value: awssdk.String("backend-sg"), }, + { + Key: awssdk.String("zzzKey"), + Value: awssdk.String("value"), + }, + }, + }, + }, + }, + deleteTagsWithContextCalls: []deleteTagsWithContextCall{ + { + req: &ec2sdk.DeleteTagsInput{ + Resources: []string{"sg-autogen"}, + Tags: []ec2types.Tag{ + { + Key: awssdk.String("tag-to-be-deleted"), + Value: awssdk.String("delete-me"), + }, }, }, }, @@ -218,10 +243,6 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { Key: awssdk.String("defaultTag"), Value: awssdk.String("specified"), }, - { - Key: awssdk.String("zzzKey"), - Value: awssdk.String("value"), - }, { Key: awssdk.String("elbv2.k8s.aws/cluster"), Value: awssdk.String(defaultClusterName), @@ -230,6 +251,10 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { Key: awssdk.String("elbv2.k8s.aws/resource"), Value: awssdk.String("backend-sg"), }, + { + Key: awssdk.String("zzzKey"), + Value: awssdk.String("value"), + }, }, }, err: &smithy.GenericAPIError{Code: "Some.Other.Error", Message: "unable to tag security group"}, @@ -416,6 +441,9 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) { for _, call := range tt.fields.createTagsWithContextCalls { ec2Client.EXPECT().CreateTagsWithContext(context.Background(), call.req).Return(call.resp, call.err) } + for _, call := range tt.fields.deleteTagsWithContextCalls { + ec2Client.EXPECT().DeleteTagsWithContext(gomock.Any(), call.req).Return(call.resp, call.err) + } k8sClient := mock_client.NewMockClient(ctrl) sgProvider := NewBackendSGProvider(defaultClusterName, tt.fields.backendSG, defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, logr.New(&log.NullLogSink{}))