HCPIE-1846: Handle DEADLINE_EXCEEDED Retries for IAM Groups and Group Members

.changelog/1140.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+Update hcp_group API calls to retry when encountering a 502, 503, or 504 error.
+```

.github/workflows/test-identity.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     paths:
       - 'internal/provider/iam/**'
+      - 'internal/clients/group.go'
+      - 'internal/clients/iam.go'
 
 jobs:
   identity_tests:

.tool-versions

@@ -0,0 +1 @@
+terraform 1.0.11

go.mod

@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/cenkalti/backoff v2.2.1+incompatible
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/go-openapi/runtime v0.28.0
 	github.com/go-openapi/strfmt v0.23.0
 	github.com/google/uuid v1.6.0

go.sum

@@ -32,6 +32,8 @@ github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZ
 github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
 github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=

internal/clients/group.go

@@ -0,0 +1,137 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package clients
+
+import (
+	"github.com/cenkalti/backoff/v4"
+
+	"github.com/hashicorp/hcp-sdk-go/clients/cloud-iam/stable/2019-12-10/client/groups_service"
+)
+
+var groupErrorCodesToRetry = [...]int{502, 503, 504}
+
+// Groups
+
+// CreateGroupRetry wraps the groups client with an exponential backoff retry mechanism.
+func CreateGroupRetry(client *Client, params *groups_service.GroupsServiceCreateGroupParams) (*groups_service.GroupsServiceCreateGroupOK, error) {
+	var res *groups_service.GroupsServiceCreateGroupOK
+	op := func() error {
+		var err error
+		res, err = client.Groups.GroupsServiceCreateGroup(params, nil)
+		return err
+	}
+
+	getCode := func(err error) (int, error) {
+		serviceErr, ok := err.(*groups_service.GroupsServiceCreateGroupDefault)
+		if !ok {
+			return 0, err
+		}
+		return serviceErr.Code(), nil
+	}
+
+	err := backoff.Retry(newBackoffOp(op, getCode), newBackoff())
+
+	return res, err
+}
+
+// UpdateGroupRetry wraps the groups client with an exponential backoff retry mechanism.
+func UpdateGroupRetry(client *Client, params *groups_service.GroupsServiceUpdateGroup2Params) (*groups_service.GroupsServiceUpdateGroup2OK, error) {
+	var res *groups_service.GroupsServiceUpdateGroup2OK
+	op := func() error {
+		var err error
+		res, err = client.Groups.GroupsServiceUpdateGroup2(params, nil)
+		return err
+	}
+
+	getCode := func(err error) (int, error) {
+		serviceErr, ok := err.(*groups_service.GroupsServiceUpdateGroup2Default)
+		if !ok {
+			return 0, err
+		}
+		return serviceErr.Code(), nil
+	}
+
+	err := backoff.Retry(newBackoffOp(op, getCode), newBackoff())
+
+	return res, err
+}
+
+// DeleteGroupRetry wraps the groups client with an exponential backoff retry mechanism.
+func DeleteGroupRetry(client *Client, params *groups_service.GroupsServiceDeleteGroupParams) (*groups_service.GroupsServiceDeleteGroupOK, error) {
+	var res *groups_service.GroupsServiceDeleteGroupOK
+	op := func() error {
+		var err error
+		res, err = client.Groups.GroupsServiceDeleteGroup(params, nil)
+		return err
+	}
+
+	getCode := func(err error) (int, error) {
+		serviceErr, ok := err.(*groups_service.GroupsServiceDeleteGroupDefault)
+		if !ok {
+			return 0, err
+		}
+		return serviceErr.Code(), nil
+	}
+
+	err := backoff.Retry(newBackoffOp(op, getCode), newBackoff())
+
+	return res, err
+}
+
+// Group Members
+
+// UpdateGroupMembersRetry wraps the groups client with an exponential backoff retry mechanism.
+func UpdateGroupMembersRetry(client *Client, params *groups_service.GroupsServiceUpdateGroupMembersParams) (*groups_service.GroupsServiceUpdateGroupMembersOK, error) {
+	var res *groups_service.GroupsServiceUpdateGroupMembersOK
+	op := func() error {
+		var err error
+		res, err = client.Groups.GroupsServiceUpdateGroupMembers(params, nil)
+		return err
+	}
+
+	getCode := func(err error) (int, error) {
+		serviceErr, ok := err.(*groups_service.GroupsServiceUpdateGroupMembersDefault)
+		if !ok {
+			return 0, err
+		}
+		return serviceErr.Code(), nil
+	}
+
+	err := backoff.Retry(newBackoffOp(op, getCode), newBackoff())
+
+	return res, err
+}
+
+// newBackoff creates a new exponential backoff with default values.
+func newBackoff() backoff.BackOff {
+	// Create a new exponential backoff with explicit default values.
+	return backoff.NewExponentialBackOff(
+		backoff.WithInitialInterval(backoff.DefaultInitialInterval),
+		backoff.WithRandomizationFactor(backoff.DefaultRandomizationFactor),
+		backoff.WithMultiplier(backoff.DefaultMultiplier),
+		backoff.WithMaxInterval(backoff.DefaultMaxInterval),
+		backoff.WithMaxElapsedTime(backoff.DefaultMaxElapsedTime),
+	)
+}
+
+func newBackoffOp(op func() error, getCode func(error) (int, error)) func() error {
+	return func() error {
+		err := op()
+
+		if err == nil {
+			return nil
+		}
+
+		code, codeErr := getCode(err)
+		if codeErr != nil {
+			return backoff.Permanent(codeErr)
+		}
+
+		if !shouldRetryErrorCode(code, groupErrorCodesToRetry[:]) {
+			return backoff.Permanent(err)
+		}
+
+		return err
+	}
+}

internal/provider/iam/resource_group.go

@@ -117,7 +117,7 @@ func (r *resourceGroup) Create(ctx context.Context, req resource.CreateRequest,
 		Description: plan.Description.ValueString(),
 	}
 
-	res, err := r.client.Groups.GroupsServiceCreateGroup(createParams, nil)
+	res, err := clients.CreateGroupRetry(r.client, createParams)
 
 	if err != nil {
 		resp.Diagnostics.AddError("Error creating group", err.Error())
@@ -198,7 +198,7 @@ func (r *resourceGroup) Update(ctx context.Context, req resource.UpdateRequest,
 	updateMaskStr := strings.Join(updateMask, ",")
 	updateParams.SetUpdateMask(&updateMaskStr)
 
-	_, err := r.client.Groups.GroupsServiceUpdateGroup2(updateParams, nil)
+	_, err := clients.UpdateGroupRetry(r.client, updateParams)
 	if err != nil {
 		resp.Diagnostics.AddError("Error updating group", err.Error())
 		return
@@ -217,7 +217,8 @@ func (r *resourceGroup) Delete(ctx context.Context, req resource.DeleteRequest,
 
 	deleteParams := groups_service.NewGroupsServiceDeleteGroupParams().WithContext(ctx)
 	deleteParams.ResourceName = state.ResourceName.ValueString()
-	_, err := r.client.Groups.GroupsServiceDeleteGroup(deleteParams, nil)
+
+	_, err := clients.DeleteGroupRetry(r.client, deleteParams)
 
 	if err != nil {
 		var getErr *groups_service.GroupsServiceDeleteGroupDefault

internal/provider/iam/resource_group_members.go

@@ -105,7 +105,7 @@ func (r *resourceGroupMembers) Create(ctx context.Context, req resource.CreateRe
 		MemberPrincipalIdsToAdd: members,
 	})
 
-	_, err = r.client.Groups.GroupsServiceUpdateGroupMembers(updateParams, nil)
+	_, err = clients.UpdateGroupMembersRetry(r.client, updateParams)
 	if err != nil {
 		resp.Diagnostics.AddError("Failed to update group members", err.Error())
 		return
@@ -187,7 +187,7 @@ func (r *resourceGroupMembers) Update(ctx context.Context, req resource.UpdateRe
 			MemberPrincipalIdsToRemove: membersToRemove,
 		})
 
-		_, err := r.client.Groups.GroupsServiceUpdateGroupMembers(updateParams, nil)
+		_, err := clients.UpdateGroupMembersRetry(r.client, updateParams)
 		if err != nil {
 			resp.Diagnostics.AddError("Failed to update group members", err.Error())
 			return
@@ -216,7 +216,7 @@ func (r *resourceGroupMembers) Delete(ctx context.Context, req resource.DeleteRe
 		MemberPrincipalIdsToRemove: members,
 	})
 
-	_, err := r.client.Groups.GroupsServiceUpdateGroupMembers(updateParams, nil)
+	_, err := clients.UpdateGroupMembersRetry(r.client, updateParams)
 	if err != nil {
 		var errResp *groups_service.GroupsServiceUpdateGroupMembersDefault
 		if errors.As(err, &errResp) && !errResp.IsCode(http.StatusNotFound) {