diff --git a/internal/resources/providers/gcplib/inventory/grpc_rate_limiter.go b/internal/resources/providers/gcplib/inventory/grpc_rate_limiter.go index 6f727ac7b7..f310b282ec 100644 --- a/internal/resources/providers/gcplib/inventory/grpc_rate_limiter.go +++ b/internal/resources/providers/gcplib/inventory/grpc_rate_limiter.go @@ -43,9 +43,9 @@ type AssetsInventoryRateLimiter struct { // https://cloud.google.com/asset-inventory/docs/quota var methods = map[string]*rate.Limiter{ - // In both 'single-account' and 'organization-account' cases, we always need to pace by project because of the consumer project (quota_project_id) - // Which is the one effectively consuming the quota - // the organization quota would be relevant if we manually send multiple requests with diff quota_project_id, which we don't do + // For both single and organization accounts, we pace by project quota tied to the consumer project ID, which consumes the quota. + // We don't manually send requests with different quota project IDs, so the organization quota isn't relevant. + // Using per-project quota suffices for both cases since it's more restrictive than per-organization quota. "/google.cloud.asset.v1.AssetService/ListAssets": rate.NewLimiter(rate.Every(time.Minute/100), 1), } @@ -64,7 +64,6 @@ func (rl *AssetsInventoryRateLimiter) Wait(ctx context.Context, method string) { rl.log.Errorf("Failed to wait for project quota on method %s, error: %w", method, err) } } - } func (rl *AssetsInventoryRateLimiter) GetInterceptorDialOption() grpc.DialOption { diff --git a/internal/resources/providers/gcplib/inventory/grpc_rate_limiter_test.go b/internal/resources/providers/gcplib/inventory/grpc_rate_limiter_test.go index e473cb50c7..360de69e28 100644 --- a/internal/resources/providers/gcplib/inventory/grpc_rate_limiter_test.go +++ b/internal/resources/providers/gcplib/inventory/grpc_rate_limiter_test.go @@ -47,20 +47,17 @@ func (s *RateLimiterTestSuite) TestRateLimiterWait() { ctx := context.Background() duration := time.Millisecond s.rateLimiter.methods = map[string]*rate.Limiter{ - "someMethod": rate.NewLimiter(rate.Every(duration/1), 1), // 1 request per second + "someMethod": rate.NewLimiter(rate.Every(duration/1), 1), // 1 request per duration } totalRequests := 5 startTime := time.Now() - for i := 0; i < totalRequests; i++ { s.rateLimiter.Wait(ctx, "someMethod") } - endTime := time.Now() + actualDuration := endTime.Sub(startTime) - // expected duration is (totalRequests-1) duration - // 1st request goes instantly, 2nd and above wait 1duration each - expectedDuration := duration * time.Duration((totalRequests - 1)) + expectedDuration := duration * time.Duration((totalRequests - 1)) // expected duration is (totalRequests-1) duration. 1st request is instant, 2nd and above wait 1duration each s.Assert().True(actualDuration >= expectedDuration, fmt.Sprintf("expected %v to be greater or equal than %v", actualDuration, expectedDuration)) } diff --git a/internal/resources/providers/gcplib/inventory/provider.go b/internal/resources/providers/gcplib/inventory/provider.go index 582cfd3cde..10947825ae 100644 --- a/internal/resources/providers/gcplib/inventory/provider.go +++ b/internal/resources/providers/gcplib/inventory/provider.go @@ -217,6 +217,7 @@ func (p *Provider) ListAllAssetTypesByName(ctx context.Context, assetTypes []str // Enrich network assets with dns policy p.enrichNetworkAssets(ctx, extendedAssets) + p.log.Infof("Listed %d assets for asset types: %v", len(extendedAssets), assetTypes) return extendedAssets, nil } @@ -323,7 +324,7 @@ func (p *Provider) enrichNetworkAssets(ctx context.Context, assets []*ExtendedGc p.log.Infof("no %s assets were listed, return original assets", DnsPolicyAssetType) return } - + p.log.Infof("Listed %d %s assets", len(dnsPolicyAssets), DnsPolicyAssetType) dnsPolicies := decodeDnsPolicies(dnsPolicyAssets) p.log.Infof("attempting to enrich %d %s assets with dns policy", len(assets), ComputeNetworkAssetType) @@ -476,7 +477,7 @@ func (p *Provider) ListProjectsAncestorsPolicies(ctx context.Context) ([]*Projec Parent: p.config.Parent, AssetTypes: []string{CrmProjectAssetType}, })) - + p.log.Infof("Listed %d project policies", len(projects)) return lo.Map(projects, func(project *assetpb.Asset, _ int) *ProjectPoliciesAsset { projectAsset := extendWithECS(ctx, p.crm, p.crmCache, []*assetpb.Asset{project})[0] // Skip first ancestor it as we already got it @@ -500,6 +501,7 @@ func getAncestorsAssets(ctx context.Context, p *Provider, ancestors []string) [] Parent: parent, AssetTypes: []string{assetType}, })) + p.log.Infof("Listed %d ancestor policies", len(assets)) return extendWithECS(ctx, p.crm, p.crmCache, assets) })) }