wip

internal/resources/providers/gcplib/inventory/grpc_rate_limiter.go

@@ -43,9 +43,9 @@ type AssetsInventoryRateLimiter struct {
 
 // https://cloud.google.com/asset-inventory/docs/quota
 var methods = map[string]*rate.Limiter{
-	// In both 'single-account' and 'organization-account' cases, we always need to pace by project because of the consumer project (quota_project_id)
-	// Which is the one effectively consuming the quota
-	// the organization quota would be relevant if we manually send multiple requests with diff quota_project_id, which we don't do
+	// For both single and organization accounts, we pace by project quota tied to the consumer project ID, which consumes the quota.
+	// We don't manually send requests with different quota project IDs, so the organization quota isn't relevant.
+	// Using per-project quota suffices for both cases since it's more restrictive than per-organization quota.
 	"/google.cloud.asset.v1.AssetService/ListAssets": rate.NewLimiter(rate.Every(time.Minute/100), 1),
 }
 
@@ -64,7 +64,6 @@ func (rl *AssetsInventoryRateLimiter) Wait(ctx context.Context, method string) {
 			rl.log.Errorf("Failed to wait for project quota on method %s, error: %w", method, err)
 		}
 	}
-
 }
 
 func (rl *AssetsInventoryRateLimiter) GetInterceptorDialOption() grpc.DialOption {

internal/resources/providers/gcplib/inventory/grpc_rate_limiter_test.go

@@ -47,20 +47,17 @@ func (s *RateLimiterTestSuite) TestRateLimiterWait() {
 	ctx := context.Background()
 	duration := time.Millisecond
 	s.rateLimiter.methods = map[string]*rate.Limiter{
-		"someMethod": rate.NewLimiter(rate.Every(duration/1), 1), // 1 request per second
+		"someMethod": rate.NewLimiter(rate.Every(duration/1), 1), // 1 request per duration
 	}
 
 	totalRequests := 5
 	startTime := time.Now()
-
 	for i := 0; i < totalRequests; i++ {
 		s.rateLimiter.Wait(ctx, "someMethod")
 	}
-
 	endTime := time.Now()
+
 	actualDuration := endTime.Sub(startTime)
-	// expected duration is (totalRequests-1) duration
-	// 1st request goes instantly, 2nd and above wait 1duration each
-	expectedDuration := duration * time.Duration((totalRequests - 1))
+	expectedDuration := duration * time.Duration((totalRequests - 1)) // expected duration is (totalRequests-1) duration. 1st request is instant, 2nd and above wait 1duration each
 	s.Assert().True(actualDuration >= expectedDuration, fmt.Sprintf("expected %v to be greater or equal than %v", actualDuration, expectedDuration))
 }

internal/resources/providers/gcplib/inventory/provider.go

@@ -217,6 +217,7 @@ func (p *Provider) ListAllAssetTypesByName(ctx context.Context, assetTypes []str
 	// Enrich network assets with dns policy
 	p.enrichNetworkAssets(ctx, extendedAssets)
 
+	p.log.Infof("Listed %d assets for asset types: %v", len(extendedAssets), assetTypes)
 	return extendedAssets, nil
 }
 
@@ -323,7 +324,7 @@ func (p *Provider) enrichNetworkAssets(ctx context.Context, assets []*ExtendedGc
 		p.log.Infof("no %s assets were listed, return original assets", DnsPolicyAssetType)
 		return
 	}
-
+	p.log.Infof("Listed %d %s assets", len(dnsPolicyAssets), DnsPolicyAssetType)
 	dnsPolicies := decodeDnsPolicies(dnsPolicyAssets)
 
 	p.log.Infof("attempting to enrich %d %s assets with dns policy", len(assets), ComputeNetworkAssetType)
@@ -476,7 +477,7 @@ func (p *Provider) ListProjectsAncestorsPolicies(ctx context.Context) ([]*Projec
 		Parent:      p.config.Parent,
 		AssetTypes:  []string{CrmProjectAssetType},
 	}))
-
+	p.log.Infof("Listed %d project policies", len(projects))
 	return lo.Map(projects, func(project *assetpb.Asset, _ int) *ProjectPoliciesAsset {
 		projectAsset := extendWithECS(ctx, p.crm, p.crmCache, []*assetpb.Asset{project})[0]
 		// Skip first ancestor it as we already got it
@@ -500,6 +501,7 @@ func getAncestorsAssets(ctx context.Context, p *Provider, ancestors []string) []
 			Parent:      parent,
 			AssetTypes:  []string{assetType},
 		}))
+		p.log.Infof("Listed %d ancestor policies", len(assets))
 		return extendWithECS(ctx, p.crm, p.crmCache, assets)
 	}))
 }