Merge pull request #81 from BloodHoundAD/BED-4599-configurable-collec…

…tion-values

BED-4599 configurable collection values

client/rest/http.go

@@ -29,13 +29,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/constants"
 )
 
 func NewHTTPClient(proxyUrl string) (*http.Client, error) {
 	transport := http.DefaultTransport.(*http.Transport).Clone()
-	transport.MaxConnsPerHost = 200
-	transport.MaxIdleConnsPerHost = 200
+	transport.MaxConnsPerHost = config.ColMaxConnsPerHost.Value().(int)
+	transport.MaxIdleConnsPerHost = config.ColMaxIdleConnsPerHost.Value().(int)
 	transport.DisableKeepAlives = false
 
 	// defaults to TLS 1.0 which is not favorable

cmd/list-app-owners.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/bloodhoundad/azurehound/v2/client"
 	"github.com/bloodhoundad/azurehound/v2/client/query"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -62,7 +63,7 @@ func listAppOwnersCmdImpl(cmd *cobra.Command, args []string) {
 func listAppOwners(ctx context.Context, client client.AzureClient, apps <-chan azureWrapper[models.App]) <-chan azureWrapper[models.AppOwners] {
 	var (
 		out     = make(chan azureWrapper[models.AppOwners])
-		streams = pipeline.Demux(ctx.Done(), apps, 25)
+		streams = pipeline.Demux(ctx.Done(), apps, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 		params  = query.GraphParams{}
 	)

cmd/list-app-role-assignments.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/bloodhoundad/azurehound/v2/client"
 	"github.com/bloodhoundad/azurehound/v2/client/query"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -65,7 +66,7 @@ func listAppRoleAssignments(ctx context.Context, client client.AzureClient, serv
 	var (
 		out         = make(chan interface{})
 		filteredSPs = make(chan models.ServicePrincipal)
-		streams     = pipeline.Demux(ctx.Done(), filteredSPs, 25)
+		streams     = pipeline.Demux(ctx.Done(), filteredSPs, config.ColStreamCount.Value().(int))
 		wg          sync.WaitGroup
 	)
 

cmd/list-automation-account-role-assignments.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -65,7 +66,7 @@ func listAutomationAccountRoleAssignments(ctx context.Context, client client.Azu
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-automation-accounts.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -63,7 +64,7 @@ func listAutomationAccounts(ctx context.Context, client client.AzureClient, subs
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-container-registries.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -68,7 +69,7 @@ func listContainerRegistries(ctx context.Context, client client.AzureClient, sub
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-container-registry-role-assignments.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -70,7 +71,7 @@ func listContainerRegistryRoleAssignments(ctx context.Context, client client.Azu
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-device-owners.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/bloodhoundad/azurehound/v2/client"
 	"github.com/bloodhoundad/azurehound/v2/client/query"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -64,7 +65,7 @@ func listDeviceOwners(ctx context.Context, client client.AzureClient, devices <-
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-function-app-role-assignments.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -65,7 +66,7 @@ func listFunctionAppRoleAssignments(ctx context.Context, client client.AzureClie
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-function-apps.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -63,7 +64,7 @@ func listFunctionApps(ctx context.Context, client client.AzureClient, subscripti
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-group-members.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/bloodhoundad/azurehound/v2/client"
 	"github.com/bloodhoundad/azurehound/v2/client/query"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -66,7 +67,7 @@ func listGroupMembers(ctx context.Context, client client.AzureClient, groups <-c
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 		params  = query.GraphParams{
 			Select: unique(listGroupMembersSelect),

cmd/list-group-owners.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/bloodhoundad/azurehound/v2/client"
 	"github.com/bloodhoundad/azurehound/v2/client/query"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -63,7 +64,7 @@ func listGroupOwners(ctx context.Context, client client.AzureClient, groups <-ch
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 		params  = query.GraphParams{}
 	)

cmd/list-key-vault-role-assignments.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -64,7 +65,7 @@ func listKeyVaultRoleAssignments(ctx context.Context, client client.AzureClient,
 	var (
 		out     = make(chan azureWrapper[models.KeyVaultRoleAssignments])
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-key-vaults.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/bloodhoundad/azurehound/v2/client"
 	"github.com/bloodhoundad/azurehound/v2/client/query"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -64,7 +65,7 @@ func listKeyVaults(ctx context.Context, client client.AzureClient, subscriptions
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-logic-app-role-assignments.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -70,7 +71,7 @@ func listLogicAppRoleAssignments(ctx context.Context, client client.AzureClient,
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-logic-apps.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -68,7 +69,7 @@ func listLogicApps(ctx context.Context, client client.AzureClient, subscriptions
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-managed-cluster-role-assignments.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -70,7 +71,7 @@ func listManagedClusterRoleAssignments(ctx context.Context, client client.AzureC
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-managed-clusters.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -68,7 +69,7 @@ func listManagedClusters(ctx context.Context, client client.AzureClient, subscri
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-management-group-descendants.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -63,7 +64,7 @@ func listManagementGroupDescendants(ctx context.Context, client client.AzureClie
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-management-group-role-assignments.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -64,7 +65,7 @@ func listManagementGroupRoleAssignments(ctx context.Context, client client.Azure
 	var (
 		out     = make(chan azureWrapper[models.ManagementGroupRoleAssignments])
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-resource-group-role-assignments.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/bloodhoundad/azurehound/v2/client"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -65,7 +66,7 @@ func listResourceGroupRoleAssignments(ctx context.Context, client client.AzureCl
 	var (
 		out     = make(chan azureWrapper[models.ResourceGroupRoleAssignments])
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 

cmd/list-resource-groups.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/bloodhoundad/azurehound/v2/client"
 	"github.com/bloodhoundad/azurehound/v2/client/query"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -64,7 +65,7 @@ func listResourceGroups(ctx context.Context, client client.AzureClient, subscrip
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)
 
@@ -98,7 +99,7 @@ func listResourceGroups(ctx context.Context, client client.AzureClient, subscrip
 					} else {
 						resourceGroup := models.ResourceGroup{
 							ResourceGroup:  item.Ok,
-							SubscriptionId: "/subscriptions/"+id,
+							SubscriptionId: "/subscriptions/" + id,
 							TenantId:       client.TenantInfo().TenantId,
 						}
 						log.V(2).Info("found resource group", "resourceGroup", resourceGroup)

cmd/list-role-assignments.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/bloodhoundad/azurehound/v2/client"
 	"github.com/bloodhoundad/azurehound/v2/client/query"
+	"github.com/bloodhoundad/azurehound/v2/config"
 	"github.com/bloodhoundad/azurehound/v2/enums"
 	"github.com/bloodhoundad/azurehound/v2/models"
 	"github.com/bloodhoundad/azurehound/v2/panicrecovery"
@@ -65,7 +66,7 @@ func listRoleAssignments(ctx context.Context, client client.AzureClient, roles <
 	var (
 		out     = make(chan interface{})
 		ids     = make(chan string)
-		streams = pipeline.Demux(ctx.Done(), ids, 25)
+		streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int))
 		wg      sync.WaitGroup
 	)