[Asset Inventory][Azure] Add storage asset fetchers (#2938)

(cherry picked from commit 57ecd03) # Conflicts: # internal/inventory/ASSETS.md # internal/inventory/asset.go # internal/inventory/cloud_assets.xlsx # tests/product/tests/data/azure_asset_inventory/test_cases.py # tests/product/tests/test_aws_asset_inventory.py # tests/product/tests/test_azure_asset_inventory.py # tests/product/tests/test_gcp_asset_inventory.py
elastic · Jan 29, 2025 · 181c560 · 181c560
1 parent d356a4e
commit 181c560
Show file tree

Hide file tree

Showing 15 changed files with 1,697 additions and 2 deletions.
diff --git a/internal/inventory/ASSETS.md b/internal/inventory/ASSETS.md
@@ -98,6 +98,7 @@ Infrastructure: 24% (18/73)
 
 ## AZURE Resources
 
+<<<<<<< HEAD
 **Progress: 31% (16/51)**
 Identity: 12% (1/8)
 Infrastructure: 34% (15/43)
@@ -157,6 +158,82 @@ Infrastructure: 34% (15/43)
 | Infrastructure | Storage | Object Storage | Azure Storage Blob Service | Yes ✅ |
 | Infrastructure | Storage | Snapshot | Azure Snapshot | Yes ✅ |
 | Infrastructure | Storage | Storage | Azure Storage Account | Yes ✅ |
+=======
+**Progress: 42% (23/54)**
+Access Management: 100% (3/3)
+Container Registry: 100% (1/1)
+Database: 100% (4/4)
+File System Service: 100% (2/2)
+Host: 100% (1/1)
+Identity: 9% (1/11)
+Infrastructure: 8% (2/23)
+Messaging Service: 100% (2/2)
+Private Endpoint: 100% (1/1)
+Service Usage Technology: 100% (2/2)
+Snapshot: 100% (1/1)
+Storage Bucket: 100% (1/1)
+Volume: 100% (1/1)
+Web Service: 100% (1/1)
+
+<details> <summary>Full table</summary>
+
+| Category | Old Type | Type | Implemented? |
+|---|---|---|---|
+| Access Management | Azure Resource Group | Azure Resource Group | Yes ✅ |
+| Access Management | Azure Subscription | Azure Subscription | Yes ✅ |
+| Access Management | Azure Tenant | Azure Tenant | Yes ✅ |
+| Container Registry | Azure Container Registry | Azure Container Registry | Yes ✅ |
+| Database | Azure Elastic Pool | Azure Elastic Pool | Yes ✅ |
+| Database | Azure SQL Database | Azure SQL Database | Yes ✅ |
+| Database | Azure SQL Server | Azure SQL Server | Yes ✅ |
+| Database | Azure Storage Table |  | Yes ✅ |
+| File System Service | Azure Storage File Service |  | Yes ✅ |
+| File System Service | Azure Storage File Share |  | Yes ✅ |
+| Host | Azure Virtual Machine | Azure Virtual Machine | Yes ✅ |
+| Identity | Access Key |  | No ❌ |
+| Identity | API Gateway Client Certificate |  | No ❌ |
+| Identity | Azure AD Application |  | No ❌ |
+| Identity | Azure AD Group |  | No ❌ |
+| Identity | Azure AD Service Principal |  | No ❌ |
+| Identity | Azure AD User |  | No ❌ |
+| Identity | Azure Principal | Azure Principal | Yes ✅ |
+| Identity | Azure Role |  | No ❌ |
+| Identity | Azure Role Assignment |  | No ❌ |
+| Identity | Azure Server AD Administrator |  | No ❌ |
+| Identity | EC2 Key Pair |  | No ❌ |
+| Infrastructure | Azure Cosmos DB Account | Azure Cosmos DB Account | Yes ✅ |
+| Infrastructure | Azure Cosmos DB Account Failover Policy |  | No ❌ |
+| Infrastructure | Azure Cosmos DB Cassandra Keyspace |  | No ❌ |
+| Infrastructure | Azure Cosmos DB Cassandra Table |  | No ❌ |
+| Infrastructure | Azure Cosmos DB CORS Policy |  | No ❌ |
+| Infrastructure | Azure Cosmos DB Location |  | No ❌ |
+| Infrastructure | Azure Cosmos DB MongoDB Collection |  | No ❌ |
+| Infrastructure | Azure Cosmos DB MongoDB Database |  | No ❌ |
+| Infrastructure | Azure Cosmos DB Private Endpoint Connection |  | No ❌ |
+| Infrastructure | Azure Cosmos DB SQL Container |  | No ❌ |
+| Infrastructure | Azure Cosmos DB SQL Database | Azure Cosmos DB SQL Database | Yes ✅ |
+| Infrastructure | Azure Cosmos DB Table Resource |  | No ❌ |
+| Infrastructure | Azure Cosmos DB Virtual Network Rule |  | No ❌ |
+| Infrastructure | Azure Data Disk |  | No ❌ |
+| Infrastructure | Azure Database Threat Detection Policy |  | No ❌ |
+| Infrastructure | Azure Failover Group |  | No ❌ |
+| Infrastructure | Azure Function |  | No ❌ |
+| Infrastructure | Azure Recoverable Database |  | No ❌ |
+| Infrastructure | Azure Replication Link |  | No ❌ |
+| Infrastructure | Azure Restorable Dropped Database |  | No ❌ |
+| Infrastructure | Azure Restore Point |  | No ❌ |
+| Infrastructure | Azure Server DNS Alias |  | No ❌ |
+| Infrastructure | Azure Transparent Data Encryption |  | No ❌ |
+| Messaging Service | Azure Storage Queue | Azure Storage Queue | Yes ✅ |
+| Messaging Service | Azure Storage Queue Service | Azure Storage Queue Service | Yes ✅ |
+| Private Endpoint | Azure Storage Account | Azure Storage Account | Yes ✅ |
+| Service Usage Technology | Azure Storage Blob Service | Azure Storage Blob Service | Yes ✅ |
+| Service Usage Technology | Azure Storage Table Service |  | Yes ✅ |
+| Snapshot | Azure Snapshot | Azure Snapshot | Yes ✅ |
+| Storage Bucket | Azure Storage Blob Container |  | Yes ✅ |
+| Volume | Azure Disk | Azure Disk | Yes ✅ |
+| Web Service | Azure App Service | Azure App Service | Yes ✅ |
+>>>>>>> 57ecd035 ([Asset Inventory][Azure] Add storage asset fetchers (#2938))
 
 </details>
 

diff --git a/internal/inventory/asset.go b/internal/inventory/asset.go
@@ -21,8 +21,36 @@ package inventory
 type AssetCategory string
 
 const (
+<<<<<<< HEAD
 	CategoryIdentity       AssetCategory = "identity"
 	CategoryInfrastructure AssetCategory = "infrastructure"
+=======
+	CategoryAccessManagement       AssetCategory = "Access Management"
+	CategoryAccount                AssetCategory = "Account"
+	CategoryContainerRegistry      AssetCategory = "Container Registry"
+	CategoryContainerService       AssetCategory = "Container Service"
+	CategoryDatabase               AssetCategory = "Database"
+	CategoryFaaS                   AssetCategory = "FaaS"
+	CategoryFileSystemService      AssetCategory = "File System Service"
+	CategoryFirewall               AssetCategory = "Firewall"
+	CategoryGateway                AssetCategory = "Gateway"
+	CategoryHost                   AssetCategory = "Host"
+	CategoryIdentity               AssetCategory = "Identity"
+	CategoryInfrastructure         AssetCategory = "Infrastructure"
+	CategoryLoadBalancer           AssetCategory = "Load Balancer"
+	CategoryMessagingService       AssetCategory = "Messaging Service"
+	CategoryNetworking             AssetCategory = "Networking"
+	CategoryOrchestrator           AssetCategory = "Orchestrator"
+	CategoryOrganization           AssetCategory = "Organization"
+	CategoryPrivateEndpoint        AssetCategory = "Private Endpoint"
+	CategoryServiceAccount         AssetCategory = "Service Account"
+	CategoryServiceUsageTechnology AssetCategory = "Service Usage Technology"
+	CategorySnapshot               AssetCategory = "Snapshot"
+	CategoryStorageBucket          AssetCategory = "Storage Bucket"
+	CategorySubnet                 AssetCategory = "Subnet"
+	CategoryVolume                 AssetCategory = "Volume"
+	CategoryWebService             AssetCategory = "Web Service"
+>>>>>>> 57ecd035 ([Asset Inventory][Azure] Add storage asset fetchers (#2938))
 )
 
 // AssetSubCategory is used to build the document index. Use only numbers, letters and dashes (-)
@@ -189,6 +217,7 @@ var (
 	AssetClassificationAwsS3Bucket                 = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryStorage, Type: TypeObjectStorage, SubType: SubTypeS3}
 	AssetClassificationAwsSnsTopic                 = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryMessaging, Type: TypeNotificationService, SubType: SubTypeSNSTopic}
 	// Azure
+<<<<<<< HEAD
 	AssetClassificationAzureAppService          = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryApplication, Type: TypeWebApplication, SubType: SubTypeAzureAppService}
 	AssetClassificationAzureContainerRegistry   = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryContainer, Type: TypeRegistry, SubType: SubTypeAzureContainerRegistry}
 	AssetClassificationAzureCosmosDBAccount     = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryDatabase, Type: TypeNoSQLDatabase, SubType: SubTypeAzureCosmosDBAccount}
@@ -207,6 +236,31 @@ var (
 	AssetClassificationAzureSubscription        = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeAzureSubscription}
 	AssetClassificationAzureTenant              = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeAzureTenant}
 	AssetClassificationAzureVirtualMachine      = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryCompute, Type: TypeVirtualMachine, SubType: SubTypeAzureVirtualMachine}
+=======
+	AssetClassificationAzureAppService           = AssetClassification{CategoryWebService, "Azure App Service"}
+	AssetClassificationAzureContainerRegistry    = AssetClassification{CategoryContainerRegistry, "Azure Container Registry"}
+	AssetClassificationAzureCosmosDBAccount      = AssetClassification{CategoryInfrastructure, "Azure Cosmos DB Account"}
+	AssetClassificationAzureCosmosDBSQLDatabase  = AssetClassification{CategoryInfrastructure, "Azure Cosmos DB SQL Database"}
+	AssetClassificationAzureDisk                 = AssetClassification{CategoryVolume, "Azure Disk"}
+	AssetClassificationAzureElasticPool          = AssetClassification{CategoryDatabase, "Azure Elastic Pool"}
+	AssetClassificationAzureResourceGroup        = AssetClassification{CategoryAccessManagement, "Azure Resource Group"}
+	AssetClassificationAzureSQLDatabase          = AssetClassification{CategoryDatabase, "Azure SQL Database"}
+	AssetClassificationAzureSQLServer            = AssetClassification{CategoryDatabase, "Azure SQL Server"}
+	AssetClassificationAzureServicePrincipal     = AssetClassification{CategoryIdentity, "Azure Principal"}
+	AssetClassificationAzureSnapshot             = AssetClassification{CategorySnapshot, "Azure Snapshot"}
+	AssetClassificationAzureStorageAccount       = AssetClassification{CategoryPrivateEndpoint, "Azure Storage Account"}
+	AssetClassificationAzureStorageBlobContainer = AssetClassification{CategoryStorageBucket, "Azure Storage Blob Container"}
+	AssetClassificationAzureStorageBlobService   = AssetClassification{CategoryServiceUsageTechnology, "Azure Storage Blob Service"}
+	AssetClassificationAzureStorageFileService   = AssetClassification{CategoryFileSystemService, "Azure Storage File Service"}
+	AssetClassificationAzureStorageFileShare     = AssetClassification{CategoryFileSystemService, "Azure Storage File Share"}
+	AssetClassificationAzureStorageQueue         = AssetClassification{CategoryMessagingService, "Azure Storage Queue"}
+	AssetClassificationAzureStorageQueueService  = AssetClassification{CategoryMessagingService, "Azure Storage Queue Service"}
+	AssetClassificationAzureStorageTable         = AssetClassification{CategoryDatabase, "Azure Storage Table"}
+	AssetClassificationAzureStorageTableService  = AssetClassification{CategoryServiceUsageTechnology, "Azure Storage Table Service"}
+	AssetClassificationAzureSubscription         = AssetClassification{CategoryAccessManagement, "Azure Subscription"}
+	AssetClassificationAzureTenant               = AssetClassification{CategoryAccessManagement, "Azure Tenant"}
+	AssetClassificationAzureVirtualMachine       = AssetClassification{CategoryHost, "Azure Virtual Machine"}
+>>>>>>> 57ecd035 ([Asset Inventory][Azure] Add storage asset fetchers (#2938))
 
 	// GCP
 	AssetClassificationGcpProject           = AssetClassification{Category: CategoryInfrastructure, SubCategory: SubCategoryManagement, Type: TypeCloudAccount, SubType: SubTypeGcpProject}

diff --git a/internal/inventory/azurefetcher/fetcher_storage.go b/internal/inventory/azurefetcher/fetcher_storage.go
@@ -35,9 +35,14 @@ type (
 	storageProviderFunc func(context.Context, []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
 	storageProvider     interface {
 		ListSubscriptions(ctx context.Context) ([]azurelib.AzureAsset, error)
+		ListStorageAccountBlobContainers(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
 		ListStorageAccountBlobServices(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
+		ListStorageAccountFileServices(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
+		ListStorageAccountFileShares(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
 		ListStorageAccountQueues(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
 		ListStorageAccountQueueServices(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
+		ListStorageAccountTables(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
+		ListStorageAccountTableServices(ctx context.Context, storageAccounts []azurelib.AzureAsset) ([]azurelib.AzureAsset, error)
 		ListStorageAccounts(ctx context.Context, storageAccountsSubscriptionsIds []string) ([]azurelib.AzureAsset, error)
 	}
 )
@@ -55,9 +60,14 @@ func (f *storageFetcher) Fetch(ctx context.Context, assetChan chan<- inventory.A
 		function       storageProviderFunc
 		classification inventory.AssetClassification
 	}{
+		{"Storage Blob Containers", f.provider.ListStorageAccountBlobContainers, inventory.AssetClassificationAzureStorageBlobContainer},
 		{"Storage Blob Services", f.provider.ListStorageAccountBlobServices, inventory.AssetClassificationAzureStorageBlobService},
+		{"Storage File Services", f.provider.ListStorageAccountFileServices, inventory.AssetClassificationAzureStorageFileService},
+		{"Storage File Shares", f.provider.ListStorageAccountFileShares, inventory.AssetClassificationAzureStorageFileShare},
 		{"Storage Queue Services", f.provider.ListStorageAccountQueueServices, inventory.AssetClassificationAzureStorageQueueService},
 		{"Storage Queues", f.provider.ListStorageAccountQueues, inventory.AssetClassificationAzureStorageQueue},
+		{"Storage Tables", f.provider.ListStorageAccountTables, inventory.AssetClassificationAzureStorageTable},
+		{"Storage Table Services", f.provider.ListStorageAccountTableServices, inventory.AssetClassificationAzureStorageTableService},
 	}
 
 	storageAccounts, err := f.listStorageAccounts(ctx)

diff --git a/internal/inventory/azurefetcher/fetcher_storage_test.go b/internal/inventory/azurefetcher/fetcher_storage_test.go
@@ -36,11 +36,26 @@ func TestStorageFetcher_Fetch(t *testing.T) {
 		Id:   "storage_account",
 		Name: "storage_account",
 	}
+	azureBlobContainer := azurelib_inventory.AzureAsset{
+		Id:          "blob_container",
+		Name:        "blob_container",
+		DisplayName: "blob_container",
+	}
 	azureBlobService := azurelib_inventory.AzureAsset{
 		Id:          "blob_service",
 		Name:        "blob_service",
 		DisplayName: "blob_service",
 	}
+	azureFileService := azurelib_inventory.AzureAsset{
+		Id:          "file_service",
+		Name:        "file_service",
+		DisplayName: "file_service",
+	}
+	azureFileShare := azurelib_inventory.AzureAsset{
+		Id:          "file_share",
+		Name:        "file_share",
+		DisplayName: "file_share",
+	}
 	azureQueueService := azurelib_inventory.AzureAsset{
 		Id:          "queue_service",
 		Name:        "queue_service",
@@ -51,8 +66,28 @@ func TestStorageFetcher_Fetch(t *testing.T) {
 		Name:        "queue",
 		DisplayName: "queue",
 	}
+	azureTable := azurelib_inventory.AzureAsset{
+		Id:          "table",
+		Name:        "table",
+		DisplayName: "table",
+	}
+	azureTableService := azurelib_inventory.AzureAsset{
+		Id:          "table_service",
+		Name:        "table_service",
+		DisplayName: "table_service",
+	}
 
 	expected := []inventory.AssetEvent{
+		inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureStorageBlobContainer,
+			azureBlobContainer.Id,
+			azureBlobContainer.Name,
+			inventory.WithRawAsset(azureBlobContainer),
+			inventory.WithCloud(inventory.Cloud{
+				Provider:    inventory.AzureCloudProvider,
+				ServiceName: "Azure",
+			}),
+		),
 		inventory.NewAssetEvent(
 			inventory.AssetClassificationAzureStorageBlobService,
 			[]string{azureBlobService.Id},
@@ -65,6 +100,26 @@ func TestStorageFetcher_Fetch(t *testing.T) {
 				},
 			}),
 		),
+		inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureStorageFileService,
+			azureFileService.Id,
+			azureFileService.Name,
+			inventory.WithRawAsset(azureFileService),
+			inventory.WithCloud(inventory.Cloud{
+				Provider:    inventory.AzureCloudProvider,
+				ServiceName: "Azure",
+			}),
+		),
+		inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureStorageFileShare,
+			azureFileShare.Id,
+			azureFileShare.Name,
+			inventory.WithRawAsset(azureFileShare),
+			inventory.WithCloud(inventory.Cloud{
+				Provider:    inventory.AzureCloudProvider,
+				ServiceName: "Azure",
+			}),
+		),
 		inventory.NewAssetEvent(
 			inventory.AssetClassificationAzureStorageQueueService,
 			[]string{azureQueueService.Id},
@@ -89,6 +144,26 @@ func TestStorageFetcher_Fetch(t *testing.T) {
 				},
 			}),
 		),
+		inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureStorageTable,
+			azureTable.Id,
+			azureTable.Name,
+			inventory.WithRawAsset(azureTable),
+			inventory.WithCloud(inventory.Cloud{
+				Provider:    inventory.AzureCloudProvider,
+				ServiceName: "Azure",
+			}),
+		),
+		inventory.NewAssetEvent(
+			inventory.AssetClassificationAzureStorageTableService,
+			azureTableService.Id,
+			azureTableService.Name,
+			inventory.WithRawAsset(azureTableService),
+			inventory.WithCloud(inventory.Cloud{
+				Provider:    inventory.AzureCloudProvider,
+				ServiceName: "Azure",
+			}),
+		),
 	}
 
 	// setup
@@ -107,12 +182,30 @@ func TestStorageFetcher_Fetch(t *testing.T) {
 		[]azurelib_inventory.AzureAsset{storageAccount}, nil,
 	)
 
+	provider.EXPECT().ListStorageAccountBlobContainers(
+		mock.Anything, mock.Anything,
+	).Return(
+		[]azurelib_inventory.AzureAsset{azureBlobContainer}, nil,
+	)
+
 	provider.EXPECT().ListStorageAccountBlobServices(
 		mock.Anything, mock.Anything,
 	).Return(
 		[]azurelib_inventory.AzureAsset{azureBlobService}, nil,
 	)
 
+	provider.EXPECT().ListStorageAccountFileServices(
+		mock.Anything, mock.Anything,
+	).Return(
+		[]azurelib_inventory.AzureAsset{azureFileService}, nil,
+	)
+
+	provider.EXPECT().ListStorageAccountFileShares(
+		mock.Anything, mock.Anything,
+	).Return(
+		[]azurelib_inventory.AzureAsset{azureFileShare}, nil,
+	)
+
 	provider.EXPECT().ListStorageAccountQueueServices(
 		mock.Anything, mock.Anything,
 	).Return(
@@ -125,6 +218,18 @@ func TestStorageFetcher_Fetch(t *testing.T) {
 		[]azurelib_inventory.AzureAsset{azureQueue}, nil,
 	)
 
+	provider.EXPECT().ListStorageAccountTables(
+		mock.Anything, mock.Anything,
+	).Return(
+		[]azurelib_inventory.AzureAsset{azureTable}, nil,
+	)
+
+	provider.EXPECT().ListStorageAccountTableServices(
+		mock.Anything, mock.Anything,
+	).Return(
+		[]azurelib_inventory.AzureAsset{azureTableService}, nil,
+	)
+
 	fetcher := newStorageFetcher(logger, provider)
 	// test & compare
 	testutil.CollectResourcesAndMatch(t, fetcher, expected)