feat!: allow retrieving all billing accounts and add billing_information column to gcp_project and gcp_organization_project tables

Makefile

@@ -1,5 +1,5 @@
 STEAMPIPE_INSTALL_DIR ?= ~/.steampipe
+STEAMPIPE_PLUGIN_VERSION ?= latest
 BUILD_TAGS = netgo
-
 install:
-	go build -o $(STEAMPIPE_INSTALL_DIR)/plugins/hub.steampipe.io/plugins/turbot/gcp@latest/steampipe-plugin-gcp.plugin -tags "${BUILD_TAGS}" *.go
+	go build -o $(STEAMPIPE_INSTALL_DIR)/plugins/hub.steampipe.io/plugins/turbot/gcp@$(STEAMPIPE_PLUGIN_VERSION)/steampipe-plugin-gcp.plugin -tags "${BUILD_TAGS}" *.go

docs/index.md

@@ -56,9 +56,9 @@ steampipe plugin install gcp
 
 | Item        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| Credentials | When running locally, you must configure your [Application Default Credentials](https://cloud.google.com/sdk/gcloud/reference/auth/application-default). If you are running in Cloud Shell or Cloud Code, [the tool uses the credentials you provided when you logged in, and manages any authorizations required](https://cloud.google.com/docs/authentication/provide-credentials-adc#cloud-based-dev).                                                                      |
-| Permissions | Assign the `Viewer` role to your user or service account. You may also need additional permissions related to IAM policies, like `pubsub.subscriptions.getIamPolicy`, `pubsub.topics.getIamPolicy`, `storage.buckets.getIamPolicy`, since these are not included in the `Viewer` role. You can grant these by creating a custom role in your project.                                                                                                                          |
-| Radius      | Each connection represents a single GCP project.                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| Credentials | When running locally, you must configure your [Application Default Credentials](https://cloud.google.com/sdk/gcloud/reference/auth/application-default). If you are running in Cloud Shell or Cloud Code, [the tool uses the credentials you provided when you logged in, and manages any authorizations required](https://cloud.google.com/docs/authentication/provide-credentials-adc#cloud-based-dev). |
+| Permissions | Assign the `Viewer` role to your user or service account. You may also need additional permissions related to IAM policies, like `pubsub.subscriptions.getIamPolicy`, `pubsub.topics.getIamPolicy`, `storage.buckets.getIamPolicy`, since these are not included in the `Viewer` role. You can grant these by creating a custom role in your project. |
+| Radius      | Each connection represents a single GCP project, except for some tables like `gcp_organization` and `gcp_organization_project` which return all resources the credentials attached to the connection have access to. |
 | Resolution  | 1. Credentials from the JSON file specified by the `credentials` parameter in your steampipe config.<br />2. Credentials from the JSON file specified by the `GOOGLE_APPLICATION_CREDENTIALS` environment variable.<br />3. Credentials from the default JSON file location (~/.config/gcloud/application_default_credentials.json). <br />4. Credentials from [the metadata server](https://cloud.google.com/docs/authentication/application-default-credentials#attached-sa) |
 
 ### Configuration

docs/tables/gcp_organization_project.md

@@ -0,0 +1,83 @@
+---
+title: "Steampipe Table: gcp_organization_project - Query Google Cloud Platform Projects using SQL"
+description: "Allows users to query Projects in Google Cloud Platform, specifically providing details about the project's ID, name, labels, and lifecycle state."
+---
+
+# Table: gcp_organization_project - Query Google Cloud Platform Projects using SQL
+
+**Note: this table is a variant of the `gcp_project` table which does not filter on the GCP project attached to connection, and thus, will return all projects that the credentials used by the connection have access to. Using this table in aggregator connections can produce unexpected duplicate results.**
+
+A Google Cloud Platform Project acts as an organizational unit within GCP where resources are allocated. It is used to group resources that belong to the same logical application or business unit. Each project is linked to a billing account and can have users, roles, and permissions assigned to it.
+
+## Table Usage Guide
+
+The `gcp_organization_project` table provides insights into Projects within Google Cloud Platform. As a DevOps engineer, explore project-specific details through this table, including ID, name, labels, and lifecycle state. Utilize it to uncover information about projects, such as their associated resources, user roles, permissions, and billing details.
+
+## Examples
+
+### Basic info
+Explore which Google Cloud Platform projects are active, by looking at their lifecycle state and creation time. This can help you manage resources effectively and keep track of ongoing projects.
+
+```sql+postgres
+select
+  name,
+  project_id,
+  project_number,
+  lifecycle_state,
+  create_time
+from
+  gcp_organization_project;
+```
+
+```sql+sqlite
+select
+  name,
+  project_id,
+  project_number,
+  lifecycle_state,
+  create_time
+from
+  gcp_organization_project;
+```
+
+### Get access approval settings for all projects
+Explore the access approval settings across your various projects. This can help you understand and manage permissions and approvals more effectively.
+
+```sql+postgres
+select
+  name,
+  jsonb_pretty(access_approval_settings) as access_approval_settings
+from
+  gcp_organization_project;
+```
+
+```sql+sqlite
+select
+  name,
+  access_approval_settings
+from
+  gcp_organization_project;
+```
+
+### Get parent and organization ID for all projects
+Get the organization ID across your various projects.
+
+```sql+postgres
+select
+  project_id,
+  parent ->> 'id' as parent_id,
+  parent ->> 'type' as parent_type,
+  case when jsonb_array_length(ancestors) > 1 then ancestors -> -1 -> 'resourceId' ->> 'id' else null end as organization_id
+from
+  gcp_project;
+```
+
+```sql+sqlite
+select
+  project_id,
+  parent ->> 'id' as parent_id,
+  parent ->> 'type' as parent_type,
+  case when json_array_length(ancestors) > 1 then ancestors -> -1 -> 'resourceId' ->> 'id' else null end as organization_id
+from
+  gcp_project;
+```

docs/tables/gcp_project.md

@@ -5,6 +5,8 @@ description: "Allows users to query Projects in Google Cloud Platform, specifica
 
 # Table: gcp_project - Query Google Cloud Platform Projects using SQL
 
+**Note: this table is a variant of the `gcp_organization_project` table which filters on the GCP project attached to connection, and thus, will only ever return details about that specific project.**
+
 A Google Cloud Platform Project acts as an organizational unit within GCP where resources are allocated. It is used to group resources that belong to the same logical application or business unit. Each project is linked to a billing account and can have users, roles, and permissions assigned to it.
 
 ## Table Usage Guide

gcp/plugin.go

@@ -122,6 +122,7 @@ func Plugin(ctx context.Context) *plugin.Plugin {
 			"gcp_monitoring_group":                                    tableGcpMonitoringGroup(ctx),
 			"gcp_monitoring_notification_channel":                     tableGcpMonitoringNotificationChannel(ctx),
 			"gcp_organization":                                        tableGcpOrganization(ctx),
+			"gcp_organization_project":                                tableGcpOrganizationProject(ctx),
 			"gcp_project":                                             tableGcpProject(ctx),
 			"gcp_project_organization_policy":                         tableGcpProjectOrganizationPolicy(ctx),
 			"gcp_project_service":                                     tableGcpProjectService(ctx),

gcp/table_gcp_billing_account.go

@@ -3,6 +3,7 @@ package gcp
 import (
 	"context"
 
+	"github.com/turbot/go-kit/types"
 	"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform"
@@ -71,13 +72,6 @@ func tableGcpBillingAccount(_ context.Context) *plugin.Table {
 				Type:        proto.ColumnType_STRING,
 				Transform:   transform.FromConstant("global"),
 			},
-			{
-				Name:        "project",
-				Description: ColumnDescriptionProject,
-				Type:        proto.ColumnType_STRING,
-				Hydrate:     getProject,
-				Transform:   transform.FromValue(),
-			},
 		},
 	}
 }
@@ -86,49 +80,52 @@ func tableGcpBillingAccount(_ context.Context) *plugin.Table {
 
 func getBillingAccount(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
 
-	var accountName string
-
 	// Create Service Connection
 	service, err := BillingService(ctx, d)
 	if err != nil {
 		plugin.Logger(ctx).Error("gcp_billing_account.getBillingAccount", "service_err", err)
 		return nil, err
 	}
 
-	// Validate whether a user has provided any input, then skip the `GetBillingInfo` api
+	// If a user has provided a billing account name, get it instead of listing all accounts
 	if d.EqualsQualString("name") != "" {
-		accountName = "billingAccounts/" + d.EqualsQualString("name")
-	} else {
+		accountName := "billingAccounts/" + d.EqualsQualString("name")
 
-		// Fetch BillingInfo for the project, to get the billing account name
-		// Get project details
-
-		projectId, err := getProject(ctx, d, h)
+		resp, err := service.BillingAccounts.Get(accountName).Do()
 		if err != nil {
-			plugin.Logger(ctx).Error("gcp_billing_account.getBillingAccount", "cache_err", err)
+			plugin.Logger(ctx).Error("gcp_billing_account.getBillingAccount.get", "api_err", err)
 			return nil, err
 		}
-		project := projectId.(string)
 
-		resp, err := service.Projects.GetBillingInfo("projects/" + project).Do()
-		if err != nil {
-			plugin.Logger(ctx).Error("gcp_billing_account.getBillingAccount.GetBillingInfo", "api_err", err)
-			return nil, err
+		d.StreamListItem(ctx, resp)
+	} else {
+		// Max limit is set as per documentation
+		pageSize := types.Int64(100)
+		limit := d.QueryContext.Limit
+		if d.QueryContext.Limit != nil {
+			if *limit < *pageSize {
+				pageSize = limit
+			}
 		}
-
-		if resp != nil && resp.BillingAccountName != "" {
-			accountName = resp.BillingAccountName
+		resp := service.BillingAccounts.List().PageSize(*pageSize)
+		if err := resp.Pages(ctx, func(page *cloudbilling.ListBillingAccountsResponse) error {
+			for _, account := range page.BillingAccounts {
+				d.StreamListItem(ctx, account)
+
+				// Check if context has been cancelled or if the limit has been hit (if specified)
+				// if there is a limit, it will return the number of rows required to reach this limit
+				if d.RowsRemaining(ctx) == 0 {
+					page.NextPageToken = ""
+					return nil
+				}
+			}
+			return nil
+		}); err != nil {
+			plugin.Logger(ctx).Error("gcp_billing_account.getBillingAccount.list", "api_err", err)
+			return nil, err
 		}
 	}
 
-	accResponse, err := service.BillingAccounts.Get(accountName).Do()
-	if err != nil {
-		plugin.Logger(ctx).Error("gcp_billing_account.getBillingAccount", "api_err", err)
-		return nil, err
-	}
-
-	d.StreamListItem(ctx, accResponse)
-
 	return nil, nil
 }
 
@@ -152,17 +149,8 @@ func getBillingAccountIamPolicy(ctx context.Context, d *plugin.QueryData, h *plu
 }
 
 func getBillingAccountAka(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
-
-	// Get project details
-
-	projectId, err := getProject(ctx, d, h)
-	if err != nil {
-		return nil, err
-	}
-
-	project := projectId.(string)
 	acc := h.Item.(*cloudbilling.BillingAccount)
-	akas := []string{"gcp://cloudbilling.googleapis.com/projects/" + project + "/" + acc.Name}
+	akas := []string{"gcp://cloudbilling.googleapis.com/" + acc.Name}
 
 	return akas, nil
 }

gcp/table_gcp_organization_project.go

@@ -0,0 +1,149 @@
+package gcp
+
+import (
+	"context"
+
+	"github.com/turbot/go-kit/types"
+	"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
+	"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
+	"github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform"
+	"google.golang.org/api/cloudresourcemanager/v1"
+)
+
+//// TABLE DEFINITION
+
+func tableGcpOrganizationProject(_ context.Context) *plugin.Table {
+	return &plugin.Table{
+		Name:        "gcp_organization_project",
+		Description: "GCP Organization Project",
+		List: &plugin.ListConfig{
+			Hydrate: listGCPOrganizationProjects,
+		},
+		Columns: []*plugin.Column{
+			{
+				Name:        "name",
+				Description: "The name of the project.",
+				Type:        proto.ColumnType_STRING,
+			},
+			{
+				Name:        "project_id",
+				Description: "An unique, user-assigned ID of the Project.",
+				Type:        proto.ColumnType_STRING,
+			},
+			{
+				Name:        "self_link",
+				Description: "Server-defined URL for the resource.",
+				Type:        proto.ColumnType_STRING,
+				Transform:   transform.From(projectSelfLink),
+			},
+			{
+				Name:        "project_number",
+				Description: "The number uniquely identifying the project.",
+				Type:        proto.ColumnType_INT,
+			},
+			{
+				Name:        "lifecycle_state",
+				Description: "Specifies the project lifecycle state.",
+				Type:        proto.ColumnType_STRING,
+			},
+			{
+				Name:        "create_time",
+				Description: "Creation time of the project.",
+				Type:        proto.ColumnType_TIMESTAMP,
+			},
+			{
+				Name:        "parent",
+				Description: "An optional reference to a parent Resource.",
+				Type:        proto.ColumnType_JSON,
+			},
+			{
+				Name:        "labels",
+				Description: "A list of labels attached to this project.",
+				Type:        proto.ColumnType_JSON,
+			},
+			{
+				Name:        "access_approval_settings",
+				Description: "The access approval settings associated with this project.",
+				Type:        proto.ColumnType_JSON,
+				Hydrate:     getProjectAccessApprovalSettings,
+				Transform:   transform.FromValue(),
+			},
+			{
+				Name:        "ancestors",
+				Description: "The ancestors of the project in the resource hierarchy, from bottom to top.",
+				Type:        proto.ColumnType_JSON,
+				Hydrate:     getProjectAncestors,
+				Transform:   transform.FromValue(),
+			},
+			{
+				Name:        "billing_information",
+				Description: "The billing information of the project.",
+				Type:        proto.ColumnType_JSON,
+				Hydrate:     getProjectBillingInfo,
+				Transform:   transform.FromValue(),
+			},
+
+			// Steampipe standard columns
+			{
+				Name:        "title",
+				Description: ColumnDescriptionTitle,
+				Type:        proto.ColumnType_STRING,
+				Transform:   transform.FromField("Name"),
+			},
+			{
+				Name:        "tags",
+				Description: ColumnDescriptionTags,
+				Type:        proto.ColumnType_JSON,
+				Transform:   transform.FromField("Labels"),
+			},
+			{
+				Name:        "akas",
+				Description: ColumnDescriptionAkas,
+				Type:        proto.ColumnType_JSON,
+				Hydrate:     getProjectAka,
+				Transform:   transform.FromValue(),
+			},
+		},
+	}
+}
+
+//// LIST FUNCTION
+
+func listGCPOrganizationProjects(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
+	// Create Service Connection
+	service, err := CloudResourceManagerService(ctx, d)
+	if err != nil {
+		plugin.Logger(ctx).Error("gcp_organization_project.listGCPOrganizationProjects", "service_err", err)
+		return nil, err
+	}
+
+	// Max limit is not documented
+	pageSize := types.Int64(500)
+	limit := d.QueryContext.Limit
+	if d.QueryContext.Limit != nil {
+		if *limit < *pageSize {
+			pageSize = limit
+		}
+	}
+
+	// List projects
+	resp := service.Projects.List().PageSize(*pageSize)
+	if err := resp.Pages(ctx, func(page *cloudresourcemanager.ListProjectsResponse) error {
+		for _, project := range page.Projects {
+			d.StreamListItem(ctx, project)
+
+			// Check if context has been cancelled or if the limit has been hit (if specified)
+			// if there is a limit, it will return the number of rows required to reach this limit
+			if d.RowsRemaining(ctx) == 0 {
+				page.NextPageToken = ""
+				return nil
+			}
+		}
+		return nil
+	}); err != nil {
+		plugin.Logger(ctx).Error("gcp_organization_project.listGCPOrganizationProjects", "api_err", err)
+		return nil, err
+	}
+
+	return nil, nil
+}