feat: support deletion of Cloud Asset Inventory feeds not in use in organization (#198)

Co-authored-by: Daniel Andrade <dandrade@ciandt.com>

Author: renato-rudnicki

modules/project_cleanup/README.md

@@ -16,13 +16,15 @@ The following services must be enabled on the project housing the cleanup functi
 - Cloud Scheduler (`cloudscheduler.googleapis.com`)
 - Cloud Resource Manager (`cloudresourcemanager.googleapis.com`)
 - Compute Engine API (`compute.googleapis.com`)
+- Cloud Asset API (`cloudasset.googleapis.com`)
 - Security Command Center API (`securitycenter.googleapis.com`)
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| clean\_up\_org\_level\_cai\_feeds | Clean up organization level Cloud Asset Inventory Feeds. | `bool` | `false` | no |
 | clean\_up\_org\_level\_scc\_notifications | Clean up organization level Security Command Center notifications. | `bool` | `false` | no |
 | clean\_up\_org\_level\_tag\_keys | Clean up organization level Tag Keys. | `bool` | `false` | no |
 | function\_timeout\_s | The amount of time in seconds allotted for the execution of the function. | `number` | `500` | no |
@@ -35,6 +37,7 @@ The following services must be enabled on the project housing the cleanup functi
 | target\_excluded\_labels | Map of project lablels that won't be deleted. | `map(string)` | `{}` | no |
 | target\_excluded\_tagkeys | List of organization Tag Key short names that won't be deleted. | `list(string)` | `[]` | no |
 | target\_folder\_id | Folder ID to delete all projects under. | `string` | `""` | no |
+| target\_included\_feeds | List of organization level Cloud Asset Inventory feeds that should be deleted. Regex example: `.*/feeds/fd-cai-monitoring-.*` | `list(string)` | `[]` | no |
 | target\_included\_labels | Map of project lablels that will be deleted. | `map(string)` | `{}` | no |
 | target\_included\_scc\_notifications | List of organization  Security Command Center notifications names regex that will be deleted. Regex example: `.*/notificationConfigs/scc-notify-.*` | `list(string)` | `[]` | no |
 | target\_tag\_name | The name of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead). | `string` | `""` | no |

modules/project_cleanup/function_source/go.mod

@@ -3,6 +3,7 @@ module github.com/terraform-google-modules/terraform-google-scheduled-function/m
 go 1.21
 
 require (
+	cloud.google.com/go/asset v1.17.2
 	cloud.google.com/go/securitycenter v1.29.0
 	golang.org/x/net v0.24.0
 	golang.org/x/oauth2 v0.20.0
@@ -11,11 +12,14 @@ require (
 
 require (
 	cloud.google.com/go v0.112.2 // indirect
+	cloud.google.com/go/accesscontextmanager v1.8.5 // indirect
 	cloud.google.com/go/auth v0.3.0 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	cloud.google.com/go/iam v1.1.7 // indirect
 	cloud.google.com/go/longrunning v0.5.6 // indirect
+	cloud.google.com/go/orgpolicy v1.12.1 // indirect
+	cloud.google.com/go/osconfig v1.12.5 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

modules/project_cleanup/function_source/go.sum

@@ -1,6 +1,10 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw=
 cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms=
+cloud.google.com/go/accesscontextmanager v1.8.5 h1:2GLNaNu9KRJhJBFTIVRoPwk6xE5mUDgD47abBq4Zp/I=
+cloud.google.com/go/accesscontextmanager v1.8.5/go.mod h1:TInEhcZ7V9jptGNqN3EzZ5XMhT6ijWxTGjzyETwmL0Q=
+cloud.google.com/go/asset v1.17.2 h1:xgFnBP3luSbUcC9RWJvb3Zkt+y/wW6PKwPHr3ssnIP8=
+cloud.google.com/go/asset v1.17.2/go.mod h1:SVbzde67ehddSoKf5uebOD1sYw8Ab/jD/9EIeWg99q4=
 cloud.google.com/go/auth v0.3.0 h1:PRyzEpGfx/Z9e8+lHsbkoUVXD0gnu4MNmm7Gp8TQNIs=
 cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w=
 cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4=
@@ -11,6 +15,10 @@ cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM=
 cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA=
 cloud.google.com/go/longrunning v0.5.6 h1:xAe8+0YaWoCKr9t1+aWe+OeQgN/iJK1fEgZSXmjuEaE=
 cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA=
+cloud.google.com/go/orgpolicy v1.12.1 h1:2JbXigqBJVp8Dx5dONUttFqewu4fP0p3pgOdIZAhpYU=
+cloud.google.com/go/orgpolicy v1.12.1/go.mod h1:aibX78RDl5pcK3jA8ysDQCFkVxLj3aOQqrbBaUL2V5I=
+cloud.google.com/go/osconfig v1.12.5 h1:Mo5jGAxOMKH/PmDY7fgY19yFcVbvwREb5D5zMPQjFfo=
+cloud.google.com/go/osconfig v1.12.5/go.mod h1:D9QFdxzfjgw3h/+ZaAb5NypM8bhOMqBzgmbhzWViiW8=
 cloud.google.com/go/securitycenter v1.29.0 h1:jKqAqF4iLwAzCe9vzncKuxGKStmxX/HL0ItvQJdFqEA=
 cloud.google.com/go/securitycenter v1.29.0/go.mod h1:1+5P3FIDLvV2lQ83UFn+aQRb5emi4ew2XYrMmd1GSHU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

modules/project_cleanup/function_source/main.go

@@ -27,6 +27,8 @@ import (
 	"strings"
 	"time"
 
+	asset "cloud.google.com/go/asset/apiv1"
+	"cloud.google.com/go/asset/apiv1/assetpb"
 	securitycenter "cloud.google.com/go/securitycenter/apiv1"
 	"cloud.google.com/go/securitycenter/apiv1/securitycenterpb"
 	"golang.org/x/net/context"
@@ -55,6 +57,8 @@ const (
 	targetFolderRegexp            = `^[0-9]+$`
 	targetOrganizationRegexp      = `^[0-9]+$`
 	SCCNotificationsPageSize      = "SCC_NOTIFICATIONS_PAGE_SIZE"
+	CleanUpCaiFeeds               = "CLEAN_UP_CAI_FEEDS"
+	TargetIncludedFeeds           = "TARGET_INCLUDED_FEEDS"
 )
 
 var (
@@ -69,6 +73,8 @@ var (
 	rootFolderId           = getCorrectFolderIdOrTerminateExecution()
 	organizationId         = getCorrectOrganizationIdOrTerminateExecution()
 	sccPageSize            = getSCCNotificationPageSizeOrTerminateExecution()
+	cleanUpCaiFeeds        = getCleanUpFeedsOrTerminateExecution()
+	includedFeedsList      = getFeedsListFromEnv(TargetIncludedFeeds)
 )
 
 type PubSubMessage struct {
@@ -172,12 +178,12 @@ func checkIfAtLeastOneLabelPresentIfAny(project *cloudresourcemanager.Project, l
 	return result
 }
 
-func checkIfSCCNotificationNameIncluded(notificationName string, includedSCCNotfis []*regexp.Regexp) bool {
-	if len(includedSCCNotfis) == 0 {
+func checkIfNameIncluded(name string, reg []*regexp.Regexp) bool {
+	if len(reg) == 0 {
 		return false
 	}
-	for _, name := range includedSCCNotfis {
-		if name.MatchString(notificationName) {
+	for _, regex := range reg {
+		if regex.MatchString(name) {
 			return true
 		}
 	}
@@ -296,6 +302,48 @@ func getSCCNotificationPageSizeOrTerminateExecution() int32 {
 	return int32(size)
 }
 
+func getFeedsListFromEnv(envVariableName string) []*regexp.Regexp {
+	var compiledRegEx []*regexp.Regexp
+	targetIncludedFeeds := os.Getenv(envVariableName)
+	logger.Println("Try to get CAI Feeds list")
+	if targetIncludedFeeds == "" {
+		logger.Printf("No CAI Feeds provided.")
+		return compiledRegEx
+	}
+
+	var caiFeeds []string
+	err := json.Unmarshal([]byte(targetIncludedFeeds), &caiFeeds)
+	if err != nil {
+		logger.Printf("Failed to get CAI Feeds list from [%s] env variable, error [%s]", envVariableName, err.Error())
+		return nil
+	} else {
+		logger.Printf("Got CAI Feeds list [%s] from [%s] env variable", caiFeeds, envVariableName)
+	}
+
+	//build Regexes
+	for _, r := range caiFeeds {
+		result, err := regexp.Compile(r)
+		if err != nil {
+			logger.Printf("Invalid regular expression [%s] for CAI Feed", r)
+		} else {
+			compiledRegEx = append(compiledRegEx, result)
+		}
+	}
+	return compiledRegEx
+}
+
+func getCleanUpFeedsOrTerminateExecution() bool {
+	cleanUpCaiFeeds, exists := os.LookupEnv(CleanUpCaiFeeds)
+	if !exists {
+		logger.Fatalf("Clean up CAI Feeds environment variable [%s] not set, set the environment variable and try again.", CleanUpCaiFeeds)
+	}
+	result, err := strconv.ParseBool(cleanUpCaiFeeds)
+	if err != nil {
+		logger.Fatalf("Invalid Clean up CAI Feeds value [%s], specify correct value for environment variable [%s] and try again.", cleanUpCaiFeeds, CleanUpCaiFeeds)
+	}
+	return result
+}
+
 func getCorrectFolderIdOrTerminateExecution() string {
 	targetFolderIdString := os.Getenv(TargetFolderId)
 	matched, err := regexp.MatchString(targetFolderRegexp, targetFolderIdString)
@@ -372,6 +420,16 @@ func getSCCNotificationServiceOrTerminateExecution(ctx context.Context, client *
 	return securitycenterClient
 }
 
+func getAssetServiceOrTerminateExecution(ctx context.Context, client *http.Client) *asset.Client {
+	logger.Println("Try to get Asset Service")
+	assetService, err := asset.NewClient(ctx)
+	if err != nil {
+		logger.Fatalf("Failed to get Asset Service with error [%s], terminate execution", err.Error())
+	}
+	logger.Println("Got Asset Service")
+	return assetService
+}
+
 func getFirewallPoliciesServiceOrTerminateExecution(ctx context.Context, client *http.Client) *compute.FirewallPoliciesService {
 	logger.Println("Try to get Firewall Policies Service")
 	computeService, err := compute.NewService(ctx, option.WithHTTPClient(client))
@@ -399,6 +457,7 @@ func invoke(ctx context.Context) {
 	tagKeyService := getTagKeysServiceOrTerminateExecution(ctx, client)
 	sccService := getSCCNotificationServiceOrTerminateExecution(ctx, client)
 	tagValuesService := getTagValuesServiceOrTerminateExecution(ctx, client)
+	feedsService := getAssetServiceOrTerminateExecution(ctx, client)
 	firewallPoliciesService := getFirewallPoliciesServiceOrTerminateExecution(ctx, client)
 	endpointService := getServiceManagementServiceOrTerminateExecution(ctx, client)
 
@@ -450,7 +509,7 @@ func invoke(ctx context.Context) {
 				break
 			}
 			projectID := strings.Split(resp.PubsubTopic, "/")[1]
-			if checkIfSCCNotificationNameIncluded(resp.Name, includedSCCNotfisList) && projectDeleteRequestedFilter(projectID) {
+			if checkIfNameIncluded(resp.Name, includedSCCNotfisList) && projectDeleteRequestedFilter(projectID) {
 				delReq := &securitycenterpb.DeleteNotificationConfigRequest{
 					Name: resp.Name,
 				}
@@ -498,6 +557,35 @@ func invoke(ctx context.Context) {
 		}
 	}
 
+	removeFeedsByName := func(organization string) {
+		logger.Printf("Try to remove feeds from organization [%s]", organization)
+
+		req := &assetpb.ListFeedsRequest{
+			Parent: fmt.Sprintf("organizations/%s", organization),
+		}
+
+		resp, err := feedsService.ListFeeds(ctx, req)
+		if err != nil {
+			logger.Printf("Failed to list Feeds, error [%s]", err.Error())
+			return
+		}
+
+		for _, feed := range resp.Feeds {
+			projectID := strings.Split(feed.FeedOutputConfig.GetPubsubDestination().Topic, "/")[1]
+			if checkIfNameIncluded(feed.Name, includedFeedsList) && projectDeleteRequestedFilter(projectID) {
+				delReq := &assetpb.DeleteFeedRequest{
+					Name: feed.Name,
+				}
+				err := feedsService.DeleteFeed(ctx, delReq)
+				if err != nil {
+					logger.Printf("Failed to remove the feed [%s], error [%s]", feed.Name, err.Error())
+				} else {
+					logger.Printf("Feed [%s] successfully removed.", feed.Name)
+				}
+			}
+		}
+	}
+
 	removeFirewallPolicies := func(folder string) {
 		logger.Printf("Try to remove Firewall Policies from folder [%s]", folder)
 		firewallPolicyList, err := firewallPoliciesService.List().ParentId(folder).Context(ctx).Do()
@@ -640,6 +728,7 @@ func invoke(ctx context.Context) {
 	} else {
 		getSubFoldersAndRemoveProjectsFoldersRecursively(rootFolder, getSubFoldersAndRemoveProjectsFoldersRecursively)
 	}
+
 	// Only Tag Keys whose values are not in use can be deleted.
 	if cleanUpTagKeys {
 		removeTagKeys(organizationId)
@@ -649,6 +738,11 @@ func invoke(ctx context.Context) {
 	if cleanUpSCCNotfi {
 		removeSCCNotifications(organizationId)
 	}
+
+	//Only delete Feeds from deleted projects
+	if cleanUpCaiFeeds {
+		removeFeedsByName(organizationId)
+	}
 }
 
 func CleanUpProjects(ctx context.Context, m PubSubMessage) error {

modules/project_cleanup/main.tf

@@ -34,6 +34,7 @@ resource "google_organization_iam_member" "main" {
     "roles/compute.orgSecurityPolicyAdmin",
     "roles/resourcemanager.tagAdmin",
     "roles/viewer",
+    "roles/cloudasset.owner",
     "roles/securitycenter.notificationConfigEditor"
   ])
 
@@ -69,5 +70,7 @@ module "scheduled_project_cleaner" {
     CLEAN_UP_SCC_NOTIFICATIONS        = var.clean_up_org_level_scc_notifications
     TARGET_INCLUDED_SCC_NOTIFICATIONS = jsonencode(var.target_included_scc_notifications)
     SCC_NOTIFICATIONS_PAGE_SIZE       = var.list_scc_notifications_page_size
+    CLEAN_UP_CAI_FEEDS                = var.clean_up_org_level_cai_feeds
+    TARGET_INCLUDED_FEEDS             = jsonencode(var.target_included_feeds)
   }
 }

modules/project_cleanup/variables.tf

@@ -25,6 +25,18 @@ variable "organization_id" {
   description = "The organization ID whose projects to clean up"
 }
 
+variable "clean_up_org_level_cai_feeds" {
+  type        = bool
+  description = "Clean up organization level Cloud Asset Inventory Feeds."
+  default     = false
+}
+
+variable "target_included_feeds" {
+  type        = list(string)
+  description = "List of organization level Cloud Asset Inventory feeds that should be deleted. Regex example: `.*/feeds/fd-cai-monitoring-.*` "
+  default     = []
+}
+
 variable "project_id" {
   type        = string
   description = "The project ID to host the scheduled function in"