feat(GO1.21)!: support deletion of Tag Keys not in use in organizations in the clean up module

modules/project_cleanup/README.md

@@ -22,13 +22,15 @@ The following services must be enabled on the project housing the cleanup functi
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| clean\_up\_org\_level\_tag\_keys | Clean up organization level Tag Keys. | `bool` | `false` | no |
 | function\_timeout\_s | The amount of time in seconds allotted for the execution of the function. | `number` | `500` | no |
 | job\_schedule | Cleaner function run frequency, in cron syntax | `string` | `"*/5 * * * *"` | no |
 | max\_project\_age\_in\_hours | The maximum number of hours that a GCP project, selected by `target_tag_name` and `target_tag_value`, can exist | `number` | `6` | no |
 | organization\_id | The organization ID whose projects to clean up | `string` | n/a | yes |
 | project\_id | The project ID to host the scheduled function in | `string` | n/a | yes |
 | region | The region the project is in (App Engine specific) | `string` | n/a | yes |
 | target\_excluded\_labels | Map of project lablels that won't be deleted. | `map(string)` | `{}` | no |
+| target\_excluded\_tagkeys | List of organization Tag Keys short names that won't be deleted. | `list(string)` | `[]` | no |
 | target\_folder\_id | Folder ID to delete all projects under. | `string` | `""` | no |
 | target\_included\_labels | Map of project lablels that will be deleted. | `map(string)` | `{}` | no |
 | target\_tag\_name | The name of a tag to filter GCP projects on for consideration by the cleanup utility (legacy, use `target_included_labels` map instead). | `string` | `""` | no |

modules/project_cleanup/function_source/main.go

@@ -31,6 +31,7 @@ import (
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/cloudresourcemanager/v1"
 	cloudresourcemanager2 "google.golang.org/api/cloudresourcemanager/v2"
+	cloudresourcemanager3 "google.golang.org/api/cloudresourcemanager/v3"
 	"google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
 	"google.golang.org/api/servicemanagement/v1"
@@ -40,17 +41,24 @@ const (
 	LifecycleStateActiveRequested = "ACTIVE"
 	TargetExcludedLabels          = "TARGET_EXCLUDED_LABELS"
 	TargetIncludedLabels          = "TARGET_INCLUDED_LABELS"
+	CleanUpTagKeys                = "CLEAN_UP_TAG_KEYS"
+	TargetExcludedTagKeys         = "TARGET_EXCLUDED_TAGKEYS"
 	TargetFolderId                = "TARGET_FOLDER_ID"
+	TargetOrganizationId          = "TARGET_ORGANIZATION_ID"
 	MaxProjectAgeHours            = "MAX_PROJECT_AGE_HOURS"
 	targetFolderRegexp            = `^[0-9]+$`
+	targetOrganizationRegexp      = `^[0-9]+$`
 )
 
 var (
 	logger                 = log.New(os.Stdout, "", 0)
 	excludedLabelsMap      = getLabelsMapFromEnv(TargetExcludedLabels)
 	includedLabelsMap      = getLabelsMapFromEnv(TargetIncludedLabels)
+	cleanUpTagKeys         = getCleanUpTagKeysOrTerminateExecution()
+	excludedTagKeysList    = getTagKeysListFromEnv(TargetExcludedTagKeys)
 	resourceCreationCutoff = getOldTime(int64(getCorrectMaxAgeInHoursOrTerminateExecution()) * 60 * 60)
 	rootFolderId           = getCorrectFolderIdOrTerminateExecution()
+	organizationId         = getCorrectOrganizationIdOrTerminateExecution()
 )
 
 type PubSubMessage struct {
@@ -154,6 +162,18 @@ func checkIfAtLeastOneLabelPresentIfAny(project *cloudresourcemanager.Project, l
 	return result
 }
 
+func checkIfTagKeyShortNameExcluded(shortName string, excludedTagKeys []string) bool {
+	if len(excludedTagKeys) == 0 {
+		return false
+	}
+	for _, name := range excludedTagKeys {
+		if shortName == name {
+			return true
+		}
+	}
+	return false
+}
+
 func getLabelsMapFromEnv(envVariableName string) map[string]string {
 	targetExcludedLabels := os.Getenv(envVariableName)
 	logger.Println("Try to get labels map")
@@ -173,6 +193,36 @@ func getLabelsMapFromEnv(envVariableName string) map[string]string {
 	return labels
 }
 
+func getTagKeysListFromEnv(envVariableName string) []string {
+	targetExcludedTagKeys := os.Getenv(envVariableName)
+	logger.Println("Try to get Tag Keys list")
+	if targetExcludedTagKeys == "" {
+		logger.Printf("No Tag Keys provided.")
+		return nil
+	}
+
+	var tagKeys []string
+	err := json.Unmarshal([]byte(targetExcludedTagKeys), &tagKeys)
+	if err != nil {
+		logger.Printf("Fail to get Tag Keys list from [%s] env variable, error [%s]", envVariableName, err.Error())
+	} else {
+		logger.Printf("Got Tag Keys list [%s] from [%s] env variable", tagKeys, envVariableName)
+	}
+	return tagKeys
+}
+
+func getCleanUpTagKeysOrTerminateExecution() bool {
+	cleanUpTagKeys, exists := os.LookupEnv(CleanUpTagKeys)
+	if !exists {
+		logger.Fatalf("Clean up Tag Keys flag not set [%s]. Specify correct value, Please.", CleanUpTagKeys)
+	}
+	result, err := strconv.ParseBool(cleanUpTagKeys)
+	if err != nil {
+		logger.Fatalf("Invalid Clean up Tag Keys flag [%s] value [%s]. Specify correct value, Please.", CleanUpTagKeys, cleanUpTagKeys)
+	}
+	return result
+}
+
 func getCorrectFolderIdOrTerminateExecution() string {
 	targetFolderIdString := os.Getenv(TargetFolderId)
 	matched, err := regexp.MatchString(targetFolderRegexp, targetFolderIdString)
@@ -182,6 +232,15 @@ func getCorrectFolderIdOrTerminateExecution() string {
 	return targetFolderIdString
 }
 
+func getCorrectOrganizationIdOrTerminateExecution() string {
+	targetOrganizationIdString := os.Getenv(TargetOrganizationId)
+	matched, err := regexp.MatchString(targetOrganizationRegexp, targetOrganizationIdString)
+	if err != nil || !matched {
+		logger.Fatalf("Invalid organization id [%s]. Specify correct value, Please.", targetOrganizationIdString)
+	}
+	return targetOrganizationIdString
+}
+
 func getServiceManagementServiceOrTerminateExecution(client *http.Client) *servicemanagement.APIService {
 	service, err := servicemanagement.New(client)
 	if err != nil {
@@ -210,6 +269,26 @@ func getFolderServiceOrTerminateExecution(client *http.Client) *cloudresourceman
 	return cloudResourceManagerService.Folders
 }
 
+func getTagKeysServiceOrTerminateExecution(client *http.Client) *cloudresourcemanager3.TagKeysService {
+	logger.Println("Try to get TagKeys Service")
+	cloudResourceManagerService, err := cloudresourcemanager3.New(client)
+	if err != nil {
+		logger.Fatalf("Fail to get TagKeys Service with error [%s], terminate execution", err.Error())
+	}
+	logger.Println("Got TagKeys Service")
+	return cloudResourceManagerService.TagKeys
+}
+
+func getTagValuesServiceOrTerminateExecution(client *http.Client) *cloudresourcemanager3.TagValuesService {
+	logger.Println("Try to get TagValues Service")
+	cloudResourceManagerService, err := cloudresourcemanager3.New(client)
+	if err != nil {
+		logger.Fatalf("Fail to get TagValues Service with error [%s], terminate execution", err.Error())
+	}
+	logger.Println("Got TagValues Service")
+	return cloudResourceManagerService.TagValues
+}
+
 func getFirewallPoliciesServiceOrTerminateExecution(client *http.Client) *compute.FirewallPoliciesService {
 	logger.Println("Try to get Firewall Policies Service")
 	computeService, err := compute.New(client)
@@ -234,6 +313,8 @@ func invoke(ctx context.Context) {
 	client := initializeGoogleClient(ctx)
 	cloudResourceManagerService := getResourceManagerServiceOrTerminateExecution(client)
 	folderService := getFolderServiceOrTerminateExecution(client)
+	tagKeyService := getTagKeysServiceOrTerminateExecution(client)
+	tagValuesService := getTagValuesServiceOrTerminateExecution(client)
 	firewallPoliciesService := getFirewallPoliciesServiceOrTerminateExecution(client)
 	endpointService := getServiceManagementServiceOrTerminateExecution(client)
 
@@ -247,6 +328,49 @@ func invoke(ctx context.Context) {
 		}
 	}
 
+	tagKeyAgeFilter := func(tagKey *cloudresourcemanager3.TagKey) bool {
+		tagKeyCreatedAt, err := time.Parse(time.RFC3339, tagKey.CreateTime)
+		if err != nil {
+			logger.Printf("Fail to parse CreateTime for tagKey [%s], skip it. Error [%s]", tagKey.Name, err.Error())
+			return false
+		}
+		return tagKeyCreatedAt.Before(resourceCreationCutoff)
+	}
+
+	removeTagValues := func(tagKey string) {
+		logger.Printf("Try to remove Tag Values from TagKey [%s]", tagKey)
+		tagValuesList, err := tagValuesService.List().Parent(tagKey).Context(ctx).Do()
+		if err != nil {
+			logger.Printf("Fail to list Tag values from TagKey [%s], error [%s]", tagKey, err.Error())
+			return
+		}
+		for _, tagValue := range tagValuesList.TagValues {
+			_, err := tagValuesService.Delete(tagValue.Name).Context(ctx).Do()
+			if err != nil {
+				logger.Printf("Fail to delete tagValue from TagKey [%s], error [%s]", tagKey, err.Error())
+			}
+		}
+	}
+
+	removeTagKeys := func(organization string) {
+		logger.Printf("Try to remove Tag Keys from organization [%s]", organization)
+		parent := fmt.Sprintf("organizations/%s", organization)
+		tagKeysList, err := tagKeyService.List().Parent(parent).Context(ctx).Do()
+		if err != nil {
+			logger.Printf("Fail to list Tag Keys from organization [%s], error [%s]", organization, err.Error())
+			return
+		}
+		for _, tagKey := range tagKeysList.TagKeys {
+			if !checkIfTagKeyShortNameExcluded(tagKey.ShortName, excludedTagKeysList) && tagKeyAgeFilter(tagKey) {
+				removeTagValues(tagKey.Name)
+				_, err := tagKeyService.Delete(tagKey.Name).Context(ctx).Do()
+				if err != nil {
+					logger.Printf("Fail to delete tagKey from organization [%s], error [%s]", organization, err.Error())
+				}
+			}
+		}
+	}
+
 	removeFirewallPolicies := func(folder string) {
 		logger.Printf("Try to remove Firewall Policies from folder [%s]", folder)
 		firewallPolicyList, err := firewallPoliciesService.List().ParentId(folder).Context(ctx).Do()
@@ -388,6 +512,10 @@ func invoke(ctx context.Context) {
 	} else {
 		getSubFoldersAndRemoveProjectsFoldersRecursively(rootFolder, getSubFoldersAndRemoveProjectsFoldersRecursively)
 	}
+	// Only Tag Keys whose values are not in use can be deleted.
+	if cleanUpTagKeys {
+		removeTagKeys(organizationId)
+	}
 }
 
 func CleanUpProjects(ctx context.Context, m PubSubMessage) error {

modules/project_cleanup/main.tf

@@ -32,6 +32,7 @@ resource "google_organization_iam_member" "main" {
     "roles/serviceusage.serviceUsageAdmin",
     "roles/compute.orgSecurityResourceAdmin",
     "roles/compute.orgSecurityPolicyAdmin",
+    "roles/resourcemanager.tagAdmin",
     "roles/viewer"
   ])
 
@@ -52,14 +53,17 @@ module "scheduled_project_cleaner" {
   topic_name                     = var.topic_name
   function_available_memory_mb   = 128
   function_description           = "Clean up GCP projects older than ${var.max_project_age_in_hours} hours matching particular tags"
-  function_runtime               = "go113"
+  function_runtime               = "go121"
   function_service_account_email = google_service_account.project_cleaner_function.email
   function_timeout_s             = var.function_timeout_s
 
   function_environment_variables = {
-    TARGET_EXCLUDED_LABELS = jsonencode(var.target_excluded_labels)
-    TARGET_FOLDER_ID       = var.target_folder_id
-    TARGET_INCLUDED_LABELS = jsonencode(local.target_included_labels)
-    MAX_PROJECT_AGE_HOURS  = var.max_project_age_in_hours
+    TARGET_ORGANIZATION_ID  = var.organization_id
+    TARGET_FOLDER_ID        = var.target_folder_id
+    TARGET_EXCLUDED_LABELS  = jsonencode(var.target_excluded_labels)
+    TARGET_INCLUDED_LABELS  = jsonencode(local.target_included_labels)
+    MAX_PROJECT_AGE_HOURS   = var.max_project_age_in_hours
+    CLEAN_UP_TAG_KEYS       = var.clean_up_org_level_tag_keys
+    TARGET_EXCLUDED_TAGKEYS = jsonencode(var.target_excluded_tagkeys)
   }
 }

modules/project_cleanup/variables.tf

@@ -77,6 +77,18 @@ variable "target_included_labels" {
   default     = {}
 }
 
+variable "clean_up_org_level_tag_keys" {
+  type        = bool
+  description = "Clean up organization level Tag Keys."
+  default     = false
+}
+
+variable "target_excluded_tagkeys" {
+  type        = list(string)
+  description = "List of organization Tag Keys short names that won't be deleted."
+  default     = []
+}
+
 variable "target_folder_id" {
   type        = string
   description = "Folder ID to delete all projects under."