tools/clean: add VPC orphan cleanup (#1450)

Author: s-urbaniak

tools/clean/atlas/atlas.go

@@ -48,7 +48,7 @@ func (pd *ProjectDependencies) Length() int {
 		len(pd.Clusters) + len(pd.ServerlessClusters) + len(pd.FederatedDatabases)
 }
 
-func (c *Cleaner) Clean(ctx context.Context, lifetime int) error {
+func (c *Cleaner) Clean(ctx context.Context, lifetimeHours int) error {
 	projects := c.listProjectsByOrg(ctx, c.orgID)
 
 	if len(projects) > 0 {
@@ -60,9 +60,8 @@ func (c *Cleaner) Clean(ctx context.Context, lifetime int) error {
 
 		fmt.Println(text.FgHiWhite.Sprintf("\tStarting deletion of project %s(%s) (created at %v)...", p.GetName(), p.GetId(), p.Created))
 
-		if time.Since(p.Created) < time.Duration(lifetime)*time.Hour {
-			fmt.Println(text.FgYellow.Sprintf("\tProject %s(%s) skipped once created less than %d hour ago", p.GetName(), p.GetId(), lifetime))
-
+		if time.Since(p.Created) < time.Duration(lifetimeHours)*time.Hour {
+			fmt.Println(text.FgYellow.Sprintf("\tProject %s(%s) skipped once created less than %d hour ago", p.GetName(), p.GetId(), lifetimeHours))
 			continue
 		}
 
@@ -92,21 +91,42 @@ func (c *Cleaner) Clean(ctx context.Context, lifetime int) error {
 		c.deleteTeam(ctx, c.orgID, &t)
 	}
 
-	return c.cleanOrphanResources(ctx, lifetime)
+	c.cleanOrphanResources(ctx, lifetimeHours)
+	return nil
 }
 
-func (c *Cleaner) cleanOrphanResources(ctx context.Context, lifetime int) error {
+func (c *Cleaner) cleanOrphanResources(ctx context.Context, lifetimeHours int) {
 	region := envOrDefault("GCP_CLEANUP_REGION", "europe-west1")
 	subnet := envOrDefault("GCP_CLEANUP_SUBNET", "atlas-operator-e2e-test-subnet1")
-	done, skipped, err := c.gcp.DeleteOrphanPrivateEndpoints(ctx, lifetime, region, subnet)
+	vpcPrefix := envOrDefault("GCP_CLEANUP_VPC_NAME_PREFIX", "network-peering-gcp-1-vpc")
+
+	var done, skipped []string
+	var errs []error
+
+	addResults := func(f func() ([]string, []string, []error)) {
+		d, s, e := f()
+		done = append(done, d...)
+		skipped = append(skipped, s...)
+		errs = append(errs, e...)
+	}
+
+	addResults(func() ([]string, []string, []error) {
+		return c.gcp.DeleteOrphanPrivateEndpoints(ctx, lifetimeHours, region, subnet)
+	})
+
+	addResults(func() ([]string, []string, []error) {
+		return c.gcp.DeleteOrphanVPCs(ctx, lifetimeHours, vpcPrefix)
+	})
+
 	for _, doneMsg := range done {
 		fmt.Println(text.FgGreen.Sprintf("%s", doneMsg))
 	}
 	for _, skippedMsg := range skipped {
 		fmt.Println(text.FgYellow.Sprintf("\t%s", skippedMsg))
 	}
-
-	return err
+	for _, err := range errs {
+		fmt.Println(text.FgRed.Sprintf(err.Error()))
+	}
 }
 
 func NewCleaner(aws *provider.AWS, gcp *provider.GCP, azure *provider.Azure) (*Cleaner, error) {

tools/clean/main.go

@@ -36,14 +36,15 @@ func main() {
 		return
 	}
 
-	lifetime, err := strconv.Atoi(os.Getenv("PROJECT_LIFETIME"))
+	lifetimeHours, err := strconv.Atoi(os.Getenv("PROJECT_LIFETIME"))
 	if err != nil {
+		err = fmt.Errorf("error parsing PROJECT_LIFETIME environment variable: %w", err)
 		fmt.Println(text.FgRed.Sprintf(err.Error()))
 
 		return
 	}
 
-	err = c.Clean(ctx, lifetime)
+	err = c.Clean(ctx, lifetimeHours)
 	if err != nil {
 		fmt.Println(text.FgRed.Sprintf(err.Error()))
 	}

tools/clean/provider/gcp.go

@@ -102,60 +102,146 @@ func (gcp *GCP) DeletePrivateEndpoint(ctx context.Context, groupName, sAttachmen
 	return nil
 }
 
-func (gcp *GCP) DeleteOrphanPrivateEndpoints(ctx context.Context, lifetime int, region string, subnet string) ([]string, []string, error) {
+func (gcp *GCP) DeleteOrphanVPCs(ctx context.Context, lifetimeHours int, vpcNamePrefix string) ([]string, []string, []error) {
+	vpcs := gcp.networkClient.List(ctx, &computepb.ListNetworksRequest{
+		Project: gcp.projectID,
+	})
+
+	var done, skipped []string
+	var errs []error
+	for {
+		vpc, err := vpcs.Next()
+		if errors.Is(err, iterator.Done) {
+			break
+		}
+		if err != nil {
+			errs = append(errs,
+				fmt.Errorf("failed iterating VPC networks in project %v: %w", gcp.projectID, err),
+			)
+			continue
+		}
+		if !strings.HasPrefix(vpc.GetName(), vpcNamePrefix) {
+			skipped = append(skipped,
+				fmt.Sprintf("VPC %s skipped\n", vpc.GetName()),
+			)
+			continue
+		}
+		createdAt, err := asTime(vpc.GetCreationTimestamp())
+		if err != nil {
+			errs = append(errs,
+				fmt.Errorf("failed parsing VPC creation timestamp %q: %w", vpc.GetCreationTimestamp(), err),
+			)
+			continue
+		}
+		if time.Since(createdAt) < time.Duration(lifetimeHours)*time.Hour {
+			skipped = append(skipped,
+				fmt.Sprintf("VPC %s skipped once created less than %d hours ago\n", vpc.GetName(), lifetimeHours),
+			)
+			continue
+		}
+
+		op, err := gcp.networkClient.Delete(ctx, &computepb.DeleteNetworkRequest{
+			Project: gcp.projectID,
+			Network: vpc.GetName(),
+		})
+		if err != nil {
+			errs = append(errs,
+				fmt.Errorf("error deleting VPC %s: %w", vpc.GetName(), err),
+			)
+			continue
+		}
+
+		err = op.Wait(ctx)
+		if err != nil {
+			errs = append(errs,
+				fmt.Errorf("error waiting for deletion of VPC %s: %w", vpc.GetName(), err),
+			)
+			continue
+		}
+		done = append(done,
+			fmt.Sprintf("Released orphan VPC %s\n", vpc.GetName()),
+		)
+	}
+
+	return done, skipped, errs
+}
+
+func (gcp *GCP) DeleteOrphanPrivateEndpoints(ctx context.Context, lifetimeHours int, region string, subnet string) ([]string, []string, []error) {
 	addresses := gcp.addressClient.List(ctx, &computepb.ListAddressesRequest{
 		Project: gcp.projectID,
 		Region:  region,
 	})
-	done := []string{}
-	skipped := []string{}
+	var done, skipped []string
+	var errs []error
 	for {
 		addr, err := addresses.Next()
 		if errors.Is(err, iterator.Done) {
 			break
 		}
 		if err != nil {
-			return nil, nil, fmt.Errorf("failed iterating addresses in project %v region %v: %w",
-				gcp.projectID, region, err)
+			errs = append(errs,
+				fmt.Errorf("failed iterating addresses in project %v region %v: %w", gcp.projectID, region, err),
+			)
+			continue
 		}
 		suffix := fmt.Sprintf("subnetworks/%s", subnet)
 		if !strings.HasSuffix(addr.GetSubnetwork(), suffix) {
 			skipped = append(skipped,
-				fmt.Sprintf("Address %s(%s) skipped, not in %s\n", addr.GetName(), addr.GetAddress(), subnet))
+				fmt.Sprintf("Address %s(%s) skipped, not in %s\n", addr.GetName(), addr.GetAddress(), subnet),
+			)
 			continue
 		}
 		createdAt, err := asTime(addr.GetCreationTimestamp())
 		if err != nil {
-			return nil, nil, fmt.Errorf("failed parsing Address creation timestamp %q: %w",
-				addr.GetCreationTimestamp(), err)
+			errs = append(errs,
+				fmt.Errorf(
+					"failed parsing Address %s(%s) creation timestamp %q: %w",
+					addr.GetCreationTimestamp(), addr.GetName(), addr.GetAddress(), err,
+				),
+			)
+			continue
 		}
-		if time.Since(createdAt) < time.Duration(lifetime)*time.Hour {
-			skipped = append(skipped, fmt.Sprintf("Address %s(%s) skipped once created less than %d hours ago\n",
-				addr.GetName(), addr.GetAddress(), lifetime))
+		if time.Since(createdAt) < time.Duration(lifetimeHours)*time.Hour {
+			skipped = append(skipped,
+				fmt.Sprintf(
+					"Address %s(%s) skipped once created less than %d hours ago\n",
+					addr.GetName(), addr.GetAddress(), lifetimeHours,
+				),
+			)
+			continue
 		}
 		frName, err := expectForwardingRule(addr.GetUsers())
 		if err != nil {
-			return nil, nil, err
+			errs = append(errs, err)
+			continue
 		}
 
 		if frName != "" {
-			if err := gcp.deleteForwardingRule(ctx, frName, region); err != nil {
-				return nil, nil, fmt.Errorf("failed deleting Forwarding Rule %q in region %q: %w", region, frName, err)
+			err := gcp.deleteForwardingRule(ctx, frName, region)
+			if err != nil {
+				errs = append(errs,
+					fmt.Errorf("failed deleting Forwarding Rule %q in region %q: %w", region, frName, err),
+				)
+				continue
 			}
-			done = append(done, fmt.Sprintf("Deleted Forwarding Rule %s for %s\n",
-				frName, addr.GetAddress()))
+			done = append(done,
+				fmt.Sprintf("Deleted Forwarding Rule %s for %s\n", frName, addr.GetAddress()),
+			)
 		} else {
 			skipped = append(skipped,
 				fmt.Sprintf("No forwarding rule using Address %s(%s)", addr.GetName(), addr.GetAddress()))
 		}
 		if err := gcp.deleteIPAddress(ctx, addr.GetName(), region); err != nil {
-			return nil, nil, fmt.Errorf("error deleting Address %s(%s) in region %q: %w",
-				region, addr.GetName(), addr.GetAddress(), err)
+			errs = append(errs,
+				fmt.Errorf("error deleting Address %s(%s) in region %q: %w",
+					region, addr.GetName(), addr.GetAddress(), err),
+			)
+			continue
 		}
 		done = append(done, fmt.Sprintf("Released orphan Address %s(%s)\n", addr.GetName(), addr.GetAddress()))
 	}
 
-	return done, skipped, nil
+	return done, skipped, errs
 }
 
 func asTime(rfc3339time string) (time.Time, error) {