[release/2.1.x]: Backport 3361 to release/2.1.x (#3499)

Author: pmalek

.github/workflows/_kongintegration_tests.yaml

@@ -56,6 +56,7 @@ jobs:
           MISE_VERBOSE: 1
           MISE_DEBUG: 1
           GOTESTSUM_JUNITFILE: ${{ env.GOTESTSUM_JUNITFILE }}
+          KONG_CUSTOM_DOMAIN: konghq.tech
           TEST_KONG_KONNECT_ACCESS_TOKEN: ${{ secrets.KONG_TEST_KONNECT_ACCESS_TOKEN }}
           KONG_LICENSE_DATA: ${{ steps.license.outputs.license }}
           TEST_KONG_ENTERPRISE: ${{ matrix.enterprise }}

hack/cleanup/konnect_system_accounts.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	sdkkonnectgo "github.com/Kong/sdk-konnect-go"
+	sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components"
+	sdkkonnectops "github.com/Kong/sdk-konnect-go/models/operations"
+	"github.com/go-logr/logr"
+	"github.com/samber/lo"
+
+	"github.com/kong/kong-operator/v2/test"
+)
+
+const (
+	systemAccountsPageSize         = int64(100)
+	timeUntilSystemAccountOrphaned = time.Hour
+)
+
+// cleanupKonnectSystemAccounts deletes orphaned system accounts created by the tests.
+func cleanupKonnectSystemAccounts(ctx context.Context, log logr.Logger) error {
+	serverURL, err := canonicalizedServerURL()
+	if err != nil {
+		return fmt.Errorf("invalid server URL %s: %w", test.KonnectServerURL(), err)
+	}
+
+	sdk := sdkkonnectgo.New(
+		sdkkonnectgo.WithSecurity(
+			sdkkonnectcomp.Security{
+				PersonalAccessToken: lo.ToPtr(test.KonnectAccessToken()),
+			},
+		),
+		sdkkonnectgo.WithServerURL(serverURL),
+	)
+
+	orphanedSAs, err := findOrphanedSystemAccounts(ctx, log, sdk.SystemAccounts)
+	if err != nil {
+		return fmt.Errorf("failed to find orphaned system accounts: %w", err)
+	}
+	if err := deleteSystemAccounts(ctx, log, sdk.SystemAccounts, orphanedSAs); err != nil {
+		return fmt.Errorf("failed to delete system accounts: %w", err)
+	}
+
+	return nil
+}
+
+// findOrphanedSystemAccounts finds system accounts that were created more than timeUntilSystemAccountOrphaned ago.
+func findOrphanedSystemAccounts(
+	ctx context.Context,
+	log logr.Logger,
+	sdk *sdkkonnectgo.SystemAccounts,
+) ([]string, error) {
+	var orphanedSystemAccounts []string
+	pageNumber := int64(1)
+
+	for {
+		response, err := sdk.GetSystemAccounts(ctx, sdkkonnectops.GetSystemAccountsRequest{
+			PageSize:   lo.ToPtr(systemAccountsPageSize),
+			PageNumber: &pageNumber,
+			Filter: &sdkkonnectops.GetSystemAccountsQueryParamFilter{
+				// Only consider non-Konnect-managed system accounts.
+				KonnectManaged: lo.ToPtr(false),
+			},
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to list system accounts (page %d): %w", pageNumber, err)
+		}
+		if response.SystemAccountCollection == nil {
+			return nil, fmt.Errorf("failed to list system accounts, response is nil (page %d)", pageNumber)
+		}
+
+		accounts := response.SystemAccountCollection.GetData()
+		if len(accounts) == 0 {
+			break
+		}
+
+		for _, sa := range accounts {
+			if sa.ID == nil {
+				log.Info("System account has no ID, skipping", "name", sa.GetName())
+				continue
+			}
+			if sa.CreatedAt == nil {
+				log.Info("System account has no creation timestamp, skipping", "id", *sa.ID, "name", sa.GetName())
+				continue
+			}
+			orphanedAfter := sa.CreatedAt.Add(timeUntilSystemAccountOrphaned)
+			if !time.Now().After(orphanedAfter) {
+				log.Info("System account is not old enough to be considered orphaned, skipping",
+					"id", *sa.ID, "name", sa.GetName(), "created_at", *sa.CreatedAt,
+				)
+				continue
+			}
+			orphanedSystemAccounts = append(orphanedSystemAccounts, *sa.ID)
+		}
+
+		if int64(len(accounts)) < systemAccountsPageSize {
+			break
+		}
+		pageNumber++
+	}
+
+	return orphanedSystemAccounts, nil
+}
+
+// deleteSystemAccounts deletes system accounts by their IDs.
+func deleteSystemAccounts(
+	ctx context.Context,
+	log logr.Logger,
+	sdk *sdkkonnectgo.SystemAccounts,
+	saIDs []string,
+) error {
+	if len(saIDs) == 0 {
+		log.Info("No system accounts to clean up")
+		return nil
+	}
+
+	var errs []error
+	for _, saID := range saIDs {
+		log.Info("Deleting system account", "id", saID)
+		if _, err := sdk.DeleteSystemAccountsID(ctx, saID); err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete system account %s: %w", saID, err))
+		}
+	}
+	return errors.Join(errs...)
+}

hack/cleanup/main.go

@@ -13,11 +13,15 @@
 // 1. It has a label `created_in_tests` with value `true`.
 // 2. It was created more than 1h ago.
 //
+// A system account is considered orphaned when all conditions are satisfied:
+// 1. It is not managed by Konnect.
+// 2. It was created more than 1h ago.
+//
 // Usage: `go run ./hack/cleanup [mode]`
 // Where `mode` is one of:
 // - `all` (default): clean up both GKE clusters and Konnect control planes
 // - `gke`: clean up only GKE clusters
-// - `konnect`: clean up only Konnect control planes
+// - `konnect`: clean up only Konnect control planes and system accounts
 package main
 
 import (
@@ -103,11 +107,13 @@ func resolveCleanupFuncs(mode string) []func(context.Context, logr.Logger) error
 	case cleanupModeKonnect:
 		return []func(context.Context, logr.Logger) error{
 			cleanupKonnectControlPlanes,
+			cleanupKonnectSystemAccounts,
 		}
 	default:
 		return []func(context.Context, logr.Logger) error{
 			cleanupGKEClusters,
 			cleanupKonnectControlPlanes,
+			cleanupKonnectSystemAccounts,
 		}
 	}
 }

ingress-controller/internal/konnect/sdk/sdk.go

@@ -1,18 +1,24 @@
 package sdk
 
 import (
+	"strings"
+
 	sdkkonnectgo "github.com/Kong/sdk-konnect-go"
 	sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components"
+	"github.com/samber/lo"
 )
 
 // New returns a new SDK instance.
 func New(token string, opts ...sdkkonnectgo.SDKOption) *sdkkonnectgo.SDK {
+	security := sdkkonnectcomp.Security{}
+	switch {
+	case strings.HasPrefix(token, "kpat_"):
+		security.PersonalAccessToken = lo.ToPtr(token)
+	case strings.HasPrefix(token, "spat_"):
+		security.SystemAccountAccessToken = lo.ToPtr(token)
+	}
 	sdkOpts := []sdkkonnectgo.SDKOption{
-		sdkkonnectgo.WithSecurity(
-			sdkkonnectcomp.Security{
-				PersonalAccessToken: sdkkonnectgo.String(token),
-			},
-		),
+		sdkkonnectgo.WithSecurity(security),
 	}
 	sdkOpts = append(sdkOpts, opts...)
 

ingress-controller/test/internal/helpers/konnect/certificates.go

@@ -8,7 +8,7 @@ import (
 
 	sdkkonnectgo "github.com/Kong/sdk-konnect-go"
 	sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components"
-	"github.com/Kong/sdk-konnect-go/retry"
+	sdkretry "github.com/Kong/sdk-konnect-go/retry"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -18,24 +18,29 @@ import (
 
 // CreateClientCertificate creates a TLS client certificate and POSTs it to Konnect Control Plane configuration API
 // so that KIC can use the certificates to authenticate against Konnect Admin API.
-func CreateClientCertificate(ctx context.Context, t *testing.T, cpID string) (certPEM string, keyPEM string) {
+func CreateClientCertificate(ctx context.Context, t *testing.T, cpID string, token ...string) (certPEM string, keyPEM string) {
 	t.Helper()
 
-	sdk := sdk.New(accessToken(), serverURLOpt(),
-		sdkkonnectgo.WithRetryConfig(retry.Config{
-			Backoff: &retry.BackoffStrategy{
-				InitialInterval: 100,
-				MaxInterval:     2000,
-				Exponent:        1.2,
-				MaxElapsedTime:  10000,
-			},
-		}),
-	)
+	retryConfig := sdkkonnectgo.WithRetryConfig(sdkretry.Config{
+		Backoff: &sdkretry.BackoffStrategy{
+			InitialInterval: 100,
+			MaxInterval:     2000,
+			Exponent:        1.2,
+			MaxElapsedTime:  10000,
+		},
+	})
+
+	var s *sdkkonnectgo.SDK
+	if len(token) == 0 {
+		s = sdk.New(accessToken(), serverURLOpt(), retryConfig)
+	} else {
+		s = sdk.New(token[0], serverURLOpt(), retryConfig)
+	}
 
 	cert, key := certificate.MustGenerateCertPEMFormat()
 
 	t.Log("creating client certificate in Konnect")
-	resp, err := sdk.DPCertificates.CreateDataplaneCertificate(ctx, cpID, &sdkkonnectcomp.DataPlaneClientCertificateRequest{
+	resp, err := s.DPCertificates.CreateDataplaneCertificate(ctx, cpID, &sdkkonnectcomp.DataPlaneClientCertificateRequest{
 		Cert: string(cert),
 	})
 	require.NoError(t, err)

ingress-controller/test/internal/helpers/konnect/control_plane.go

@@ -9,11 +9,11 @@ import (
 	"testing"
 	"time"
 
+	sdkkonnectgo "github.com/Kong/sdk-konnect-go"
 	sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components"
 	"github.com/avast/retry-go/v4"
 	"github.com/google/uuid"
 	"github.com/samber/lo"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/kong/kong-operator/v2/ingress-controller/internal/adminapi"
@@ -25,14 +25,18 @@ import (
 
 // CreateTestControlPlane creates a control plane to be used in tests. It returns the created control plane's ID.
 // It also sets up a cleanup function for it to be deleted.
-func CreateTestControlPlane(ctx context.Context, t *testing.T) string {
+func CreateTestControlPlane(ctx context.Context, t *testing.T, token ...string) string {
 	t.Helper()
 
-	sdk := sdk.New(accessToken(), serverURLOpt())
-
+	var s *sdkkonnectgo.SDK
+	if len(token) == 0 {
+		s = sdk.New(accessToken(), serverURLOpt())
+	} else {
+		s = sdk.New(token[0], serverURLOpt())
+	}
 	var cpID string
 	createRgErr := retry.Do(func() error {
-		createResp, err := sdk.ControlPlanes.CreateControlPlane(ctx,
+		createResp, err := s.ControlPlanes.CreateControlPlane(ctx,
 			sdkkonnectcomp.CreateControlPlaneRequest{
 				Name:        uuid.NewString(),
 				Description: lo.ToPtr(generateTestKonnectControlPlaneDescription(t)),
@@ -63,20 +67,29 @@ func CreateTestControlPlane(ctx context.Context, t *testing.T) string {
 
 		cpID = createResp.ControlPlane.ID
 		return nil
-	}, retry.Attempts(5), retry.Delay(time.Second))
+	},
+		retry.Attempts(5),
+		retry.Delay(time.Second),
+		retry.OnRetry(func(_ uint, err error) {
+			t.Logf("failed to create control plane, retrying: %v", err)
+		}),
+	)
 	require.NoError(t, createRgErr)
 
 	t.Cleanup(func() {
-		ctx = context.Background()
-		t.Logf("deleting test Konnect Control Plane: %q", cpID)
+		fmt.Printf("deleting test Konnect Control Plane: %q\n", cpID)
 		err := retry.Do(
 			func() error { //nolint:contextcheck
-				_, err := sdk.ControlPlanes.DeleteControlPlane(context.Background(), cpID)
+				_, err := s.ControlPlanes.DeleteControlPlane(context.Background(), cpID)
 				return err
 			},
 			retry.Attempts(5), retry.Delay(time.Second),
 		)
-		assert.NoErrorf(t, err, "failed to cleanup a control plane: %q", cpID)
+		if err != nil {
+			// Don't fail the test if cleanup fails, just log the error.
+			// Cleanup job will eventually clean up the control plane.
+			fmt.Printf("failed to delete control plane %q: %v\n", cpID, err)
+		}
 
 		// Since Konnect authorization v2 supports cleanup of roles after control plane deleted, we do not need to delete them manually.
 	})