Allow only including / excluding specific extension images

stbenjam · stbenjam · commit 01212f363562 · 2025-08-01T05:44:49.000-04:00
When developing on my Mac, I point origin at locally-built binaries so I
can run it directly locally.  Now that we have many extension binaries,
I would have to override each one to make it work as the
payload-extracted images are all Linux executables. This adds
environment variables for only specifically including or excluding
particular image tags, making it easier to iterate on just one
extension.

Also migrates docs to the `docs` folder.
diff --git a/docs/test_extensions.md b/docs/test_extensions.md
@@ -1,25 +1,34 @@
-# External Binaries
+# OpenShift Test Extensions
 
-This package includes the code used for working with external test binaries.
-It's intended to house the implementation of the openshift-tests side of the
-[openshift-tests extension interface](https://github.com/openshift/enhancements/pull/1676), which is only
-partially implemented here for the moment.
+`openshift-tests` can be extended using the [openshift-tests extension
+interface](https://github.com/openshift/enhancements/pull/1676).
 
-There is a registry defined in binary.go, that lists the release image tag, and
-path to each external test binary.  These binaries should implement the OTE
-interface defined in the enhancement, and implemented by the vendorable
-[openshift-tests-extension](https://github.com/openshift-eng/openshift-tests-extension).
+There is a registry of binaries defined in
+`pkg/test/extensions/binary.go`, that lists the release image tag, and
+path to each external test binary.  These binaries should implement the
+OTE interface defined in the enhancement, and implemented by the
+vendorable [openshift-tests-extension](https://github.com/openshift-eng/openshift-tests-extension)
+framework.
 
-## Requirements
+## Local Development
 
 If the architecture of your local system where `openshift-tests` will run
 differs from the cluster under test, you should override the release payload
 with a payload of the architecture of your own system, as it is where the
-binaries will execute. Note, your OS must still be Linux. That means on Apple
-Silicon, you'll still need to run this in a Linux environment, such as a
-virtual machine, or x86 podman container.
+binaries will execute. Note, your OS must still be Linux to run any extracted images
+from the payload.
 
-## Overrides
+Alternatively, you can point origin to locally-built binaries.  An
+example workflow for Mac would be to override both the payload and
+specific image binaries:
+
+```
+export EXTENSIONS_PAYLOAD_OVERRIDE=registry.ci.openshift.org/ocp-arm64/release-arm64:4.18.0-0.nightly-arm64-2024-11-15-135718
+export EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS=tests,hyperkube
+export EXTENSION_BINARY_OVERRIDE_HYPERKUBE=$HOME/go/src/github.com/kubernetes/kubernetes/_output/bin/k8s-tests-ext"
+```
+
+## Environment Variables
 
 A number of environment variables for overriding the behavior of external
 binaries are available, but in general this should "just work". A complex set
@@ -28,7 +37,19 @@ credentials to use are found in this code, and extensively documented in code
 comments.  The following environment variables are available to force certain
 behaviors:
 
-### Extension Binary
+### Extension Binary Filtering
+
+Filter which extension binaries are extracted by image tag:
+
+```bash
+# Exclude specific extensions
+export EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS="hyperkube,machine-api-operator"
+
+# Include only specific extensions
+export EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS="tests,hyperkube"
+```
+
+### Extension Binary Override
 
 When developing locally, you may want to use a locally built extension
 binary. You can override the binary from the registry by setting:
diff --git a/pkg/test/extensions/binary.go b/pkg/test/extensions/binary.go
@@ -414,6 +414,9 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB
 		return nil, nil, errors.New("parallelism must be greater than zero")
 	}
 
+	// Filter extension binaries based on environment variables
+	filteredBinaries := filterExtensionBinariesByTags(extensionBinaries)
+
 	releaseImage, err := DetermineReleasePayloadImage()
 	if err != nil {
 		return nil, nil, errors.WithMessage(err, "couldn't determine release image")
@@ -441,14 +444,14 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB
 		binaries []*TestBinary
 		mu       sync.Mutex
 		wg       sync.WaitGroup
-		errCh    = make(chan error, len(extensionBinaries))
+		errCh    = make(chan error, len(filteredBinaries))
 		jobCh    = make(chan TestBinary)
 	)
 
 	// Producer: sends jobs to the jobCh channel
 	go func() {
 		defer close(jobCh)
-		for _, b := range extensionBinaries {
+		for _, b := range filteredBinaries {
 			select {
 			case <-ctx.Done():
 				return // Exit if context is cancelled
@@ -747,6 +750,77 @@ func safeComponentPath(c *extension.Component) string {
 	)
 }
 
+// filterExtensionBinariesByTags filters the extension binaries based on environment variables:
+// - EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS: comma-separated list of image tags to exclude
+// - EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS: comma-separated list of image tags to include (use only these)
+func filterExtensionBinariesByTags(binaries []TestBinary) []TestBinary {
+	excludeTags := os.Getenv("EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS")
+	includeTags := os.Getenv("EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS")
+
+	// If neither environment variable is set, return all binaries
+	if excludeTags == "" && includeTags == "" {
+		return binaries
+	}
+
+	var filtered []TestBinary
+
+	// Parse exclude tags
+	var excludeSet sets.Set[string]
+	if excludeTags != "" {
+		excludeList := strings.Split(excludeTags, ",")
+		excludeSet = sets.New[string]()
+		for _, tag := range excludeList {
+			tag = strings.TrimSpace(tag)
+			if tag != "" {
+				excludeSet.Insert(tag)
+			}
+		}
+		logrus.Infof("Excluding extension binaries with image tags: %v", excludeSet.UnsortedList())
+	}
+
+	// Parse include tags
+	var includeSet sets.Set[string]
+	if includeTags != "" {
+		includeList := strings.Split(includeTags, ",")
+		includeSet = sets.New[string]()
+		for _, tag := range includeList {
+			tag = strings.TrimSpace(tag)
+			if tag != "" {
+				includeSet.Insert(tag)
+			}
+		}
+		logrus.Infof("Including only extension binaries with image tags: %v", includeSet.UnsortedList())
+	}
+
+	// Filter binaries
+	for _, binary := range binaries {
+		imageTag := binary.imageTag
+
+		// If include tags are specified, only include binaries with those tags
+		if includeSet != nil {
+			if includeSet.Has(imageTag) {
+				filtered = append(filtered, binary)
+				logrus.Infof("Including extension binary with image tag: %s", imageTag)
+			} else {
+				logrus.Infof("Excluding extension binary with image tag: %s (not in include list)", imageTag)
+			}
+			continue
+		}
+
+		// If exclude tags are specified, exclude binaries with those tags
+		if excludeSet != nil && excludeSet.Has(imageTag) {
+			logrus.Infof("Excluding extension binary with image tag: %s", imageTag)
+			continue
+		}
+
+		// Include the binary
+		filtered = append(filtered, binary)
+	}
+
+	logrus.Infof("Filtered extension binaries: %d out of %d binaries will be processed", len(filtered), len(binaries))
+	return filtered
+}
+
 // filterToApplicableEnvironmentFlags filters the provided envFlags to only those that are applicable to the
 // APIVersion of OTE within the external binary.
 func (b *TestBinary) filterToApplicableEnvironmentFlags(envFlags EnvironmentFlags) EnvironmentFlags {
diff --git a/pkg/test/extensions/binary_filter_test.go b/pkg/test/extensions/binary_filter_test.go
@@ -0,0 +1,132 @@
+package extensions
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterExtensionBinariesByTags(t *testing.T) {
+	// Test data
+	testBinaries := []TestBinary{
+		{imageTag: "tests", binaryPath: "/usr/bin/openshift-tests"},
+		{imageTag: "hyperkube", binaryPath: "/usr/bin/k8s-tests-ext.gz"},
+		{imageTag: "machine-api-operator", binaryPath: "/machine-api-tests-ext.gz"},
+		{imageTag: "custom-operator", binaryPath: "/custom-tests-ext.gz"},
+	}
+
+	tests := []struct {
+		name              string
+		excludeTags       string
+		includeTags       string
+		expectedImageTags []string
+		expectedCount     int
+	}{
+		{
+			name:              "no environment variables set",
+			excludeTags:       "",
+			includeTags:       "",
+			expectedImageTags: []string{"tests", "hyperkube", "machine-api-operator", "custom-operator"},
+			expectedCount:     4,
+		},
+		{
+			name:              "exclude single tag",
+			excludeTags:       "hyperkube",
+			includeTags:       "",
+			expectedImageTags: []string{"tests", "machine-api-operator", "custom-operator"},
+			expectedCount:     3,
+		},
+		{
+			name:              "exclude multiple tags",
+			excludeTags:       "hyperkube,machine-api-operator",
+			includeTags:       "",
+			expectedImageTags: []string{"tests", "custom-operator"},
+			expectedCount:     2,
+		},
+		{
+			name:              "exclude with spaces",
+			excludeTags:       " hyperkube , machine-api-operator ",
+			includeTags:       "",
+			expectedImageTags: []string{"tests", "custom-operator"},
+			expectedCount:     2,
+		},
+		{
+			name:              "include single tag",
+			excludeTags:       "",
+			includeTags:       "tests",
+			expectedImageTags: []string{"tests"},
+			expectedCount:     1,
+		},
+		{
+			name:              "include multiple tags",
+			excludeTags:       "",
+			includeTags:       "tests,hyperkube",
+			expectedImageTags: []string{"tests", "hyperkube"},
+			expectedCount:     2,
+		},
+		{
+			name:              "include with spaces",
+			excludeTags:       "",
+			includeTags:       " tests , hyperkube ",
+			expectedImageTags: []string{"tests", "hyperkube"},
+			expectedCount:     2,
+		},
+		{
+			name:              "include takes precedence over exclude",
+			excludeTags:       "tests,hyperkube",
+			includeTags:       "tests",
+			expectedImageTags: []string{"tests"},
+			expectedCount:     1,
+		},
+		{
+			name:              "include non-existent tag",
+			excludeTags:       "",
+			includeTags:       "non-existent",
+			expectedImageTags: []string{},
+			expectedCount:     0,
+		},
+		{
+			name:              "exclude all tags",
+			excludeTags:       "tests,hyperkube,machine-api-operator,custom-operator",
+			includeTags:       "",
+			expectedImageTags: []string{},
+			expectedCount:     0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Set environment variables
+			if tt.excludeTags != "" {
+				os.Setenv("EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS", tt.excludeTags)
+			} else {
+				os.Unsetenv("EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS")
+			}
+
+			if tt.includeTags != "" {
+				os.Setenv("EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS", tt.includeTags)
+			} else {
+				os.Unsetenv("EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS")
+			}
+
+			// Call the function
+			result := filterExtensionBinariesByTags(testBinaries)
+
+			// Verify the count
+			assert.Equal(t, tt.expectedCount, len(result), "Expected %d binaries, got %d", tt.expectedCount, len(result))
+
+			// Verify the image tags
+			var actualImageTags []string
+			for _, binary := range result {
+				actualImageTags = append(actualImageTags, binary.imageTag)
+			}
+
+			assert.ElementsMatch(t, tt.expectedImageTags, actualImageTags, "Expected image tags %v, got %v", tt.expectedImageTags, actualImageTags)
+
+			// Clean up environment variables
+			os.Unsetenv("EXTENSION_BINARY_OVERRIDE_EXCLUDE_TAGS")
+			os.Unsetenv("EXTENSION_BINARY_OVERRIDE_INCLUDE_TAGS")
+		})
+	}
+}
