Update configuration for experimental/standard manifests

* Move feature overlays to be components.
* Create experimental and e2e overlays and manifests.
* Use e2e manifests for testing.
* Create `test-experimental-e2e` target.
* Remove unused webhook manifests.
* Add run of experimental-e2e

This is a first step, mostly moving things around, and there's a lot
of duplication in the overlays.

This also fixes the issue with the current test-e2e modifying the
standard manifest with e2e parameters, leaving a dirty workspace. This
changes the manifest used buy the e2e test to be it's own standard-e2e
manifest.

Signed-off-by: Todd Short <[email protected]>

Author: tmshort

.github/workflows/e2e.yaml

@@ -48,6 +48,26 @@ jobs:
           flags: e2e
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  experimental-e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Run e2e tests
+        run: ARTIFACT_PATH=/tmp/artifacts make test-experimental-e2e
+
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: experimental-e2e-artifacts
+          path: /tmp/artifacts/
+
   upgrade-e2e:
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -71,7 +71,10 @@ else
 $(warning Could not find docker or podman in path! This may result in targets requiring a container runtime failing!)
 endif
 
-KUSTOMIZE_BUILD_DIR := config/overlays/cert-manager
+KUSTOMIZE_STANDARD_OVERLAY := config/overlays/standard
+KUSTOMIZE_STANDARD_E2E_OVERLAY := config/overlays/standard-e2e
+KUSTOMIZE_EXPERIMENTAL_OVERLAY := config/overlays/experimental
+KUSTOMIZE_EXPERIMENTAL_E2E_OVERLAY := config/overlays/experimental-e2e
 
 export RELEASE_MANIFEST := operator-controller.yaml
 export RELEASE_INSTALL := install.sh
@@ -80,8 +83,14 @@ export RELEASE_CATALOGS := default-catalogs.yaml
 # List of manifests that are checked in
 MANIFEST_HOME := ./manifests
 STANDARD_MANIFEST := ./manifests/standard.yaml
+STANDARD_E2E_MANIFEST := ./manifests/standard-e2e.yaml
+EXPERIMENTAL_MANIFEST := ./manifests/experimental.yaml
+EXPERIMENTAL_E2E_MANIFEST := ./manifests/experimental-e2e.yaml
 CATALOGS_MANIFEST := ./manifests/default-catalogs.yaml
 
+# Manifest used by kind-deploy, which may be overridden by other targets
+SOURCE_MANIFEST := $(STANDARD_MANIFEST)
+
 # Disable -j flag for make
 .NOTPARALLEL:
 
@@ -147,7 +156,10 @@ manifests: $(CONTROLLER_GEN) $(KUSTOMIZE) #EXHELP Generate WebhookConfiguration,
 	$(CONTROLLER_GEN) --load-build-tags=$(GO_BUILD_TAGS) webhook paths="./internal/catalogd/..." output:webhook:artifacts:config=$(KUSTOMIZE_CATD_WEBHOOKS_DIR)
 	# Generate manifests stored in source-control
 	mkdir -p $(MANIFEST_HOME)
-	$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) > $(STANDARD_MANIFEST)
+	$(KUSTOMIZE) build $(KUSTOMIZE_STANDARD_OVERLAY) > $(STANDARD_MANIFEST)
+	$(KUSTOMIZE) build $(KUSTOMIZE_STANDARD_E2E_OVERLAY) > $(STANDARD_E2E_MANIFEST)
+	$(KUSTOMIZE) build $(KUSTOMIZE_EXPERIMENTAL_OVERLAY) > $(EXPERIMENTAL_MANIFEST)
+	$(KUSTOMIZE) build $(KUSTOMIZE_EXPERIMENTAL_E2E_OVERLAY) > $(EXPERIMENTAL_E2E_MANIFEST)
 
 .PHONY: generate
 generate: $(CONTROLLER_GEN) #EXHELP Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -247,11 +259,17 @@ image-registry: ## Build the testdata catalog used for e2e tests and push it to
 #
 # for example: ARTIFACT_PATH=/tmp/artifacts make test-e2e
 .PHONY: test-e2e
+test-e2e: SOURCE_MANIFEST := $(STANDARD_E2E_MANIFEST)
 test-e2e: KIND_CLUSTER_NAME := operator-controller-e2e
-test-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/e2e
 test-e2e: GO_BUILD_EXTRA_FLAGS := -cover
 test-e2e: run image-registry prometheus e2e e2e-metrics e2e-coverage kind-clean #HELP Run e2e test suite on local kind cluster
 
+.PHONY: test-experimental-e2e
+test-experimental-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_E2E_MANIFEST)
+test-experimental-e2e: KIND_CLUSTER_NAME := operator-controller-e2e
+test-experimental-e2e: GO_BUILD_EXTRA_FLAGS := -cover
+test-experimental-e2e: run image-registry prometheus e2e e2e-metrics e2e-coverage kind-clean #HELP Run experimental e2e test suite on local kind cluster
+
 .PHONY: prometheus
 prometheus: PROMETHEUS_NAMESPACE := olmv1-system
 prometheus: PROMETHEUS_VERSION := v0.83.0
@@ -270,7 +288,6 @@ e2e-metrics: #HELP Request metrics from prometheus; place in ARTIFACT_PATH if se
 	http://localhost:30900/api/v1/query > $(if $(ARTIFACT_PATH),$(ARTIFACT_PATH),.)/metrics.out
 
 .PHONY: extension-developer-e2e
-extension-developer-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/cert-manager
 extension-developer-e2e: KIND_CLUSTER_NAME := operator-controller-ext-dev-e2e
 extension-developer-e2e: export INSTALL_DEFAULT_CATALOGS := false
 extension-developer-e2e: run image-registry test-ext-dev-e2e kind-clean #EXHELP Run extension-developer e2e on local kind cluster
@@ -308,7 +325,8 @@ kind-load: $(KIND) #EXHELP Loads the currently constructed images into the KIND
 kind-deploy: export MANIFEST := $(RELEASE_MANIFEST)
 kind-deploy: export DEFAULT_CATALOG := $(RELEASE_CATALOGS)
 kind-deploy: manifests
-	sed "s/cert-git-version/cert-$(VERSION)/g" $(STANDARD_MANIFEST) > $(MANIFEST)
+	@echo -e "\n\U1F4D8 Using $(SOURCE_MANIFEST) as source manifest\n"
+	sed "s/cert-git-version/cert-$(VERSION)/g" $(SOURCE_MANIFEST) > $(MANIFEST)
 	cp $(CATALOGS_MANIFEST) $(DEFAULT_CATALOG)
 	envsubst '$$DEFAULT_CATALOG,$$CERT_MGR_VERSION,$$INSTALL_DEFAULT_CATALOGS,$$MANIFEST' < scripts/install.tpl.sh | bash -s
 

config/README.md

@@ -7,7 +7,7 @@ This includes basic support for an insecure OLMv1 deployment. This configuration
 * config/base/operator-controller
 * config/base/common
 
-## config/overlays/cert-manager
+## config/overlays/standard
 
 This includes support for a secure (i.e. with TLS) configuration of OLMv1. This configuration uses:
 * config/base/catalogd
@@ -19,18 +19,22 @@ This includes support for a secure (i.e. with TLS) configuration of OLMv1. This
 
 This configuration requires cert-manager.
 
-## config/overlays/e2e
+## config/overlays/standard-e2e
 
-This provides additional configuration support for end-to-end testing, including code coverage. This configuration uses:
-* config/base/catalogd
-* config/base/operator-controller
-* config/base/common
-* config/components/coverage
-* config/components/tls/catalogd
-* config/components/tls/operator-controller
-* config/components/tls/ca
+This provides additional configuration support for end-to-end testing, including code coverage. This configuration is based on **standard**, but also includes:
+* config/components/e2e/coverage
+* config/components/e2e/registries-conf
 
-This configuration requires cert-manager.
+## config/overlays/experimental
+
+This provides additional configuration experimental features. This configuration is based on **standard**, but also includes:
+* config/components/features/*
+
+## config/overlays/experimental-e2e
+
+This provides additional configuration support for end-to-end testing, including code coverage, and experimental features. This configuration is based on **experimental**, but also includes:
+* config/components/e2e/coverage
+* config/components/e2e/registries-conf
 
 ## Base Configuration
 
@@ -52,6 +56,10 @@ This provides common components to both operator-controller and catalogd, i.e. n
 
 Each of the `kustomization.yaml` files specify a `Component`, rather than an overlay, and thus, can be used within the overlays.
 
+### config/components/features
+
+This is the location for feature-gated configuration.
+
 ### config/components/tls/catalogd
 
 This provides a basic configuration of catalogd with TLS support.
@@ -72,6 +80,10 @@ This component _does not_ specify a namespace, and _must_ be included last.
 
 This component requires cert-manager.
 
-### config/components/coverage
+### config/components/e2e/coverage
+
+Provides code coverage configuration for end-to-end testing.
+
+### config/components/e2e/registries-conf
 
-Provides configuration for code coverage.
+Provides registry configuration for for end-to-end testing.

config/base/catalogd/crd/kustomization.yaml renamed to config/base/catalogd/crd/experimental/kustomization.yaml

@@ -2,5 +2,5 @@
 # since it depends on service name and namespace that are out of this kustomize package.
 # It should be run by config/default
 resources:
-- standard/olm.operatorframework.io_clustercatalogs.yaml
+- olm.operatorframework.io_clustercatalogs.yaml
 #+kubebuilder:scaffold:crdkustomizeresource

config/base/catalogd/crd/standard/kustomization.yaml

@@ -0,0 +1,6 @@
+# This kustomization.yaml is not intended to be run by itself,
+# since it depends on service name and namespace that are out of this kustomize package.
+# It should be run by config/default
+resources:
+- olm.operatorframework.io_clustercatalogs.yaml
+#+kubebuilder:scaffold:crdkustomizeresource

config/base/catalogd/kustomization.yaml

@@ -3,6 +3,5 @@ kind: Kustomization
 namespace: olmv1-system
 namePrefix: catalogd-
 resources:
-- crd
 - rbac
 - manager

config/base/operator-controller/crd/kustomization.yaml renamed to config/base/operator-controller/crd/experimental/kustomization.yaml

@@ -2,7 +2,7 @@
 # since it depends on service name and namespace that are out of this kustomize package.
 # It should be run by config/default
 resources:
-- standard/olm.operatorframework.io_clusterextensions.yaml
+- olm.operatorframework.io_clusterextensions.yaml
 
 # the following config is for teaching kustomize how to do kustomization for CRDs.
 configurations:

config/base/operator-controller/crd/kustomizeconfig.yaml renamed to config/base/operator-controller/crd/experimental/kustomizeconfig.yaml

@@ -0,0 +1,9 @@
+# This kustomization.yaml is not intended to be run by itself,
+# since it depends on service name and namespace that are out of this kustomize package.
+# It should be run by config/default
+resources:
+- olm.operatorframework.io_clusterextensions.yaml
+
+# the following config is for teaching kustomize how to do kustomization for CRDs.
+configurations:
+- kustomizeconfig.yaml

config/base/operator-controller/crd/standard/kustomization.yaml

@@ -0,0 +1,19 @@
+# This file is for teaching kustomize how to substitute name and namespace reference in CRD
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: CustomResourceDefinition
+    version: v1
+    group: apiextensions.k8s.io
+    path: spec/conversion/webhook/clientConfig/service/name
+
+namespace:
+- kind: CustomResourceDefinition
+  version: v1
+  group: apiextensions.k8s.io
+  path: spec/conversion/webhook/clientConfig/service/namespace
+  create: false
+
+varReference:
+- path: metadata/annotations