operator controller and rework kustomize to support bindata

Dockerfile is updated to include new 'operator' binary
and bindata directory. Bindata will contain
all the CRDs, RBAC, and deployment files needed
to deploy all the operators.

Add Makefile targets for bindata, and run-operator.

Forklift bindata impl from old CNOSP compute_node_operator.

Implement new controllers/operator/openstack_controller.go
which process files in /bindata.

Calling 'make bindata' during a Renovate sync should keep
things in sync.

Jira: OSPRH-11244

Author: dprince

.github/workflows/build-openstack-operator.yaml

@@ -18,7 +18,6 @@ jobs:
       go_version: 1.21.x
       operator_sdk_version: 1.31.0
       bundle_dockerfile: ./bundle.Dockerfile
-      catalog_extra_bundles_script: ./hack/pin-bundle-images.sh
     secrets:
       IMAGENAMESPACE: ${{ secrets.IMAGENAMESPACE }}
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}

.pre-commit-config.yaml

@@ -30,6 +30,11 @@ repos:
       language: system
       entry: make
       args: ['crd-schema-check']
+    - id: make-bindata
+      name: make-bindata
+      language: system
+      entry: make
+      args: ['bindata']
       pass_filenames: false
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -47,6 +52,7 @@ repos:
     - id: destroyed-symlinks
     - id: check-yaml
       args: [-m]
+      exclude: '^bindata/operator|^config/operator'
     - id: check-json
     - id: detect-private-key
     - id: end-of-file-fixer

Dockerfile

@@ -26,8 +26,10 @@ RUN if [ ! -f $CACHITO_ENV_FILE ]; then go mod download ; fi
 
 # Build manager
 RUN if [ -f $CACHITO_ENV_FILE ] ; then source $CACHITO_ENV_FILE ; fi ; env ${GO_BUILD_EXTRA_ENV_ARGS} go build ${GO_BUILD_EXTRA_ARGS} -a -o ${DEST_ROOT}/manager main.go
+RUN if [ -f $CACHITO_ENV_FILE ] ; then source $CACHITO_ENV_FILE ; fi ; env ${GO_BUILD_EXTRA_ENV_ARGS} go build ${GO_BUILD_EXTRA_ARGS} -a -o ${DEST_ROOT}/operator cmd/operator/main.go
 
 RUN cp -r config/services ${DEST_ROOT}/services
+RUN cp -r bindata ${DEST_ROOT}/bindata
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -65,9 +67,11 @@ WORKDIR /
 
 # Install operator binary to WORKDIR
 COPY --from=builder ${DEST_ROOT}/manager .
+COPY --from=builder ${DEST_ROOT}/operator .
 
 # Install services
 COPY --from=builder ${DEST_ROOT}/services ${OPERATOR_SERVICES}
+COPY --from=builder ${DEST_ROOT}/bindata /bindata
 
 USER $USER_ID
 

Makefile

@@ -131,15 +131,30 @@ help: ## Display this help.
 
 ##@ Development
 
+# (dprince) FIXME: controller-gen crd didn't seem to like multiple paths so I didn't split it. So we can continue using kubebuilder
+# I did split out the rbac for both binaries so we can use separate roles
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd$(CRDDESC_OVERRIDE) webhook paths="./..." output:crd:artifacts:config=config/crd/bases && \
+	mkdir -p config/operator/rbac && \
+	$(CONTROLLER_GEN) crd$(CRDDESC_OVERRIDE) output:crd:artifacts:config=config/crd/bases webhook paths="./..." && \
+	$(CONTROLLER_GEN) rbac:roleName=manager-role paths="{./apis/client/...,./apis/core/...,./apis/dataplane/...,./controllers/client/...,./controllers/core/...,./controllers/dataplane/...,./pkg/...}" output:dir=config/rbac && \
+	$(CONTROLLER_GEN) rbac:roleName=operator-role paths="./controllers/operator/..." paths="./apis/operator/..." output:dir=config/operator/rbac && \
 	rm -f apis/bases/* && cp -a config/crd/bases apis/
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
+.PHONY: bindata
+bindata: kustomize yq ## Build docker image with the manager.
+	mkdir -p bindata/crds bindata/rbac bindata/operator
+	$(KUSTOMIZE) build config/crd > bindata/crds/crds.yaml
+	$(KUSTOMIZE) build config/default > bindata/operator/operator.yaml
+	cp config/operator/managers.yaml bindata/operator/
+	cp config/operator/rabbit.yaml bindata/operator/
+	$(KUSTOMIZE) build config/rbac > bindata/rbac/rbac.yaml
+	/bin/bash hack/sync-bindata.sh
+
 .PHONY: fmt
 fmt: ## Run go fmt against code.
 	go fmt ./...
@@ -207,8 +222,18 @@ run: manifests generate fmt vet ## Run a controller from your host.
 	source hack/export_related_images.sh && \
 	go run ./main.go -metrics-bind-address ":$(METRICS_PORT)" -health-probe-bind-address ":$(HEALTH_PORT)"
 
+.PHONY: run-operator
+run-operator: export METRICS_PORT?=8080
+run-operator: export HEALTH_PORT?=8081
+run-operator: export ENABLE_WEBHOOKS?=false
+run-operator: export BASE_BINDATA?=bindata
+run-operator: export OPERATOR_IMAGE_URL=${IMG}
+run-operator: manifests generate fmt vet ## Run a controller from your host.
+	source hack/export_operator_related_images.sh && \
+	go run ./cmd/operator/main.go -metrics-bind-address ":$(METRICS_PORT)" -health-probe-bind-address ":$(HEALTH_PORT)"
+
 .PHONY: docker-build
-docker-build:  ## Build docker image with the manager.
+docker-build: ## Build docker image with the manager.
 	podman build -t ${IMG} . ${DOCKER_BUILD_ARGS}
 
 .PHONY: docker-push
@@ -273,7 +298,7 @@ GINKGO_TESTS ?= ./tests/... ./apis/client/... ./apis/core/... ./apis/dataplane/.
 KUTTL ?= $(LOCALBIN)/kubectl-kuttl
 
 ## Tool Versions
-KUSTOMIZE_VERSION ?= v3.8.7
+KUSTOMIZE_VERSION ?= v5.5.0 #(dprince: bumped to aquire new features like --load-restrictor)
 CONTROLLER_TOOLS_VERSION ?= v0.14.0
 CRD_MARKDOWN_VERSION ?= v0.0.3
 KUTTL_VERSION ?= 0.17.0
@@ -341,11 +366,10 @@ endif
 .PHONY: bundle
 bundle: build manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files.
 	$(OPERATOR_SDK) generate kustomize manifests -q
-	cd config/manager && \
-	$(KUSTOMIZE) edit set image controller=$(IMG) && \
-	$(KUSTOMIZE) edit add patch --kind Deployment --name controller-manager --namespace system --patch "[{\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/env/0\", \"value\": {\"name\": \"OPENSTACK_RELEASE_VERSION\", \"value\": \"$(OPENSTACK_RELEASE_VERSION)\"}}]"
-	$(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS)
-	cp dependencies.yaml ./bundle/metadata
+	cd config/operator/deployment/ && $(KUSTOMIZE) edit set image controller=$(IMG) && \
+	$(KUSTOMIZE) edit add patch --kind Deployment --name controller-operator --namespace system --patch "[{\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/env/0\", \"value\": {\"name\": \"OPENSTACK_RELEASE_VERSION\", \"value\": \"$(OPENSTACK_RELEASE_VERSION)\"}}]"
+	sed -i -e 's|operator:image|$(IMG)|' config/operator/deployment/deployment.yaml
+	$(KUSTOMIZE) build config/operator --load-restrictor='LoadRestrictionsNone' | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS)
 	$(OPERATOR_SDK) bundle validate ./bundle
 
 .PHONY: bundle-build

apis/bases/operator.openstack.org_openstacks.yaml

@@ -15,7 +15,15 @@ spec:
     singular: openstack
   scope: Namespaced
   versions:
-  - name: v1beta1
+  - additionalPrinterColumns:
+    - jsonPath: .status.deployedOperatorCount
+      name: Deployed Operator Count
+      type: integer
+    - description: Status
+      jsonPath: .status.conditions[0].status
+      name: Status
+      type: string
+    name: v1beta1
     schema:
       openAPIV3Schema:
         properties:
@@ -26,11 +34,36 @@ spec:
           metadata:
             type: object
           spec:
-            properties:
-              foo:
-                type: string
             type: object
           status:
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastTransitionTime:
+                      format: date-time
+                      type: string
+                    message:
+                      type: string
+                    reason:
+                      type: string
+                    severity:
+                      type: string
+                    status:
+                      type: string
+                    type:
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - status
+                  - type
+                  type: object
+                type: array
+              deployedOperatorCount:
+                type: integer
+              observedGeneration:
+                format: int64
+                type: integer
             type: object
         type: object
     served: true

apis/operator/v1beta1/conditions.go

@@ -0,0 +1,46 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
+)
+
+// OpenStack Condition Types used by API objects.
+const (
+	// OpenStackOperatorReadyCondition Status=True condition which indicates if operators have been deployed
+	OpenStackOperatorReadyCondition condition.Type = "OpenStackOperatorReadyCondition"
+)
+
+// Common Messages used by Openstack operator
+const (
+	//
+	// OpenStackOperator condition messages
+	//
+
+	// OpenStackOperatorErrorMessage
+	OpenStackOperatorErrorMessage = "OpenStackOperator error occured %s"
+
+	// OpenStackOperatorReadyInitMessage
+	OpenStackOperatorReadyInitMessage = "OpenStackOperator not started"
+
+	// OpenStackOperatorReadyRunningMessage
+	OpenStackOperatorReadyRunningMessage = "OpenStackOperator in progress"
+
+	// OpenStackOperatorReadyMessage
+	OpenStackOperatorReadyMessage = "OpenStackOperator completed"
+
+)

apis/operator/v1beta1/openstack_types.go

@@ -17,30 +17,32 @@ limitations under the License.
 package v1beta1
 
 import (
+	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
-// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
-
 // OpenStackSpec defines the desired state of OpenStack
 type OpenStackSpec struct {
-	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
-
-	// Foo is an example field of OpenStack. Edit openstack_types.go to remove/update
-	Foo string `json:"foo,omitempty"`
 }
 
 // OpenStackStatus defines the observed state of OpenStack
 type OpenStackStatus struct {
-	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-	// Important: Run "make" to regenerate code after modifying this file
-}
 
-//+kubebuilder:object:root=true
-//+kubebuilder:subresource:status
+	// +operator-sdk:csv:customresourcedefinitions:type=status,xDescriptors={"urn:alm:descriptor:io.kubernetes.conditions"}
+	// Conditions
+	Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"`
+
+	// DeployedOperatorCount - the number of operators deployed
+	DeployedOperatorCount *int `json:"deployedOperatorCount,omitempty"`
+
+	// ObservedGeneration - the most recent generation observed for this object.
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"` // no spec yet so maybe we don't need this
+}
 
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Deployed Operator Count",type=integer,JSONPath=`.status.deployedOperatorCount`
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status"
 // OpenStack is the Schema for the openstacks API
 type OpenStack struct {
 	metav1.TypeMeta   `json:",inline"`

apis/operator/v1beta1/zz_generated.deepcopy.go

@@ -44,7 +44,6 @@ import (
 
 	operatorv1beta1 "github.com/openstack-k8s-operators/openstack-operator/apis/operator/v1beta1"
 	operatorcontrollers "github.com/openstack-k8s-operators/openstack-operator/controllers/operator"
-	// +kubebuilder:scaffold:imports
 )
 
 var (
@@ -55,7 +54,6 @@ var (
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(operatorv1beta1.AddToScheme(scheme))
-	// +kubebuilder:scaffold:scheme
 }
 
 func main() {
@@ -96,7 +94,7 @@ func main() {
 		},
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
-		LeaderElectionID:       "40ba705e.openstack.org",
+		LeaderElectionID:       "20ca801f.openstack.org",
 		WebhookServer: webhook.NewServer(
 			webhook.Options{
 				Port:    9443,
@@ -147,7 +145,8 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "OpenStack")
 		os.Exit(1)
 	}
-	// +kubebuilder:scaffold:builder
+	operatorcontrollers.SetupEnv()
+
 	if err := mgr.AddHealthzCheck("healthz", checker); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)