Expose structured configuration

Author: astefanutti

controllers/appwrapper_controller.go

@@ -18,28 +18,30 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"os"
-	"strconv"
 	"strings"
 	"time"
 
 	mapiclientset "github.com/openshift/client-go/machine/clientset/versioned"
 	machineinformersv1beta1 "github.com/openshift/client-go/machine/informers/externalversions"
 	"github.com/openshift/client-go/machine/listers/machine/v1beta1"
-
-	appwrapperClientSet "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned"
+	"github.com/project-codeflare/instascale/pkg/config"
 
 	arbv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+	appwrapperClientSet "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned"
+	arbinformersFactory "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/informers/externalversions"
 	appwrapperlisters "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/listers/controller/v1beta1"
 
-	arbinformersFactory "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/informers/externalversions"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog"
+
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
@@ -48,9 +50,8 @@ import (
 // AppWrapperReconciler reconciles a AppWrapper object
 type AppWrapperReconciler struct {
 	client.Client
-	Scheme             *runtime.Scheme
-	ConfigsNamespace   string
-	OcmSecretNamespace string
+	Scheme *runtime.Scheme
+	Config config.InstaScaleConfiguration
 }
 
 var (
@@ -137,7 +138,6 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error {
-
 	kubeconfig := os.Getenv("KUBECONFIG")
 	cb, err := NewClientBuilder(kubeconfig)
 	if err != nil {
@@ -150,31 +150,23 @@ func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	machineClient = cb.MachineClientOrDie("machine-shared-informer")
 	kubeClient, _ = kubernetes.NewForConfig(restConfig)
-	instascaleConfigmap, err := kubeClient.CoreV1().ConfigMaps(r.ConfigsNamespace).Get(context.Background(), "instascale-config", metav1.GetOptions{})
-	if err != nil {
-		klog.Infof("Config map named instascale-config is not available in namespace %v", r.ConfigsNamespace)
-		return err
-	}
 
-	maxScaleNodesAllowed, err = strconv.Atoi(instascaleConfigmap.Data["maxScaleoutAllowed"])
-	if err != nil {
-		klog.Warningf("Error converting %v to int. Setting maxScaleNodesAllowed to 3", maxScaleNodesAllowed)
-		maxScaleNodesAllowed = 3
-	}
-	if instascaleConfigmap != nil {
-		klog.Errorf("Got config map named %v from namespace %v that configures max nodes in cluster to value %v", instascaleConfigmap.Name, instascaleConfigmap.Namespace, maxScaleNodesAllowed)
-	}
+	maxScaleNodesAllowed = int(r.Config.MaxScaleoutAllowed)
 
 	useMachineSets = true
-	ocmSecretExists := ocmSecretExists(r.OcmSecretNamespace)
-	if ocmSecretExists {
-		machinePoolsExists := machinePoolExists()
-
-		if machinePoolsExists {
-			useMachineSets = false
-			klog.Infof("Using machine pools %v", machinePoolsExists)
+	if ocmSecretRef := r.Config.OCMSecretRef; ocmSecretRef != nil {
+		if ocmSecret, err := getOCMSecret(ocmSecretRef); err != nil {
+			return fmt.Errorf("error reading OCM Secret from ref %q: %w", ocmSecretRef, err)
+		} else if token := ocmSecret.Data["token"]; len(token) > 0 {
+			ocmToken = string(token)
 		} else {
-			klog.Infof("Setting useMachineSets to %v", useMachineSets)
+			return fmt.Errorf("token is missing from OCM Secret %q", ocmSecretRef)
+		}
+		if ok, err := machinePoolExists(); err != nil {
+			return err
+		} else if ok {
+			useMachineSets = false
+			klog.Info("Using machine pools for cluster auto-scaling")
 		}
 	}
 
@@ -183,17 +175,8 @@ func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func ocmSecretExists(namespace string) bool {
-	instascaleOCMSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), "instascale-ocm-secret", metav1.GetOptions{})
-	if err != nil {
-		klog.Errorf("Error getting instascale-ocm-secret from namespace %v: %v", namespace, err)
-		klog.Infof("If you are looking to use OCM, ensure that the 'instascale-ocm-secret' secret is available on the cluster within namespace %v", namespace)
-		klog.Infof("Setting useMachineSets to %v.", useMachineSets)
-		return false
-	}
-
-	ocmToken = string(instascaleOCMSecret.Data["token"])
-	return true
+func getOCMSecret(secretRef *corev1.SecretReference) (*corev1.Secret, error) {
+	return kubeClient.CoreV1().Secrets(secretRef.Namespace).Get(context.Background(), secretRef.Name, metav1.GetOptions{})
 }
 
 func addAppwrappersThatNeedScaling() {

controllers/machinepools.go

@@ -3,14 +3,16 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"os"
+	"strings"
+
 	ocmsdk "github.com/openshift-online/ocm-sdk-go"
 	cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
 	configv1 "github.com/openshift/api/config/v1"
 	arbv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog"
-	"os"
-	"strings"
 )
 
 func createOCMConnection() (*ocmsdk.Connection, error) {
@@ -82,17 +84,15 @@ func deleteMachinePool(aw *arbv1.AppWrapper) {
 	})
 }
 
-// Check if machine pools exist
-func machinePoolExists() bool {
+func machinePoolExists() (bool, error) {
 	connection, err := createOCMConnection()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating OCM connection: %v", err)
+		return false, fmt.Errorf("error creating OCM connection: %w", err)
 	}
 	defer connection.Close()
 
 	machinePools := connection.ClustersMgmt().V1().Clusters().Cluster(ocmClusterID).MachinePools()
-	klog.Infof("Machine pools: %v", machinePools)
-	return machinePools != nil
+	return machinePools != nil, nil
 }
 
 // getOCMClusterID determines the internal clusterID to be used for OCM API calls

main.go

@@ -19,21 +19,28 @@ package main
 import (
 	"flag"
 	"os"
-
-	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
-	// to ensure that exec-entrypoint and run can make use of them.
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"strconv"
 
 	configv1 "github.com/openshift/api/config/v1"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
+	// to ensure that exec-entrypoint and run can make use of them.
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"github.com/project-codeflare/instascale/controllers"
-	//+kubebuilder:scaffold:imports
+	"github.com/project-codeflare/instascale/pkg/config"
+	// +kubebuilder:scaffold:imports
 	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
 )
 
@@ -46,7 +53,7 @@ func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(configv1.Install(scheme))
 
-	//+kubebuilder:scaffold:scheme
+	// +kubebuilder:scaffold:scheme
 	_ = mcadv1beta1.AddToScheme(scheme)
 }
 
@@ -71,7 +78,47 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	ctx := ctrl.SetupSignalHandler()
+
+	restConfig := ctrl.GetConfigOrDie()
+	kubeClient, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		setupLog.Error(err, "Error creating Kubernetes client")
+		os.Exit(1)
+	}
+
+	// InstaScale configuration
+	cfg := config.InstaScaleConfiguration{
+		MaxScaleoutAllowed: 3,
+	}
+
+	// Source InstaScale ConfigMap
+	if InstaScaleConfigMap, err := kubeClient.CoreV1().ConfigMaps(configsNamespace).Get(ctx, "instascale-config", metav1.GetOptions{}); err != nil {
+		setupLog.Error(err, "Error reading 'instascale-config' ConfigMap")
+		os.Exit(1)
+	} else if maxScaleoutAllowed := InstaScaleConfigMap.Data["maxScaleoutAllowed"]; maxScaleoutAllowed != "" {
+		if max, err := strconv.Atoi(maxScaleoutAllowed); err != nil {
+			setupLog.Error(err, "Error converting 'maxScaleoutAllowed' to integer")
+			os.Exit(1)
+		} else {
+			cfg.MaxScaleoutAllowed = int32(max)
+		}
+	}
+
+	// Source OCM Secret optionally
+	if OCMSecret, err := kubeClient.CoreV1().Secrets(ocmSecretNamespace).Get(ctx, "instascale-ocm-secret", metav1.GetOptions{}); apierrors.IsNotFound(err) {
+		setupLog.Info("If you are looking to use OCM, ensure the 'instascale-ocm-secret' Secret has been created")
+	} else if err != nil {
+		setupLog.Error(err, "Error checking if OCM Secret exists")
+		os.Exit(1)
+	} else {
+		cfg.OCMSecretRef = &corev1.SecretReference{
+			Namespace: OCMSecret.Namespace,
+			Name:      OCMSecret.Name,
+		}
+	}
+
+	mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
 		Scheme:                 scheme,
 		MetricsBindAddress:     metricsAddr,
 		Port:                   9443,
@@ -85,15 +132,14 @@ func main() {
 	}
 
 	if err = (&controllers.AppWrapperReconciler{
-		Client:             mgr.GetClient(),
-		Scheme:             mgr.GetScheme(),
-		ConfigsNamespace:   configsNamespace,
-		OcmSecretNamespace: ocmSecretNamespace,
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+		Config: cfg,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "AppWrapper")
 		os.Exit(1)
 	}
-	//+kubebuilder:scaffold:builder
+	// +kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
@@ -105,7 +151,7 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

pkg/config/config.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2022.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import corev1 "k8s.io/api/core/v1"
+
+// InstaScaleConfiguration defines the InstaScale configuration.
+type InstaScaleConfiguration struct {
+	// ocmSecretRef is reference to the authentication Secret for connecting to OCM.
+	// If provided, MachinePools are used to auto-scale the cluster.
+	// +optional
+	OCMSecretRef *corev1.SecretReference `json:"ocmSecretRef,omitempty"`
+	// maxScaleoutAllowed defines the upper limit for the number of cluster nodes
+	// that can be scaled out by InstaScale.
+	MaxScaleoutAllowed int32 `json:"maxScaleoutAllowed"`
+}