slight refactor to simplify workload controller (#49)

* slight refactor to simplify workload controller
* add workload rbacs to appwrapper_contoller

Author: dgrove-oss

internal/controller/appwrapper_controller.go

@@ -67,10 +67,15 @@ type podStatusSummary struct {
 	failed    int32
 }
 
+// permission to fully control appwrappers
 //+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/finalizers,verbs=update
 
+// permission to manipulate workloads controlling appwrapper components to enable admitting them to our pseudo-clusterqueue
+// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads,verbs=get
+// +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloads/status,verbs=get;update;patch
+
 // permission to edit wrapped resources: pods, services, jobs, podgroups, pytorchjobs, rayclusters
 
 //+kubebuilder:rbac:groups="",resources=pods;services,verbs=get;list;watch;create;update;patch;delete

internal/controller/appwrapper_webhook.go

@@ -185,7 +185,7 @@ func (w *AppWrapperWebhook) validateAppWrapperCreate(ctx context.Context, aw *wo
 			if ps.Path == "" {
 				allErrors = append(allErrors, field.Required(podSetPath.Child("path"), "podspec must specify path"))
 			}
-			if _, err := getPodTemplateSpec(unstruct, ps.Path); err != nil {
+			if _, err := GetPodTemplateSpec(unstruct, ps.Path); err != nil {
 				allErrors = append(allErrors, field.Invalid(podSetPath.Child("path"), ps.Path,
 					fmt.Sprintf("path does not refer to a v1.PodSpecTemplate: %v", err)))
 			}

internal/controller/utils.go

@@ -23,13 +23,10 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-
-	workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2"
-	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
 )
 
-// getPodTemplateSpec extracts a Kueue-compatible PodTemplateSpec at the given path within obj
-func getPodTemplateSpec(obj *unstructured.Unstructured, path string) (*v1.PodTemplateSpec, error) {
+// GetPodTemplateSpec extracts a Kueue-compatible PodTemplateSpec at the given path within obj
+func GetPodTemplateSpec(obj *unstructured.Unstructured, path string) (*v1.PodTemplateSpec, error) {
 	candidatePTS, err := getRawTemplate(obj.UnstructuredContent(), path)
 	if err != nil {
 		return nil, err
@@ -59,26 +56,6 @@ func getPodTemplateSpec(obj *unstructured.Unstructured, path string) (*v1.PodTem
 	return template, nil
 }
 
-func getKueuePodSets(obj *unstructured.Unstructured, component *workloadv1beta2.AppWrapperComponent, awName string, componentIdx int) ([]kueue.PodSet, error) {
-	podSets := []kueue.PodSet{}
-	for psIdx, podSet := range component.PodSets {
-		replicas := int32(1)
-		if podSet.Replicas != nil {
-			replicas = *podSet.Replicas
-		}
-		template, err := getPodTemplateSpec(obj, podSet.Path)
-		if err != nil {
-			return nil, err
-		}
-		podSets = append(podSets, kueue.PodSet{
-			Name:     fmt.Sprintf("%s-%v-%v", awName, componentIdx, psIdx),
-			Template: *template,
-			Count:    replicas,
-		})
-	}
-	return podSets, nil
-}
-
 // return the subobject found at the given path, or nil if the path is invalid
 func getRawTemplate(obj map[string]interface{}, path string) (map[string]interface{}, error) {
 	parts := strings.Split(path, ".")

internal/controller/workload_controller.go

@@ -17,6 +17,8 @@ limitations under the License.
 package controller
 
 import (
+	"fmt"
+
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -74,9 +76,18 @@ func (aw *AppWrapper) PodSets() []kueue.PodSet {
 			if _, _, err := unstructured.UnstructuredJSONScheme.Decode(component.Template.Raw, nil, obj); err != nil {
 				continue // Should be unreachable; Template.Raw validated by our AdmissionController
 			}
-			toAdd, err := getKueuePodSets(obj, &component, aw.Name, componentIdx)
-			if err == nil {
-				podSets = append(podSets, toAdd...)
+			for psIdx, podSet := range component.PodSets {
+				replicas := int32(1)
+				if podSet.Replicas != nil {
+					replicas = *podSet.Replicas
+				}
+				if template, err := GetPodTemplateSpec(obj, podSet.Path); err == nil {
+					podSets = append(podSets, kueue.PodSet{
+						Name:     fmt.Sprintf("%s-%v-%v", aw.Name, componentIdx, psIdx),
+						Template: *template,
+						Count:    replicas,
+					})
+				}
 			}
 		}
 	}