diff --git a/PROJECT b/PROJECT index c9cee1a4b..c62ed2011 100644 --- a/PROJECT +++ b/PROJECT @@ -51,4 +51,13 @@ resources: kind: GuardrailsOrchestrator path: github.com/trustyai-explainability/trustyai-service-operator/api/gorch/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: opendatahub.io + group: trustyai + kind: NemoGuardrails + path: github.com/trustyai-explainability/trustyai-service-operator/api/nemo/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/common/ca_bundle.go b/api/common/ca_bundle.go new file mode 100644 index 000000000..f0e698ec8 --- /dev/null +++ b/api/common/ca_bundle.go @@ -0,0 +1,38 @@ +package common + +// CABundleConfig defines the CA bundle configuration for custom certificates +type CABundleConfig struct { + // ConfigMapName is the name of the ConfigMap containing CA bundle certificates + ConfigMapName string `json:"configMapName"` + // ConfigMapNamespace is the namespace of the ConfigMap (defaults to the same namespace as the CR) + // +optional + ConfigMapNamespace string `json:"configMapNamespace,omitempty"` + // ConfigMapKeys specifies multiple keys within the ConfigMap containing CA bundle data + // All certificates from these keys will be concatenated into a single CA bundle file + // If not specified, defaults to [DefaultCABundleKey] + // +optional + // +kubebuilder:validation:MaxItems=50 + // +kubebuilder:validation:Items:Pattern="^[a-zA-Z0-9]([a-zA-Z0-9\\-_.]*[a-zA-Z0-9])?$" + // +kubebuilder:validation:Items:MaxLength=253 + ConfigMapKeys []string `json:"configMapKeys,omitempty"` +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CABundleConfig) DeepCopyInto(out *CABundleConfig) { + *out = *in + if in.ConfigMapKeys != nil { + in, out := &in.ConfigMapKeys, &out.ConfigMapKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CABundleConfig. +func (in *CABundleConfig) DeepCopy() *CABundleConfig { + if in == nil { + return nil + } + out := new(CABundleConfig) + in.DeepCopyInto(out) + return out +} diff --git a/api/common/condition.go b/api/common/condition.go new file mode 100644 index 000000000..cabf9c0f9 --- /dev/null +++ b/api/common/condition.go @@ -0,0 +1,27 @@ +package common + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Condition struct { + Type string `json:"type" description:"type of condition ie. Available|Progressing|Degraded."` + + Status corev1.ConditionStatus `json:"status" description:"status of the condition, one of True, False, Unknown"` + + // +optional + Reason string `json:"reason,omitempty" description:"one-word CamelCase reason for the condition's last transition"` + + // +optional + Message string `json:"message,omitempty" description:"human-readable message indicating details about last transition"` + + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime" description:"last time the condition transit from one status to another"` +} + +// DeepCopyInto copies all properties of this object into another object of the same type. +func (in *Condition) DeepCopyInto(out *Condition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} diff --git a/api/nemo/v1alpha1/groupversion_info.go b/api/nemo/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..6b3582135 --- /dev/null +++ b/api/nemo/v1alpha1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the trustyai v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=trustyai.opendatahub.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "trustyai.opendatahub.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/nemo/v1alpha1/nemoguardrails_types.go b/api/nemo/v1alpha1/nemoguardrails_types.go new file mode 100644 index 000000000..94b648025 --- /dev/null +++ b/api/nemo/v1alpha1/nemoguardrails_types.go @@ -0,0 +1,87 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "github.com/trustyai-explainability/trustyai-service-operator/api/common" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// NemoGuardrailsSpec defines the desired state of NemoGuardrails +type NemoGuardrailsSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // NemoConfig should be the name of the configmap containing the NeMO server configuration + NemoConfig string `json:"nemoConfig,omitempty"` + CABundleConfig *common.CABundleConfig `json:"caBundleConfig,omitempty"` + // Define Env information for the main container + // +optional + Env []corev1.EnvVar `json:"env,omitempty"` +} + +type CAStatus struct { + ODHTrustedCAFound bool `json:"odhTrustedCAFound"` + ODHTrustedCAError string `json:"odhTrustedCAError,omitempty"` + OpenshiftServingCAFound bool `json:"openshiftServingCAFound"` + OpenshiftServingCAError string `json:"openshiftServingCAError,omitempty"` + UserCAFound bool `json:"userCAFound,omitempty"` + UserCAError string `json:"userCAError,omitempty"` +} + +// NemoGuardrailStatus defines the observed state of NemoGuardrails +type NemoGuardrailStatus struct { + Phase string `json:"phase,omitempty"` + + // Conditions describes the state of the NemoGuardrails resource. + // +optional + Conditions []common.Condition `json:"conditions,omitempty"` + // CA describes the status of the CA configmaps + // +optional + CA *CAStatus `json:"ca,omitempty"` + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// NemoGuardrails is the Schema for the nemoguardrails API +type NemoGuardrails struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NemoGuardrailsSpec `json:"spec,omitempty"` + Status NemoGuardrailStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// NemoGuardrailsList contains a list of NemoGuardrails +type NemoGuardrailsList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NemoGuardrails `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NemoGuardrails{}, &NemoGuardrailsList{}) +} diff --git a/api/nemo/v1alpha1/zz_generated.deepcopy.go b/api/nemo/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..fc35375bd --- /dev/null +++ b/api/nemo/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,154 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "github.com/trustyai-explainability/trustyai-service-operator/api/common" + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CAStatus) DeepCopyInto(out *CAStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CAStatus. +func (in *CAStatus) DeepCopy() *CAStatus { + if in == nil { + return nil + } + out := new(CAStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoGuardrailStatus) DeepCopyInto(out *NemoGuardrailStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]common.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.CA != nil { + in, out := &in.CA, &out.CA + *out = new(CAStatus) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoGuardrailStatus. +func (in *NemoGuardrailStatus) DeepCopy() *NemoGuardrailStatus { + if in == nil { + return nil + } + out := new(NemoGuardrailStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoGuardrails) DeepCopyInto(out *NemoGuardrails) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoGuardrails. +func (in *NemoGuardrails) DeepCopy() *NemoGuardrails { + if in == nil { + return nil + } + out := new(NemoGuardrails) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NemoGuardrails) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoGuardrailsList) DeepCopyInto(out *NemoGuardrailsList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NemoGuardrails, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoGuardrailsList. +func (in *NemoGuardrailsList) DeepCopy() *NemoGuardrailsList { + if in == nil { + return nil + } + out := new(NemoGuardrailsList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NemoGuardrailsList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NemoGuardrailsSpec) DeepCopyInto(out *NemoGuardrailsSpec) { + *out = *in + if in.CABundleConfig != nil { + in, out := &in.CABundleConfig, &out.CABundleConfig + *out = (*in).DeepCopy() + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NemoGuardrailsSpec. +func (in *NemoGuardrailsSpec) DeepCopy() *NemoGuardrailsSpec { + if in == nil { + return nil + } + out := new(NemoGuardrailsSpec) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/main.go b/cmd/main.go index c637abc29..f201cb942 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -19,6 +19,7 @@ package main import ( "flag" "fmt" + nemov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/nemo/v1alpha1" "os" kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" @@ -67,6 +68,7 @@ func init() { utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) utilruntime.Must(kueuev1beta1.AddToScheme(scheme)) utilruntime.Must(gorchv1alpha1.AddToScheme(scheme)) + utilruntime.Must(nemov1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/config/base/params.env b/config/base/params.env index 9b8581d0b..3ebb43a6b 100644 --- a/config/base/params.env +++ b/config/base/params.env @@ -13,4 +13,5 @@ lmes-allow-online=true lmes-allow-code-execution=true guardrails-orchestrator-image=quay.io/trustyai/ta-guardrails-orchestrator:latest guardrails-built-in-detector-image=quay.io/trustyai/guardrails-detector-built-in:latest -guardrails-sidecar-gateway-image=quay.io/trustyai/guardrails-sidecar-gateway:latest \ No newline at end of file +guardrails-sidecar-gateway-image=quay.io/trustyai/guardrails-sidecar-gateway:latest +nemo-guardrails-image=quay.io/trustyai/nemo-guardrails-server:latest \ No newline at end of file diff --git a/config/crd/bases/trustyai.opendatahub.io_nemoguardrails.yaml b/config/crd/bases/trustyai.opendatahub.io_nemoguardrails.yaml new file mode 100644 index 000000000..ab06fa9d2 --- /dev/null +++ b/config/crd/bases/trustyai.opendatahub.io_nemoguardrails.yaml @@ -0,0 +1,231 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nemoguardrails.trustyai.opendatahub.io +spec: + group: trustyai.opendatahub.io + names: + kind: NemoGuardrails + listKind: NemoGuardrailsList + plural: nemoguardrails + singular: nemoguardrails + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NemoGuardrails is the Schema for the nemoguardrails API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NemoGuardrailsSpec defines the desired state of NemoGuardrails + properties: + caBundleConfig: + description: CABundleConfig defines the CA bundle configuration for + custom certificates + properties: + configMapKeys: + description: |- + ConfigMapKeys specifies multiple keys within the ConfigMap containing CA bundle data + All certificates from these keys will be concatenated into a single CA bundle file + If not specified, defaults to [DefaultCABundleKey] + items: + type: string + maxItems: 50 + type: array + configMapName: + description: ConfigMapName is the name of the ConfigMap containing + CA bundle certificates + type: string + configMapNamespace: + description: ConfigMapNamespace is the namespace of the ConfigMap + (defaults to the same namespace as the CR) + type: string + required: + - configMapName + type: object + env: + description: Define Env information for the main container + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + nemoConfig: + description: NemoConfig should be the name of the configmap containing + the NeMO server configuration + type: string + type: object + status: + description: NemoGuardrailStatus defines the observed state of NemoGuardrails + properties: + ca: + description: CA describes the status of the CA configmaps + properties: + odhTrustedCAError: + type: string + odhTrustedCAFound: + type: boolean + openshiftServingCAError: + type: string + openshiftServingCAFound: + type: boolean + userCAError: + type: string + userCAFound: + type: boolean + required: + - odhTrustedCAFound + - openshiftServingCAFound + type: object + conditions: + description: Conditions describes the state of the NemoGuardrails + resource. + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + phase: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 9857e5394..9c4ac8a87 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,10 +2,12 @@ resources: - bases/trustyai.opendatahub.io_trustyaiservices.yaml - bases/trustyai.opendatahub.io_lmevaljobs.yaml - bases/trustyai.opendatahub.io_guardrailsorchestrators.yaml + - bases/trustyai.opendatahub.io_nemoguardrails.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: [] #+kubebuilder:scaffold:crdkustomizewebhookpatch +#- path: patches/cainjection_in_nemoguardrails.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch configurations: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 163fd4527..701cb2e41 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -33,7 +33,7 @@ spec: args: - --leader-elect - --enable-services - - "TAS,LMES,GORCH" + - "TAS,LMES,GORCH,NEMO-GUARDRAILS" image: $(trustyaiOperatorImage) name: manager securityContext: diff --git a/config/overlays/odh/params.env b/config/overlays/odh/params.env index 91c3ea58e..b19c68bb5 100644 --- a/config/overlays/odh/params.env +++ b/config/overlays/odh/params.env @@ -13,4 +13,5 @@ lmes-allow-online=true lmes-allow-code-execution=true guardrails-orchestrator-image=quay.io/opendatahub/ta-guardrails-orchestrator:latest guardrails-built-in-detector-image=quay.io/opendatahub/odh-built-in-detector:latest -guardrails-sidecar-gateway-image=quay.io/opendatahub/vllm-orchestrator-gateway:latest \ No newline at end of file +guardrails-sidecar-gateway-image=quay.io/opendatahub/vllm-orchestrator-gateway:latest +nemo-guardrails-image=quay.io/trustyai/nemo-guardrails-server:latest \ No newline at end of file diff --git a/config/overlays/rhoai/params.env b/config/overlays/rhoai/params.env index 957a8c0ff..dde8a767d 100644 --- a/config/overlays/rhoai/params.env +++ b/config/overlays/rhoai/params.env @@ -13,4 +13,5 @@ lmes-allow-online=false lmes-allow-code-execution=false guardrails-orchestrator-image=quay.io/trustyai/ta-guardrails-orchestrator:latest guardrails-built-in-detector-image=quay.io/trustyai/guardrails-detector-built-in:latest -guardrails-sidecar-gateway-image=quay.io/trustyai/guardrails-sidecar-gateway:latest \ No newline at end of file +guardrails-sidecar-gateway-image=quay.io/trustyai/guardrails-sidecar-gateway:latest +nemo-guardrails-image=quay.io/trustyai/nemo-guardrails-server:latest \ No newline at end of file diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index f63567866..cd35c343b 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -8,6 +8,8 @@ resources: - auth_proxy_role.yaml - auth_proxy_role_binding.yaml - auth_proxy_client_clusterrole.yaml + - nemoguardrail_editor_role.yaml + - nemoguardrail_viewer_role.yaml - trustyaiservice_editor_role.yaml - trustyaiservice_viewer_role.yaml - non_admin_lmeval_role.yaml diff --git a/config/rbac/nemoguardrail_editor_role.yaml b/config/rbac/nemoguardrail_editor_role.yaml new file mode 100644 index 000000000..41a1bca13 --- /dev/null +++ b/config/rbac/nemoguardrail_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit nemoguardrails. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: trustyai-service-operator + app.kubernetes.io/managed-by: kustomize + name: nemoguardrail-editor-role +rules: +- apiGroups: + - trustyai.opendatahub.io + resources: + - nemoguardrails + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - trustyai.opendatahub.io + resources: + - nemoguardrails/status + verbs: + - get diff --git a/config/rbac/nemoguardrail_viewer_role.yaml b/config/rbac/nemoguardrail_viewer_role.yaml new file mode 100644 index 000000000..3efc74e9b --- /dev/null +++ b/config/rbac/nemoguardrail_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view nemoguardrails. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: trustyai-service-operator + app.kubernetes.io/managed-by: kustomize + name: nemoguardrail-viewer-role +rules: +- apiGroups: + - trustyai.opendatahub.io + resources: + - nemoguardrails + verbs: + - get + - list + - watch +- apiGroups: + - trustyai.opendatahub.io + resources: + - nemoguardrails/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c675a57c8..eed50bf4e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -221,6 +221,7 @@ rules: resources: - guardrailsorchestrators - lmevaljobs + - nemoguardrails - trustyaiservices verbs: - create @@ -235,6 +236,7 @@ rules: resources: - guardrailsorchestrators/finalizers - lmevaljobs/finalizers + - nemoguardrails/finalizers - trustyaiservices/finalizers verbs: - update @@ -243,6 +245,7 @@ rules: resources: - guardrailsorchestrators/status - lmevaljobs/status + - nemoguardrails/status - trustyaiservices/status verbs: - get diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 000000000..2c8f9096a --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples of your project ## +resources: +- trustyai_v1alpha1_nemoguardrail.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/trustyai_v1alpha1_nemoguardrail.yaml b/config/samples/trustyai_v1alpha1_nemoguardrail.yaml new file mode 100644 index 000000000..429a184d6 --- /dev/null +++ b/config/samples/trustyai_v1alpha1_nemoguardrail.yaml @@ -0,0 +1,9 @@ +apiVersion: trustyai.opendatahub.io/v1alpha1 +kind: NemoGuardrail +metadata: + labels: + app.kubernetes.io/name: trustyai-service-operator + app.kubernetes.io/managed-by: kustomize + name: nemoguardrail-sample +spec: + # TODO(user): Add fields here diff --git a/controllers/nemo.go b/controllers/nemo.go new file mode 100644 index 000000000..43edcfac8 --- /dev/null +++ b/controllers/nemo.go @@ -0,0 +1,25 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "github.com/trustyai-explainability/trustyai-service-operator/controllers/nemo" +) + +func init() { + registerService(nemo.ServiceName, nemo.ControllerSetUp) +} diff --git a/controllers/nemo/ca.go b/controllers/nemo/ca.go new file mode 100644 index 000000000..838b34d51 --- /dev/null +++ b/controllers/nemo/ca.go @@ -0,0 +1,142 @@ +package nemo + +import ( + "context" + "github.com/go-logr/logr" + nemov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/nemo/v1alpha1" + "github.com/trustyai-explainability/trustyai-service-operator/controllers/utils" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +// information about writing the new collated certificate +const ( + caBundleInitContainerName = "ca-bundle-initializer" + caBundleTransferVolumeName = "ca-bundle-transfer" + caBundleTransferDirName = "/tmp/ca-bundle-transfer/" + caBundleFileName = "ca-certificates.crt" + caBundleMountPath = "/etc/ssl/certs/" +) + +// information about certificate sources to be consumed +const ( + odhTrustedCABundle = "odh-trusted-ca-bundle" + odhTrustedCASourceDir = "/tmp/odh-trusted-ca-bundle/" + odhTrustedCABundleKey = "ca-bundle.crt" + openshiftServingCABundleKey = "service-ca.crt" + openshiftServingCASourceDir = "/tmp/openshift-serving-ca-bundle/" + userCASourceDir = "/tmp/user-ca-bundle/" +) + +func GetCABundleVolumeName(configmap corev1.ConfigMap) string { + return configmap.Name + "-vol" +} + +// LoadCAConfigs grabs the default CA certificates from odh-trusted-ca, the Openshift serving ca, and user-specified configmaps. Any configmaps that are not found are skipped +func (r *NemoGuardrailsReconciler) LoadCAConfigs(ctx context.Context, logger logr.Logger, nemoGuardrails nemov1alpha1.NemoGuardrails) (utils.CABundleInitContainerConfig, []corev1.ConfigMap, *nemov1alpha1.CAStatus) { + // === CA handling ===== + caBundleInitContainerConfig := utils.CABundleInitContainerConfig{ + CABundleInitName: caBundleInitContainerName, + CABundleSources: make([]utils.CABundleSourceVolume, 0), + CABundleTransferVolumeName: caBundleTransferVolumeName, + CABundleTransferDir: caBundleTransferDirName, + CABundleTransferFileName: caBundleFileName, + } + + var configMapsToMount []corev1.ConfigMap + caStatus := &nemov1alpha1.CAStatus{} + + odhTrustedConfigMap, err := utils.GetConfigMapByName(ctx, r.Client, odhTrustedCABundle, nemoGuardrails.Namespace) + if err != nil { + logger.Error(err, "Could not find or load ODH trusted CA bundle configmap, so will not be mounted.") + caStatus.ODHTrustedCAFound = false + caStatus.ODHTrustedCAError = err.Error() + } else { + configMapsToMount = append(configMapsToMount, *odhTrustedConfigMap) + caBundleInitContainerConfig.CABundleSources = append(caBundleInitContainerConfig.CABundleSources, utils.CABundleSourceVolume{ + CABundleSourceVolumeName: GetCABundleVolumeName(*odhTrustedConfigMap), + CABundleSourceDir: odhTrustedCASourceDir, + CABundleFileNames: []string{odhTrustedCABundleKey}, + }) + caStatus.ODHTrustedCAFound = true + } + + openshiftServingConfigMap, err := utils.GetConfigMapByName(ctx, r.Client, nemoGuardrails.Name+"-ca-bundle", nemoGuardrails.Namespace) + if err != nil { + logger.Error(err, "Could not find or load ODH trusted CA bundle configmap, so will not be mounted.") + caStatus.OpenshiftServingCAFound = false + caStatus.OpenshiftServingCAError = err.Error() + } else { + configMapsToMount = append(configMapsToMount, *openshiftServingConfigMap) + caBundleInitContainerConfig.CABundleSources = append(caBundleInitContainerConfig.CABundleSources, utils.CABundleSourceVolume{ + CABundleSourceVolumeName: GetCABundleVolumeName(*openshiftServingConfigMap), + CABundleSourceDir: openshiftServingCASourceDir, + CABundleFileNames: []string{openshiftServingCABundleKey}, + }) + caStatus.OpenshiftServingCAFound = true + } + + if nemoGuardrails.Spec.CABundleConfig != nil { + userCAConfigMap, err := utils.GetConfigMapByName(ctx, r.Client, nemoGuardrails.Spec.CABundleConfig.ConfigMapName, nemoGuardrails.Spec.CABundleConfig.ConfigMapNamespace) + if err != nil { + logger.Error(err, "Could not find or load the user-specified CA bundle configmap, so will not be mounted.") + caStatus.UserCAFound = false + caStatus.UserCAError = err.Error() + } else { + configMapsToMount = append(configMapsToMount, *userCAConfigMap) + caBundleInitContainerConfig.CABundleSources = append(caBundleInitContainerConfig.CABundleSources, utils.CABundleSourceVolume{ + CABundleSourceVolumeName: GetCABundleVolumeName(*userCAConfigMap), + CABundleSourceDir: userCASourceDir, + CABundleFileNames: nemoGuardrails.Spec.CABundleConfig.ConfigMapKeys, + }) + caStatus.UserCAFound = true + } + } + + return caBundleInitContainerConfig, configMapsToMount, caStatus +} + +// AddCAToDeployment modifies the NemoDeployment to contain the CA bundle init container and all necessary volumes +func (r *NemoGuardrailsReconciler) AddCAToDeployment(logger logr.Logger, deployment *appsv1.Deployment, caBundleInitContainerConfig utils.CABundleInitContainerConfig, nemoGuardrailsImage string, configMapsToMount []corev1.ConfigMap) error { + caBundleInitContainerConfig.CABundleInitImage = nemoGuardrailsImage + + // mount CA configmap volumes + for _, cm := range configMapsToMount { + utils.MountConfigMapToDeployment(&cm, GetCABundleVolumeName(cm), deployment) + } + + // create CA initcontainer + initContainer, err := utils.CreateCABundleInitContainer(caBundleInitContainerConfig) + if err != nil { + logger.Error(err, "Failed to create ca bundle init container") + return err + } + + // Add the volume to the deployment spec + volume := corev1.Volume{ + Name: caBundleInitContainerConfig.CABundleTransferVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + // mount transfer volume to main container + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, volume) + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( + deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + Name: caBundleInitContainerConfig.CABundleTransferVolumeName, + MountPath: caBundleMountPath, + ReadOnly: true, + }, + ) + // Set Python CA env var + deployment.Spec.Template.Spec.Containers[0].Env = append( + deployment.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{ + Name: "SSL_CERT_FILE", + Value: caBundleMountPath + caBundleFileName, + }, + ) + deployment.Spec.Template.Spec.InitContainers = []corev1.Container{initContainer} + return nil +} diff --git a/controllers/nemo/constants.go b/controllers/nemo/constants.go new file mode 100644 index 000000000..84282bcd8 --- /dev/null +++ b/controllers/nemo/constants.go @@ -0,0 +1,9 @@ +package nemo + +const ( + nemoGuardrailsName = "nemo-guardrail" + finalizerName = "trustyai.opendatahub.io/nemo-finalizer" + ServiceName = "NEMO-GUARDRAILS" + nemoImageKey = "nemo-guardrails-image" + oauthProxyImageKey = "oauthProxyImage" +) diff --git a/controllers/nemo/deployment.go b/controllers/nemo/deployment.go new file mode 100644 index 000000000..26043084b --- /dev/null +++ b/controllers/nemo/deployment.go @@ -0,0 +1,63 @@ +package nemo + +import ( + "context" + nemov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/nemo/v1alpha1" + "github.com/trustyai-explainability/trustyai-service-operator/controllers/constants" + templateParser "github.com/trustyai-explainability/trustyai-service-operator/controllers/nemo/templates" + "github.com/trustyai-explainability/trustyai-service-operator/controllers/utils" + appsv1 "k8s.io/api/apps/v1" + "reflect" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type ContainerImages struct { + NemoGuardrailsImage string + OAuthProxyImage string +} + +type DeploymentConfig struct { + NemoGuardrails *nemov1alpha1.NemoGuardrails + ContainerImages ContainerImages +} + +const deploymentTemplateFilename = "deployment.tmpl.yaml" + +func (r *NemoGuardrailsReconciler) createDeployment(ctx context.Context, nemoGuardrails *nemov1alpha1.NemoGuardrails) (*appsv1.Deployment, *string, error) { + var containerImages ContainerImages + + // get nemo guardrails image from trustyai configmap + nemoGuardrailsImage, err := utils.GetImageFromConfigMap(ctx, r.Client, nemoImageKey, constants.ConfigMap, r.Namespace) + if nemoGuardrailsImage == "" || err != nil { + log.FromContext(ctx).Error(err, "Error getting nemo-guardrails container image from ConfigMap.") + return nil, nil, err + } + containerImages.NemoGuardrailsImage = nemoGuardrailsImage + log.FromContext(ctx).Info("using NemoGuardrailsImage " + nemoGuardrailsImage + " " + "from config map " + r.Namespace + ":" + constants.ConfigMap) + + // get oauth image from trustyai configmap + oauthImage, err := utils.GetImageFromConfigMap(ctx, r.Client, oauthProxyImageKey, constants.ConfigMap, r.Namespace) + if err != nil { + log.FromContext(ctx).Error(err, "Error getting oauth container image from ConfigMap.") + return nil, nil, err + } + containerImages.OAuthProxyImage = oauthImage + log.FromContext(ctx).Info("using OauthProxyImage " + oauthImage + " " + "from config map " + r.Namespace + ":" + constants.ConfigMap) + + deploymentConfig := DeploymentConfig{ + NemoGuardrails: nemoGuardrails, + ContainerImages: containerImages, + } + var deployment *appsv1.Deployment + + deployment, err = templateParser.ParseResource[appsv1.Deployment](deploymentTemplateFilename, deploymentConfig, reflect.TypeOf(&appsv1.Deployment{})) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to parse deployment template") + } + if err := controllerutil.SetControllerReference(nemoGuardrails, deployment, r.Scheme); err != nil { + log.FromContext(ctx).Error(err, "Failed to set controller reference for deployment") + return nil, nil, err + } + return deployment, &nemoGuardrailsImage, nil +} diff --git a/controllers/nemo/nemoguardrail_controller.go b/controllers/nemo/nemoguardrail_controller.go new file mode 100644 index 000000000..3bc0a9230 --- /dev/null +++ b/controllers/nemo/nemoguardrail_controller.go @@ -0,0 +1,159 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nemo + +import ( + "context" + nemov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/nemo/v1alpha1" + templateParser "github.com/trustyai-explainability/trustyai-service-operator/controllers/nemo/templates" + "github.com/trustyai-explainability/trustyai-service-operator/controllers/utils" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/manager" + "time" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// NemoGuardrailReconciler reconciles a NemoGuardrails object +type NemoGuardrailsReconciler struct { + client.Client + Scheme *runtime.Scheme + Namespace string + Recorder record.EventRecorder +} + +const ( + serviceTemplate = "service.tmpl.yaml" + caBundleTemplate = "ca-bundle-configmap.tmpl.yaml" + routeTemplate = "route.tmpl.yaml" + routePort = "oauth-proxy" +) + +//+kubebuilder:rbac:groups=trustyai.opendatahub.io,resources=nemoguardrails,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=trustyai.opendatahub.io,resources=nemoguardrails/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=trustyai.opendatahub.io,resources=nemoguardrails/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the NemoGuardrails object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.3/pkg/reconcile +func (r *NemoGuardrailsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + // fetch instance of NemoGuardrails CR + nemoGuardrails := &nemov1alpha1.NemoGuardrails{} + err := r.Get(context.TODO(), req.NamespacedName, nemoGuardrails) + if err != nil { + if errors.IsNotFound(err) { + logger.Info("NemoGuardrails resource not found. Ignoring since object must be deleted.") + return ctrl.Result{}, nil + } + logger.Error(err, "Failed to get NemoGuardrails.") + return ctrl.Result{}, err + } + + _, err = utils.ReconcileConfigMap(ctx, r.Client, nemoGuardrails, nemoGuardrails.Name+"-ca-bundle", caBundleTemplate, templateParser.ParseResource) + if err != nil { + logger.Error(err, "Failed to reconcile service") + return ctrl.Result{}, err + } + + caBundleInitContainerConfig, configMapsToMount, caStatus := r.LoadCAConfigs(ctx, logger, *nemoGuardrails) + nemoGuardrails.Status.CA = caStatus + if err := r.Status().Update(ctx, nemoGuardrails); err != nil { + logger.Error(err, "Failed to update CA status") + return ctrl.Result{}, err + } + + existingDeployment := &appsv1.Deployment{} + err = r.Get(ctx, types.NamespacedName{Name: nemoGuardrails.Name, Namespace: nemoGuardrails.Namespace}, existingDeployment) + if err != nil && errors.IsNotFound(err) { + // Create a new deployment + deployment, nemoGuardrailsImage, err := r.createDeployment(ctx, nemoGuardrails) + if err != nil { + return ctrl.Result{}, err + } + + err = r.AddCAToDeployment(logger, deployment, caBundleInitContainerConfig, *nemoGuardrailsImage, configMapsToMount) + if err != nil { + return ctrl.Result{}, err + } + + // add user environment variables + if nemoGuardrails.Spec.Env != nil && len(nemoGuardrails.Spec.Env) > 0 { + logger.Info("Updating NemoGuardrails env with user-provided environment variables") + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, nemoGuardrails.Spec.Env...) + } + + logger.Info("Creating a new Deployment", "Deployment.Namespace", deployment.Namespace, "Deployment.Name", deployment.Name) + err = r.Create(ctx, deployment) + if err != nil { + logger.Error(err, "Failed to create new Deployment", "Deployment.Namespace", deployment.Namespace, "Deployment.Name", deployment.Name) + return ctrl.Result{}, err + } + } else if err != nil { + logger.Error(err, "Failed to get Deployment") + return ctrl.Result{}, err + } + + _, err = utils.ReconcileService(ctx, r.Client, nemoGuardrails, serviceTemplate, templateParser.ParseResource) + if err != nil { + logger.Error(err, "Failed to reconcile service") + return ctrl.Result{}, err + } + + _, err = utils.ReconcileDefaultRoute(ctx, r.Client, nemoGuardrails, routeTemplate, templateParser.ParseResource) + if err != nil { + logger.Error(err, "Failed to reconcile service") + return ctrl.Result{}, err + } + + // Finalize reconcilation + _, updateErr := r.reconcileStatuses(ctx, nemoGuardrails) + if updateErr != nil { + return ctrl.Result{}, updateErr + } + return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NemoGuardrailsReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&nemov1alpha1.NemoGuardrails{}). + Complete(r) +} + +// The registered function to set up NEMO-GUARDRAILS controller +func ControllerSetUp(mgr manager.Manager, ns, configmap string, recorder record.EventRecorder) error { + return (&NemoGuardrailsReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Namespace: ns, + Recorder: recorder, + }).SetupWithManager(mgr) +} diff --git a/controllers/nemo/nemoguardrail_controller_test.go b/controllers/nemo/nemoguardrail_controller_test.go new file mode 100644 index 000000000..85c8e6937 --- /dev/null +++ b/controllers/nemo/nemoguardrail_controller_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nemo + +// +//var _ = Describe("NemoGuardrails Controller", func() { +// Context("When reconciling a resource", func() { +// const resourceName = "test-resource" +// +// ctx := context.Background() +// +// typeNamespacedName := types.NamespacedName{ +// Name: resourceName, +// Namespace: "default", // TODO(user):Modify as needed +// } +// nemoguardrail := &trustyaiv1alpha1.NemoGuardrails{} +// +// BeforeEach(func() { +// By("creating the custom resource for the Kind NemoGuardrails") +// err := k8sClient.Get(ctx, typeNamespacedName, nemoguardrail) +// if err != nil && errors.IsNotFound(err) { +// resource := &trustyaiv1alpha1.NemoGuardrails{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: resourceName, +// Namespace: "default", +// }, +// // TODO(user): Specify other spec details if needed. +// } +// Expect(k8sClient.Create(ctx, resource)).To(Succeed()) +// } +// Expect(err).NotTo(HaveOccurred()) +// }) +// +// AfterEach(func() { +// // TODO(user): Cleanup logic after each test, like removing the resource instance. +// resource := &trustyaiv1alpha1.NemoGuardrails{} +// err := k8sClient.Get(ctx, typeNamespacedName, resource) +// Expect(err).NotTo(HaveOccurred()) +// +// By("Cleanup the specific resource instance NemoGuardrails") +// Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) +// }) +// It("should successfully reconcile the resource", func() { +// By("Reconciling the created resource") +// controllerReconciler := &NemoGuardrailsReconciler{ +// Client: k8sClient, +// Scheme: k8sClient.Scheme(), +// } +// +// _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ +// NamespacedName: typeNamespacedName, +// }) +// Expect(err).NotTo(HaveOccurred()) +// // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. +// // Example: If you expect a certain status condition after reconciliation, verify it here. +// }) +// }) +//}) diff --git a/controllers/nemo/status.go b/controllers/nemo/status.go new file mode 100644 index 000000000..bd76ac7bc --- /dev/null +++ b/controllers/nemo/status.go @@ -0,0 +1,57 @@ +package nemo + +import ( + "context" + nemov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/nemo/v1alpha1" + "github.com/trustyai-explainability/trustyai-service-operator/controllers/utils" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +func (r *NemoGuardrailsReconciler) updateStatus(ctx context.Context, original *nemov1alpha1.NemoGuardrails, update func(saved *nemov1alpha1.NemoGuardrails)) (*nemov1alpha1.NemoGuardrails, error) { + saved := original.DeepCopy() + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := r.Client.Get(ctx, client.ObjectKeyFromObject(original), saved) + if err != nil { + return err + } + update(saved) + err = r.Client.Status().Update(ctx, saved) + return err + }) + return saved, err +} + +func (r *NemoGuardrailsReconciler) reconcileStatuses(ctx context.Context, nemoGuardrails *nemov1alpha1.NemoGuardrails) (ctrl.Result, error) { + deploymentReady, _ := utils.CheckDeploymentReady(ctx, r.Client, nemoGuardrails.Name, nemoGuardrails.Namespace) + routeReady, _ := utils.CheckRouteReady(ctx, r.Client, nemoGuardrails.Name, nemoGuardrails.Namespace, "") + + if deploymentReady && routeReady { + _, updateErr := r.updateStatus(ctx, nemoGuardrails, func(saved *nemov1alpha1.NemoGuardrails) { + utils.SetResourceCondition(&saved.Status.Conditions, "Deployment", "DeploymentReady", "Deployment is ready", corev1.ConditionTrue) + utils.SetResourceCondition(&saved.Status.Conditions, "Route", "RouteReady", "Route is ready", corev1.ConditionTrue) + utils.SetCompleteCondition(&saved.Status.Conditions, corev1.ConditionTrue, utils.ReconcileCompleted, utils.ReconcileCompletedMessage) + saved.Status.Phase = utils.PhaseReady + }) + if updateErr != nil { + log.FromContext(ctx).Error(updateErr, "Failed to update status") + return ctrl.Result{}, updateErr + } + } else { + _, updateErr := r.updateStatus(ctx, nemoGuardrails, func(saved *nemov1alpha1.NemoGuardrails) { + + utils.SetStatus(&saved.Status.Conditions, "Deployment", deploymentReady) + utils.SetStatus(&saved.Status.Conditions, "Route", routeReady) + utils.SetCompleteCondition(&saved.Status.Conditions, corev1.ConditionFalse, utils.ReconcileFailed, utils.ReconcileFailedMessage) + }) + if updateErr != nil { + log.FromContext(ctx).Error(updateErr, "Failed to update status") + return ctrl.Result{}, updateErr + } + } + return ctrl.Result{}, nil +} diff --git a/controllers/nemo/suite_test.go b/controllers/nemo/suite_test.go new file mode 100644 index 000000000..e777fb6db --- /dev/null +++ b/controllers/nemo/suite_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nemo + +import ( + "fmt" + trustyaiv1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/nemo/v1alpha1" + "path/filepath" + "runtime" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s", + fmt.Sprintf("1.29.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = trustyaiv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/controllers/nemo/templates/ca-bundle-configmap.tmpl.yaml b/controllers/nemo/templates/ca-bundle-configmap.tmpl.yaml new file mode 100644 index 000000000..5356b9585 --- /dev/null +++ b/controllers/nemo/templates/ca-bundle-configmap.tmpl.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.ConfigMapName}} + namespace: {{.Owner.Namespace}} + annotations: + service.beta.openshift.io/inject-cabundle: 'true' + labels: + app: {{.Owner.Name}} + component: {{.Owner.Name}} \ No newline at end of file diff --git a/controllers/nemo/templates/deployment.tmpl.yaml b/controllers/nemo/templates/deployment.tmpl.yaml new file mode 100644 index 000000000..22a364787 --- /dev/null +++ b/controllers/nemo/templates/deployment.tmpl.yaml @@ -0,0 +1,113 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.NemoGuardrails.Name}} + namespace: {{.NemoGuardrails.Namespace}} + labels: + app: {{.NemoGuardrails.Name}} + component: {{.NemoGuardrails.Name}} + deploy-name: {{.NemoGuardrails.Name}} + app.kubernetes.io/instance: {{.NemoGuardrails.Name}} + app.kubernetes.io/name: {{.NemoGuardrails.Name}} + app.kubernetes.io/part-of: trustyai +spec: + replicas: 1 + selector: + matchLabels: + app: {{.NemoGuardrails.Name}} + template: + metadata: + labels: + app: {{.NemoGuardrails.Name}} + component: {{.NemoGuardrails.Name}} + deploy-name: {{.NemoGuardrails.Name}} + app.kubernetes.io/instance: {{.NemoGuardrails.Name}} + app.kubernetes.io/name: {{.NemoGuardrails.Name}} + app.kubernetes.io/part-of: trustyai + spec: + containers: + - name: nemo-guardrails + image: {{.ContainerImages.NemoGuardrailsImage}} + env: + - name: CONFIG_ID + value: rails-custom-action + readinessProbe: + httpGet: + path: / + port: 8000 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 10 + periodSeconds: 20 + successThreshold: 1 + failureThreshold: 3 + ports: + - containerPort: 8000 + volumeMounts: + - name: config-volume + mountPath: /app/config/rails-custom-action + - name: oauth-proxy + image: {{ .ContainerImages.OAuthProxyImage }} + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 100m + memory: 64Mi + readinessProbe: + httpGet: + path: /oauth/healthz + port: oauth-proxy + scheme: HTTPS + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /oauth/healthz + port: oauth-proxy + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + ports: + - name: oauth-proxy + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: {{.NemoGuardrails.Name}}-tls + mountPath: /etc/tls/private + args: + - '--cookie-secret=SECRET' + - '--https-address=:8443' + - '--email-domain=*' + - '--openshift-service-account={{ .NemoGuardrails.Name }}-proxy' + - '--provider=openshift' + - '--tls-cert=/etc/tls/private/tls.crt' + - '--tls-key=/etc/tls/private/tls.key' + - '--upstream=http://localhost:8000' + - '--skip-auth-regex=''(^/apis/v1beta1/healthz)''' + - >- + --openshift-sar={"namespace":"{{ .NemoGuardrails.Namespace }}","resource":"pods","verb":"get"} + - >- + --openshift-delegate-urls={"/": {"namespace": "{{ .NemoGuardrails.Namespace }}", "resource": + "pods", "verb": "get"}} + volumes: + - name: config-volume + configMap: + name: {{.NemoGuardrails.Spec.NemoConfig}} + - name: {{.NemoGuardrails.Name}}-tls + secret: + secretName: {{.NemoGuardrails.Name}}-tls + defaultMode: 420 diff --git a/controllers/nemo/templates/parser.go b/controllers/nemo/templates/parser.go new file mode 100644 index 000000000..d4c356a20 --- /dev/null +++ b/controllers/nemo/templates/parser.go @@ -0,0 +1,14 @@ +package templates + +import ( + "embed" + "github.com/trustyai-explainability/trustyai-service-operator/controllers/utils" + "reflect" +) + +//go:embed *.tmpl.yaml +var templateFS embed.FS + +func ParseResource[T any](templatePath string, data interface{}, outType reflect.Type) (*T, error) { + return utils.ParseResourceFromFS[T](templatePath, data, outType, templateFS) +} diff --git a/controllers/nemo/templates/route.tmpl.yaml b/controllers/nemo/templates/route.tmpl.yaml new file mode 100644 index 000000000..8ed6aeedc --- /dev/null +++ b/controllers/nemo/templates/route.tmpl.yaml @@ -0,0 +1,19 @@ +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: {{.Owner.Name}} + namespace: {{.Owner.Namespace}} + labels: + app: {{.Owner.Name}} + component: {{.Owner.Name}} +spec: + to: + kind: Service + name: {{.Owner.Name}}-service + weight: 100 + port: + targetPort: oauth-proxy + tls: + termination: passthrough + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/controllers/nemo/templates/service.tmpl.yaml b/controllers/nemo/templates/service.tmpl.yaml new file mode 100644 index 000000000..069802d0e --- /dev/null +++ b/controllers/nemo/templates/service.tmpl.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.Owner.Name}}-service + namespace: {{.Owner.Namespace}} + labels: + app: {{.Owner.Name}} + component: {{.Owner.Name}} + annotations: + service.beta.openshift.io/serving-cert-secret-name: {{.Owner.Name}}-tls +spec: + ipFamilies: + - IPv4 + ports: + - name: oauth-proxy + protocol: TCP + port: 443 + targetPort: 8443 + internalTrafficPolicy: Cluster + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app: {{.Owner.Name}} + component: {{.Owner.Name}} diff --git a/controllers/utils/ca.go b/controllers/utils/ca.go new file mode 100644 index 000000000..2bff00c25 --- /dev/null +++ b/controllers/utils/ca.go @@ -0,0 +1,143 @@ +package utils + +// modified from https://github.com/opendatahub-io/llama-stack-k8s-operator/blob/odh/controllers/resource_helper.go + +import ( + "errors" + "fmt" + corev1 "k8s.io/api/core/v1" + "regexp" + "strings" +) + +// Constants for validation limits. +const ( + // maxConfigMapKeyLength defines the maximum allowed length for ConfigMap keys + // based on Kubernetes DNS subdomain name limits. + maxConfigMapKeyLength = 253 +) + +// validConfigMapKeyRegex defines allowed characters for ConfigMap keys. +// Kubernetes ConfigMap keys must be valid DNS subdomain names or data keys. +var validConfigMapKeyRegex = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9\-_.]*[a-zA-Z0-9])?$`) + +// validateConfigMapKeys validates that all ConfigMap keys contain only safe characters. +// Note: This function validates key names only. PEM content validation is performed +// separately in the controller's reconcileCABundleConfigMap function. +func validateConfigMapKeys(keys []string) error { + for _, key := range keys { + if key == "" { + return errors.New("ConfigMap key cannot be empty") + } + if len(key) > maxConfigMapKeyLength { + return fmt.Errorf("failed to validate ConfigMap key '%s': too long (max %d characters)", key, maxConfigMapKeyLength) + } + if !validConfigMapKeyRegex.MatchString(key) { + return fmt.Errorf("failed to validate ConfigMap key '%s': contains invalid characters. Only alphanumeric characters, hyphens, underscores, and dots are allowed", key) + } + // Additional security check: prevent path traversal attempts + if strings.Contains(key, "..") || strings.Contains(key, "/") { + return fmt.Errorf("failed to validate ConfigMap key '%s': contains invalid path characters", key) + } + } + return nil +} + +type CABundleSourceVolume struct { + // the volume that contains the CA files to be concatenated + CABundleSourceVolumeName string + // the directory to use on the source volume + CABundleSourceDir string + // the CA bundle config - contains the file names that contain CA info + CABundleFileNames []string +} + +type CABundleInitContainerConfig struct { + // the name of the ca bundle init container + CABundleInitName string + // the container image to use for the ca bundle init container + CABundleInitImage string + + // the volumes that contain the CA bundles + CABundleSources []CABundleSourceVolume + + // the name of the volume used to transfer the created CA file to the containers + CABundleTransferVolumeName string + // the directory to use on the transfer volume + CABundleTransferDir string + // the filename to use on the transfer volume in the transfer directory + CABundleTransferFileName string +} + +// CreateCABundleInitContainer creates an InitContainer that concatenates multiple CA bundle keys +// from a number of source ConfigMap into a single file in the ca-bundle "transfer" volume. +func CreateCABundleInitContainer(caBundleInitContainerConfig CABundleInitContainerConfig) (corev1.Container, error) { + + var volumeMounts = []corev1.VolumeMount{ + { + Name: caBundleInitContainerConfig.CABundleTransferVolumeName, + MountPath: caBundleInitContainerConfig.CABundleTransferDir, + }, + } + + var fileListBuilder strings.Builder + for i, caBundleSource := range caBundleInitContainerConfig.CABundleSources { + // Validate ConfigMap keys for security + if err := validateConfigMapKeys(caBundleSource.CABundleFileNames); err != nil { + return corev1.Container{}, fmt.Errorf("failed to validate ConfigMap keys: %w", err) + } + + // Build the file list as a shell array embedded in the script + // This ensures the arguments are properly passed to the script + + for j, key := range caBundleSource.CABundleFileNames { + if i > 0 || j > 0 { + fileListBuilder.WriteString(" ") + } + // Quote each key to handle any special characters safely + fileListBuilder.WriteString(fmt.Sprintf("%q", caBundleSource.CABundleSourceDir+"/"+key)) + } + + // add the CA bundle source volume to the initcontainer volume mounts + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: caBundleSource.CABundleSourceVolumeName, + MountPath: caBundleSource.CABundleSourceDir, + }) + } + fileList := fileListBuilder.String() + + // Use a secure script approach that embeds the file list directly + // This eliminates the issue with arguments not being passed to sh -c + + script := fmt.Sprintf(`#!/bin/sh +set -e +output_file="%s/%s" + +# Clear the output file +> "$output_file" + +# Process each validated key file (keys are pre-validated) +for file_path in %s; do + if [ -f "$file_path" ]; then + cat "$file_path" >> "$output_file" + echo >> "$output_file" # Add newline between certificates + else + echo "Warning: Certificate file $file_path not found" >&2 + fi +done`, caBundleInitContainerConfig.CABundleTransferDir, caBundleInitContainerConfig.CABundleTransferFileName, fileList) + + return corev1.Container{ + Name: caBundleInitContainerConfig.CABundleInitName, + Image: caBundleInitContainerConfig.CABundleInitImage, + Command: []string{"/bin/sh", "-c", script}, + // No Args needed since we embed the file list in the script + VolumeMounts: volumeMounts, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &[]bool{false}[0], + RunAsNonRoot: &[]bool{false}[0], + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }, nil +} diff --git a/controllers/utils/configmap.go b/controllers/utils/configmap.go new file mode 100644 index 000000000..b944992e5 --- /dev/null +++ b/controllers/utils/configmap.go @@ -0,0 +1,111 @@ +package utils + +import ( + "context" + "fmt" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "reflect" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ConfigMapConfig struct { + Owner metav1.Object + ConfigMapName string +} + +// GetImageFromConfigMap retrieves a value from a ConfigMap by key. +// Can be used by any reconciler by passing in the controller-runtime client. +func GetImageFromConfigMap(ctx context.Context, c client.Client, configMapKey, configMapName, namespace string) (string, error) { + configMap := &corev1.ConfigMap{} + err := c.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: namespace}, configMap) + if err != nil { + if errors.IsNotFound(err) { + return "", err + } + return "", fmt.Errorf("error reading configmap %s in namespace %s: %w", configMapName, namespace, err) + } + + value, ok := configMap.Data[configMapKey] + if !ok { + return "", fmt.Errorf("configmap %s in namespace %s does not contain key %s", configMapName, namespace, configMapKey) + } + return value, nil +} + +func createConfigMap(ctx context.Context, c client.Client, owner metav1.Object, configMapName string, configMapTemplatePath string, parser ResourceParserFunc[corev1.ConfigMap]) (*corev1.ConfigMap, error) { + configMapConfig := ConfigMapConfig{ + Owner: owner, + ConfigMapName: configMapName, + } + var configMap *corev1.ConfigMap + configMap, err := parser(configMapTemplatePath, configMapConfig, reflect.TypeOf(&corev1.ConfigMap{})) + + if err != nil { + log.FromContext(ctx).Error(err, "failed to parse configmap template") + return nil, err + } + err = controllerutil.SetControllerReference(owner, configMap, c.Scheme()) + if err != nil { + log.FromContext(ctx).Error(err, "failed to set controller reference") + return nil, err + } + return configMap, nil +} + +func GetConfigMapByName(ctx context.Context, c client.Client, configMapName, namespace string) (*corev1.ConfigMap, error) { + configMap := &corev1.ConfigMap{} + err := c.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: namespace}, configMap) + if err != nil { + if errors.IsNotFound(err) { + return nil, fmt.Errorf("could not find configmap %s in namespace %s: %w", configMapName, namespace, err) + } + return nil, fmt.Errorf("error reading configmap %s in namespace %s: %w", configMapName, namespace, err) + } + return configMap, nil +} + +func ReconcileConfigMap(ctx context.Context, c client.Client, owner metav1.Object, configMapName string, templatePath string, parserFunc ResourceParserFunc[corev1.ConfigMap]) (ctrl.Result, error) { + existingConfigMap := &corev1.ConfigMap{} + err := c.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: owner.GetNamespace()}, existingConfigMap) + if err != nil && errors.IsNotFound(err) { + // Define a new configmap + cm, err := createConfigMap(ctx, c, owner, configMapName, templatePath, parserFunc) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to define configmap", "configmap", owner.GetName(), "namespace", owner.GetNamespace()) + return ctrl.Result{}, err + } + log.FromContext(ctx).Info("Creating a new ConfigMap", "ConfigMap.Namespace", cm.Namespace, "ConfigMap.Name", cm.Name) + err = c.Create(ctx, cm) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to create new ConfigMap", "ConfigMap.Namespace", cm.Namespace, "ConfigMap.Name", cm.Name) + return ctrl.Result{}, err + } + } else if err != nil { + log.FromContext(ctx).Error(err, "Failed to get ConfigMap") + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +func MountConfigMapToDeployment(configMap *corev1.ConfigMap, volumeName string, deployment *appsv1.Deployment) { + // Add the volume to the deployment spec + volume := corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMap.Name, + }, + }, + }, + } + deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, volume) +} diff --git a/controllers/utils/deployment.go b/controllers/utils/deployment.go new file mode 100644 index 000000000..b5c218c1f --- /dev/null +++ b/controllers/utils/deployment.go @@ -0,0 +1,56 @@ +package utils + +import ( + "context" + "fmt" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" +) + +func CheckDeploymentReady(ctx context.Context, c client.Client, name string, namespace string) (bool, error) { + var err error + var deployment *appsv1.Deployment + err = retry.OnError( + wait.Backoff{ + Duration: 5 * time.Second, + }, + func(err error) bool { + return errors.IsNotFound(err) || err != nil + }, + func() error { + err = c.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, deployment) + if err != nil { + return err + } + for _, condition := range deployment.Status.Conditions { + if condition.Type == appsv1.DeploymentAvailable && condition.Status == "True" { + err = CheckPodsReady(ctx, c, *deployment) + } + } + return fmt.Errorf("deployment %s is not ready", deployment.Name) + }, + ) + return true, nil +} + +func CheckPodsReady(ctx context.Context, c client.Client, deployment appsv1.Deployment) error { + podList := &corev1.PodList{} + if err := c.List(ctx, podList, client.InNamespace(deployment.Namespace)); err != nil { + return err + } + // check if all pods are all ready + for _, pod := range podList.Items { + for _, cs := range pod.Status.ContainerStatuses { + if !cs.Ready { + return fmt.Errorf("pod %s is not ready", pod.Name) + } + } + } + return nil +} diff --git a/controllers/utils/parser.go b/controllers/utils/parser.go new file mode 100644 index 000000000..4ac37ceea --- /dev/null +++ b/controllers/utils/parser.go @@ -0,0 +1,46 @@ +package utils + +import ( + "bytes" + "embed" + "reflect" + "text/template" + + "sigs.k8s.io/yaml" +) + +// executeTemplate parses the template file and executes it with the provided data. +func executeTemplate(templatePath string, data interface{}, templateFS embed.FS) (bytes.Buffer, error) { + var processed bytes.Buffer + tmpl, err := template.ParseFS(templateFS, templatePath) + if err != nil { + return processed, err + } + err = tmpl.Execute(&processed, data) + return processed, err +} + +// unmarshallResource unmarshal YAML bytes into the specified Kubernetes resource. +func unmarshallResource(yamlBytes []byte, outType reflect.Type) (interface{}, error) { + outValue := reflect.New(outType.Elem()).Interface() + err := yaml.Unmarshal(yamlBytes, outValue) + return outValue, err +} + +// ParseResource parses templates and return a provided Kubernetes resource. +func ParseResourceFromFS[T any](templatePath string, data interface{}, outType reflect.Type, templateFS embed.FS) (*T, error) { + processed, err := executeTemplate(templatePath, data, templateFS) + if err != nil { + return nil, err + } + + // Convert the processed bytes into the provided Kubernetes resource. + result, err := unmarshallResource(processed.Bytes(), outType) + if err != nil { + return nil, err + } + + return result.(*T), nil +} + +type ResourceParserFunc[T any] func(templatePath string, data interface{}, outType reflect.Type) (*T, error) diff --git a/controllers/utils/route.go b/controllers/utils/route.go new file mode 100644 index 000000000..70812d994 --- /dev/null +++ b/controllers/utils/route.go @@ -0,0 +1,109 @@ +package utils + +import ( + "context" + "fmt" + routev1 "github.com/openshift/api/route/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "reflect" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "time" +) + +type RouteConfig struct { + Owner metav1.Object + RouteName string + RoutePort string +} + +func createRoute(ctx context.Context, c client.Client, owner metav1.Object, routeName string, portName string, routeTemplatePath string, parser ResourceParserFunc[routev1.Route]) (*routev1.Route, error) { + routeConfig := RouteConfig{ + Owner: owner, + RouteName: routeName, + RoutePort: portName, + } + var route *routev1.Route + route, err := parser(routeTemplatePath, routeConfig, reflect.TypeOf(&routev1.Route{})) + + if err != nil { + log.FromContext(ctx).Error(err, "failed to parse route template") + return nil, err + } + err = controllerutil.SetControllerReference(owner, route, c.Scheme()) + if err != nil { + log.FromContext(ctx).Error(err, "failed to set controller reference") + return nil, err + } + return route, nil +} + +func CheckRouteReady(ctx context.Context, c client.Client, name string, namespace string, portName string) (bool, error) { + // Retry logic for getting the route and checking its readiness + var existingRoute *routev1.Route + err := retry.OnError( + wait.Backoff{ + Duration: time.Second * 5, + }, + func(err error) bool { + // Retry on transient errors, such as network errors or resource not found + return errors.IsNotFound(err) || err != nil + }, + func() error { + // Fetch the Route resource + typedNamespaceName := types.NamespacedName{Name: name + portName, Namespace: namespace} + existingRoute = &routev1.Route{} + err := c.Get(ctx, typedNamespaceName, existingRoute) + if err != nil { + return err + } + + for _, ingress := range existingRoute.Status.Ingress { + for _, condition := range ingress.Conditions { + if condition.Type == routev1.RouteAdmitted && condition.Status == "True" { + return nil + } + } + } + // Route is not admitted yet, return an error to retry + return fmt.Errorf("route %s is not admitted", name) + }, + ) + if err != nil { + return false, err + } + return true, nil +} + +func ReconcileDefaultRoute(ctx context.Context, c client.Client, owner metav1.Object, templatePath string, parserFunc ResourceParserFunc[routev1.Route]) (ctrl.Result, error) { + return ReconcileRoute(ctx, c, owner, owner.GetName(), "", templatePath, parserFunc) +} + +func ReconcileRoute(ctx context.Context, c client.Client, owner metav1.Object, routeName string, portName string, templatePath string, parserFunc ResourceParserFunc[routev1.Route]) (ctrl.Result, error) { + existingRoute := &routev1.Route{} + err := c.Get(ctx, types.NamespacedName{Name: routeName, Namespace: owner.GetNamespace()}, existingRoute) + if err != nil && errors.IsNotFound(err) { + // Define a new route + route, err := createRoute(ctx, c, owner, routeName, portName, templatePath, parserFunc) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to define route", "route", owner.GetName(), "namespace", owner.GetNamespace()) + return ctrl.Result{}, err + } + log.FromContext(ctx).Info("Creating a new Route", "Route.Namespace", route.Namespace, "Route.Name", route.Name) + err = c.Create(ctx, route) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to create new Route", "Route.Namespace", route.Namespace, "Route.Name", route.Name) + return ctrl.Result{}, err + } + } else if err != nil { + log.FromContext(ctx).Error(err, "Failed to get Route") + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} diff --git a/controllers/utils/service.go b/controllers/utils/service.go new file mode 100644 index 000000000..6b66f3f22 --- /dev/null +++ b/controllers/utils/service.go @@ -0,0 +1,66 @@ +package utils + +import ( + "context" + "github.com/trustyai-explainability/trustyai-service-operator/controllers/constants" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "reflect" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type ServiceConfig struct { + Owner *metav1.Object + Name string + Namespace string + Version string +} + +func CreateService(ctx context.Context, c client.Client, owner metav1.Object, templatePath string, parser ResourceParserFunc[corev1.Service]) (*corev1.Service, error) { + serviceConfig := ServiceConfig{ + Owner: &owner, + Name: owner.GetName(), + Namespace: owner.GetNamespace(), + Version: constants.Version, + } + var service *corev1.Service + service, err := parser(templatePath, serviceConfig, reflect.TypeOf(&corev1.Service{})) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to parse service template") + return nil, err + } + err = controllerutil.SetControllerReference(owner, service, c.Scheme()) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to set controller reference") + return nil, err + } + return service, nil +} + +func ReconcileService(ctx context.Context, c client.Client, owner metav1.Object, templatePath string, parserFunc ResourceParserFunc[corev1.Service]) (ctrl.Result, error) { + existingService := &corev1.Service{} + err := c.Get(ctx, types.NamespacedName{Name: owner.GetName() + "-service", Namespace: owner.GetNamespace()}, existingService) + if err != nil && errors.IsNotFound(err) { + // Define a new service + service, err := CreateService(ctx, c, owner, templatePath, parserFunc) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to define service", "service", owner.GetName(), "namespace", owner.GetNamespace()) + return ctrl.Result{}, err + } + log.FromContext(ctx).Info("Creating a new Service", "Service.Namespace", owner.GetNamespace(), "Service.Name", owner.GetName()) + err = c.Create(ctx, service) + if err != nil { + log.FromContext(ctx).Error(err, "Failed to create new Service", "Service.Namespace", owner.GetNamespace(), "Service.Name", owner.GetName()) + return ctrl.Result{}, err + } + } else if err != nil { + log.FromContext(ctx).Error(err, "Failed to get Service") + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} diff --git a/controllers/utils/status.go b/controllers/utils/status.go new file mode 100644 index 000000000..060a5dd22 --- /dev/null +++ b/controllers/utils/status.go @@ -0,0 +1,108 @@ +package utils + +import ( + "github.com/trustyai-explainability/trustyai-service-operator/api/common" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + ConditionReconcileComplete = "ReconcileComplete" + ConditionProgessing = "Progressing" +) + +const ( + PhaseProgressing = "Progressing" + PhaseReady = "Ready" +) + +const ( + ReconcileFailed = "ReconcileFailed" + ReconcileInit = "ReconcileInit" + ReconcileCompleted = "ReconcileCompleted" + ReconcileCompletedMessage = "Reconcile completed successfully" + ReconcileFailedMessage = "Reconcile failed" +) + +func SetStatusCondition(conditions *[]common.Condition, newCondition common.Condition) bool { + if conditions == nil { + conditions = &[]common.Condition{} + } + existingCondition := GetStatusCondition(*conditions, newCondition.Type) + if existingCondition == nil { + newCondition.LastTransitionTime = metav1.NewTime(time.Now()) + *conditions = append(*conditions, newCondition) + return true + } + + changed := updateCondition(existingCondition, newCondition) + + return changed +} + +func GetStatusCondition(conditions []common.Condition, conditionType string) *common.Condition { + for i := range conditions { + if conditions[i].Type == conditionType { + return &conditions[i] + } + } + return nil +} + +func updateCondition(existingCondition *common.Condition, newCondition common.Condition) bool { + changed := false + if existingCondition.Status != newCondition.Status { + changed = true + existingCondition.Status = newCondition.Status + existingCondition.LastTransitionTime = metav1.NewTime(time.Now()) + } + if existingCondition.Reason != newCondition.Reason { + changed = true + existingCondition.Reason = newCondition.Reason + + } + if existingCondition.Message != newCondition.Message { + changed = true + existingCondition.Message = newCondition.Message + } + return changed +} + +func SetProgressingCondition(conditions *[]common.Condition, reason string, message string) { + SetStatusCondition(conditions, common.Condition{ + Type: ConditionProgessing, + Status: corev1.ConditionTrue, + Reason: reason, + Message: message, + }) + +} + +func SetResourceCondition(conditions *[]common.Condition, component string, reason string, message string, status corev1.ConditionStatus) { + condtype := component + "Ready" + SetStatusCondition(conditions, common.Condition{ + Type: condtype, + Status: status, + Reason: reason, + Message: message, + }) +} + +func SetCompleteCondition(conditions *[]common.Condition, status corev1.ConditionStatus, reason, message string) { + SetStatusCondition(conditions, common.Condition{ + Type: ConditionReconcileComplete, + Status: status, + Reason: reason, + Message: message, + }) +} + +func SetStatus(conditions *[]common.Condition, resourceKind string, isReady bool) { + if isReady { + SetResourceCondition(conditions, resourceKind, resourceKind+"Ready", resourceKind+"is ready", corev1.ConditionTrue) + } else { + SetResourceCondition(conditions, resourceKind, resourceKind+"NotReady", resourceKind+" is not ready", corev1.ConditionFalse) + } +}