Skip to content

Commit de86917

Browse files
HumairAKHumairAK
authored
Merge pull request #37 from rimolive/rmartine
Create DSPA Status Condition fields
2 parents ac4ad69 + 9ee02a8 commit de86917

File tree

5 files changed

+155
-6
lines changed

5 files changed

+155
-6
lines changed

‎config/samples/dspa_all_fields.yaml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -157,11 +157,11 @@ status:
157157
lastTransitionTime: '2023-02-02T21:00:00Z'
158158
reason: MinimumReplicasAvailable
159159
message: 'some message'
160-
- type: MLPIpelinesUIReady
160+
- type: UserInterfaceReady
161161
status: "True"
162162
observedGeneration: 4
163163
lastTransitionTime: '2023-02-02T21:00:00Z'
164-
reason: MinimumReplicasAvailable # DeploymentDisabled
164+
reason: MinimumReplicasAvailable
165165
message: 'some message'
166166
- type: PersistenceAgentReady
167167
status: "True"
@@ -182,11 +182,11 @@ status:
182182
status: "True"
183183
observedGeneration: 4
184184
lastTransitionTime: '2023-02-02T21:00:00Z'
185-
reason: DataBaseUnreachable # DataBaseFailedToDeploy, DataBaseReady
186-
message: '500 gateway error received'
185+
reason: DataBaseReady
186+
message: ''
187187
- type: ObjectStorageReady
188188
status: "True"
189189
observedGeneration: 4
190190
lastTransitionTime: '2023-02-02T21:00:00Z'
191-
reason: ObjectStorageUnreachable # ObjectStorageFailedToDeploy, ObjectStorageReady
192-
message: 'host unreachable'
191+
reason: ObjectStorageReady
192+
message: ''

‎controllers/config/defaults.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,19 @@ const (
6363
MariaDBImagePath = "Images.MariaDB"
6464
)
6565

66+
const (
67+
DatabaseReady = "DatabaseReady"
68+
DataBaseFailedToDeploy = "DataBaseFailedToDeploy"
69+
ObjectStorageReady = "ObjectStorageReady"
70+
ObjectStorageFailedToDeploy = "ObjectStorageFailedToDeploy"
71+
APIServerReady = "APIServerReady"
72+
PersistenceAgentReady = "PersistenceAgentReady"
73+
ScheduledWorkflowReady = "ScheduledWorkflowReady"
74+
UserInterfaceReady = "UserInterfaceReady"
75+
CrReady = "Ready"
76+
MinimumReplicasAvailable = "MinimumReplicasAvailable"
77+
)
78+
6679
// Any required Configmap paths can be added here,
6780
// they will be automatically included for required
6881
// validation check

‎controllers/dspipeline_controller.go

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package controllers
1919
import (
2020
"context"
2121
"fmt"
22+
2223
"github.com/go-logr/logr"
2324
mf "github.com/manifestival/manifestival"
2425
dspav1alpha1 "github.com/opendatahub-io/data-science-pipelines-operator/api/v1alpha1"
@@ -28,6 +29,7 @@ import (
2829
corev1 "k8s.io/api/core/v1"
2930
rbacv1 "k8s.io/api/rbac/v1"
3031
apierrs "k8s.io/apimachinery/pkg/api/errors"
32+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3133
"k8s.io/apimachinery/pkg/runtime"
3234
"k8s.io/apimachinery/pkg/types"
3335
ctrl "sigs.k8s.io/controller-runtime"
@@ -106,6 +108,37 @@ func (r *DSPAReconciler) DeleteResourceIfItExists(ctx context.Context, obj clien
106108
return err
107109
}
108110

111+
func (r *DSPAReconciler) buildCondition(conditionType string, dspa *dspav1alpha1.DataSciencePipelinesApplication, reason string) metav1.Condition {
112+
condition := metav1.Condition{}
113+
condition.Type = conditionType
114+
condition.ObservedGeneration = dspa.Generation
115+
condition.Status = metav1.ConditionFalse
116+
condition.Reason = reason
117+
118+
if dspa.Status.Conditions == nil {
119+
condition.LastTransitionTime = metav1.Now()
120+
}
121+
122+
return condition
123+
}
124+
125+
func (r *DSPAReconciler) isDeploymentAvailable(ctx context.Context, dspa *dspav1alpha1.DataSciencePipelinesApplication, name string) bool {
126+
found := &appsv1.Deployment{}
127+
128+
// Every Deployment in DSPA is the name followed by the DSPA CR name
129+
component := name + "-" + dspa.Name
130+
131+
err := r.Get(ctx, types.NamespacedName{Name: component, Namespace: dspa.Namespace}, found)
132+
if err == nil {
133+
for _, s := range found.Status.Conditions {
134+
if s.Type == "Available" && s.Status == corev1.ConditionTrue {
135+
return true
136+
}
137+
}
138+
}
139+
return false
140+
}
141+
109142
//+kubebuilder:rbac:groups=datasciencepipelinesapplications.opendatahub.io,resources=datasciencepipelinesapplications,verbs=get;list;watch;create;update;patch;delete
110143
//+kubebuilder:rbac:groups=datasciencepipelinesapplications.opendatahub.io,resources=datasciencepipelinesapplications/status,verbs=get;update;patch
111144
//+kubebuilder:rbac:groups=datasciencepipelinesapplications.opendatahub.io,resources=datasciencepipelinesapplications/finalizers,verbs=update
@@ -189,46 +222,118 @@ func (r *DSPAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
189222
return ctrl.Result{}, nil
190223
}
191224

225+
// Initialize conditions
226+
var conditions []metav1.Condition
227+
228+
databaseReady := r.buildCondition(config.DatabaseReady, dspa, config.DatabaseReady)
192229
err = r.ReconcileDatabase(ctx, dspa, params)
193230
if err != nil {
194231
return ctrl.Result{}, err
195232
}
196233

234+
if r.isDeploymentAvailable(ctx, dspa, "mariadb") {
235+
databaseReady.Status = metav1.ConditionTrue
236+
}
237+
conditions = append(conditions, databaseReady)
238+
239+
objectStorageReady := r.buildCondition(config.ObjectStorageReady, dspa, config.ObjectStorageReady)
197240
err = r.ReconcileStorage(ctx, dspa, params)
198241
if err != nil {
199242
return ctrl.Result{}, err
200243
}
201244

245+
if r.isDeploymentAvailable(ctx, dspa, "minio") {
246+
objectStorageReady.Status = metav1.ConditionTrue
247+
}
248+
conditions = append(conditions, objectStorageReady)
249+
202250
err = r.ReconcileCommon(dspa, params)
203251
if err != nil {
204252
return ctrl.Result{}, err
205253
}
206254

255+
apiServerReady := r.buildCondition(config.APIServerReady, dspa, config.MinimumReplicasAvailable)
207256
err = r.ReconcileAPIServer(ctx, dspa, params)
208257
if err != nil {
209258
return ctrl.Result{}, err
210259
}
211260

261+
if r.isDeploymentAvailable(ctx, dspa, "ds-pipeline") {
262+
apiServerReady.Status = metav1.ConditionTrue
263+
}
264+
conditions = append(conditions, apiServerReady)
265+
266+
persistenceAgentReady := r.buildCondition(config.PersistenceAgentReady, dspa, config.MinimumReplicasAvailable)
212267
err = r.ReconcilePersistenceAgent(dspa, params)
213268
if err != nil {
214269
return ctrl.Result{}, err
215270
}
216271

272+
if r.isDeploymentAvailable(ctx, dspa, "ds-pipeline-persistenceagent") {
273+
persistenceAgentReady.Status = metav1.ConditionTrue
274+
}
275+
conditions = append(conditions, persistenceAgentReady)
276+
277+
scheduledWorkflowReady := r.buildCondition(config.ScheduledWorkflowReady, dspa, config.MinimumReplicasAvailable)
217278
err = r.ReconcileScheduledWorkflow(dspa, params)
218279
if err != nil {
219280
return ctrl.Result{}, err
220281
}
221282

283+
if r.isDeploymentAvailable(ctx, dspa, "ds-pipeline-scheduledworkflow") {
284+
scheduledWorkflowReady.Status = metav1.ConditionTrue
285+
}
286+
conditions = append(conditions, scheduledWorkflowReady)
287+
288+
userInterfaceReady := r.buildCondition(config.UserInterfaceReady, dspa, config.MinimumReplicasAvailable)
222289
err = r.ReconcileUI(dspa, params)
223290
if err != nil {
224291
return ctrl.Result{}, err
225292
}
226293

294+
if r.isDeploymentAvailable(ctx, dspa, "ds-pipeline-ui") {
295+
userInterfaceReady.Status = metav1.ConditionTrue
296+
}
297+
conditions = append(conditions, userInterfaceReady)
298+
227299
err = r.ReconcileViewerCRD(dspa, params)
228300
if err != nil {
229301
return ctrl.Result{}, err
230302
}
231303

304+
log.Info("Updating CR status")
305+
306+
crReady := r.buildCondition(config.CrReady, dspa, config.MinimumReplicasAvailable)
307+
crReady.Type = config.CrReady
308+
309+
// Compute Ready Logic for the CR
310+
if ((apiServerReady.Status == metav1.ConditionTrue) &&
311+
(persistenceAgentReady.Status == metav1.ConditionTrue) &&
312+
(scheduledWorkflowReady.Status == metav1.ConditionTrue) &&
313+
(databaseReady.Status == metav1.ConditionTrue) &&
314+
(objectStorageReady.Status == metav1.ConditionTrue)) &&
315+
(userInterfaceReady.Status == metav1.ConditionTrue || (userInterfaceReady.Status == metav1.ConditionFalse && !dspa.Spec.MlPipelineUI.Deploy)) {
316+
crReady.Status = metav1.ConditionTrue
317+
} else {
318+
crReady.Status = metav1.ConditionFalse
319+
}
320+
conditions = append(conditions, crReady)
321+
322+
for i, condition := range dspa.Status.Conditions {
323+
if condition.Status != conditions[i].Status {
324+
conditions[i].LastTransitionTime = metav1.Now()
325+
} else {
326+
conditions[i].LastTransitionTime = condition.LastTransitionTime
327+
}
328+
}
329+
dspa.Status.Conditions = conditions
330+
331+
err = r.Status().Update(ctx, dspa)
332+
if err != nil {
333+
log.Info(err.Error())
334+
return ctrl.Result{}, err
335+
}
336+
232337
return ctrl.Result{}, nil
233338
}
234339

‎tests/basictests/dsp-operator.sh

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,32 @@ function create_and_verify_data_science_pipelines_resources() {
3232
os::cmd::try_until_text "oc get crd -n ${DSPAPROJECT} datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io" "datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io" $odhdefaulttimeout $odhdefaultinterval
3333
os::cmd::try_until_text "oc -n ${DSPAPROJECT} get pods -l app=ds-pipeline-sample -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}'" "True" $odhdefaulttimeout $odhdefaultinterval
3434
running_pods=$(oc get pods -n ${DSPAPROJECT} -l component=data-science-pipelines --field-selector='status.phase=Running' -o jsonpath='{$.items[*].metadata.name}' | wc -w)
35+
echo "Sleeping for 5 minutes for the DSPO CR settle up "
36+
sleep 5m
3537
os::cmd::expect_success "if [ "$running_pods" -gt "0" ]; then exit 0; else exit 1; fi"
3638
}
3739

40+
function check_custom_resource_conditions() {
41+
header "Testing Data Science Pipelines Application CR conditions"
42+
43+
# Check if all CR conditions are good
44+
os::cmd::try_until_text "oc get -n ${DSPAPROJECT} datasciencepipelinesapplication sample -o jsonpath='{.status.conditions[6].status}'" "True" $odhdefaulttimeout $odhdefaultinterval
45+
46+
## Given general condition is good, is expected that other component conditions are good
47+
# DataBaseReady
48+
os::cmd::try_until_text "oc get -n ${DSPAPROJECT} datasciencepipelinesapplication sample -o jsonpath='{.status.conditions[0].status}'" "True" $odhdefaulttimeout $odhdefaultinterval
49+
# ObjectStorageReady
50+
os::cmd::try_until_text "oc get -n ${DSPAPROJECT} datasciencepipelinesapplication sample -o jsonpath='{.status.conditions[1].status}'" "True" $odhdefaulttimeout $odhdefaultinterval
51+
# ApiServerReady
52+
os::cmd::try_until_text "oc get -n ${DSPAPROJECT} datasciencepipelinesapplication sample -o jsonpath='{.status.conditions[2].status}'" "True" $odhdefaulttimeout $odhdefaultinterval
53+
# PersistenceAgentReady
54+
os::cmd::try_until_text "oc get -n ${DSPAPROJECT} datasciencepipelinesapplication sample -o jsonpath='{.status.conditions[3].status}'" "True" $odhdefaulttimeout $odhdefaultinterval
55+
# ScheduledWorkflowReady
56+
os::cmd::try_until_text "oc get -n ${DSPAPROJECT} datasciencepipelinesapplication sample -o jsonpath='{.status.conditions[4].status}'" "True" $odhdefaulttimeout $odhdefaultinterval
57+
# UserInterfaceReady
58+
os::cmd::try_until_text "oc get -n ${DSPAPROJECT} datasciencepipelinesapplication sample -o jsonpath='{.status.conditions[5].status}'" "True" $odhdefaulttimeout $odhdefaultinterval
59+
}
60+
3861
function check_data_science_pipeline_route() {
3962
header "Checking Routes of Data Science Pipeline availability"
4063

@@ -127,6 +150,7 @@ echo "Testing Data Science Pipelines Operator functionality"
127150
128151
verify_data_science_pipelines_operator_install
129152
create_and_verify_data_science_pipelines_resources
153+
check_custom_resource_conditions
130154
check_data_science_pipeline_route
131155
setup_monitoring
132156
test_metrics

‎tests/scripts/installandtest.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,18 @@ fi
4242

4343
echo "Saving the dump of the pods logs in the artifacts directory"
4444
oc get pods -o yaml -n ${ODHPROJECT} > ${ARTIFACT_DIR}/${ODHPROJECT}.pods.yaml
45+
oc get pods -o yaml -n data-science-pipelines-test > ${ARTIFACT_DIR}/data-science-pipelines-test.pods.yaml
4546
oc get pods -o yaml -n openshift-operators > ${ARTIFACT_DIR}/openshift-operators.pods.yaml
4647
echo "Saving the events in the artifacts directory"
4748
oc get events --sort-by='{.lastTimestamp}' > ${ARTIFACT_DIR}/${ODHPROJECT}.events.txt
49+
oc get events --sort-by='{.lastTimestamp}' -n data-science-pipelines-test > ${ARTIFACT_DIR}/data-science-pipelines-test.events.txt
4850
echo "Saving the logs from the opendatahub-operator pod in the artifacts directory"
4951
oc logs -n openshift-operators $(oc get pods -n openshift-operators -l name=opendatahub-operator -o jsonpath="{$.items[*].metadata.name}") > ${ARTIFACT_DIR}/opendatahub-operator.log 2> /dev/null || echo "No logs for openshift-operators/opendatahub-operator"
52+
echo "Saving the logs from the Data Science Pipelines Operator pods in the artifacts directory"
53+
for pod in $(oc get pods -n opendatahub -l control-plane=controller-manager -o jsonpath="{$.items[*].metadata.name}");
54+
do
55+
oc logs -n opendatahub $pod > ${ARTIFACT_DIR}/$pod.log;
56+
done
5057

5158
if [ "$success" -ne 1 ]; then
5259
exit 1

0 commit comments

Comments
 (0)