Skip to content

Commit bade8c4

Browse files
RHOAIENG-26179: Deploy CMA / KEDA during E2E tests setup (#635)
As part of testing the CMA / KEDA integration, we need to install the KEDA operator (Custom Metrics Autoscaler operator) during the setup E2E tests script. RHOAIENG-26179: https://issues.redhat.com/browse/RHOAIENG-26179 Signed-off-by: Pierangelo Di Pilato <[email protected]>
1 parent 01338f3 commit bade8c4

File tree

6 files changed

+220
-52
lines changed

6 files changed

+220
-52
lines changed

test/scripts/openshift-ci/common.sh

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
2+
# Helper function to wait for a pod with a given label to be created
3+
wait_for_pod_labeled() {
4+
local ns=${1:?namespace is required}
5+
local podlabel=${2:?pod label is required}
6+
7+
echo "Waiting for pod -l \"$podlabel\" in namespace \"$ns\" to be created..."
8+
until oc get pod -n "$ns" -l "$podlabel" -o=jsonpath='{.items[0].metadata.name}' >/dev/null 2>&1; do
9+
sleep 2
10+
done
11+
echo "Pod -l \"$podlabel\" in namespace \"$ns\" found."
12+
}
13+
14+
# Helper function to wait for a pod with a given label to become ready
15+
wait_for_pod_ready() {
16+
local ns=${1:?namespace is required}
17+
local podlabel=${2:?pod label is required}
18+
local timeout=${3:-600s} # Default timeout 600s
19+
20+
wait_for_pod_labeled "$ns" "$podlabel"
21+
sleep 5 # Brief pause to allow K8s to fully register pod status
22+
23+
echo "Current pods for -l \"$podlabel\" in namespace \"$ns\":"
24+
oc get pod -n "$ns" -l "$podlabel"
25+
26+
echo "Waiting up to $timeout for pod(s) -l \"$podlabel\" in namespace \"$ns\" to become ready..."
27+
if ! oc wait --for=condition=ready --timeout="$timeout" pod -n "$ns" -l "$podlabel"; then
28+
echo "ERROR: Pod(s) -l \"$podlabel\" in namespace \"$ns\" did not become ready in time."
29+
echo "Describing pod(s):"
30+
oc describe pod -n "$ns" -l "$podlabel"
31+
32+
# Try to get logs from the first pod matching the label if any exist
33+
local first_pod_name
34+
first_pod_name=$(oc get pod -n "$ns" -l "$podlabel" -o=jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
35+
36+
if [ -n "$first_pod_name" ]; then
37+
echo "Logs for pod \"$first_pod_name\" in namespace \"$ns\" (last 100 lines per container):"
38+
oc logs -n "$ns" "$first_pod_name" --all-containers --tail=100 || echo "Could not retrieve logs for $first_pod_name."
39+
else
40+
echo "No pods found matching -l \"$podlabel\" in namespace \"$ns\" to retrieve logs from."
41+
fi
42+
return 1 # Indicate failure
43+
fi
44+
echo "Pod(s) -l \"$podlabel\" in namespace \"$ns\" are ready."
45+
}
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
set -eu # Exit on error and undefined variables
16+
17+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
18+
source "${SCRIPT_DIR}/common.sh"
19+
20+
: "${SUBSCRIPTION_NAME:=openshift-custom-metrics-autoscaler-operator}"
21+
: "${KEDA_NAMESPACE:=openshift-keda}"
22+
: "${KEDA_OPERATOR_POD_LABEL:=app=keda-operator}"
23+
: "${KEDA_METRICS_API_SERVER_POD_LABEL:=app=keda-metrics-apiserver}"
24+
: "${KEDA_WEBHOOK_POD_LABEL:=app=keda-admission-webhooks}"
25+
26+
echo "Creating namespace openshift-keda..."
27+
cat <<EOF | oc apply -f -
28+
apiVersion: v1
29+
kind: Namespace
30+
metadata:
31+
name: ${KEDA_NAMESPACE}
32+
labels:
33+
openshift.io/cluster-monitoring: "true"
34+
EOF
35+
echo "Namespace openshift-keda created/ensured."
36+
echo "---"
37+
38+
echo "Creating OperatorGroup openshift-keda..."
39+
cat <<EOF | oc apply -f -
40+
apiVersion: operators.coreos.com/v1
41+
kind: OperatorGroup
42+
metadata:
43+
name: openshift-keda
44+
namespace: ${KEDA_NAMESPACE}
45+
spec:
46+
targetNamespaces:
47+
- openshift-keda
48+
upgradeStrategy: Default
49+
EOF
50+
echo "OperatorGroup openshift-keda created/ensured."
51+
echo "---"
52+
53+
echo "Creating Subscription for openshift-custom-metrics-autoscaler-operator..."
54+
cat <<EOF | oc apply -f -
55+
apiVersion: operators.coreos.com/v1alpha1
56+
kind: Subscription
57+
metadata:
58+
labels:
59+
operators.coreos.com/${SUBSCRIPTION_NAME}.${KEDA_NAMESPACE}: ""
60+
name: ${SUBSCRIPTION_NAME}
61+
namespace: ${KEDA_NAMESPACE}
62+
spec:
63+
channel: stable
64+
installPlanApproval: Automatic
65+
name: ${SUBSCRIPTION_NAME}
66+
source: redhat-operators
67+
sourceNamespace: openshift-marketplace
68+
EOF
69+
echo "Subscription ${SUBSCRIPTION_NAME} created/ensured."
70+
echo "---"
71+
72+
echo "Waiting for Custom Metrics Autoscaler Operator CSV to be ready..."
73+
CSV_NAME=""
74+
# Wait for the CSV to be installed and report success (typically up to 5-10 minutes for operator installs)
75+
for i in $(seq 1 120); do # Wait up to 10 minutes (120 * 5s = 600s)
76+
CSV_NAME=$(oc get subscriptions "$SUBSCRIPTION_NAME" -n "$KEDA_NAMESPACE" -o=jsonpath='{.status.installedCSV}' 2>/dev/null || true)
77+
if [ -n "$CSV_NAME" ]; then
78+
CSV_PHASE=$(oc get csv "$CSV_NAME" -n "$KEDA_NAMESPACE" -o=jsonpath='{.status.phase}' 2>/dev/null || true)
79+
if [ "$CSV_PHASE" == "Succeeded" ]; then
80+
echo "CSV $CSV_NAME is ready (Phase: $CSV_PHASE)."
81+
break
82+
else
83+
echo "CSV $CSV_NAME found, but not yet Succeeded (Phase: $CSV_PHASE). Waiting... (Attempt $i/120)"
84+
fi
85+
else
86+
echo "Waiting for CSV to be installed for subscription $SUBSCRIPTION_NAME... (Attempt $i/120)"
87+
fi
88+
sleep 5
89+
CSV_NAME="" # Reset for next loop iteration if not found or not ready
90+
done
91+
92+
if [ -z "$CSV_NAME" ]; then
93+
echo "ERROR: Could not find installed CSV for $SUBSCRIPTION_NAME in namespace $KEDA_NAMESPACE after waiting."
94+
echo "Status of subscription:"
95+
oc get subscription "$SUBSCRIPTION_NAME" -n "$KEDA_NAMESPACE" -o yaml
96+
exit 1
97+
fi
98+
99+
echo "Custom Metrics Autoscaler Operator is ready."
100+
echo "---"
101+
102+
# 5. Apply KedaController Custom Resource
103+
echo "Applying KedaController custom resource..."
104+
cat <<EOF | oc apply -f -
105+
apiVersion: keda.sh/v1alpha1
106+
kind: KedaController
107+
metadata:
108+
name: keda
109+
namespace: ${KEDA_NAMESPACE}
110+
spec:
111+
watchNamespace: "" # watch all namespaces
112+
operator:
113+
logLevel: info
114+
logEncoder: console
115+
metricsServer:
116+
logLevel: "0"
117+
admissionWebhooks:
118+
logLevel: info
119+
logEncoder: console
120+
EOF
121+
echo "KedaController custom resource applied."
122+
echo "---"
123+
124+
# These components are deployed based on the KedaController CR.
125+
# It might take a moment for the operator to process the KedaController CR and create these deployments.
126+
echo "Allowing time for KEDA components to be provisioned by the operator ..."
127+
sleep 10
128+
129+
echo "Waiting for KEDA Operator pod (selector: \"$KEDA_OPERATOR_POD_LABEL\") to be ready in namespace $KEDA_NAMESPACE..."
130+
if ! wait_for_pod_ready "$KEDA_NAMESPACE" "$KEDA_OPERATOR_POD_LABEL" 120s; then
131+
echo "ERROR: KEDA Operator pod failed to become ready."
132+
exit 1
133+
fi
134+
echo "KEDA Operator pod is ready."
135+
136+
echo "Waiting for KEDA Metrics API Server pod (selector: \"$KEDA_METRICS_API_SERVER_POD_LABEL\") to be ready in namespace $KEDA_NAMESPACE..."
137+
if ! wait_for_pod_ready "$KEDA_NAMESPACE" "$KEDA_METRICS_API_SERVER_POD_LABEL" 120s; then
138+
echo "ERROR: KEDA Metrics API Server pod failed to become ready."
139+
exit 1
140+
fi
141+
echo "KEDA Metrics API Server pod is ready."
142+
143+
echo "Waiting for KEDA Webhook pod (selector: \"$KEDA_WEBHOOK_POD_LABEL\") to be ready in namespace $KEDA_NAMESPACE..."
144+
if ! wait_for_pod_ready "$KEDA_NAMESPACE" "$KEDA_WEBHOOK_POD_LABEL" 120s; then
145+
echo "ERROR: KEDA Webhook pod failed to become ready."
146+
exit 1
147+
fi
148+
echo "KEDA Webhook pod is ready."
149+
150+
echo "---"
151+
echo "✅ KEDA deployment script finished successfully."

test/scripts/openshift-ci/deploy.ossm.sh

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,8 @@
1414

1515
set -eu
1616

17-
waitforpodlabeled() {
18-
local ns=${1?namespace is required}; shift
19-
local podlabel=${1?pod label is required}; shift
20-
21-
echo "Waiting for pod -l $podlabel to be created"
22-
until oc get pod -n "$ns" -l $podlabel -o=jsonpath='{.items[0].metadata.name}' >/dev/null 2>&1; do
23-
sleep 1
24-
done
25-
}
26-
27-
waitpodready() {
28-
local ns=${1?namespace is required}; shift
29-
local podlabel=${1?pod label is required}; shift
30-
31-
waitforpodlabeled "$ns" "$podlabel"
32-
echo "Waiting for pod -l $podlabel to become ready"
33-
oc wait --for=condition=ready --timeout=180s pod -n $ns -l $podlabel
34-
}
17+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
18+
source "${SCRIPT_DIR}/common.sh"
3519

3620
# Deploy OSSM operator
3721
cat <<EOF | oc apply -f -
@@ -50,7 +34,7 @@ spec:
5034
sourceNamespace: openshift-marketplace
5135
EOF
5236

53-
waitpodready "openshift-operators" "name=istio-operator"
37+
wait_for_pod_ready "openshift-operators" "name=istio-operator"
5438

5539
# Create new namespace
5640
oc new-project istio-system
@@ -101,6 +85,6 @@ spec:
10185
EOF
10286

10387
# Waiting for OSSM minimum start
104-
waitpodready "istio-system" "app=istiod"
88+
wait_for_pod_ready "istio-system" "app=istiod"
10589

10690
echo -e "\n OSSM has partially started and should be fully ready soon."

test/scripts/openshift-ci/deploy.serverless.sh

Lines changed: 13 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,8 @@
1414

1515
set -eu
1616

17-
waitforpodlabeled() {
18-
local ns=${1?namespace is required}; shift
19-
local podlabel=${1?pod label is required}; shift
20-
21-
echo "Waiting for pod -l $podlabel to be created"
22-
until oc get pod -n "$ns" -l $podlabel -o=jsonpath='{.items[0].metadata.name}' >/dev/null 2>&1; do
23-
sleep 1
24-
done
25-
}
26-
27-
waitpodready() {
28-
local ns=${1?namespace is required}; shift
29-
local podlabel=${1?pod label is required}; shift
30-
31-
waitforpodlabeled "$ns" "$podlabel"
32-
sleep 2
33-
oc get pod -n $ns -l $podlabel
34-
35-
echo "Waiting for pod -l $podlabel to become ready"
36-
oc wait --for=condition=ready --timeout=600s pod -n $ns -l $podlabel || (oc get pod -n $ns -l $podlabel && false)
37-
}
17+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
18+
source "${SCRIPT_DIR}/common.sh"
3819

3920
# Create namespaces(openshift-serverless)
4021
oc create ns openshift-serverless
@@ -69,9 +50,9 @@ spec:
6950
sourceNamespace: openshift-marketplace
7051
EOF
7152

72-
waitpodready "openshift-serverless" "name=knative-openshift"
73-
waitpodready "openshift-serverless" "name=knative-openshift-ingress"
74-
waitpodready "openshift-serverless" "name=knative-operator"
53+
wait_for_pod_ready "openshift-serverless" "name=knative-openshift"
54+
wait_for_pod_ready "openshift-serverless" "name=knative-openshift-ingress"
55+
wait_for_pod_ready "openshift-serverless" "name=knative-operator"
7556

7657
# Install KNative
7758
cat <<EOF | oc apply -f -
@@ -167,18 +148,18 @@ EOF
167148

168149

169150
# Apparently, as part of KNative installation, deployments can be restarted because
170-
# of configuration changes, leading to waitpodready to fail sometimes.
151+
# of configuration changes, leading to wait_for_pod_ready to fail sometimes.
171152
# Let's sleep 2minutes to let the KNative operator to stabilize the installation before
172153
# checking for the readiness of KNative stack.
173154
sleep 120
174155

175-
waitpodready "knative-serving" "app=controller"
176-
waitpodready "knative-serving" "app=net-istio-controller"
177-
waitpodready "knative-serving" "app=net-istio-webhook"
178-
waitpodready "knative-serving" "app=autoscaler-hpa"
179-
waitpodready "knative-serving" "app=webhook"
180-
waitpodready "knative-serving" "app=activator"
181-
waitpodready "knative-serving" "app=autoscaler"
156+
wait_for_pod_ready "knative-serving" "app=controller"
157+
wait_for_pod_ready "knative-serving" "app=net-istio-controller"
158+
wait_for_pod_ready "knative-serving" "app=net-istio-webhook"
159+
wait_for_pod_ready "knative-serving" "app=autoscaler-hpa"
160+
wait_for_pod_ready "knative-serving" "app=webhook"
161+
wait_for_pod_ready "knative-serving" "app=activator"
162+
wait_for_pod_ready "knative-serving" "app=autoscaler"
182163

183164
export secret_name=$(oc get IngressController default -n openshift-ingress-operator -o yaml -o=jsonpath='{ .spec.defaultCertificate.name}')
184165
if [ -z "$secret_name" ]; then

test/scripts/openshift-ci/setup-e2e-tests.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ pushd $PROJECT_ROOT/python/kserve >/dev/null
6666
poetry install --with=test --no-interaction
6767
popd
6868

69+
$MY_PATH/deploy.cma.sh
70+
6971
# Install KServe stack
7072
if [ "$1" != "raw" ]; then
7173
echo "Installing OSSM"

test/scripts/openshift-ci/teardown-e2e-setup.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,4 +150,9 @@ spec:
150150
- Egress
151151
EOF
152152

153+
echo "Delete CMA / KEDA operator"
154+
oc delete kedacontroller -n openshift-keda keda --ignore-not-found
155+
oc delete subscription -n openshift-keda openshift-custom-metrics-autoscaler-operator --ignore-not-found
156+
oc delete namespace openshift-keda --ignore-not-found
157+
153158
echo "Teardown complete"

0 commit comments

Comments
 (0)