Skip to content

Commit 1ed92fc

Browse files
authored
Updated versions of Grafana, Prometheus, added extra debugging (#1623)
* added more debug info * added more debug info1 * added debug to deploy app * changed version prom * added debug * fixstyle * added sed * fixed typo * fixed logic * added debug for grafana * fixed bash * fixed order * separated scripts * updated grafana ver * updated grafana ver1 * fixed redeploy * corrected scripts * added extra check * added extra class just for Promdebug * added more debug * added more debug1 * fixed typo * TEMP cleanup * removed TEMP * fixed typo * delete debug * more debug * added view logs * deleted testfile * added log collection * added more debug info * deleted commented out blocks * fixed typo * reverted deploywebapp.py to dev version, needed only for extra debugging * replaced exec
1 parent ff7af18 commit 1ed92fc

File tree

9 files changed

+375
-186
lines changed

9 files changed

+375
-186
lines changed

integration-tests/src/test/java/oracle/kubernetes/operator/ItMonitoringExporter.java

Lines changed: 126 additions & 133 deletions
Large diffs are not rendered by default.

integration-tests/src/test/resources/OperatorIT.properties

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ OCR_SERVER=container-registry.oracle.com
1818
DOMAIN_API_VERSION = weblogic.oracle/v7
1919
CRD_VERSION = v7
2020
WDT_VERSION=1.7.2
21-
PROMETHEUS_CHART_VERSION=9.1.1
22-
GRAFANA_CHART_VERSION=3.12.0
21+
PROMETHEUS_CHART_VERSION=11.1.5
22+
GRAFANA_CHART_VERSION=5.0.20
2323
MONITORING_EXPORTER_VERSION=1.1.2
2424
weblogicImageTagWIT = 12.2.1.4.0
2525
weblogicImageDevTagWIT = 12.2.1.4.0-dev

integration-tests/src/test/resources/exporter/createGrafana.sh

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,57 @@ monitoringExporterDir=$1
55
resourceExporterDir=$2
66
grafanaVersionArgs=$3
77
monitoringExporterEndToEndDir=${monitoringExporterDir}/src/samples/kubernetes/end2end
8-
8+
CHARTNS=monitortestns
9+
CHARTNAME=grafana
910
HELM_VERSION=$(helm version --short --client)
1011

12+
function checkPod() {
13+
max=20
14+
count=0
15+
echo "Checking pods $CHARTNAME"
16+
grafanapod=$(kubectl get po -n $CHARTNS | grep $CHARTNAME | awk '{print $1 }')
17+
while test $count -lt $max; do
18+
if test "$(kubectl get po -n $CHARTNS | grep $CHARTNAME | awk '{ print $2 }')" = 1/1; then
19+
echo "$CHARTNAME pod is running now."
20+
echo "Finished - [createGrafana.sh] ..."
21+
exit 0;
22+
fi
23+
count=`expr $count + 1`
24+
sleep 5
25+
done
26+
echo "ERROR: $CHARTNAME pod failed to start."
27+
echo "createGrafana.sh returned: 1"
28+
kubectl describe pod/${grafanapod} -n $CHARTNS
29+
kubectl logs pod/${grafanapod} -n $CHARTNS -c init-chown-data
30+
kubectl logs pod/${grafanapod} -n $CHARTNS -c grafana
31+
kubectl describe pod/${grafanapod} -n $CHARTNS
32+
exit 1
33+
}
34+
1135
helm repo update
1236
export appname=grafana
13-
for p in `kubectl get po -l app=$appname -o name -n monitortestns `;do echo $p; kubectl delete ${p} -n monitoring --force --grace-period=0 --ignore-not-found; done
37+
for p in `kubectl get po -l app.kubernetes.io/name=$appname -o name -n $CHARTNS `;do echo deleting $p; kubectl delete ${p} -n $CHARTNS --force --grace-period=0 --ignore-not-found; done
1438

15-
kubectl --namespace monitortestns create secret generic grafana-secret --from-literal=username=admin --from-literal=password=12345678
39+
kubectl -n $CHARTNS create secret generic grafana-secret --from-literal=username=admin --from-literal=password=12345678
40+
sed -i "s/monitoring/${CHARTNS}/g" ${monitoringExporterEndToEndDir}/grafana/persistence.yaml
41+
sed -i "s/pv-grafana/pv-testgrafana/g" ${monitoringExporterEndToEndDir}/grafana/persistence.yaml
1642

1743
kubectl apply -f ${monitoringExporterEndToEndDir}/grafana/persistence.yaml
1844
echo "Detected Helm Version [${HELM_VERSION}]"
19-
if [[ "$HELM_VERSION" =~ "v2" ]]; then
20-
helm install --wait --name grafana --namespace monitortestns --values ${monitoringExporterEndToEndDir}/grafana/values.yaml stable/grafana --version ${grafanaVersionArgs}
21-
elif [[ "$HELM_VERSION" =~ "v3" ]]; then
22-
helm install grafana --wait --namespace monitortestns --values ${monitoringExporterEndToEndDir}/grafana/values.yaml stable/grafana --version ${grafanaVersionArgs}
45+
if [[ "$HELM_VERSION" =~ "v3" ]]; then
46+
helm install $CHARTNAME --namespace $CHARTNS --values ${monitoringExporterEndToEndDir}/grafana/values.yaml stable/grafana --version ${grafanaVersionArgs}
47+
script_status=$?
48+
echo "status $script_status "
49+
if [ $script_status != 0 ]; then
50+
echo "createGrafana.sh returned: $script_status"
51+
echo "ERROR: createGrafana.sh failed"
52+
kubectl get events -n $CHARTNS --sort-by='.metadata.creationTimestamp'
53+
helm uninstall $CHARTNAME --namespace $CHARTNS
54+
exit $script_status
55+
fi
2356
else
2457
echo "Detected Unsuppoted Helm Version [${HELM_VERSION}]"
2558
exit 1
2659
fi
27-
echo "Finished - [createGrafana.sh] ..."
60+
echo "Wait until $CHARTNAME pod is running."
61+
checkPod

integration-tests/src/test/resources/exporter/createProm.sh

Lines changed: 67 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,35 +3,87 @@
33
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upload
44
monitoringExporterDir=$1
55
domainNS=$4
6-
domainNS1=$5
76
resourceExporterDir=$2
87
promVersionArgs=$3
98
monitoringExporterEndToEndDir=${monitoringExporterDir}/src/samples/kubernetes/end2end
9+
CHARTNAME=prometheus
10+
CHARTNS=monitortestns
1011

11-
12+
function checkPod() {
13+
max=15
14+
count=0
15+
echo "Checking pods $CHARTNAME-server and $CHARTNAME-alertmanager"
16+
serverpod=$(kubectl get po -n $CHARTNS | grep $CHARTNAME-server | awk '{print $1 }')
17+
alertpod=$(kubectl get po -n $CHARTNS | grep $CHARTNAME-alertmanager | awk '{print $1 }')
18+
while test $count -lt $max; do
19+
kubectl get po -n $CHARTNS | grep $CHARTNAME-server
20+
kubectl get po -n $CHARTNS | grep $CHARTNAME-alertmanager
21+
kubectl logs ${alertpod} -n $CHARTNS -c prometheus-alertmanager
22+
if test "$(kubectl get po -n $CHARTNS | grep $CHARTNAME-server | awk '{ print $2 }')" = 2/2; then
23+
echo "$CHARTNAME-server pod is running now."
24+
if test "$(kubectl get po -n $CHARTNS | grep $CHARTNAME-alertmanager | awk '{ print $2 }')" = 2/2; then
25+
echo "$CHARTNAME-alertmanager pod is running now."
26+
echo "Finished - [createProm.sh] ..."
27+
exit 0;
28+
fi
29+
fi
30+
count=`expr $count + 1`
31+
sleep 5
32+
done
33+
echo "ERROR: $CHARTNAME pods failed to start."
34+
echo
35+
kubectl describe pod/${alertpod} -n $CHARTNS
36+
kubectl describe pod/${serverpod} -n $CHARTNS
37+
kubectl get events -n $CHARTNS --sort-by='.metadata.creationTimestamp'
38+
kubectl logs ${alertpod} -n $CHARTNS -c prometheus-alertmanager
39+
kubectl logs ${prompod} -n $CHARTNS -c prometheus-server
40+
kubectl logs ${alertpod} -n $CHARTNS -c prometheus-alertmanager-configmap-reload
41+
kubectl logs ${prompod} -n $CHARTNS -c prometheus-server-configmap-reload
42+
echo "createProm.sh returned: 1"
43+
exit 1
44+
}
1245
sed -i "s/default;domain1/${domainNS};${domainNS}/g" ${monitoringExporterEndToEndDir}/prometheus/promvalues.yaml
1346

14-
kubectl create ns monitortestns
47+
kubectl create ns $CHARTNS
48+
49+
sed -i "s/monitoring/${CHARTNS}/g" ${monitoringExporterEndToEndDir}/prometheus/persistence.yaml
50+
sed -i "s/monitoring/${CHARTNS}/g" ${monitoringExporterEndToEndDir}/prometheus/alert-persistence.yaml
51+
sed -i "s/pv-prom/pv-testprom/g" ${monitoringExporterEndToEndDir}/prometheus/persistence.yaml
52+
sed -i "s/pv-alert/pv-testalert/g" ${monitoringExporterEndToEndDir}/prometheus/alert-persistence.yaml
53+
cat ${monitoringExporterEndToEndDir}/prometheus/alert-persistence.yaml
54+
cat ${monitoringExporterEndToEndDir}/prometheus/persistence.yaml
55+
1556

1657
kubectl apply -f ${monitoringExporterEndToEndDir}/prometheus/persistence.yaml
1758
kubectl apply -f ${monitoringExporterEndToEndDir}/prometheus/alert-persistence.yaml
18-
kubectl get pv -n monitortestns
19-
kubectl get pvc -n monitortestns
59+
kubectl get pv -n $CHARTNS
60+
kubectl get pvc -n $CHARTNS
2061

2162
HELM_VERSION=$(helm version --short --client)
63+
if [[ "$HELM_VERSION" =~ "v2" ]]; then
64+
echo "Detected unsupported Helm version [${HELM_VERSION}]"
65+
echo "createProm.sh returned: 1"
66+
exit 1
67+
fi
2268

2369
helm repo update
2470

2571
export appname=prometheus
26-
for p in `kubectl get po -l app=$appname -o name -n monitortestns `;do echo $p; kubectl delete ${p} -n monitoring --force --grace-period=0 --ignore-not-found; done
72+
for p in `kubectl get po -l app=$appname -o name -n $CHARTNS `;do echo "deleting $p "; kubectl delete ${p} -n $CHARTNS --force --grace-period=0 --ignore-not-found; done
73+
sleep 15
2774

28-
if [[ "$HELM_VERSION" =~ "v2" ]]; then
29-
helm install --wait --name prometheus --namespace monitortestns --values ${monitoringExporterEndToEndDir}/prometheus/promvalues.yaml stable/prometheus --version ${promVersionArgs}
30-
elif [[ "$HELM_VERSION" =~ "v3" ]]; then
31-
helm install prometheus --wait --namespace monitortestns --values ${monitoringExporterEndToEndDir}/prometheus/promvalues.yaml stable/prometheus --version ${promVersionArgs}
32-
POD_NAME=$(kubectl get pod -l app=prometheus -n monitortestns -o jsonpath="{.items[0].metadata.name}")
33-
else
34-
echo "Detected Unsuppoted Helm Version [${HELM_VERSION}]"
35-
exit 1
75+
echo "Installing $CHARTNAME helm chart."
76+
helm install $CHARTNAME stable/$CHARTNAME --namespace $CHARTNS \
77+
--values ${monitoringExporterEndToEndDir}/$CHARTNAME/promvalues.yaml --version ${promVersionArgs}
78+
script_status=$?
79+
echo "status $script_status "
80+
if [ $script_status != 0 ]; then
81+
echo "createProm.sh returned: $script_status"
82+
echo "ERROR: createProm.sh failed"
83+
helm status $CHARTNAME --namespace $CHARTNS
84+
kubectl get events -n $CHARTNS --sort-by='.metadata.creationTimestamp'
85+
helm uninstall $CHARTNAME --namespace $CHARTNS
86+
exit $script_status
3687
fi
37-
echo "Finished - [createProm.sh] ..."
88+
echo "Wait until $CHARTNAME pod is running."
89+
checkPod

integration-tests/src/test/resources/exporter/createPromGrafanaMySqlCoordWebhook.sh

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,32 @@ cp $resourceExporterDir/create*.sh $scriptDir
1414
chmod 777 $scriptDir/*.sh
1515
echo "Installing Prometheus ..."
1616
sh $scriptDir/createProm.sh $monitoringExporterDir $resourceExporterDir $promVersionArgs $domainNS $domainNS1
17-
echo "Installing Grafana ..."
18-
sh $scriptDir/createGrafana.sh $monitoringExporterDir $resourceExporterDir $grafanaVersionArgs
17+
script_status=$?
18+
echo "status $script_status "
19+
if [ $script_status != 0 ]; then
20+
echo "createProm.sh returned: $script_status"
21+
exit $script_status
22+
fi
23+
1924
echo "Installing mysql"
2025
sh $scriptDir/createMySql.sh $monitoringExporterDir $resourceExporterDir $domainNS1
26+
script_status=$?
27+
if [ $script_status != 0 ]; then
28+
echo "createMySql.sh returned: $script_status"
29+
exit $script_status
30+
fi
2131
echo "Installing webhook"
2232
sh $scriptDir/createWebhook.sh $monitoringExporterDir $resourceExporterDir
33+
script_status=$?
34+
if [ $script_status != 0 ]; then
35+
echo "createWebhook.sh returned: $script_status"
36+
exit $script_status
37+
fi
2338
echo "Installing coordinator"
2439
sh $scriptDir/createCoord.sh $monitoringExporterDir $resourceExporterDir $domainNS
40+
script_status=$?
41+
if [ $script_status != 0 ]; then
42+
echo "createCoordinator.sh returned: $script_status"
43+
exit $script_status
44+
fi
2545
echo "Completed [createPromGrafanaMySqlCoordWebhook.sh]"

integration-tests/src/test/resources/exporter/createWebhook.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ if [ ${SHARED_CLUSTER} = "true" ]; then
1515
docker push $REPO_REGISTRY/weblogick8s/webhook-log:$tag
1616
if [ ! "$?" = "0" ] ; then
1717
echo "Error: Could not push the image to $REPO_REGISTRY".
18-
#exit 1
18+
exit 1
1919
fi
2020
sed -i "s/webhook-log:1.0/$REPO_REGISTRY\/weblogick8s\/webhook-log:$tag/g" ${resourceExporterDir}/server.yaml
2121
sed -i "s/IfNotPresent/Always/g" ${resourceExporterDir}/server.yaml

integration-tests/src/test/resources/exporter/deletePromGrafanaMySqlCoordWebhook.sh

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,8 @@ kubectl delete ns webhook
1212
HELM_VERSION=$(helm version --short --client)
1313
echo "Detected Helm Version [${HELM_VERSION}]"
1414
#delete grafana, prometheus
15-
if [[ "$HELM_VERSION" =~ "v2" ]]; then
16-
helm delete --purge grafana
17-
helm delete --purge prometheus
18-
elif [[ "$HELM_VERSION" =~ "v3" ]]; then
15+
16+
if [[ "$HELM_VERSION" =~ "v3" ]]; then
1917
helm uninstall grafana --namespace monitortestns
2018
helm uninstall prometheus --namespace monitortestns
2119
else
@@ -27,20 +25,19 @@ kubectl -n monitortestns delete secret grafana-secret --ignore-not-found
2725
kubectl delete -f ${monitoringExporterEndToEndDir}/grafana/persistence.yaml --ignore-not-found
2826

2927
export appname=grafana
30-
for p in `kubectl get po -l app=$appname -o name -n monitoring `;do echo $p; kubectl delete ${p} -n monitortestns --force --grace-period=0 --ignore-not-found; done
28+
for p in `kubectl get po -l app.kubernetes.io/name=$appname -o name -n monitortestns `;do echo $p; kubectl delete ${p} -n monitortestns --force --grace-period=0 --ignore-not-found; done
3129

3230
kubectl delete -f ${monitoringExporterEndToEndDir}/prometheus/persistence.yaml --ignore-not-found
3331
kubectl delete -f ${monitoringExporterEndToEndDir}/prometheus/alert-persistence.yaml --ignore-not-found
3432

3533
export appname=prometheus
36-
for p in `kubectl get po -l app=$appname -o name -n monitoring `;do echo $p; kubectl delete ${p} -n monitortestns --force --grace-period=0 --ignore-not-found; done
34+
for p in `kubectl get po -l app=$appname -o name -n monitortestns `;do echo $p; kubectl delete ${p} -n monitortestns --force --grace-period=0 --ignore-not-found; done
3735

3836
#delete coordinator
3937
kubectl delete -f ${resourceExporterDir}/coordinator_${domainNS}.yaml
4038

4139
#delete database
4240
kubectl delete -f ${monitoringExporterEndToEndDir}/mysql/mysql.yaml --ignore-not-found
4341
kubectl delete -f ${monitoringExporterEndToEndDir}/mysql/persistence.yaml --ignore-not-found
44-
#kubectl delete -f ${monitoringExporterEndToEndDir}/mysql/mysql-secret.yaml
4542
kubectl delete -f ${monitoringExporterEndToEndDir}/util/curl.yaml
4643
echo "Run the script [deletePromGrafanaMySqlCoordWebhook.sh] ..."

integration-tests/src/test/resources/exporter/promvalues.yaml

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,21 @@ alertmanager:
1515
service:
1616
type: NodePort
1717
nodePort: 32500
18+
# securityContext:
19+
# runAsNonRoot: false
20+
# runAsUser: 0
21+
image:
22+
repository: prom/alertmanager
23+
tag: v0.20.0
24+
pullPolicy: Always
1825
securityContext:
19-
runAsNonRoot: false
20-
runAsUser: 0
26+
runAsUser: 65534
27+
runAsNonRoot: true
28+
runAsGroup: 65534
29+
fsGroup: 65534
30+
31+
nodeExporter:
32+
enabled: false
2133

2234
server:
2335
persistentVolume:
@@ -28,10 +40,15 @@ server:
2840
securityContext:
2941
runAsNonRoot: false
3042
runAsUser: 0
43+
3144
global:
3245
evaluation_interval: 1m
3346
scrape_interval: 10s
3447
scrape_timeout: 5s
48+
image:
49+
repository: prom/prometheus
50+
tag: v2.16.0
51+
pullPolicy: Always
3552

3653
alertmanagerFiles:
3754
alertmanager.yml:

0 commit comments

Comments
 (0)