Skip to content

Commit f050a25

Browse files
marinakogmarinakog
authored
Added automated test for custom metrics HPA (#3624)
* Added automated test for custom metrics HPA
1 parent a6c5cc1 commit f050a25

File tree

8 files changed

+602
-10
lines changed

8 files changed

+602
-10
lines changed

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java

Lines changed: 421 additions & 0 deletions
Large diffs are not rendered by default.

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,7 @@ public interface TestConstants {
284284
getNonEmptySystemProperty("wko.it.grafana.chart.version", "6.38.6");
285285
public static final String PROMETHEUS_REPO_NAME = "prometheus-community";
286286
public static final String PROMETHEUS_REPO_URL = "https://prometheus-community.github.io/helm-charts";
287+
287288
public static final String GRAFANA_REPO_NAME = "grafana";
288289
public static final String GRAFANA_REPO_URL = "https://grafana.github.io/helm-charts";
289290

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import java.io.IOException;
77
import java.time.OffsetDateTime;
8+
import java.util.HashMap;
89
import java.util.List;
910
import java.util.Map;
1011
import java.util.concurrent.Callable;
@@ -1044,6 +1045,21 @@ public static Callable<Boolean> isPrometheusReady(String namespace, String relea
10441045
return Prometheus.isReady(namespace, releaseName);
10451046
}
10461047

1048+
/**
1049+
* Check if the prometheus pods are running in a given namespace.
1050+
* @param namespace in which to check for the prometheus pods
1051+
* @param podName prometheus adapter pod name
1052+
* @return true if found and running otherwise false
1053+
*/
1054+
public static Callable<Boolean> isPrometheusAdapterReady(String namespace, String podName) {
1055+
Map<String,String> labelMapPromSvc = new HashMap<>();
1056+
labelMapPromSvc.put("name", "prometheus-adapter");
1057+
1058+
return () -> {
1059+
return (Kubernetes.isPodReady(namespace, (Map<String, String>) null, podName));
1060+
};
1061+
}
1062+
10471063
/**
10481064
* Check if Grafana is running.
10491065
*

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/MonitoringUtils.java

Lines changed: 95 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import java.nio.file.StandardCopyOption;
1111
import java.util.ArrayList;
1212
import java.util.Collections;
13+
import java.util.HashMap;
1314
import java.util.List;
1415
import java.util.Map;
1516
import java.util.concurrent.Callable;
@@ -39,10 +40,12 @@
3940
import oracle.weblogic.kubernetes.actions.impl.PrometheusParams;
4041
import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
4142
import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams;
43+
import oracle.weblogic.kubernetes.actions.impl.primitive.Helm;
4244
import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams;
4345
import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes;
4446
import oracle.weblogic.kubernetes.assertions.impl.ClusterRole;
4547
import oracle.weblogic.kubernetes.assertions.impl.ClusterRoleBinding;
48+
import oracle.weblogic.kubernetes.assertions.impl.RoleBinding;
4649
import oracle.weblogic.kubernetes.logging.LoggingFacade;
4750
import org.apache.commons.io.FileUtils;
4851

@@ -75,6 +78,7 @@
7578
import static oracle.weblogic.kubernetes.assertions.TestAssertions.callTestWebAppAndCheckForServerNameInResponse;
7679
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isGrafanaReady;
7780
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isHelmReleaseDeployed;
81+
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isPrometheusAdapterReady;
7882
import static oracle.weblogic.kubernetes.assertions.TestAssertions.isPrometheusReady;
7983
import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndCheckForServerNameInResponse;
8084
import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterAndVerify;
@@ -91,6 +95,7 @@
9195
import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret;
9296
import static oracle.weblogic.kubernetes.utils.ImageUtils.dockerLoginAndPushImageToRegistry;
9397
import static oracle.weblogic.kubernetes.utils.PodUtils.execInPod;
98+
import static oracle.weblogic.kubernetes.utils.PodUtils.getPodName;
9499
import static oracle.weblogic.kubernetes.utils.PodUtils.setPodAntiAffinity;
95100
import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword;
96101
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
@@ -431,6 +436,63 @@ public static PrometheusParams installAndVerifyPrometheus(String promReleaseSuff
431436
return prometheusParams;
432437
}
433438

439+
/**
440+
* Install Prometheus adapter and wait up to five minutes until the prometheus adapter pods are ready.
441+
*
442+
* @param promAdapterReleaseName the prometheus adapter release name
443+
* @param promAdapterNamespace the prometheus adapter namespace
444+
* @return the prometheus adapter Helm installation parameters
445+
*/
446+
public static HelmParams installAndVerifyPrometheusAdapter(String promAdapterReleaseName,
447+
String promAdapterNamespace,
448+
String prometheusHost,
449+
int prometheusPort) {
450+
LoggingFacade logger = getLogger();
451+
Path srcPromAdapterFile = Paths.get(RESOURCE_DIR, "exporter", "promadaptervalues.yaml");
452+
453+
// Helm install parameters
454+
HelmParams promAdapterHelmParams = new HelmParams()
455+
.releaseName(promAdapterReleaseName)
456+
.namespace(promAdapterNamespace)
457+
.repoUrl(PROMETHEUS_REPO_URL)
458+
.repoName(PROMETHEUS_REPO_NAME)
459+
.chartName("prometheus-adapter")
460+
.chartValuesFile(srcPromAdapterFile.toString());
461+
462+
463+
// install prometheus adapter
464+
logger.info("Installing prometheus adapter in namespace {0}", promAdapterNamespace);
465+
Map<String, Object> chartValues = new HashMap<>();
466+
chartValues.put("prometheus.url", "http://" + prometheusHost);
467+
chartValues.put("prometheus.port", prometheusPort);
468+
assertTrue(Helm.install(promAdapterHelmParams, chartValues),
469+
String.format("Failed to install prometheus adapter in namespace %s", promAdapterNamespace));
470+
logger.info("Prometheus Adapter installed in namespace {0}", promAdapterNamespace);
471+
472+
// list Helm releases matching operator release name in operator namespace
473+
logger.info("Checking prometheus adapter release {0} status in namespace {1}",
474+
promAdapterReleaseName, promAdapterNamespace);
475+
assertTrue(isHelmReleaseDeployed(promAdapterReleaseName, promAdapterNamespace),
476+
String.format("Prometheus release %s is not in deployed status in namespace %s",
477+
promAdapterReleaseName, promAdapterNamespace));
478+
logger.info("Prometheus adapter release {0} status is deployed in namespace {1}",
479+
promAdapterReleaseName, promAdapterNamespace);
480+
481+
// wait for the promethues adapter pod to be ready
482+
logger.info("Wait for the promethues adapter pod is ready in namespace {0}", promAdapterNamespace);
483+
String podName = assertDoesNotThrow(() -> getPodName(promAdapterNamespace, "prometheus-adapter"),
484+
"Can't find prometheus-adapter pod");
485+
testUntil(
486+
assertDoesNotThrow(() -> isPrometheusAdapterReady(promAdapterNamespace, podName),
487+
"prometheusAdapterIsReady failed with ApiException"),
488+
logger,
489+
"prometheus adapter to be running in namespace {0}",
490+
promAdapterNamespace);
491+
492+
return promAdapterHelmParams;
493+
}
494+
495+
434496
/**
435497
* Install Grafana and wait up to five minutes until the grafana pod is ready.
436498
*
@@ -570,6 +632,39 @@ public static void cleanupPromGrafanaClusterRoles(String prometheusReleaseName,
570632
}
571633
}
572634

635+
/**
636+
* Extra clean up for Prometheus Adapter artifacts.
637+
*
638+
*/
639+
public static void cleanupPrometheusAdapterClusterRoles() {
640+
//extra cleanup
641+
String prometheusAdapterReleaseName = "prometheus-adapter";
642+
try {
643+
if (ClusterRole.clusterRoleExists(prometheusAdapterReleaseName + "-resource-reader")) {
644+
Kubernetes.deleteClusterRole(prometheusAdapterReleaseName + "-resource-reader");
645+
}
646+
if (ClusterRole.clusterRoleExists(prometheusAdapterReleaseName + "-server-resources")) {
647+
Kubernetes.deleteClusterRole(prometheusAdapterReleaseName + "-server-resources");
648+
}
649+
if (ClusterRoleBinding.clusterRoleBindingExists(prometheusAdapterReleaseName + "-hpa-controller")) {
650+
Kubernetes.deleteClusterRoleBinding(prometheusAdapterReleaseName + "-hpa-controller");
651+
}
652+
if (ClusterRoleBinding.clusterRoleBindingExists(prometheusAdapterReleaseName + "-resource-reader")) {
653+
Kubernetes.deleteClusterRoleBinding(prometheusAdapterReleaseName + "-resource-reader");
654+
}
655+
if (ClusterRoleBinding.clusterRoleBindingExists(prometheusAdapterReleaseName + "-system-auth-delegator")) {
656+
Kubernetes.deleteClusterRoleBinding(prometheusAdapterReleaseName + "-system-auth-delegator");
657+
}
658+
if (RoleBinding.roleBindingExists(prometheusAdapterReleaseName + "-auth-reader", "kube-system")) {
659+
Kubernetes.deleteNamespacedRoleBinding("kube-system", prometheusAdapterReleaseName + "-auth-reader");
660+
}
661+
662+
} catch (Exception ex) {
663+
//ignoring
664+
logger.info("getting exception during delete artifacts for prometheus adapter");
665+
}
666+
}
667+
573668
/**
574669
* Download src from monitoring exporter github project and build webapp.
575670
*
@@ -937,15 +1032,6 @@ public static void verifyMonExpAppAccessThroughNginx(String nginxHost, int repli
9371032
logger,
9381033
"Verify NGINX can access the monitoring exporter metrics \n"
9391034
+ "from all managed servers in the domain via http");
940-
/*
941-
assertThat(callWebAppAndCheckForServerNameInResponse(curlCmd, managedServerNames, 100))
942-
.as("Verify NGINX can access the monitoring exporter metrics "
943-
+ "from all managed servers in the domain via http")
944-
.withFailMessage("NGINX can not access the monitoring exporter metrics "
945-
+ "from one or more of the managed servers via http")
946-
.isTrue();
947-
948-
*/
9491035
}
9501036

9511037
/**

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/PodUtils.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import io.kubernetes.client.openapi.models.V1Pod;
1919
import io.kubernetes.client.openapi.models.V1PodAffinityTerm;
2020
import io.kubernetes.client.openapi.models.V1PodAntiAffinity;
21+
import io.kubernetes.client.openapi.models.V1PodList;
2122
import io.kubernetes.client.openapi.models.V1WeightedPodAffinityTerm;
2223
import oracle.weblogic.domain.ClusterResource;
2324
import oracle.weblogic.domain.DomainResource;
@@ -501,4 +502,25 @@ public static boolean checkInUncompletedIntroPodLogContainsRegex(String regex, S
501502

502503
return matcher.find();
503504
}
505+
506+
/**
507+
* Get pod name with given prefix.
508+
* @param namespace namespace where pod exists
509+
* @return pod name
510+
* @throws ApiException if Kubernetes client API call fails
511+
*/
512+
public static String getPodName(String namespace, String podPrefix) throws ApiException {
513+
String podName = null;
514+
V1PodList pods = null;
515+
pods = Kubernetes.listPods(namespace, null);
516+
if (pods.getItems().size() != 0) {
517+
for (V1Pod pod : pods.getItems()) {
518+
if (pod != null && pod.getMetadata().getName().startsWith(podPrefix)) {
519+
podName = pod.getMetadata().getName();
520+
break;
521+
}
522+
}
523+
}
524+
return podName;
525+
}
504526
}

‎integration-tests/src/test/resources/exporter/customhpa.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright (c) 2022, Oracle and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
apiVersion: autoscaling/v2
4+
kind: HorizontalPodAutoscaler
5+
metadata:
6+
name: custommetrics-hpa
7+
namespace: default
8+
spec:
9+
scaleTargetRef:
10+
apiVersion: weblogic.oracle/v1
11+
kind: Cluster
12+
name: hpacustomcluster
13+
behavior:
14+
scaleDown:
15+
stabilizationWindowSeconds: 60
16+
scaleUp:
17+
stabilizationWindowSeconds: 60
18+
minReplicas: 2
19+
maxReplicas: 3
20+
metrics:
21+
- type: Pods
22+
pods:
23+
metric:
24+
name: total_opened_sessions_myear_app
25+
target:
26+
type: AverageValue
27+
averageValue: 15

‎integration-tests/src/test/resources/exporter/promadaptervalues.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Copyright (c) 2022, Oracle and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
rules:
4+
default: false
5+
custom:
6+
- seriesQuery: '{__name__=~"wls_webapp_config_sessions_opened_total_count",pod!="",namespace!="", app="myear"}'
7+
resources:
8+
overrides:
9+
namespace:
10+
resource: namespace
11+
pod:
12+
resource: pod
13+
name:
14+
matches: ^(.*)
15+
as: "total_opened_sessions_myear_app"
16+
metricsQuery: wls_webapp_config_sessions_opened_total_count{<<.LabelMatchers>>,app='myear'}

‎integration-tests/src/test/resources/exporter/promvalues.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,10 @@ extraScrapeConfigs: |
115115
regex: __meta_kubernetes_pod_label_(.+)
116116
- source_labels: [__meta_kubernetes_pod_name]
117117
action: replace
118-
target_label: pod_name
118+
target_label: pod
119+
- source_labels: [__meta_kubernetes_namespace]
120+
action: replace
121+
target_label: namespace
119122
basic_auth:
120123
username: weblogic
121124
password: welcome1

0 commit comments

Comments
 (0)