Merge branch 'monexpfixmain' into 'main'

forward changes from 4.2 for OWLS-127302

See merge request weblogic-cloud/weblogic-kubernetes-operator!4946

Author: rjeberhard

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMonitoringExporterSamples.java

@@ -107,6 +107,7 @@
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.createIngressForDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyNginx;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.checkMetricsViaPrometheus;
+import static oracle.weblogic.kubernetes.utils.MonitoringUtils.checkPrometheusAlert;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.cleanupPromGrafanaClusterRoles;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.createAndVerifyDomain;
 import static oracle.weblogic.kubernetes.utils.MonitoringUtils.createAndVerifyMiiImage;
@@ -376,7 +377,7 @@ void testEndToEndViaChart() throws Exception {
     }
   }
 
-  private void fireAlert() throws ApiException {
+  private void fireAlert() throws Exception {
     // scale domain2
     logger.info("Scaling cluster {0} of domain {1} in namespace {2} to {3} servers.",
         cluster1Name, domain2Uid, domain2Namespace, 1);
@@ -385,6 +386,16 @@ private void fireAlert() throws ApiException {
         domain2Uid + "-" + MANAGED_SERVER_NAME_BASE, replicaCount, managedServersCount,
         null, null);
 
+    logger.info("Wait for the prometheus  to create alert ");
+    checkPrometheusAlert("ClusterWarning", hostPortPrometheus,
+        prometheusReleaseName
+            + "." + monitoringNS);
+
+    logger.info("Wait for the prometheus  to fire alert ");
+    checkPrometheusAlert("firing", hostPortPrometheus,
+        prometheusReleaseName
+            + "." + monitoringNS);
+
     //check webhook log for firing alert
     List<V1Pod> pods = listPods(webhookNS, "app=webhook").getItems();
     assertNotNull((pods), "No pods are running in namespace : " + webhookNS);
@@ -399,7 +410,7 @@ private void fireAlert() throws ApiException {
 
     testUntil(withLongRetryPolicy,
         assertDoesNotThrow(() -> searchPodLogForKey(pod,
-            "Some WLS cluster has only one running server for more than 1 minutes"),
+            "Some WLS cluster has only one running server for more than 15 secs"),
             "webhook failed to fire alert"),
         logger,
         "webhook to fire alert");

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/MonitoringUtils.java

@@ -318,6 +318,33 @@ public static void checkMetricsViaPrometheus(String searchKey, String expectedVa
         expectedVal);
   }
 
+  /**
+   * Check metrics using Prometheus.
+   *
+   * @param expectedVal        - expected alert data to search
+   * @param hostPortPrometheus host:nodePort for prometheus
+   * @throws Exception if command to check metrics fails
+   */
+  public static void checkPrometheusAlert(String expectedVal,
+                                          String hostPortPrometheus, String ingressHost)
+      throws Exception {
+
+    LoggingFacade logger = getLogger();
+    // url
+    String curlCmd =
+        String.format("curl -g --silent --show-error --noproxy '*' -v  -H 'host: %s'"
+                + " http://%s/api/v1/alerts",
+            ingressHost, hostPortPrometheus);
+
+    logger.info("Executing Curl cmd {0}", curlCmd);
+    logger.info(" expected Value {0} ", expectedVal);
+    testUntil(
+        searchForKey(curlCmd, expectedVal),
+        logger,
+        "Check prometheus alert against expected {0}",
+        expectedVal);
+  }
+
   /**
    * Check output of the command against expected output.
    *
@@ -1333,9 +1360,19 @@ public static boolean verifyMonExpAppAccess(String uri, String searchKey, Boolea
       podName = domainUid + "-managed-server1";
     }
     // access metrics
-    final String command = String.format("%s exec -n %s %s -- curl -k %s://%s:%s@%s:%s/%s",
-        KUBERNETES_CLI, domainNS, podName, protocol, ADMIN_USERNAME_DEFAULT, ADMIN_PASSWORD_DEFAULT,
-        podName, port, uri);
+    final String command = String.format(
+        "%s exec -n %s %s -- curl -k %s://%s:%s@%s:%s/%s",
+        KUBERNETES_CLI,
+        domainNS,
+        podName,
+        protocol,
+        ADMIN_USERNAME_DEFAULT,
+        ADMIN_PASSWORD_DEFAULT,
+        podName,
+        port,
+        uri
+    );
+
     logger.info("accessing managed server exporter via " + command);
 
     boolean isFound = false;

integration-tests/src/test/resources/exporter/promvalues.yaml

@@ -147,7 +147,7 @@ serverFiles:
             labels:
               severity: page
             annotations:
-              description: 'Some WLS cluster has only one running server for more than 1 minutes.'
+              description: 'Some WLS cluster has only one running server for more than 15 secs.'
               summary: 'Some wls cluster is in warning state.'
 
 extraScrapeConfigs: |

integration-tests/src/test/resources/exporter/promvaluesoke.yaml

@@ -151,7 +151,7 @@ serverFiles:
             labels:
               severity: page
             annotations:
-              description: 'Some WLS cluster has only one running server for more than 1 minutes.'
+              description: 'Some WLS cluster has only one running server for more than 15 secs.'
               summary: 'Some wls cluster is in warning state.'
 
 extraScrapeConfigs: |