Port forward changes from release/4.2 for oke runs

marinakog · jshum2479 · commit 48d9c4a88d71 · 2024-10-08T22:33:51.000Z
diff --git a/Jenkinsfile.oke b/Jenkinsfile.oke
@@ -3,7 +3,9 @@
 //
 
 CRON_SETTINGS = '''H 3 * * 0-4 % MAVEN_PROFILE_NAME=oke-gate;CLUSTER_NAME=seqone;PARALLEL_RUN=false
+                   H 1 * * 0-4 % MAVEN_PROFILE_NAME=oke-sequential;CLUSTER_NAME=seqtwo;PARALLEL_RUN=false
                    H 2 * * 0-4 % MAVEN_PROFILE_NAME=oke-parallel;CLUSTER_NAME=parone;PARALLEL_RUN=true'''
+
 pipeline {
     agent { label 'large' }
     options {
@@ -113,9 +115,10 @@ pipeline {
         )
 
                 choice(name: 'MAVEN_PROFILE_NAME',
-                        description: 'Profile to use in mvn command to run the tests.  Possible values are oke-gate,oke-parallel. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.',
+                        description: 'Profile to use in mvn command to run the tests.  Possible values are oke-gate,oke-parallel, oke-sequential. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.',
                         choices: [
                                 'oke-gate',
+                                'oke-sequential',
                                 'oke-parallel'
                         ]
                 )
@@ -528,6 +531,8 @@ EOF
                                 export OCI_CLI_CONFIG_FILE=${jenkins_home_directory}/.oci/config
                                 export OCI_CLI_PROFILE=${oci_profile}
                                 compartment_ocid=${compartment_id}
+                                export COMPARTMENT_OCID=${compartment_id}
+                                echo "COMPARTMENT_OCID : ${COMPARTMENT_OCID}"
                                 mkdir -m777 -p "${WORKSPACE}/.mvn"
                                 touch ${WORKSPACE}/.mvn/maven.config
                                 export KUBECONFIG=${kubeconfig_file}
@@ -599,6 +604,7 @@ EOF
                                 echo "-Dwko.it.monitoring.exporter.branch=\"${MONITORING_EXPORTER_BRANCH}\""                 >> ${WORKSPACE}/.mvn/maven.config
                                 echo "-Dwko.it.monitoring.exporter.webapp.version=\"${MONITORING_EXPORTER_WEBAPP_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config
                                 echo "-Dwko.it.prometheus.chart.version=\"${PROMETHEUS_CHART_VERSION}\""                     >> ${WORKSPACE}/.mvn/maven.config
+                                echo "-Dwko.it.oci.compartment.ocid=\"${COMPARTMENT_OCID}\""                                 >> ${WORKSPACE}/.mvn/maven.config
                                 echo "-Dwko.it.grafana.chart.version=\"${GRAFANA_CHART_VERSION}\""                           >> ${WORKSPACE}/.mvn/maven.config
                                 echo "-Dwko.it.collect.logs.on.success=\"${COLLECT_LOGS_ON_SUCCESS}\""                       >> ${WORKSPACE}/.mvn/maven.config
                                 echo "-Dwko.it.remoteconsole.version=\"${REMOTECONSOLE_VERSION}\""                           >> ${WORKSPACE}/.mvn/maven.config
@@ -608,6 +614,10 @@ EOF
                                 echo "${WORKSPACE}/.mvn/maven.config contents:"
                                 cat "${WORKSPACE}/.mvn/maven.config"
                                 cp "${WORKSPACE}/.mvn/maven.config" "${result_root}"
+                                echo "${HTTP_PROXY} HTTP_PROXY settings"
+                                echo "${HTTPS_PROXY} HTTPS_PROXY settings"
+                                echo "${http_proxy} http_proxy settings"
+                                echo "${https_proxy} https_proxy settings"
                             '''
                         }
                         withMaven(globalMavenSettingsConfig: 'wkt-maven-settings-xml', publisherStrategy: 'EXPLICIT') {
@@ -620,12 +630,15 @@ EOF
                                     export OKE_CLUSTER_PRIVATEIP="true"
                                     export OKD="false"
                                     export KUBECONFIG=${kubeconfig_file}
+                                    export COMPARTMENT_OCID=${compartment_id}
                                     export BASE_IMAGES_REPO_USERNAME="${OCIR_USER}"
                                     export BASE_IMAGES_REPO_PASSWORD="${OCIR_PASS}"
                                     export BASE_IMAGES_REPO_EMAIL="noreply@oracle.com"
                                     export TEST_IMAGES_REPO_USERNAME="${OCIR_USER}"
                                     export TEST_IMAGES_REPO_PASSWORD="${OCIR_PASS}"
                                     export TEST_IMAGES_REPO_EMAIL="noreply@oracle.com"
+                                    export no_proxy="${NO_PROXY},localhost,127.0.0.1,.us.oracle.com,.oraclecorp.com,login.oracle.com"
+                                    export NO_PROXY="${no_proxy}"
                                     
                                     if ! mvn -pl integration-tests -P ${MAVEN_PROFILE_NAME} verify 2>&1 | tee "${result_root}/oketest.log"; then
                                         echo "integration-tests failed"
@@ -670,7 +683,7 @@ EOF
                             ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN}
                         fi
 
-                        if [ "${MAVEN_PROFILE_NAME}" = "oke-gate" ] && [ "${BRANCH}" = "main" ]; then
+                        if [ "${MAVEN_PROFILE_NAME}" = "oke-gate" ] && [ "${BRANCH}" = "release/4.2" ]; then
                             compname="wkt"
                             wkt_compartment_ocid=$(oci iam compartment list --compartment-id-in-subtree true --all | jq --arg compname "$compname" '.data[] | select(."name"==$compname)' | jq -r ."id")
                             sec_list_id=$(oci network security-list list --compartment-id="$wkt_compartment_ocid" --display-name=Security-List-wktiso1 | jq -r '.data[] | ."id"')
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
@@ -48,6 +48,7 @@
         <wko.it.grafana.chart.version>${env.GRAFANA_CHART_VERSION}</wko.it.grafana.chart.version>
         <wko.it.istio.version>${env.ISTIO_VERSION}</wko.it.istio.version>
         <wko.it.oke.cluster>${env.OKE_CLUSTER}</wko.it.oke.cluster>
+        <wko.it.oci.compartment.ocid>${env.COMPARTMENT_OCID}</wko.it.oci.compartment.ocid>
         <wko.it.arm.cluster>${env.ARM}</wko.it.arm.cluster>
         <wko.it.oke.cluster.privateip>${env.OKE_CLUSTER_PRIVATEIP}</wko.it.oke.cluster.privateip>
         <wko.it.nfs.server>${env.NFS_SERVER}</wko.it.nfs.server>
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItAddNewDynamicClusterUsingWlst.java
@@ -57,7 +57,7 @@
 @IntegrationTest
 @Tag("kind-parallel")
 @Tag("olcne-mrg")
-@Tag("oke-gate")
+@Tag("oke-parallel")
 
 class ItAddNewDynamicClusterUsingWlst {
 
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItCrossDomainTransaction.java
@@ -107,7 +107,7 @@
  */
 @DisplayName("Verify cross domain transaction is successful")
 @IntegrationTest
-@Tag("oke-gate")
+@Tag("oke-sequential")
 @Tag("kind-parallel")
 @Tag("okd-wls-srg")
 class ItCrossDomainTransaction {
@@ -666,7 +666,7 @@ private static void createNginxIngressPathRoutingRules() {
 
     ingressRules.add(ingressRule);
 
-    createIngressAndRetryIfFail(60, false, ingressName, domain1Namespace, null, ingressClassName, ingressRules, null);
+    createIngressAndRetryIfFail(20, false, ingressName, domain1Namespace, null, ingressClassName, ingressRules, null);
 
     // check the ingress was found in the domain namespace
     assertThat(assertDoesNotThrow(() -> listIngresses(domain1Namespace)))
@@ -681,6 +681,6 @@ private static void createNginxIngressPathRoutingRules() {
         + "/weblogic/ready --write-out %{http_code} -o /dev/null";
 
     logger.info("Executing curl command {0}", curlCmd);
-    assertTrue(callWebAppAndWaitTillReady(curlCmd, 60));
+    assertTrue(callWebAppAndWaitTillReady(curlCmd, 20));
   }
 }
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItFmwDynamicDomainInPV.java
@@ -78,7 +78,7 @@
  */
 @DisplayName("Test to creat a FMW dynamic domain in persistent volume using WLST")
 @IntegrationTest
-@Tag("oke-gate")
+@Tag("oke-sequential")
 @Tag("kind-sequential")
 @Tag("okd-fmw-cert")
 class ItFmwDynamicDomainInPV {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java
@@ -177,7 +177,7 @@
 @Tag("olcne-srg")
 @Tag("kind-parallel")
 @Tag("okd-wls-mrg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
 @Tag("oke-arm")
 class ItIntrospectVersion {
 
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIstioMonitoringExporter.java
@@ -77,7 +77,7 @@
 
 @DisplayName("Test the monitoring WebLogic Domain via istio provided Prometheus")
 @IntegrationTest
-@Tag("oke-gate")
+@Tag("oke-parallel")
 @Tag("kind-parallel")
 @Tag("olcne-mrg")
 class ItIstioMonitoringExporter {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java
@@ -150,7 +150,7 @@
 @Tag("oke-arm")
 @IntegrationTest
 @Tag("olcne-srg")
-@Tag("oke-parallel")
+@Tag("oke-sequential")
 class ItKubernetesDomainEvents {
 
   private static String opNamespace = null;
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiAuxiliaryImage.java
@@ -121,7 +121,7 @@
 @Tag("toolkits-srg")
 @Tag("okd-wls-srg")
 @Tag("oke-arm")
-@Tag("oke-gate")
+@Tag("oke-parallel")
 class ItMiiAuxiliaryImage {
 
   private static String domainNamespace = null;
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiClusterResource.java
@@ -104,7 +104,7 @@
 @Tag("kind-parallel")
 @Tag("okd-wls-srg")
 @Tag("oke-arm")
-@Tag("oke-gate")
+@Tag("oke-sequential")
 class ItMiiClusterResource {
 
   private static String opNamespace = null;
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart1.java
@@ -66,7 +66,7 @@
 @Tag("kind-parallel")
 @Tag("toolkits-srg")
 @Tag("okd-wls-mrg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
 @Tag("oke-arm")
 @Tag("olcne-srg")
 class ItMiiDynamicUpdatePart1 {
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart2.java
@@ -65,7 +65,7 @@
 @Tag("kind-parallel")
 @Tag("toolkits-srg")
 @Tag("okd-wls-mrg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
 @Tag("oke-arm")
 class ItMiiDynamicUpdatePart2 {
 
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiDynamicUpdatePart3.java
@@ -78,7 +78,7 @@
 @DisplayName("Test dynamic updates to a model in image domain, part3")
 @IntegrationTest
 @Tag("olcne-mrg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
 @Tag("kind-parallel")
 @Tag("toolkits-srg")
 @Tag("okd-wls-mrg")
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMiiUpdateDomainConfig.java
@@ -119,7 +119,7 @@
 @Tag("kind-parallel")
 @Tag("toolkits-srg")
 @Tag("okd-wls-srg")
-@Tag("oke-gate")
+@Tag("oke-sequential")
 class ItMiiUpdateDomainConfig {
 
   private static String opNamespace = null;
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsScale.java
@@ -121,7 +121,7 @@
     + "rolling restart behavior in a multi-cluster MII domain and "
     + "the sample application can be accessed via NGINX ingress controller")
 @Tag("kind-sequential")
-@Tag("oke-gate")
+@Tag("oke-sequential")
 @IntegrationTest
 class ItMultiDomainModelsScale {
 
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/ItOCILoadBalancer.java
@@ -4,15 +4,12 @@
 package oracle.weblogic.kubernetes;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
-import io.kubernetes.client.openapi.models.V1LoadBalancerIngress;
-import io.kubernetes.client.openapi.models.V1Service;
 import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes;
 import oracle.weblogic.kubernetes.annotations.IntegrationTest;
 import oracle.weblogic.kubernetes.annotations.Namespaces;
+import oracle.weblogic.kubernetes.extensions.InitializationTasks;
 import oracle.weblogic.kubernetes.logging.LoggingFacade;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -25,10 +22,11 @@
 import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_NAME;
 import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG;
 import static oracle.weblogic.kubernetes.TestConstants.SKIP_CLEANUP;
-import static oracle.weblogic.kubernetes.assertions.impl.Kubernetes.getService;
 import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndCheckForServerNameInResponse;
 import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createMiiDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret;
+import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.deleteLoadBalancer;
+import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.getLoadBalancerIP;
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyOCILoadBalancer;
 import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
 import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
@@ -45,7 +43,7 @@
     + "all managed servers in the domain through OCI Load Balancer")
 @IntegrationTest
 @Tag("oke-arm")
-@Tag("oke-parallel")
+@Tag("oke-gate")
 class ItOCILoadBalancer {
   // domain constants
   private static final int replicaCount = 2;
@@ -88,6 +86,7 @@ public void tearDownAll() {
     if (!SKIP_CLEANUP) {
       Kubernetes.deleteService(OCI_LB_NAME, domainNamespace);
     }
+    deleteLoadBalancer(loadBalancerIP);
   }
 
   /**
@@ -118,37 +117,14 @@ void testOCILoadBalancer() throws Exception {
     assertDoesNotThrow(() -> installAndVerifyOCILoadBalancer(domainNamespace,
         clusterHttpPort, clusterName, domainUid, OCI_LB_NAME),
         "Installation of OCI Load Balancer failed");
-    loadBalancerIP = getLoadBalancerIP(domainNamespace,OCI_LB_NAME);
+    loadBalancerIP = getLoadBalancerIP(domainNamespace,OCI_LB_NAME, true);
     assertNotNull(loadBalancerIP, "External IP for Load Balancer is undefined");
+    InitializationTasks.registerLoadBalancerExternalIP(loadBalancerIP);
+
     logger.info("LoadBalancer IP is " + loadBalancerIP);
     verifyWebAppAccessThroughOCILoadBalancer(loadBalancerIP, 2, clusterHttpPort);
   }
 
-  /**
-   * Retreive external IP from OCI LoadBalancer.
-   */
-  private static String getLoadBalancerIP(String namespace, String lbName) throws Exception {
-    Map<String, String> labels = new HashMap<>();
-    labels.put("loadbalancer", lbName);
-    V1Service service = getService(lbName, labels, namespace);
-    assertNotNull(service, "Can't find service with name " + lbName);
-    logger.info("Found service with name {0} in {1} namespace ", lbName, namespace);
-    assertNotNull(service.getStatus(), "service status is null");
-    assertNotNull(service.getStatus().getLoadBalancer(), "service loadbalancer is null");
-    List<V1LoadBalancerIngress> ingress = service.getStatus().getLoadBalancer().getIngress();
-    if (ingress != null) {
-      logger.info("LoadBalancer Ingress " + ingress.toString());
-      V1LoadBalancerIngress lbIng = ingress.stream().filter(c ->
-          c.getIp() != null && !c.getIp().equals("pending")
-      ).findAny().orElse(null);
-      if (lbIng != null) {
-        logger.info("OCI LoadBalancer is created with external ip" + lbIng.getIp());
-        return lbIng.getIp();
-      }
-    }
-    return null;
-  }
-
   /**
    * Verify the sample-app app can be accessed from all managed servers in the domain through OCI Load Balancer.
    */
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/TestConstants.java
@@ -370,6 +370,7 @@ public interface TestConstants {
       Boolean.parseBoolean(getNonEmptySystemProperty("wko.it.oke.cluster.privateip", "false"));
   public static final String NFS_SERVER = System.getProperty("wko.it.nfs.server", "");
   public static final String NODE_IP = System.getProperty("wko.it.node.ip", "");
+  public static final String COMPARTMENT_OCID = System.getProperty("wko.it.oci.compartment.ocid", "");
   public static final String [] FSS_DIR = System.getProperty("wko.it.fss.dir","").split(",");
   public static final String IMAGE_PULL_POLICY = "IfNotPresent";
 
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/extensions/InitializationTasks.java
@@ -11,9 +11,11 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -113,6 +115,7 @@
 import static oracle.weblogic.kubernetes.utils.FileUtils.cleanupDirectory;
 import static oracle.weblogic.kubernetes.utils.IstioUtils.installIstio;
 import static oracle.weblogic.kubernetes.utils.IstioUtils.uninstallIstio;
+import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.deleteLoadBalancer;
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyTraefik;
 import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
 import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
@@ -133,6 +136,7 @@ public class InitializationTasks implements BeforeAllCallback, ExtensionContext.
   private static String wdtBasicImage;
 
   private static Collection<String> pushedImages = new ArrayList<>();
+  private static Set<String> lbIPs = new HashSet<>();
   private static boolean isInitializationSuccessful = false;
 
   ConditionFactory withVeryLongRetryPolicy
@@ -355,6 +359,15 @@ public static void registerPushedImage(String imageName) {
     pushedImages.add(imageName);
   }
 
+  /**
+   * Called when load balancer is created in OCI, allowing conditional cleanup of load balancers.
+   *
+   * @param lbIP external IP of load balancer.
+   */
+  public static void registerLoadBalancerExternalIP(String lbIP) {
+    lbIPs.add(lbIP);
+  }
+
   @Override
   public void close() {
     LoggingFacade logger = getLogger();
@@ -395,6 +408,13 @@ public void close() {
       for (String image : pushedImages) {
         deleteImage(image);
       }
+      if (OKE_CLUSTER) {
+        logger.info("Cleanup created in OCI load balancers after all test suites are run");
+        // delete all load balancers in OCI
+        for (String ip : lbIPs) {
+          deleteLoadBalancer(ip);
+        }
+      }
     }
 
     // delete images from TEST_IMAGES_REPO, if necessary
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/IstioUtils.java
diff --git a/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java b/integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java
diff --git a/integration-tests/src/test/resources/bash-scripts/delete_loadbalancer.sh b/integration-tests/src/test/resources/bash-scripts/delete_loadbalancer.sh
