Fix Github CI integration tests

fhanik · web-flow · commit eaef25829055 · 2025-11-13T08:00:52.000-08:00
Better use of memory, consistent runs of integration tests
diff --git a/build.gradle b/build.gradle
@@ -105,7 +105,8 @@ subprojects {
         // gradle might stop the test run due to the failFast but still concludes with BUILD SUCCESSFUL (if the retry is successful)
         failFast = false
         useJUnitPlatform()
-        jvmArgs += ["-Xmx1024m",
+        // Reduced from 1024m to 640m - unit tests don't need as much as integration tests
+        jvmArgs += ["-Xmx640m",
                     "-XX:+StartAttachListener",
                     "-XX:+HeapDumpOnOutOfMemoryError",
                     "-XX:HeapDumpPath=/var/log/uaa-tests.hprof"
@@ -120,8 +121,21 @@ subprojects {
     }
 
     tasks.register('integrationTest', Test) {
+        dependsOn subprojects.integrationTest
+        
         useJUnitPlatform()
 
+        // This prevents integrationTests from hanging indefinitely
+        timeout = Duration.ofMinutes(180)
+        
+        // Integration test workers need same memory as unit tests
+        // Actual CI configuration is controlled via integration_tests.sh script
+        jvmArgs += ["-Xmx640m",
+                    "-XX:+StartAttachListener",
+                    "-XX:+HeapDumpOnOutOfMemoryError",
+                    "-XX:HeapDumpPath=/var/log/uaa-tests.hprof"
+        ]
+
         // Enable JaCoCo for integration tests
         jacoco {
             enabled = true
@@ -198,10 +212,6 @@ tasks.register('manifests', Copy) {
     into("build/sample-manifests")
 }
 
-tasks.register('integrationTest', Test) {
-    dependsOn subprojects.integrationTest
-}
-
 tasks.register('cleanBootTomcatDir') {
     String tomcatBase = file("scripts/boot/tomcat/").getAbsolutePath()
     delete(java.nio.file.Path.of(tomcatBase))
diff --git a/dependencies.gradle b/dependencies.gradle
@@ -5,7 +5,7 @@ ext {
 
 // Versions shared between multiple dependencies
 versions.apacheDsVersion = "2.0.0.AM27"
-versions.bouncyCastleFipsVersion = "2.1.1"
+versions.bouncyCastleFipsVersion = "2.1.2"
 versions.bouncyCastlePkixFipsVersion = "2.1.9"
 versions.bouncyCastleTlsFipsVersion = "2.1.20"
 versions.springBootVersion = "3.5.7"
diff --git a/gradle.properties b/gradle.properties
@@ -1,9 +1,23 @@
 version=0.0.0
 
 # Required for LdapMockMvcTests when asserting it can find a user in a different language
-org.gradle.jvmargs=-Dfile.encoding=utf8 -XX:+StartAttachListener -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/uaa-tests.hprof
+# Optimized for Gradle 9.0 + Kotlin 2.2 to prevent test hanging and classloading deadlocks
+# CICompilerCount=2 and ProcessReaper flag prevent thread contention in containers
+org.gradle.jvmargs=-Dfile.encoding=utf8 \
+  -XX:+StartAttachListener \
+  -XX:+HeapDumpOnOutOfMemoryError \
+  -XX:HeapDumpPath=/var/log/uaa-tests.hprof \
+  -Xmx1024m \
+  -Xms1024m \
+  -XX:MaxMetaspaceSize=256m \
+  -XX:+UseG1GC \
+  -XX:MaxGCPauseMillis=100 \
+  -XX:ParallelGCThreads=2 \
+  -XX:CICompilerCount=2 \
+  -Djdk.lang.processReaperUseDefaultStackSize=true
 
-org.gradle.parallel=true
+# Disable parallel execution to prevent classloading deadlocks with Kotlin 2.2
+org.gradle.parallel=false
 
 signing.keyId=
 signing.password=
@@ -12,4 +26,5 @@ ossrhUsername=
 ossrhPassword=
 group=org.cloudfoundry.identity
 archivesBaseName="uaa"
-org.gradle.workers.max=6
+# Limit workers to 2 for memory-constrained environments (CI containers)
+org.gradle.workers.max=2
diff --git a/scripts/integration_tests.sh b/scripts/integration_tests.sh
@@ -6,6 +6,10 @@ set -eu
 # Global env vars:
 #   UAA_GRADLE_INT_TEST_COMMAND: Gradle command to run integration tests (default: integrationTest)
 #       this could include :cloudfoundry-identity-server:integrationTest --tests to run specific tests
+#   jvm_heap: JVM heap size for UAA boot server (default: 640m)
+#   jvm_metaspace: JVM metaspace size for UAA boot server (default: 192m)
+#   gradle_heap: JVM heap size for Gradle daemon (default: 1024m)
+#   gradle_test_heap: JVM heap size for Gradle test workers (default: 640m)
 #######################################
 function main() {
   local script_dir; script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -24,18 +28,38 @@ function main() {
 
     local wd launch_boot assemble_code integration_test_code
     wd=$(pwd)
-    echo "Setting heap to ${jvm_heap:=768m}"
-    echo "Setting metaspace to ${jvm_metaspace:=256m}"
+    temp_dir=${script_dir}/tmp
+    mkdir -p "${temp_dir}"
+    
+    # Memory settings optimized for Gradle 9.0 with Kotlin 2.2
+    # Boot server needs enough memory to handle test requests without crashing
+    # Increased Gradle daemon heap to 1GB to prevent hanging with 2 workers
+    # --no-configuration-cache prevents stale Kotlin compiler state reuse between daemon processes
+    # logging.manager is set to org.apache.logging.log4j.jul.LogManager to prevent log4j2 from using java.util.logging
+    echo "Setting boot heap to ${jvm_heap:=640m}"
+    echo "Setting boot metaspace to ${jvm_metaspace:=192m}"
+    echo "Setting Gradle daemon heap to ${gradle_heap:=1024m}"
+    echo "Setting test worker heap to ${gradle_test_heap:=640m}"
 
     readonly launch_boot="nohup java \
-               -XX:+UseParallelGC \
-               -Xmx${jvm_heap} \
+               -XX:+UseG1GC \
+               -XX:G1HeapRegionSize=1m \
+               -Xms64m -Xmx${jvm_heap} \
                -XX:MaxMetaspaceSize=${jvm_metaspace} \
+               -XX:MetaspaceSize=${jvm_metaspace} \
+               -XX:+UseStringDeduplication \
+               -XX:MaxGCPauseMillis=200 \
                -XX:+HeapDumpOnOutOfMemoryError \
                -XX:HeapDumpPath=${wd} \
                -DCLOUDFOUNDRY_CONFIG_PATH=${wd}/scripts/boot \
+               -Dlogging.config=${wd}/scripts/boot/log4j2.properties \
+               -Dlog4j.configurationFile=${wd}/scripts/boot/log4j2.properties \
+               -Dlog4j2.formatMsgNoLookups=true \
+               -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
                -DSECRETS_DIR=${wd}/scripts/boot \
                -Djava.security.egd=file:/dev/./urandom \
+               -Djava.io.tmpdir=${temp_dir} \
+               -Dorg.bouncycastle.native.loader.install_dir=${temp_dir} \
                -Dmetrics.perRequestMetrics=true \
                -Dserver.servlet.context-path=/uaa \
                -Dserver.tomcat.basedir=${wd}/scripts/boot/tomcat \
@@ -50,46 +74,89 @@ function main() {
                -jar ${wd}/uaa/build/libs/cloudfoundry-identity-uaa-0.0.0.war \
                > boot.log 2>&1 &"
 
+    # Explicit Gradle daemon memory for Kotlin 2.2 with additional GC tuning
     readonly assemble_code="./gradlew '-Dspring.profiles.active=${test_profile}' \
                 '-Djava.security.egd=file:/dev/./urandom' \
+                '-Dorg.gradle.jvmargs=-Dfile.encoding=utf8 -Xms64m -Xmx${gradle_heap} -XX:MaxMetaspaceSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=100' \
                 assemble \
+                --no-watch-fs \
                 --no-daemon \
-                --max-workers=4 \
+                --no-configuration-cache \
+                --max-workers=2 \
                 --stacktrace \
                 --console=plain"
 
+    # Explicit memory limits for test JVMs with GC tuning and classloader fixes
+    # All flags required to prevent classloading deadlocks and thread starvation during test init
+    # --no-configuration-cache prevents stale Kotlin compiler state reuse between daemon processes
+    readonly compile_test_code="./gradlew \
+                '-Dspring.profiles.active=${test_profile}' \
+                '-Djava.security.egd=file:/dev/./urandom' \
+                '-DskipUaaAutoStart=true' \
+                '-Dorg.gradle.jvmargs=-Dfile.encoding=utf8 -Xms64m -Xmx${gradle_test_heap} -XX:MaxMetaspaceSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:ParallelGCThreads=2 -XX:CICompilerCount=2 -Djdk.lang.processReaperUseDefaultStackSize=true' \
+                '-Dorg.gradle.daemon.idletimeout=300000' \
+                '-Dorg.gradle.parallel=false' \
+                '-Dorg.gradle.workers.max=2' \
+                clean assemble compileTestJava \
+                --no-watch-fs \
+                --no-daemon \
+                --no-configuration-cache \
+                --max-workers=2 \
+                --stacktrace \
+                --console=plain"
+
+    # Explicit memory limits for test JVMs with GC tuning and classloader fixes
+    # All flags required to prevent classloading deadlocks and thread starvation during test init
     readonly integration_test_code="./gradlew \
                 '-Dspring.profiles.active=${test_profile}' \
                 '-Djava.security.egd=file:/dev/./urandom' \
                 '-DskipUaaAutoStart=true' \
+                '-Dorg.gradle.jvmargs=-Dfile.encoding=utf8 -Xms64m -Xmx${gradle_test_heap} -XX:MaxMetaspaceSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:ParallelGCThreads=2 -XX:CICompilerCount=2 -Djdk.lang.processReaperUseDefaultStackSize=true' \
+                '-Dorg.gradle.daemon.idletimeout=300000' \
+                '-Dorg.gradle.parallel=false' \
+                '-Dorg.gradle.workers.max=2' \
                 ${UAA_GRADLE_INT_TEST_COMMAND:-integrationTest} \
-                --stacktrace \
+                --no-watch-fs \
                 --no-daemon \
+                --no-configuration-cache \
+                --max-workers=2 \
+                --stacktrace \
                 --console=plain"
 
-    set -x
     if [[ "${RUN_TESTS:-true}" = 'true' ]]; then
+      set -x
       eval "$assemble_code"
 
-      # Always start the boot server before running integration tests
+      # Start and ensure the boot server is running before integration tests
       eval "$launch_boot"
       echo $! > boot.pid
+      { set +x; } 2>/dev/null
+
       if is_boot_running ; then
         echo "Boot started. Can continue to run tests."
       else
-        echo "Boot did not start - failing"
+        echo "Boot did not start, failing"
         cat boot.log
         exit 1
       fi
 
+      if [[ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ]]; then
+        export DBUS_SESSION_BUS_ADDRESS=/dev/null
+      fi
+      set -x
+      eval "$compile_test_code"
       eval "$integration_test_code"
-
+      { set +x; } 2>/dev/null
+      
       # Clean up: kill the boot server
       if [[ -f boot.pid ]]; then
-        kill -9 "$(cat boot.pid)" || true
+        local pid; pid=$(cat boot.pid)
+        echo "Sending SIGKILL (kill -9) to UAA process (pid=${pid})"
+        kill -9 "${pid}" || true
         rm boot.pid
       fi
     else
+      set -x
       echo "$integration_test_code"
       bash
     fi
diff --git a/scripts/lib_util_helper.sh b/scripts/lib_util_helper.sh
@@ -2,19 +2,26 @@
 set -eu
 
 ########################################
-# Check if Boot has started by looking for a specific line in the log file
+# Check if Boot has started by checking if the port is responding
 ##########################################
 function is_boot_running() {
-  local log_file="boot.log"
-  local target_line="Started UaaBootApplication"
+  local port=${PORT:-8080}
   local timeout=600 # Timeout in seconds
 
   local start_time
   start_time=$(date +%s)
 
+  echo
+  echo "Waiting for the UAA server to start, only partial log messages will be shown as it progresses:"
   while true; do
-    if grep "$target_line" "$log_file"; then
-      echo "Boot Start was found in the log file."
+    # Use curl to check if the port is responding
+    # Any HTTP response (even 4xx/5xx) indicates the server is running
+    if curl -ks --max-time 5 -o /dev/null --connect-timeout 2 -u "admin:adminsecret" \
+            --data "client_id=admin&grant_type=client_credentials" \
+            -X POST "http://localhost:${port}/uaa/oauth/token" 2>/dev/null; then
+      echo
+      echo "Boot is running on port ${port}."
+      grep "Started UaaBootApplication" boot.log
       return 0
     fi
 
@@ -23,15 +30,40 @@ function is_boot_running() {
     elapsed_time=$((current_time - start_time))
 
     if [[ "$elapsed_time" -ge "$timeout" ]]; then
-      echo "Timeout reached. Boot did not start"
+      echo
+      echo "Timeout reached. Boot did not start on port ${port}"
+      curl -ksS --max-time 5 --connect-timeout 2 -u "admin:adminsecret" \
+              --data "client_id=admin&grant_type=client_credentials" \
+              -X POST "http://localhost:${port}/uaa/info" || true
+
+      thread_dump_on_boot_pid
       return 1
     fi
 
-    tail -n 1 "$log_file"
+    tail -n 1 boot.log
     sleep 1 # Check every second
   done
 }
 
+########################################
+# thread_dump_on_boot_pid
+# Display Memeory info and Request a thread dump on the pid in boot.pid
+##########################################
+function thread_dump_on_boot_pid() {
+  local pid
+  pid=$(cat boot.pid)
+  if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
+    echo "Collecting JVM diagnostics..."
+    jstat -gccause "${pid}" 2>/dev/null || true
+    jstat -gccapacity "${pid}" 2>/dev/null || true
+    jstat -gcmetacapacity "${pid}" 2>/dev/null || true
+    echo "Sending SIGQUIT (kill -3) to Thread Dump on UAA process (pid=${pid})"
+    kill -3 "$pid" || true
+    # Wait a moment for the thread dump to be written
+    sleep 2
+  fi
+}
+
 ########################################
 # setup_hosts_file
 # Appends test-zone and other necessary host entries to /etc/hosts
diff --git a/scripts/unit_tests.sh b/scripts/unit_tests.sh
@@ -22,6 +22,14 @@ function main() {
     start_ldap
 
     set -x
+    
+    ./gradlew "-Dspring.profiles.active=${test_profile}" \
+                "-Djava.security.egd=file:/dev/./urandom" \
+                clean assemble compileTestJava \
+                --stacktrace  \
+                --no-daemon \
+                --console=plain
+
     ./gradlew "-Dspring.profiles.active=${test_profile}" \
             "-Djava.security.egd=file:/dev/./urandom" \
             ${UAA_GRADLE_UNIT_TEST_COMMAND:-test} \
diff --git a/uaa/src/test/java/org/cloudfoundry/identity/uaa/integration/feature/DefaultIntegrationTestConfig.java b/uaa/src/test/java/org/cloudfoundry/identity/uaa/integration/feature/DefaultIntegrationTestConfig.java
@@ -72,19 +72,45 @@ public UaaWebDriver webDriver() {
     private static ChromeOptions getChromeOptions() {
         ChromeOptions options = new ChromeOptions();
         options.addArguments(
-                "--verbose",
                 // Comment the following line to run selenium test browser in Headed Mode
-                "--headless",
+                "--headless=new", // Use new headless mode (more stable)
                 "--guest", //attempt to disable password checkups that disrupt the flow
                 "--disable-web-security",
                 "--ignore-certificate-errors",
                 "--allow-running-insecure-content",
                 "--allow-insecure-localhost",
-                "--no-sandbox",
+                "--no-sandbox", // Required for Docker/CI environments
                 "--disable-gpu",
-                "--remote-allow-origins=*"
+                "--remote-allow-origins=*",
+                "--disable-dev-shm-usage", // Overcome limited resource problems in Docker
+                // Additional stability flags
+                "--disable-extensions",
+                "--disable-software-rasterizer",
+                "--disable-background-timer-throttling",
+                "--disable-backgrounding-occluded-windows",
+                "--disable-renderer-backgrounding",
+                "--disable-blink-features=AutomationControlled",
+                "--disable-features=TranslateUI",
+                // Hang detection and renderer stability flags
+                "--disable-hang-monitor", // Prevents Chrome from killing "hung" renderer processes (useful for slow backend responses)
+                "--disable-background-networking", // Reduces background network activity that could interfere with test requests
+                "--disable-features=RendererScheduling", // Disables aggressive renderer scheduling that might cause timeouts
+                "--run-all-compositor-stages-before-draw", // Ensures all rendering stages complete before drawing (prevents partial renders)
+                "--disable-prompt-on-repost",
+                "--disable-sync",
+                "--disable-component-extensions-with-background-pages",
+                "--force-color-profile=srgb",
+                "--no-first-run",
+                "--no-default-browser-check",
+                "--disable-default-apps",
+                "--disable-popup-blocking",
+                "--test-type",
+                "--disable-infobars"
         );
         options.setAcceptInsecureCerts(true);
+        
+        // Set page load strategy to 'normal' to ensure proper page load detection
+        options.setPageLoadStrategy(org.openqa.selenium.PageLoadStrategy.NORMAL);
 
         return options;
     }
