8357826: Avoid running some jtreg tests when asan is configured

Reviewed-by: rschmelter
Backport-of: d7aa34982053bad37b3b726539f1245d054258f4

Author: MBaesken

src/hotspot/share/prims/whitebox.cpp

@@ -1049,6 +1049,22 @@ bool WhiteBox::validate_cgroup(const char* proc_cgroups,
 }
 #endif
 
+bool WhiteBox::is_asan_enabled() {
+#ifdef ADDRESS_SANITIZER
+  return true;
+#else
+  return false;
+#endif
+}
+
+bool WhiteBox::is_ubsan_enabled() {
+#ifdef UNDEFINED_BEHAVIOR_SANITIZER
+  return true;
+#else
+  return false;
+#endif
+}
+
 bool WhiteBox::compile_method(Method* method, int comp_level, int bci, JavaThread* THREAD) {
   // Screen for unavailable/bad comp level or null method
   AbstractCompiler* comp = CompileBroker::compiler(comp_level);
@@ -1832,6 +1848,14 @@ WB_ENTRY(jboolean, WB_IsMonitorInflated(JNIEnv* env, jobject wb, jobject obj))
   return (jboolean) obj_oop->mark().has_monitor();
 WB_END
 
+WB_ENTRY(jboolean, WB_IsAsanEnabled(JNIEnv* env))
+  return (jboolean) WhiteBox::is_asan_enabled();
+WB_END
+
+WB_ENTRY(jboolean, WB_IsUbsanEnabled(JNIEnv* env))
+  return (jboolean) WhiteBox::is_ubsan_enabled();
+WB_END
+
 WB_ENTRY(jboolean, WB_DeflateIdleMonitors(JNIEnv* env, jobject wb))
   log_info(monitorinflation)("WhiteBox initiated DeflateIdleMonitors");
   return ObjectSynchronizer::request_deflate_idle_monitors_from_wb();
@@ -2758,6 +2782,8 @@ static JNINativeMethod methods[] = {
                                                       (void*)&WB_AddModuleExportsToAll },
   {CC"deflateIdleMonitors", CC"()Z",                  (void*)&WB_DeflateIdleMonitors },
   {CC"isMonitorInflated0", CC"(Ljava/lang/Object;)Z", (void*)&WB_IsMonitorInflated  },
+  {CC"isAsanEnabled", CC"()Z",                        (void*)&WB_IsAsanEnabled },
+  {CC"isUbsanEnabled", CC"()Z",                       (void*)&WB_IsUbsanEnabled },
   {CC"forceSafepoint",     CC"()V",                   (void*)&WB_ForceSafepoint     },
   {CC"forceClassLoaderStatsSafepoint", CC"()V",       (void*)&WB_ForceClassLoaderStatsSafepoint },
   {CC"getConstantPool0",   CC"(Ljava/lang/Class;)J",  (void*)&WB_GetConstantPool    },

src/hotspot/share/prims/whitebox.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -71,6 +71,9 @@ class WhiteBox : public AllStatic {
 #ifdef LINUX
   static bool validate_cgroup(const char* proc_cgroups, const char* proc_self_cgroup, const char* proc_self_mountinfo, u1* cg_flags);
 #endif
+  // provide info about enabling of Address Sanitizer / Undefined Behavior Sanitizer
+  static bool is_asan_enabled();
+  static bool is_ubsan_enabled();
 };
 
 #endif // SHARE_PRIMS_WHITEBOX_HPP

test/hotspot/jtreg/TEST.ROOT

@@ -83,6 +83,8 @@ requires.properties= \
     vm.compiler1.enabled \
     vm.compiler2.enabled \
     vm.musl \
+    vm.asan \
+    vm.ubsan \
     vm.flagless \
     docker.support \
     systemd.support \

test/hotspot/jtreg/gc/arguments/TestUseCompressedOopsFlagsWithUlimit.java

@@ -31,6 +31,8 @@
  * @library /test/lib
  * @library /
  * @requires vm.bits == "64"
+ * @comment ulimit clashes with the memory requirements of ASAN
+ * @requires !vm.asan
  * @requires os.family == "linux"
  * @requires vm.gc != "Z"
  * @requires vm.opt.UseCompressedOops == null

test/hotspot/jtreg/runtime/Thread/TestBreakSignalThreadDump.java

@@ -36,6 +36,8 @@
  * @bug 8292695
  * @summary Check that Ctrl-\ causes HotSpot VM to print a full thread dump when signal chaining is used.
  * @requires os.family != "windows" & os.family != "aix"
+ * @comment loading of the jsig lib does currently not work well with ASAN lib
+ * @requires !vm.asan
  * @library /vmTestbase
  *          /test/lib
  * @run driver TestBreakSignalThreadDump load_libjsig

test/hotspot/jtreg/runtime/XCheckJniJsig/XCheckJSig.java

@@ -29,6 +29,8 @@
  * @modules java.base/jdk.internal.misc
  *          java.management
  * @requires os.family == "linux" | os.family == "mac"
+ * @comment loading of the jsig lib does currently not work well with ASAN lib
+ * @requires !vm.asan
  * @run driver XCheckJSig
  */
 

test/hotspot/jtreg/serviceability/dcmd/vm/SystemDumpMapTest.java

@@ -36,6 +36,8 @@
  * @summary Test of diagnostic command System.map
  * @library /test/lib
  * @requires (os.family=="linux")
+ * @comment ASAN changes the memory map dump slightly, but the test has rather strict requirements
+ * @requires !vm.asan
  * @modules java.base/jdk.internal.misc
  *          java.compiler
  *          java.management

test/hotspot/jtreg/serviceability/dcmd/vm/SystemMapTest.java

@@ -32,6 +32,8 @@
  * @summary Test of diagnostic command System.map
  * @library /test/lib
  * @requires (os.family=="linux")
+ * @comment ASAN changes the memory map dump slightly, but the test has rather strict requirements
+ * @requires !vm.asan
  * @modules java.base/jdk.internal.misc
  *          java.compiler
  *          java.management

test/jdk/TEST.ROOT

@@ -93,6 +93,8 @@ requires.properties= \
     vm.cds.write.archived.java.heap \
     vm.continuations \
     vm.musl \
+    vm.asan \
+    vm.ubsan \
     vm.debug \
     vm.hasSA \
     vm.hasJFR \

test/jdk/build/AbsPathsInImage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,8 @@
  * @bug 8226346
  * @summary Check all output files for absolute path fragments
  * @requires !vm.debug
+ * @comment ASAN keeps the 'unwanted' paths in the binaries because of its build options
+ * @requires !vm.asan
  * @run main/othervm -Xmx900m AbsPathsInImage
  */
 public class AbsPathsInImage {