Do not allow interleaved recursive locks to be virtualized.

Author: mur47x111

compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/MonitorPEATest.java

@@ -234,4 +234,44 @@ public static void snippet6() {
     public void testSnippet6() {
         test("snippet6");
     }
+
+    public static void snippet7() {
+        B b = new B();
+        C c = new C(b);
+        synchronized (c) {
+            synchronized (c) {
+                synchronized (b) {
+                    synchronized (b) {
+                        synchronized (B.class) {
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testSnippet7() {
+        test("snippet7");
+    }
+
+    public static void snippet8() {
+        B b = new B();
+        C c = new C(b);
+        synchronized (c) {
+            synchronized (b) {
+                synchronized (c) {
+                    synchronized (b) {
+                        synchronized (B.class) {
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testSnippet8() {
+        test("snippet8");
+    }
 }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/java/MonitorEnterNode.java

@@ -104,7 +104,7 @@ public void virtualize(VirtualizerTool tool) {
         ValueNode alias = tool.getAlias(object());
         if (alias instanceof VirtualObjectNode) {
             VirtualObjectNode virtual = (VirtualObjectNode) alias;
-            if (virtual.hasIdentity()) {
+            if (virtual.hasIdentity() && tool.canVirtualizeLock(virtual, getMonitorId())) {
                 tool.addLock(virtual, getMonitorId());
                 if (!tool.getPlatformConfigurationProvider().areLocksSideEffectFree()) {
                     // Ensure that the locks appear to have been acquired in the nearest FrameState.

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/spi/VirtualizerTool.java

@@ -30,12 +30,11 @@
 import jdk.graal.compiler.debug.GraalError;
 import jdk.graal.compiler.graph.Node;
 import jdk.graal.compiler.graph.NodeSourcePosition;
-import jdk.graal.compiler.nodes.java.MonitorIdNode;
-import jdk.graal.compiler.nodes.virtual.VirtualObjectNode;
 import jdk.graal.compiler.nodes.ValueNode;
 import jdk.graal.compiler.nodes.WithExceptionNode;
+import jdk.graal.compiler.nodes.java.MonitorIdNode;
+import jdk.graal.compiler.nodes.virtual.VirtualObjectNode;
 import jdk.graal.compiler.options.OptionValues;
-
 import jdk.vm.ci.meta.JavaKind;
 
 /**
@@ -97,6 +96,8 @@ default void setVirtualEntry(VirtualObjectNode virtualObject, int index, ValueNo
 
     ValueNode getEntry(VirtualObjectNode virtualObject, int index);
 
+    boolean canVirtualizeLock(VirtualObjectNode virtualObject, MonitorIdNode monitorId);
+
     void addLock(VirtualObjectNode virtualObject, MonitorIdNode monitorId);
 
     MonitorIdNode removeLock(VirtualObjectNode virtualObject);

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/virtual/phases/ea/PartialEscapeBlockState.java

@@ -172,6 +172,14 @@ public void escape(int object, ValueNode materialized) {
         getObjectStateForModification(object).escape(materialized);
     }
 
+    public boolean isNonImmediateRecursiveLock(int object, MonitorIdNode monitorId) {
+        ObjectState state = getObjectStateForModification(object);
+        if (state.hasLocks()) {
+            return state.getLocks().monitorId.getLockDepth() < monitorId.getLockDepth() - 1;
+        }
+        return false;
+    }
+
     public void addLock(int object, MonitorIdNode monitorId) {
         getObjectStateForModification(object).addLock(monitorId);
     }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/virtual/phases/ea/VirtualizerToolImpl.java

@@ -364,6 +364,15 @@ public boolean ensureMaterialized(VirtualObjectNode virtualObject) {
         return closure.ensureMaterialized(state, virtualObject.getObjectId(), position, effects, PartialEscapeClosure.COUNTER_MATERIALIZATIONS_UNHANDLED);
     }
 
+    @Override
+    public boolean canVirtualizeLock(VirtualObjectNode virtualObject, MonitorIdNode monitorId) {
+        if (getPlatformConfigurationProvider().requiresStrictLockOrder()) {
+            int id = virtualObject.getObjectId();
+            return !state.isNonImmediateRecursiveLock(id, monitorId);
+        }
+        return true;
+    }
+
     @Override
     public void addLock(VirtualObjectNode virtualObject, MonitorIdNode monitorId) {
         int id = virtualObject.getObjectId();