[GR-59329] Add option to outline SerialGC post-write barrier snippets

PullRequest: graal/19148

Author: c-refice

substratevm/mx.substratevm/mx_substratevm.py

@@ -1568,6 +1568,10 @@ def prevent_build_path_in_libgraal():
 
     # No need for container support in libgraal as HotSpot already takes care of it
     '-H:-UseContainerSupport',
+
+    # Reduce image size by outlining all write barriers.
+    # Benchmarking showed no performance degradation.
+    '-H:+OutlineWriteBarriers',
 ] + ([
     # Force page size to support libgraal on AArch64 machines with a page size up to 64K.
     '-H:PageSize=64K'

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialGCOptions.java

@@ -113,6 +113,10 @@ public Integer getValue(OptionValues values) {
     @Option(help = "Ignore the maximum heap size while in VM-internal code.", type = OptionType.Expert)//
     public static final HostedOptionKey<Boolean> IgnoreMaxHeapSizeWhileInVMOperation = new HostedOptionKey<>(false, SerialGCOptions::serialGCOnly);
 
+    @Option(help = "Determines whether to always (if true) or never (if false) outline write barrier code to a separate function, " +
+                    "trading reduced image size for (potentially) worse performance. Serial GC only.", type = OptionType.Expert) //
+    public static final HostedOptionKey<Boolean> OutlineWriteBarriers = new HostedOptionKey<>(null, SerialGCOptions::serialGCOnly);
+
     /** Query these options only through an appropriate method. */
     public static class ConcealedOptions {
         @Option(help = "Collect old generation by compacting in-place instead of copying.", type = OptionType.Expert) //

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java

@@ -24,26 +24,35 @@
  */
 package com.oracle.svm.core.genscavenge.graal;
 
+import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT;
+
 import java.util.Map;
 
 import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.UnsignedWord;
 
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.genscavenge.ObjectHeaderImpl;
 import com.oracle.svm.core.genscavenge.SerialGCOptions;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
+import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
 import com.oracle.svm.core.graal.snippets.NodeLoweringProvider;
 import com.oracle.svm.core.graal.snippets.SubstrateTemplates;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.StoredContinuation;
+import com.oracle.svm.core.snippets.SnippetRuntime;
+import com.oracle.svm.core.snippets.SubstrateForeignCallTarget;
 
 import jdk.graal.compiler.api.replacements.Snippet;
 import jdk.graal.compiler.api.replacements.Snippet.ConstantParameter;
+import jdk.graal.compiler.core.common.GraalOptions;
+import jdk.graal.compiler.core.common.spi.ForeignCallDescriptor;
 import jdk.graal.compiler.graph.Node;
 import jdk.graal.compiler.nodes.BreakpointNode;
 import jdk.graal.compiler.nodes.NamedLocationIdentity;
 import jdk.graal.compiler.nodes.extended.BranchProbabilityNode;
 import jdk.graal.compiler.nodes.extended.FixedValueAnchorNode;
+import jdk.graal.compiler.nodes.extended.ForeignCallNode;
 import jdk.graal.compiler.nodes.gc.SerialArrayRangeWriteBarrierNode;
 import jdk.graal.compiler.nodes.gc.SerialWriteBarrierNode;
 import jdk.graal.compiler.nodes.gc.WriteBarrierNode;
@@ -63,6 +72,10 @@ public class BarrierSnippets extends SubstrateTemplates implements Snippets {
     /** A LocationIdentity to distinguish card locations from other locations. */
     public static final LocationIdentity CARD_REMEMBERED_SET_LOCATION = NamedLocationIdentity.mutable("CardRememberedSet");
 
+    private static final SnippetRuntime.SubstrateForeignCallDescriptor POST_WRITE_BARRIER = SnippetRuntime.findForeignCall(BarrierSnippets.class, "postWriteBarrierStub",
+                    NO_SIDE_EFFECT,
+                    CARD_REMEMBERED_SET_LOCATION);
+
     private final SnippetInfo postWriteBarrierSnippet;
 
     BarrierSnippets(OptionValues options, Providers providers) {
@@ -78,8 +91,26 @@ public void registerLowerings(MetaAccessProvider metaAccess, Map<Class<? extends
         lowerings.put(SerialArrayRangeWriteBarrierNode.class, lowering);
     }
 
+    public static void registerForeignCalls(SubstrateForeignCallsProvider provider) {
+        provider.register(POST_WRITE_BARRIER);
+    }
+
+    @SubstrateForeignCallTarget(stubCallingConvention = false, fullyUninterruptible = true)
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static void postWriteBarrierStub(Object object) {
+        UnsignedWord objectHeader = ObjectHeader.readHeaderFromObject(object);
+        if (ObjectHeaderImpl.isUnalignedHeader(objectHeader)) {
+            RememberedSet.get().dirtyCardForUnalignedObject(object, false);
+        } else {
+            RememberedSet.get().dirtyCardForAlignedObject(object, false);
+        }
+    }
+
+    @Node.NodeIntrinsic(ForeignCallNode.class)
+    private static native void callPostWriteBarrierStub(@Node.ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
+
     @Snippet
-    public static void postWriteBarrierSnippet(Object object, @ConstantParameter boolean alwaysAlignedChunk, @ConstantParameter boolean verifyOnly) {
+    public static void postWriteBarrierSnippet(Object object, @ConstantParameter boolean shouldOutline, @ConstantParameter boolean alwaysAlignedChunk, @ConstantParameter boolean verifyOnly) {
         Object fixedObject = FixedValueAnchorNode.getObject(object);
         UnsignedWord objectHeader = ObjectHeader.readHeaderFromObject(fixedObject);
 
@@ -94,10 +125,10 @@ public static void postWriteBarrierSnippet(Object object, @ConstantParameter boo
             if (BranchProbabilityNode.probability(BranchProbabilityNode.SLOW_PATH_PROBABILITY, ObjectHeaderImpl.isUnalignedHeader(objectHeader))) {
                 BreakpointNode.breakpoint();
             }
-            if (BranchProbabilityNode.probability(BranchProbabilityNode.SLOW_PATH_PROBABILITY, object == null)) {
+            if (BranchProbabilityNode.probability(BranchProbabilityNode.SLOW_PATH_PROBABILITY, fixedObject == null)) {
                 BreakpointNode.breakpoint();
             }
-            if (BranchProbabilityNode.probability(BranchProbabilityNode.SLOW_PATH_PROBABILITY, object.getClass().isArray())) {
+            if (BranchProbabilityNode.probability(BranchProbabilityNode.SLOW_PATH_PROBABILITY, fixedObject.getClass().isArray())) {
                 BreakpointNode.breakpoint();
             }
         }
@@ -107,6 +138,11 @@ public static void postWriteBarrierSnippet(Object object, @ConstantParameter boo
             return;
         }
 
+        if (shouldOutline && !verifyOnly) {
+            callPostWriteBarrierStub(POST_WRITE_BARRIER, fixedObject);
+            return;
+        }
+
         if (!alwaysAlignedChunk) {
             boolean unaligned = ObjectHeaderImpl.isUnalignedHeader(objectHeader);
             if (BranchProbabilityNode.probability(BranchProbabilityNode.NOT_LIKELY_PROBABILITY, unaligned)) {
@@ -143,12 +179,20 @@ public void lower(WriteBarrierNode barrier, LoweringTool tool) {
             boolean alwaysAlignedChunk = baseType != null && !baseType.isArray() && !baseType.isJavaLangObject() && !baseType.isInterface();
 
             args.add("object", address.getBase());
+            args.addConst("shouldOutline", shouldOutline(barrier));
             args.addConst("alwaysAlignedChunk", alwaysAlignedChunk);
             args.addConst("verifyOnly", getVerifyOnly(barrier));
 
             template(tool, barrier, args).instantiate(tool.getMetaAccess(), barrier, SnippetTemplate.DEFAULT_REPLACER, args);
         }
 
+        private static boolean shouldOutline(WriteBarrierNode barrier) {
+            if (SerialGCOptions.OutlineWriteBarriers.getValue() != null) {
+                return SerialGCOptions.OutlineWriteBarriers.getValue();
+            }
+            return GraalOptions.ReduceCodeSize.getValue(barrier.getOptions());
+        }
+
         private static boolean getVerifyOnly(WriteBarrierNode barrier) {
             if (barrier instanceof SerialWriteBarrierNode) {
                 return ((SerialWriteBarrierNode) barrier).getVerifyOnly();

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeGCFeature.java

@@ -142,6 +142,7 @@ public void beforeCompilation(BeforeCompilationAccess access) {
 
     @Override
     public void registerForeignCalls(SubstrateForeignCallsProvider foreignCalls) {
+        BarrierSnippets.registerForeignCalls(foreignCalls);
         GenScavengeAllocationSupport.registerForeignCalls(foreignCalls);
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java

@@ -128,12 +128,14 @@ public boolean hasRememberedSet(UnsignedWord header) {
 
     @Override
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public void dirtyCardForAlignedObject(Object object, boolean verifyOnly) {
         AlignedChunkRememberedSet.dirtyCardForObject(object, verifyOnly);
     }
 
     @Override
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) {
         UnalignedChunkRememberedSet.dirtyCardForObject(object, verifyOnly);
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java

@@ -124,11 +124,13 @@ public boolean hasRememberedSet(UnsignedWord header) {
     }
 
     @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public void dirtyCardForAlignedObject(Object object, boolean verifyOnly) {
         throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport
     }
 
     @Override
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) {
         throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java

@@ -115,6 +115,7 @@ static RememberedSet get() {
      * (from old generation to young generation, or from image heap to runtime heap).
      */
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void dirtyCardForAlignedObject(Object object, boolean verifyOnly);
 
     /**
@@ -123,6 +124,7 @@ static RememberedSet get() {
      * (from old generation to young generation, or from image heap to runtime heap).
      */
     @AlwaysInline("GC performance")
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     void dirtyCardForUnalignedObject(Object object, boolean verifyOnly);
 
     /**