|
40 | 40 | import static jdk.graal.compiler.hotspot.HotSpotBackend.SHAREDRUNTIME_NOTIFY_JVMTI_VTHREAD_UNMOUNT;
|
41 | 41 | import static jdk.graal.compiler.hotspot.HotSpotBackend.UPDATE_BYTES_CRC32;
|
42 | 42 | import static jdk.graal.compiler.hotspot.HotSpotBackend.UPDATE_BYTES_CRC32C;
|
| 43 | +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_CONTINUATION_ENTRY_PIN_COUNT; |
| 44 | +import static jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil.HOTSPOT_JAVA_THREAD_CONT_ENTRY; |
43 | 45 | import static jdk.graal.compiler.java.BytecodeParserOptions.InlineDuringParsing;
|
44 | 46 | import static jdk.graal.compiler.nodes.ConstantNode.forBoolean;
|
| 47 | +import static jdk.graal.compiler.nodes.ProfileData.BranchProbabilityData.injected; |
45 | 48 | import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
|
| 49 | +import static jdk.vm.ci.meta.DeoptimizationReason.TypeCheckedInliningViolated; |
46 | 50 | import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE;
|
47 | 51 |
|
48 | 52 | import java.lang.annotation.Annotation;
|
|
98 | 102 | import jdk.graal.compiler.nodes.MergeNode;
|
99 | 103 | import jdk.graal.compiler.nodes.NodeView;
|
100 | 104 | import jdk.graal.compiler.nodes.PiNode;
|
101 |
| -import jdk.graal.compiler.nodes.ProfileData; |
102 |
| -import jdk.graal.compiler.nodes.ProfileData.BranchProbabilityData; |
103 | 105 | import jdk.graal.compiler.nodes.StructuredGraph;
|
104 | 106 | import jdk.graal.compiler.nodes.ValueNode;
|
105 | 107 | import jdk.graal.compiler.nodes.ValuePhiNode;
|
@@ -245,6 +247,7 @@ public void run() {
|
245 | 247 | registerSystemPlugins(invocationPlugins);
|
246 | 248 | registerThreadPlugins(invocationPlugins, config, replacements);
|
247 | 249 | registerVirtualThreadPlugins(invocationPlugins, config, replacements);
|
| 250 | + registerContinuationPlugins(invocationPlugins, config, replacements); |
248 | 251 | registerCallSitePlugins(invocationPlugins);
|
249 | 252 | registerReflectionPlugins(invocationPlugins, replacements, config);
|
250 | 253 | registerAESPlugins(invocationPlugins, config, replacements, target.arch);
|
@@ -732,8 +735,7 @@ private static void inlineNativeNotifyJvmtiFunctions(GraalHotSpotVMConfig config
|
732 | 735 | BeginNode trueSuccessor = graph.add(new BeginNode());
|
733 | 736 | BeginNode falseSuccessor = graph.add(new BeginNode());
|
734 | 737 |
|
735 |
| - ProfileData.BranchProbabilityData probability = ProfileData.BranchProbabilityData.injected(NOT_FREQUENT_PROBABILITY); |
736 |
| - b.add(new IfNode(testNotifyJvmtiEnabled, trueSuccessor, falseSuccessor, probability)); |
| 738 | + b.add(new IfNode(testNotifyJvmtiEnabled, trueSuccessor, falseSuccessor, injected(NOT_FREQUENT_PROBABILITY))); |
737 | 739 |
|
738 | 740 | // if notifyJvmti enabled then make a call to the given runtime function
|
739 | 741 | ForeignCallNode runtimeCall = graph.add(new ForeignCallNode(descriptor, virtualThread, hide, javaThread));
|
@@ -761,6 +763,62 @@ private static void inlineNativeNotifyJvmtiFunctions(GraalHotSpotVMConfig config
|
761 | 763 | }
|
762 | 764 | }
|
763 | 765 |
|
| 766 | + // @formatter:off |
| 767 | + @SyncPort(from = "https://github.com/openjdk/jdk/blob/20d8f58c92009a46dfb91b951e7d87b4cb8e8b41/src/hotspot/share/opto/library_call.cpp#L3741-L3824", |
| 768 | + sha1 = "d65356dbc0235df26aa56b233bcd100462a5dab4") |
| 769 | + // @formatter:on |
| 770 | + private static class ContinuationPinningPlugin extends InvocationPlugin { |
| 771 | + |
| 772 | + private final GraalHotSpotVMConfig config; |
| 773 | + private final boolean pin; |
| 774 | + |
| 775 | + ContinuationPinningPlugin(GraalHotSpotVMConfig config, boolean pin) { |
| 776 | + super(pin ? "pin" : "unpin"); |
| 777 | + this.config = config; |
| 778 | + this.pin = pin; |
| 779 | + } |
| 780 | + |
| 781 | + @Override |
| 782 | + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { |
| 783 | + try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) { |
| 784 | + // @formatter:off |
| 785 | + // if (currentThread::_cont_entry != null) |
| 786 | + // uint pin_count = currentThread::_cont_entry::_pin_count; |
| 787 | + // if (pin_count == (pin ? 0xFFFFFFFFUL : 0)) deoptimize; |
| 788 | + // else currentThread::_cont_entry::_pin_count = pin_count + (pin ? 1 : -1); |
| 789 | + // @formatter:on |
| 790 | + StructuredGraph graph = b.getGraph(); |
| 791 | + CurrentJavaThreadNode javaThread = graph.addOrUniqueWithInputs(new CurrentJavaThreadNode(helper.getWordKind())); |
| 792 | + |
| 793 | + GraalError.guarantee(config.contEntry != -1, "JavaThread::_cont_entry is not exported"); |
| 794 | + OffsetAddressNode lastContinuationAddr = graph.addOrUniqueWithInputs(new OffsetAddressNode(javaThread, helper.asWord(config.contEntry))); |
| 795 | + ValueNode lastContinuation = b.add(new JavaReadNode(JavaKind.Object, lastContinuationAddr, HOTSPOT_JAVA_THREAD_CONT_ENTRY, BarrierType.NONE, MemoryOrderMode.PLAIN, false)); |
| 796 | + ValueNode nonNullLastContinuation = helper.emitNullReturnGuard(lastContinuation, null, NOT_FREQUENT_PROBABILITY); |
| 797 | + |
| 798 | + GraalError.guarantee(config.pinCount != -1, "ContinuationEntry::_pin_count is not exported"); |
| 799 | + OffsetAddressNode pinCountAddr = graph.addOrUniqueWithInputs(new OffsetAddressNode(nonNullLastContinuation, helper.asWord(config.pinCount))); |
| 800 | + ValueNode pinCount = b.add(new JavaReadNode(JavaKind.Int, pinCountAddr, HOTSPOT_CONTINUATION_ENTRY_PIN_COUNT, BarrierType.NONE, MemoryOrderMode.PLAIN, false)); |
| 801 | + |
| 802 | + LogicNode overFlow = IntegerEqualsNode.create(pinCount, pin ? ConstantNode.forInt(-1) : ConstantNode.forInt(0), NodeView.DEFAULT); |
| 803 | + // TypeCheckedInliningViolated (Reason_type_checked_inlining) is aliasing |
| 804 | + // Reason_intrinsic |
| 805 | + b.append(new FixedGuardNode(overFlow, TypeCheckedInliningViolated, DeoptimizationAction.None, true)); |
| 806 | + ValueNode newPinCount = b.add(AddNode.create(pinCount, pin ? ConstantNode.forInt(1) : ConstantNode.forInt(-1), NodeView.DEFAULT)); |
| 807 | + b.add(new JavaWriteNode(JavaKind.Int, pinCountAddr, HOTSPOT_CONTINUATION_ENTRY_PIN_COUNT, newPinCount, BarrierType.NONE, false)); |
| 808 | + helper.emitFinalReturn(JavaKind.Void, null); |
| 809 | + } |
| 810 | + return true; |
| 811 | + } |
| 812 | + } |
| 813 | + |
| 814 | + private static void registerContinuationPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, Replacements replacements) { |
| 815 | + Registration r = new Registration(plugins, "jdk.internal.vm.Continuation", replacements); |
| 816 | + if (JavaVersionUtil.JAVA_SPEC >= 24) { |
| 817 | + r.register(new ContinuationPinningPlugin(config, true)); |
| 818 | + r.register(new ContinuationPinningPlugin(config, false)); |
| 819 | + } |
| 820 | + } |
| 821 | + |
764 | 822 | private static void registerVirtualThreadPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, Replacements replacements) {
|
765 | 823 | Registration r = new Registration(plugins, "java.lang.VirtualThread", replacements);
|
766 | 824 | if (config.supportJVMTIVThreadNotification()) {
|
@@ -1374,7 +1432,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec
|
1374 | 1432 | ValueNode layoutHelperLong = b.add(SignExtendNode.create(layoutHelper, JavaKind.Long.getBitCount(), NodeView.DEFAULT));
|
1375 | 1433 | ValueNode instanceSizeLong = b.add(AndNode.create(layoutHelperLong, ConstantNode.forLong(-((long) JavaKind.Long.getByteCount())), NodeView.DEFAULT));
|
1376 | 1434 |
|
1377 |
| - b.add(new IfNode(isArray, arrayLengthNode, instanceBranch, BranchProbabilityData.injected(0.5D))); |
| 1435 | + b.add(new IfNode(isArray, arrayLengthNode, instanceBranch, injected(0.5D))); |
1378 | 1436 | MergeNode merge = b.append(new MergeNode());
|
1379 | 1437 | merge.addForwardEnd(arrayBranch);
|
1380 | 1438 | merge.addForwardEnd(instanceBranch);
|
|
0 commit comments