Skip to content

Commit 71e8fe2

Browse files
kittyoraclekittyoracle
committed
AbstractMask::checkIndexFailed prevents intrinsification
1 parent 4dc6019 commit 71e8fe2

File tree

2 files changed

+174
-0
lines changed

2 files changed

+174
-0
lines changed

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/replacements/vectorapi/VectorAPIBoxingUtils.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import jdk.graal.compiler.core.common.type.StampFactory;
3737
import jdk.graal.compiler.debug.GraalError;
3838
import jdk.graal.compiler.nodes.ConstantNode;
39+
import jdk.graal.compiler.nodes.FixedNode;
3940
import jdk.graal.compiler.nodes.FixedWithNextNode;
4041
import jdk.graal.compiler.nodes.NamedLocationIdentity;
4142
import jdk.graal.compiler.nodes.NodeView;
@@ -45,9 +46,16 @@
4546
import jdk.graal.compiler.nodes.calc.NarrowNode;
4647
import jdk.graal.compiler.nodes.calc.NotNode;
4748
import jdk.graal.compiler.nodes.calc.SignExtendNode;
49+
import jdk.graal.compiler.nodes.extended.MembarNode;
50+
import jdk.graal.compiler.nodes.extended.PublishWritesNode;
4851
import jdk.graal.compiler.nodes.java.LoadFieldNode;
52+
import jdk.graal.compiler.nodes.java.NewArrayNode;
53+
import jdk.graal.compiler.nodes.java.NewInstanceNode;
54+
import jdk.graal.compiler.nodes.java.StoreFieldNode;
4955
import jdk.graal.compiler.nodes.memory.ReadNode;
56+
import jdk.graal.compiler.nodes.memory.WriteNode;
5057
import jdk.graal.compiler.nodes.memory.address.AddressNode;
58+
import jdk.graal.compiler.nodes.memory.address.IndexAddressNode;
5159
import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode;
5260
import jdk.graal.compiler.nodes.spi.CoreProviders;
5361
import jdk.graal.compiler.nodes.spi.ValueProxy;
@@ -205,6 +213,56 @@ static VectorAPIType asUnboxableVectorType(ValueNode value, CoreProviders provid
205213
return null;
206214
}
207215

216+
/**
217+
* Box the given value by allocating an appropriate Java instance and storing the payload of the
218+
* value into the {@code payload} field of the box instance. The allocation will be inserted
219+
* before {@code successor}.
220+
*/
221+
static ValueNode boxVector(ResolvedJavaType boxType, FixedNode successor, ValueNode vector, CoreProviders providers) {
222+
VectorAPIType typeMeta = VectorAPIType.ofType(boxType, providers);
223+
GraalError.guarantee(typeMeta != null, "unexpected vector type %s", boxType);
224+
StructuredGraph graph = vector.graph();
225+
226+
// Allocate the payload array
227+
ResolvedJavaType payloadElementType = providers.getMetaAccess().lookupJavaType(typeMeta.payloadKind.toJavaClass());
228+
NewArrayNode payloadArray = graph.add(new NewArrayNode(payloadElementType, ConstantNode.forInt(typeMeta.vectorLength, graph), false));
229+
graph.addBeforeFixed(successor, payloadArray);
230+
231+
// Store the value into the payload array
232+
ValueNode payload;
233+
if (typeMeta.isMask) {
234+
payload = graph.addOrUniqueWithInputs(logicAsBooleans(vector, VectorAPIUtils.vectorArchitecture(providers)));
235+
} else {
236+
payload = vector;
237+
}
238+
AddressNode payloadArrayAddress = graph.addOrUnique(new IndexAddressNode(payloadArray, ConstantNode.forInt(0, graph), payloadElementType.getJavaKind()));
239+
WriteNode storeToPayload = graph.add(new WriteNode(payloadArrayAddress, LocationIdentity.INIT_LOCATION, payload, BarrierType.NONE, MemoryOrderMode.PLAIN));
240+
graph.addBeforeFixed(successor, storeToPayload);
241+
242+
// Publish the allocated array
243+
graph.addBeforeFixed(successor, graph.add(MembarNode.forInitialization()));
244+
PublishWritesNode publishedPayloadArray = graph.add(new PublishWritesNode(payloadArray));
245+
graph.addBeforeFixed(successor, publishedPayloadArray);
246+
247+
// Allocate the box instance, fillContents must be true because the field is an oop
248+
NewInstanceNode box = graph.add(new NewInstanceNode(boxType, true));
249+
graph.addBeforeFixed(successor, box);
250+
251+
// Store the allocated payload array into the corresponding field of the box instance
252+
ResolvedJavaField[] boxFields = boxType.getInstanceFields(true);
253+
GraalError.guarantee(boxFields.length == 1, "unexpected field count in %s", boxType);
254+
ResolvedJavaField payloadField = boxFields[0];
255+
GraalError.guarantee(payloadField.getName().equals("payload"), "unexpected field %s %s in %s", payloadField.getDeclaringClass(), payloadField.getName(), boxType);
256+
StoreFieldNode storeToBox = graph.add(new StoreFieldNode(box, payloadField, publishedPayloadArray));
257+
graph.addBeforeFixed(successor, storeToBox);
258+
259+
// Publish the allocated box instance
260+
graph.addBeforeFixed(successor, graph.add(MembarNode.forInitialization()));
261+
PublishWritesNode publishedBox = graph.add(new PublishWritesNode(box));
262+
graph.addBeforeFixed(successor, publishedBox);
263+
return publishedBox;
264+
}
265+
208266
/**
209267
* Unbox the given value (which must have been checked to be {@linkplain #asUnboxableVectorType
210268
* unboxable}. Unboxing it means first loading the vector object's primitive array payload

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/replacements/vectorapi/VectorAPIExpansionPhase.java‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,16 @@
5050
import jdk.graal.compiler.nodes.FixedWithNextNode;
5151
import jdk.graal.compiler.nodes.FrameState;
5252
import jdk.graal.compiler.nodes.GraphState;
53+
import jdk.graal.compiler.nodes.Invoke;
5354
import jdk.graal.compiler.nodes.NodeView;
55+
import jdk.graal.compiler.nodes.ReturnNode;
5456
import jdk.graal.compiler.nodes.StructuredGraph;
5557
import jdk.graal.compiler.nodes.ValueNode;
5658
import jdk.graal.compiler.nodes.ValuePhiNode;
5759
import jdk.graal.compiler.nodes.ValueProxyNode;
5860
import jdk.graal.compiler.nodes.calc.MinMaxNode;
5961
import jdk.graal.compiler.nodes.extended.FixedValueAnchorNode;
62+
import jdk.graal.compiler.nodes.java.MethodCallTargetNode;
6063
import jdk.graal.compiler.nodes.spi.CoreProviders;
6164
import jdk.graal.compiler.nodes.spi.SimplifierTool;
6265
import jdk.graal.compiler.nodes.type.StampTool;
@@ -72,7 +75,9 @@
7275
import jdk.graal.compiler.vector.nodes.simd.SimdStamp;
7376
import jdk.graal.compiler.vector.replacements.vectorapi.nodes.VectorAPIMacroNode;
7477
import jdk.graal.compiler.vector.replacements.vectorapi.nodes.VectorAPISinkNode;
78+
import jdk.vm.ci.meta.JavaKind;
7579
import jdk.vm.ci.meta.ResolvedJavaType;
80+
import org.graalvm.collections.Equivalence;
7681

7782
/**
7883
* Expands {@link VectorAPIMacroNode}s to SIMD operations if they are supported by the target
@@ -167,6 +172,36 @@ private static class ConnectedComponent {
167172
private ArrayList<ConstantNode> constants;
168173
/** Unboxable vector values as inputs to other nodes in this component. */
169174
private ArrayList<ValueNode> unboxes;
175+
176+
/**
177+
* Record locations at which nodes in this component need to be materialized due to
178+
* unexpected usage. This cuts the usages off the component, allows the connected component
179+
* to be expanded. For example, a macro node {@code v} is used as an argument to a call:
180+
*
181+
* <pre>
182+
* {@code
183+
* IntVector v;
184+
* consume(v);
185+
* }
186+
* </pre>
187+
*
188+
* In general, the escape of {@code v} disallows its expansion. However, if it seems that
189+
* the call {@code consume(v)} happens rarely, we may manually box the vector instance, so
190+
* that the pseudocode snippet changes to:
191+
*
192+
* <pre>
193+
* {@code
194+
* IntVector v;
195+
* IntVector v1 = new IntVector;
196+
* v.intoArray(v1.payload);
197+
* consume(v1);
198+
* }
199+
* </pre>
200+
*
201+
* This disconnects {@code v} from the call to {@code consume}, allows it to be expanded.
202+
*/
203+
private ArrayList<VectorAPIMacroNode> boxes;
204+
170205
/**
171206
* A map from each node in this component to the corresponding SIMD stamp. The keys of this
172207
* map can be used to iterate over all the nodes in this component, i.e., the union of
@@ -187,6 +222,7 @@ private static class ConnectedComponent {
187222
this.proxies = new ArrayList<>();
188223
this.constants = new ArrayList<>();
189224
this.unboxes = new ArrayList<>();
225+
this.boxes = new ArrayList<>();
190226
this.simdStamps = EconomicMap.create();
191227
}
192228

@@ -400,6 +436,11 @@ private static Iterable<ConnectedComponent> buildConnectedComponents(StructuredG
400436
* on the heap.
401437
*/
402438
continue;
439+
} else if (node instanceof VectorAPIMacroNode macro && shouldBox(macro, usage, context)) {
440+
// Manually box the vector node to disconnect the unexpected usage from the
441+
// ConnectedComponent
442+
component.boxes.add(macro);
443+
continue;
403444
} else {
404445
/*
405446
* Some other usage, this would force materialization of the vector object.
@@ -495,6 +536,8 @@ private static void expandComponents(StructuredGraph graph, HighTierContext cont
495536

496537
/* Expand unboxing operations that are inputs to the component. */
497538
unboxComponentInputs(graph, context, component, expanded);
539+
/* Box the nodes that escape to make the component expandable */
540+
boxComponentOutputs(graph, context, component, expanded, vectorArch);
498541
/* Expand, starting from sinks and recursing upwards through inputs. */
499542
for (VectorAPISinkNode sink : component.sinks) {
500543
expandRecursivelyUpwards(graph, context, expanded, component.simdStamps, sink, vectorArch);
@@ -554,6 +597,79 @@ private static void unboxComponentInputs(StructuredGraph graph, CoreProviders pr
554597
component.unboxes.clear();
555598
}
556599

600+
private static void boxComponentOutputs(StructuredGraph graph, CoreProviders providers, ConnectedComponent component, NodeMap<ValueNode> expanded, VectorArchitecture vectorArch) {
601+
GraalError.guarantee(component.canExpand, "should only place box nodes once we know the component can expand");
602+
for (VectorAPIMacroNode macro : component.boxes) {
603+
expandRecursivelyUpwards(graph, providers, expanded, component.simdStamps, macro, vectorArch);
604+
ValueNode expandedDef = expanded.get(macro);
605+
GraalError.guarantee(expandedDef != null, "must be expanded %s", macro);
606+
ResolvedJavaType boxType = macro.stamp(NodeView.DEFAULT).javaType(providers.getMetaAccess());
607+
EconomicSet<Node> uses = EconomicSet.create(Equivalence.DEFAULT);
608+
uses.addAll(macro.usages());
609+
for (Node use : uses) {
610+
// For a use, this operation might replace it with a clone that has the macro input
611+
// fixed. As a result, we need to collect the uses here instead of recording them
612+
// while constructing the connected components.
613+
if (!shouldBox(macro, use, providers)) {
614+
continue;
615+
}
616+
617+
if (use instanceof FixedNode successor) {
618+
// If the usage is a FixedNode, box the macro there and replace the macro input
619+
// with the allocated box instance
620+
ValueNode boxedMacro = VectorAPIBoxingUtils.boxVector(boxType, successor, expandedDef, providers);
621+
successor.replaceAllInputs(macro, boxedMacro);
622+
} else {
623+
// The pattern here looks similar to macro -> MethodCallTarget -> Invoke. As a
624+
// result, we need to clone a MethodCallTarget for each of its Invoke output,
625+
// then replace the macro in the cloned MethodCallTarget with a boxed vector
626+
// instance.
627+
GraalError.guarantee(use instanceof MethodCallTargetNode, "unexpected use %s", use);
628+
EconomicSet<Node> successors = EconomicSet.create();
629+
successors.addAll(use.usages());
630+
for (Node successor : successors) {
631+
FixedNode fixedSuccessor = (FixedNode) successor;
632+
ValueNode useCloned = (ValueNode) use.copyWithInputs();
633+
ValueNode boxedMacro = VectorAPIBoxingUtils.boxVector(boxType, fixedSuccessor, expandedDef, providers);
634+
useCloned.replaceAllInputs(macro, boxedMacro);
635+
useCloned = graph.addOrUniqueWithInputs(useCloned);
636+
fixedSuccessor.replaceAllInputs(use, useCloned);
637+
}
638+
}
639+
}
640+
641+
graph.getDebug().dump(DebugContext.VERY_DETAILED_LEVEL, graph, "after boxing %s for %s", macro, component);
642+
}
643+
644+
graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "after boxing all escapes of %s", component);
645+
}
646+
647+
/**
648+
* If a {@link VectorAPIMacroNode} has a usage {@code use} that cannot be expanded. We try to
649+
* see if it is profitable to box {@code macro} at {@code use}, disconnecting {@code use} from
650+
* the {@link ConnectedComponent}, allowing it to be expanded.
651+
*/
652+
private static boolean shouldBox(VectorAPIMacroNode macro, Node use, CoreProviders providers) {
653+
if (use instanceof MethodCallTargetNode method) {
654+
/*
655+
* If a macro node is used in a method call that appears to be uncommon, we can manually
656+
* box the vector, disconnecting the method call from the connected component. The
657+
* conservative heuristics now is that a method returning a throwable is likely
658+
* uncommon. Revisit and expand the heuristic if the need arises.
659+
*/
660+
ResolvedJavaType throwableType = providers.getMetaAccess().lookupJavaType(Throwable.class);
661+
if (method.returnKind() == JavaKind.Object && throwableType.isAssignableFrom(method.returnStamp().getTrustedStamp().javaType(providers.getMetaAccess())) &&
662+
method.usages().filter(n -> !(n instanceof Invoke)).isEmpty()) {
663+
return true;
664+
}
665+
} else if (use instanceof ReturnNode returnNode && returnNode.result().equals(macro)) {
666+
// If a macro node is returned, we can also box the vector there
667+
return true;
668+
}
669+
670+
return false;
671+
}
672+
557673
private static void anchorAndUnboxInput(StructuredGraph graph, CoreProviders providers, ValueNode usage, ValueNode unboxableInput, FixedNode insertionPoint, NodeMap<ValueNode> expanded) {
558674
anchorAndUnboxInput(graph, providers, usage, unboxableInput, insertionPoint, expanded, -1);
559675
}

0 commit comments

Comments
 (0)