Skip to content

Commit 7b970e2

Browse files
committed
Optimized nodes for the common pattern of memory access to GEP.
1 parent 275b0d9 commit 7b970e2

File tree

3 files changed

+421
-1
lines changed

3 files changed

+421
-1
lines changed

sulong/projects/com.oracle.truffle.llvm.parser.factories/src/com/oracle/truffle/llvm/parser/factories/BasicNodeFactory.java

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2016, 2024, Oracle and/or its affiliates.
2+
* Copyright (c) 2016, 2025, Oracle and/or its affiliates.
33
*
44
* All rights reserved.
55
*
@@ -72,6 +72,7 @@
7272
import com.oracle.truffle.llvm.runtime.memory.VarargsAreaStackAllocationNode;
7373
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMControlFlowNode;
7474
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMExpressionNode;
75+
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMInstrumentableNode;
7576
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMLoadNode;
7677
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMStatementNode;
7778
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMStoreNode;
@@ -205,12 +206,18 @@
205206
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMCompareExchangeNode;
206207
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMFenceExpressionNodeGen;
207208
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMFenceNodeGen;
209+
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMGetElementPtrNode;
208210
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMGetElementPtrNodeGen;
209211
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMInsertValueNodeGen;
210212
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMNativeVarargsAreaStackAllocationNodeGen;
211213
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMStructByValueNodeGen;
212214
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMVarArgCompoundAddressNodeGen;
213215
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMVectorizedGetElementPtrNodeGen;
216+
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMWithElementPtrLoadNodeFactory.LLVMWithElementPtrLoadI16NodeGen;
217+
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMWithElementPtrLoadNodeFactory.LLVMWithElementPtrLoadI32NodeGen;
218+
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMWithElementPtrLoadNodeFactory.LLVMWithElementPtrLoadI64NodeGen;
219+
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMWithElementPtrLoadNodeFactory.LLVMWithElementPtrLoadI8NodeGen;
220+
import com.oracle.truffle.llvm.runtime.nodes.memory.LLVMWithElementPtrStoreNodeGen;
214221
import com.oracle.truffle.llvm.runtime.nodes.memory.NativeMemSetNodeGen;
215222
import com.oracle.truffle.llvm.runtime.nodes.memory.NativeProfiledMemMoveNodeGen;
216223
import com.oracle.truffle.llvm.runtime.nodes.memory.ProtectReadOnlyGlobalsBlockNodeGen;
@@ -474,13 +481,49 @@ public LLVMExpressionNode createShuffleVector(Type llvmType, LLVMExpressionNode
474481
throw new AssertionError(resultType + " is not supported for shufflevector");
475482
}
476483

484+
private static <T extends LLVMInstrumentableNode> T transferStatementTag(T target, LLVMInstrumentableNode source) {
485+
target.setHasStatementTag(source.hasStatementTag());
486+
return target;
487+
}
488+
477489
@Override
478490
public LLVMExpressionNode createLoad(Type resolvedResultType, LLVMExpressionNode loadTarget) {
491+
if (loadTarget instanceof LLVMGetElementPtrNode) {
492+
LLVMGetElementPtrNode ptr = (LLVMGetElementPtrNode) loadTarget;
493+
if (resolvedResultType instanceof PrimitiveType) {
494+
switch (((PrimitiveType) resolvedResultType).getPrimitiveKind()) {
495+
case I8:
496+
return transferStatementTag(LLVMWithElementPtrLoadI8NodeGen.create(ptr.getTypeWidth(), ptr.getBase(), ptr.getOffset()), ptr);
497+
case I16:
498+
return transferStatementTag(LLVMWithElementPtrLoadI16NodeGen.create(ptr.getTypeWidth(), ptr.getBase(), ptr.getOffset()), ptr);
499+
case I32:
500+
return transferStatementTag(LLVMWithElementPtrLoadI32NodeGen.create(ptr.getTypeWidth(), ptr.getBase(), ptr.getOffset()), ptr);
501+
case I64:
502+
return transferStatementTag(LLVMWithElementPtrLoadI64NodeGen.create(ptr.getTypeWidth(), ptr.getBase(), ptr.getOffset()), ptr);
503+
case FLOAT:
504+
case DOUBLE:
505+
case X86_FP80:
506+
// fallthrough
507+
}
508+
}
509+
}
479510
return CommonNodeFactory.createLoad(resolvedResultType, loadTarget);
480511
}
481512

482513
@Override
483514
public LLVMStatementNode createStore(LLVMExpressionNode pointerNode, LLVMExpressionNode valueNode, Type type) {
515+
if (pointerNode instanceof LLVMGetElementPtrNode) {
516+
LLVMGetElementPtrNode ptr = (LLVMGetElementPtrNode) pointerNode;
517+
LLVMOffsetStoreNode offsetStore;
518+
try {
519+
offsetStore = createOffsetMemoryStore(type, valueNode);
520+
} catch (TypeOverflowException e) {
521+
return Type.handleOverflowStatement(e);
522+
}
523+
if (offsetStore != null) {
524+
return transferStatementTag(LLVMWithElementPtrStoreNodeGen.create(offsetStore, ptr.getTypeWidth(), ptr.getBase(), ptr.getOffset()), pointerNode);
525+
}
526+
}
484527
try {
485528
return createMemoryStore(pointerNode, valueNode, type);
486529
} catch (TypeOverflowException e) {
Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
/*
2+
* Copyright (c) 2025, Oracle and/or its affiliates.
3+
*
4+
* All rights reserved.
5+
*
6+
* Redistribution and use in source and binary forms, with or without modification, are
7+
* permitted provided that the following conditions are met:
8+
*
9+
* 1. Redistributions of source code must retain the above copyright notice, this list of
10+
* conditions and the following disclaimer.
11+
*
12+
* 2. Redistributions in binary form must reproduce the above copyright notice, this list of
13+
* conditions and the following disclaimer in the documentation and/or other materials provided
14+
* with the distribution.
15+
*
16+
* 3. Neither the name of the copyright holder nor the names of its contributors may be used to
17+
* endorse or promote products derived from this software without specific prior written
18+
* permission.
19+
*
20+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
21+
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
22+
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
23+
* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24+
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
25+
* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26+
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
27+
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
28+
* OF THE POSSIBILITY OF SUCH DAMAGE.
29+
*/
30+
package com.oracle.truffle.llvm.runtime.nodes.memory;
31+
32+
import com.oracle.truffle.api.dsl.ImportStatic;
33+
import com.oracle.truffle.api.dsl.NodeChild;
34+
import com.oracle.truffle.api.dsl.Specialization;
35+
import com.oracle.truffle.api.nodes.UnexpectedResultException;
36+
import com.oracle.truffle.llvm.runtime.nodes.api.LLVMExpressionNode;
37+
import com.oracle.truffle.llvm.runtime.nodes.memory.load.LLVMDoubleLoadNode.LLVMDoubleOffsetLoadNode;
38+
import com.oracle.truffle.llvm.runtime.nodes.memory.load.LLVMFloatLoadNode.LLVMFloatOffsetLoadNode;
39+
import com.oracle.truffle.llvm.runtime.nodes.memory.load.LLVMI16LoadNode.LLVMI16OffsetLoadNode;
40+
import com.oracle.truffle.llvm.runtime.nodes.memory.load.LLVMI32LoadNode.LLVMI32OffsetLoadNode;
41+
import com.oracle.truffle.llvm.runtime.nodes.memory.load.LLVMI64LoadNode.LLVMI64OffsetLoadNode;
42+
import com.oracle.truffle.llvm.runtime.nodes.memory.load.LLVMI8LoadNode.LLVMI8OffsetLoadNode;
43+
import com.oracle.truffle.llvm.runtime.pointer.LLVMManagedPointer;
44+
import com.oracle.truffle.llvm.runtime.pointer.LLVMNativePointer;
45+
import com.oracle.truffle.llvm.runtime.pointer.LLVMPointer;
46+
47+
@NodeChild(value = "base", type = LLVMExpressionNode.class)
48+
@NodeChild(value = "offset", type = LLVMExpressionNode.class)
49+
public abstract class LLVMWithElementPtrLoadNode extends LLVMExpressionNode {
50+
51+
final long typeWidth;
52+
53+
protected LLVMWithElementPtrLoadNode(long typeWidth) {
54+
this.typeWidth = typeWidth;
55+
}
56+
57+
public final long getTypeWidth() {
58+
return typeWidth;
59+
}
60+
61+
@ImportStatic(LLVMWithElementPtrStoreNode.class)
62+
public abstract static class LLVMWithElementPtrLoadI8Node extends LLVMWithElementPtrLoadNode {
63+
@Child private LLVMI8OffsetLoadNode load = LLVMI8OffsetLoadNode.create();
64+
65+
protected LLVMWithElementPtrLoadI8Node(long typeWidth) {
66+
super(typeWidth);
67+
}
68+
69+
@Specialization(guards = "isNegated(addr.getObject(), element.getObject())")
70+
protected byte doPointerDiff(LLVMManagedPointer addr, LLVMManagedPointer element) {
71+
return load.executeWithTarget(LLVMNativePointer.create(addr.getOffset() + element.getOffset()), 0);
72+
}
73+
74+
@Specialization(guards = "isNegated(element.getObject(), addr.getObject())")
75+
protected byte doPointerDiffRev(LLVMManagedPointer addr, LLVMManagedPointer element) {
76+
return load.executeWithTarget(LLVMNativePointer.create(element.getOffset() + addr.getOffset()), 0);
77+
}
78+
79+
@Specialization
80+
protected byte doInt(LLVMPointer addr, int element) {
81+
return load.executeWithTarget(addr, typeWidth * element);
82+
}
83+
84+
@Specialization
85+
protected byte doLong(LLVMPointer addr, long element) {
86+
return load.executeWithTarget(addr, typeWidth * element);
87+
}
88+
89+
@Specialization
90+
protected byte doNativePointer(LLVMPointer addr, LLVMNativePointer element) {
91+
return load.executeWithTarget(addr, typeWidth * element.asNative());
92+
}
93+
}
94+
95+
@ImportStatic(LLVMWithElementPtrStoreNode.class)
96+
public abstract static class LLVMWithElementPtrLoadI16Node extends LLVMWithElementPtrLoadNode {
97+
@Child private LLVMI16OffsetLoadNode load = LLVMI16OffsetLoadNode.create();
98+
99+
protected LLVMWithElementPtrLoadI16Node(long typeWidth) {
100+
super(typeWidth);
101+
}
102+
103+
@Specialization(guards = "isNegated(addr.getObject(), element.getObject())")
104+
protected short doPointerDiff(LLVMManagedPointer addr, LLVMManagedPointer element) {
105+
return load.executeWithTarget(LLVMNativePointer.create(addr.getOffset() + element.getOffset()), 0);
106+
}
107+
108+
@Specialization(guards = "isNegated(element.getObject(), addr.getObject())")
109+
protected short doPointerDiffRev(LLVMManagedPointer addr, LLVMManagedPointer element) {
110+
return load.executeWithTarget(LLVMNativePointer.create(element.getOffset() + addr.getOffset()), 0);
111+
}
112+
113+
@Specialization
114+
protected short doInt(LLVMPointer addr, int element) {
115+
return load.executeWithTarget(addr, typeWidth * element);
116+
}
117+
118+
@Specialization
119+
protected short doLong(LLVMPointer addr, long element) {
120+
return load.executeWithTarget(addr, typeWidth * element);
121+
}
122+
123+
@Specialization
124+
protected short doNativePointer(LLVMPointer addr, LLVMNativePointer element) {
125+
return load.executeWithTarget(addr, typeWidth * element.asNative());
126+
}
127+
}
128+
129+
@ImportStatic(LLVMWithElementPtrStoreNode.class)
130+
public abstract static class LLVMWithElementPtrLoadI32Node extends LLVMWithElementPtrLoadNode {
131+
@Child private LLVMI32OffsetLoadNode load = LLVMI32OffsetLoadNode.create();
132+
133+
protected LLVMWithElementPtrLoadI32Node(long typeWidth) {
134+
super(typeWidth);
135+
}
136+
137+
@Specialization(guards = "isNegated(addr.getObject(), element.getObject())")
138+
protected int doPointerDiff(LLVMManagedPointer addr, LLVMManagedPointer element) {
139+
return load.executeWithTarget(LLVMNativePointer.create(addr.getOffset() + element.getOffset()), 0);
140+
}
141+
142+
@Specialization(guards = "isNegated(element.getObject(), addr.getObject())")
143+
protected int doPointerDiffRev(LLVMManagedPointer addr, LLVMManagedPointer element) {
144+
return load.executeWithTarget(LLVMNativePointer.create(element.getOffset() + addr.getOffset()), 0);
145+
}
146+
147+
@Specialization
148+
protected int doInt(LLVMPointer addr, int element) {
149+
return load.executeWithTarget(addr, typeWidth * element);
150+
}
151+
152+
@Specialization
153+
protected int doLong(LLVMPointer addr, long element) {
154+
return load.executeWithTarget(addr, typeWidth * element);
155+
}
156+
157+
@Specialization
158+
protected int doNativePointer(LLVMPointer addr, LLVMNativePointer element) {
159+
return load.executeWithTarget(addr, typeWidth * element.asNative());
160+
}
161+
}
162+
163+
@ImportStatic(LLVMWithElementPtrStoreNode.class)
164+
public abstract static class LLVMWithElementPtrLoadI64Node extends LLVMWithElementPtrLoadNode {
165+
@Child private LLVMI64OffsetLoadNode load = LLVMI64OffsetLoadNode.create();
166+
167+
protected LLVMWithElementPtrLoadI64Node(long typeWidth) {
168+
super(typeWidth);
169+
}
170+
171+
@Specialization(guards = "isNegated(addr.getObject(), element.getObject())", rewriteOn = UnexpectedResultException.class)
172+
protected long doPointerDiff(LLVMManagedPointer addr, LLVMManagedPointer element) throws UnexpectedResultException {
173+
return load.executeWithTarget(LLVMNativePointer.create(addr.getOffset() + element.getOffset()), 0);
174+
}
175+
176+
@Specialization(guards = "isNegated(element.getObject(), addr.getObject())", rewriteOn = UnexpectedResultException.class)
177+
protected long doPointerDiffRev(LLVMManagedPointer addr, LLVMManagedPointer element) throws UnexpectedResultException {
178+
return load.executeWithTarget(LLVMNativePointer.create(element.getOffset() + addr.getOffset()), 0);
179+
}
180+
181+
@Specialization(rewriteOn = UnexpectedResultException.class)
182+
protected long doInt(LLVMPointer addr, int element) throws UnexpectedResultException {
183+
return load.executeWithTarget(addr, typeWidth * element);
184+
}
185+
186+
@Specialization(rewriteOn = UnexpectedResultException.class)
187+
protected long doLong(LLVMPointer addr, long element) throws UnexpectedResultException {
188+
return load.executeWithTarget(addr, typeWidth * element);
189+
}
190+
191+
@Specialization(rewriteOn = UnexpectedResultException.class)
192+
protected long doNativePointer(LLVMPointer addr, LLVMNativePointer element) throws UnexpectedResultException {
193+
return load.executeWithTarget(addr, typeWidth * element.asNative());
194+
}
195+
196+
@Specialization(guards = "isNegated(addr.getObject(), element.getObject())", replaces = "doPointerDiff")
197+
protected Object doPointerDiffGeneric(LLVMManagedPointer addr, LLVMManagedPointer element) {
198+
return load.executeWithTargetGeneric(LLVMNativePointer.create(addr.getOffset() + element.getOffset()), 0);
199+
}
200+
201+
@Specialization(guards = "isNegated(element.getObject(), addr.getObject())", replaces = "doPointerDiffRev")
202+
protected Object doPointerDiffRevGeneric(LLVMManagedPointer addr, LLVMManagedPointer element) {
203+
return load.executeWithTargetGeneric(LLVMNativePointer.create(element.getOffset() + addr.getOffset()), 0);
204+
}
205+
206+
@Specialization(replaces = "doInt")
207+
protected Object doIntGeneric(LLVMPointer addr, int element) {
208+
return load.executeWithTargetGeneric(addr, typeWidth * element);
209+
}
210+
211+
@Specialization(replaces = "doLong")
212+
protected Object doLongGeneric(LLVMPointer addr, long element) {
213+
return load.executeWithTargetGeneric(addr, typeWidth * element);
214+
}
215+
216+
@Specialization(replaces = "doNativePointer")
217+
protected Object doNativePointerGeneric(LLVMPointer addr, LLVMNativePointer element) {
218+
return load.executeWithTargetGeneric(addr, typeWidth * element.asNative());
219+
}
220+
}
221+
222+
@ImportStatic(LLVMWithElementPtrStoreNode.class)
223+
public abstract static class LLVMWithElementPtrLoadDoubleNode extends LLVMWithElementPtrLoadNode {
224+
@Child private LLVMDoubleOffsetLoadNode load = LLVMDoubleOffsetLoadNode.create();
225+
226+
protected LLVMWithElementPtrLoadDoubleNode(long typeWidth) {
227+
super(typeWidth);
228+
}
229+
230+
@Specialization(guards = "isNegated(addr.getObject(), element.getObject())")
231+
protected double doPointerDiff(LLVMManagedPointer addr, LLVMManagedPointer element) {
232+
return load.executeWithTarget(LLVMNativePointer.create(addr.getOffset() + element.getOffset()), 0);
233+
}
234+
235+
@Specialization(guards = "isNegated(element.getObject(), addr.getObject())")
236+
protected double doPointerDiffRev(LLVMManagedPointer addr, LLVMManagedPointer element) {
237+
return load.executeWithTarget(LLVMNativePointer.create(element.getOffset() + addr.getOffset()), 0);
238+
}
239+
240+
@Specialization
241+
protected double doInt(LLVMPointer addr, int element) {
242+
return load.executeWithTarget(addr, typeWidth * element);
243+
}
244+
245+
@Specialization
246+
protected double doLong(LLVMPointer addr, long element) {
247+
return load.executeWithTarget(addr, typeWidth * element);
248+
}
249+
250+
@Specialization
251+
protected double doNativePointer(LLVMPointer addr, LLVMNativePointer element) {
252+
return load.executeWithTarget(addr, typeWidth * element.asNative());
253+
}
254+
}
255+
256+
@ImportStatic(LLVMWithElementPtrStoreNode.class)
257+
public abstract static class LLVMWithElementPtrLoadFloatNode extends LLVMWithElementPtrLoadNode {
258+
@Child private LLVMFloatOffsetLoadNode load = LLVMFloatOffsetLoadNode.create();
259+
260+
protected LLVMWithElementPtrLoadFloatNode(long typeWidth) {
261+
super(typeWidth);
262+
}
263+
264+
@Specialization(guards = "isNegated(addr.getObject(), element.getObject())")
265+
protected float doPointerDiff(LLVMManagedPointer addr, LLVMManagedPointer element) {
266+
return load.executeWithTarget(LLVMNativePointer.create(addr.getOffset() + element.getOffset()), 0);
267+
}
268+
269+
@Specialization(guards = "isNegated(element.getObject(), addr.getObject())")
270+
protected float doPointerDiffRev(LLVMManagedPointer addr, LLVMManagedPointer element) {
271+
return load.executeWithTarget(LLVMNativePointer.create(element.getOffset() + addr.getOffset()), 0);
272+
}
273+
274+
@Specialization
275+
protected float doInt(LLVMPointer addr, int element) {
276+
return load.executeWithTarget(addr, typeWidth * element);
277+
}
278+
279+
@Specialization
280+
protected float doLong(LLVMPointer addr, long element) {
281+
return load.executeWithTarget(addr, typeWidth * element);
282+
}
283+
284+
@Specialization
285+
protected float doNativePointer(LLVMPointer addr, LLVMNativePointer element) {
286+
return load.executeWithTarget(addr, typeWidth * element.asNative());
287+
}
288+
}
289+
}

0 commit comments

Comments
 (0)