Skip to content

Commit 9fa1512

Browse files
timfeltimfel
committed
basic implementation of _lsprof using truffle sampling profiler
1 parent caa821e commit 9fa1512

File tree

7 files changed

+308
-2
lines changed

7 files changed

+308
-2
lines changed

‎docs/user/TOOLS.md

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,17 @@ only line counts and called functions.
3434

3535
### Profiling
3636

37-
We are going to support the `cProfile` module API, but using the Truffle
38-
*cpusampler* tool
37+
We implement the `_lsprof` built-in module using the Truffle *cpusampler*
38+
tool. Not all profiling features are currently supported, but basic profiling
39+
works:
40+
41+
$ graalpython -m cProfile -s sort -m ginstall --help
42+
43+
The interactive exploration of a stats output file also works:
44+
45+
$ graalpython -m cProfile -o ginstall.profile -m ginstall --help
46+
$ graalpython -m pstats ginstall.profile
47+
ginstall.profile%
48+
callers
49+
[...]
50+

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import com.oracle.graal.python.builtins.modules.JavaModuleBuiltins;
6666
import com.oracle.graal.python.builtins.modules.LZMAModuleBuiltins;
6767
import com.oracle.graal.python.builtins.modules.LocaleModuleBuiltins;
68+
import com.oracle.graal.python.builtins.modules.LsprofModuleBuiltins;
6869
import com.oracle.graal.python.builtins.modules.MMapModuleBuiltins;
6970
import com.oracle.graal.python.builtins.modules.MarshalModuleBuiltins;
7071
import com.oracle.graal.python.builtins.modules.MathModuleBuiltins;
@@ -254,6 +255,8 @@ private static final String[] initializeCoreFiles() {
254255
"resource",
255256
"_contextvars",
256257
"pip_hook",
258+
"_lsprof",
259+
"marshal",
257260
"_lzma"));
258261
// must be last
259262
coreFiles.add("final_patches");
@@ -377,6 +380,8 @@ private static final PythonBuiltins[] initializeBuiltins() {
377380
new MultiprocessingModuleBuiltins(),
378381
new SemLockBuiltins(),
379382
new TraceModuleBuiltins(),
383+
new LsprofModuleBuiltins(),
384+
LsprofModuleBuiltins.newProfilerBuiltins(),
380385
new GraalPythonModuleBuiltins()));
381386
if (!TruffleOptions.AOT) {
382387
ServiceLoader<PythonBuiltins> providers = ServiceLoader.load(PythonBuiltins.class, Python3Core.class.getClassLoader());

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ public enum PythonBuiltinClassType implements LazyPythonClass {
118118
PDirEntry("DirEntry", "posix"),
119119
PLZMACompressor("LZMACompressor", "_lzma"),
120120
PLZMADecompressor("LZMADecompressor", "_lzma"),
121+
LsprofProfiler("Profiler", "_lsprof"),
121122

122123
// Errors and exceptions:
123124

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/LsprofModuleBuiltins.java

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
package com.oracle.graal.python.builtins.modules;
2+
3+
import java.util.ArrayList;
4+
import java.util.Collection;
5+
import java.util.List;
6+
import java.util.Map;
7+
8+
import com.oracle.graal.python.builtins.Builtin;
9+
import com.oracle.graal.python.builtins.CoreFunctions;
10+
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
11+
import com.oracle.graal.python.builtins.PythonBuiltins;
12+
import com.oracle.graal.python.builtins.objects.PNone;
13+
import com.oracle.graal.python.builtins.objects.function.PKeyword;
14+
import com.oracle.graal.python.builtins.objects.list.PList;
15+
import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject;
16+
import com.oracle.graal.python.builtins.objects.type.LazyPythonClass;
17+
import com.oracle.graal.python.nodes.SpecialMethodNames;
18+
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
19+
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
20+
import com.oracle.truffle.api.InstrumentInfo;
21+
import com.oracle.truffle.api.TruffleLanguage.Env;
22+
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
23+
import com.oracle.truffle.api.dsl.NodeFactory;
24+
import com.oracle.truffle.api.dsl.Specialization;
25+
import com.oracle.truffle.api.instrumentation.SourceSectionFilter;
26+
import com.oracle.truffle.api.source.SourceSection;
27+
import com.oracle.truffle.tools.profiler.CPUSampler;
28+
import com.oracle.truffle.tools.profiler.CPUSampler.Payload;
29+
import com.oracle.truffle.tools.profiler.ProfilerNode;
30+
import com.oracle.truffle.tools.profiler.impl.CPUSamplerInstrument;
31+
32+
@CoreFunctions(defineModule = "_lsprof")
33+
public class LsprofModuleBuiltins extends PythonBuiltins {
34+
@Override
35+
protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
36+
return LsprofModuleBuiltinsFactory.getFactories();
37+
}
38+
39+
public static PythonBuiltins newProfilerBuiltins() {
40+
return new ProfilerBuiltins();
41+
}
42+
43+
@Builtin(name = "Profiler", minNumOfPositionalArgs = 1, takesVarArgs = true, takesVarKeywordArgs = true, constructsClass = PythonBuiltinClassType.LsprofProfiler)
44+
@GenerateNodeFactory
45+
abstract static class LsprofNew extends PythonBuiltinNode {
46+
@Specialization
47+
Profiler doit(LazyPythonClass cls, @SuppressWarnings("unused") Object[] args, @SuppressWarnings("unused") PKeyword[] kwargs) {
48+
Env env = getContext().getEnv();
49+
Map<String, InstrumentInfo> instruments = env.getInstruments();
50+
InstrumentInfo instrumentInfo = instruments.get(CPUSamplerInstrument.ID);
51+
if (instrumentInfo != null) {
52+
CPUSampler sampler = env.lookup(instrumentInfo, CPUSampler.class);
53+
if (sampler != null) {
54+
return factory().trace(new Profiler(cls, sampler));
55+
}
56+
}
57+
throw raise(PythonBuiltinClassType.NotImplementedError, "coverage tracker not available");
58+
}
59+
}
60+
}
61+
62+
class Profiler extends PythonBuiltinObject {
63+
boolean subcalls;
64+
boolean builtins;
65+
double timeunit;
66+
Object externalTimer;
67+
double time;
68+
final CPUSampler sampler;
69+
70+
public Profiler(LazyPythonClass cls, CPUSampler sampler) {
71+
super(cls);
72+
this.sampler = sampler;
73+
this.sampler.setFilter(SourceSectionFilter.newBuilder().includeInternal(true).build());
74+
this.sampler.setMode(CPUSampler.Mode.ROOTS);
75+
this.sampler.setPeriod(1);
76+
}
77+
}
78+
79+
@CoreFunctions(extendClasses = PythonBuiltinClassType.LsprofProfiler)
80+
class ProfilerBuiltins extends PythonBuiltins {
81+
@Override
82+
protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
83+
return ProfilerBuiltinsFactory.getFactories();
84+
}
85+
86+
@Builtin(name = SpecialMethodNames.__INIT__, minNumOfPositionalArgs = 1, parameterNames = {"$self", "timer", "timeunit", "subcalls", "builtins"})
87+
@GenerateNodeFactory
88+
abstract static class Init extends PythonBuiltinNode {
89+
@Specialization
90+
PNone doit(Profiler self, Object timer, double timeunit, long subcalls, long builtins) {
91+
self.subcalls = subcalls > 0;
92+
self.builtins = builtins > 0;
93+
self.timeunit = timeunit;
94+
self.externalTimer = timer;
95+
return PNone.NONE;
96+
}
97+
98+
@Specialization
99+
@SuppressWarnings("unused")
100+
PNone doit(Profiler self, Object timer, PNone timeunit, PNone subcalls, PNone builtins) {
101+
self.subcalls = true;
102+
self.builtins = true;
103+
self.timeunit = -1;
104+
self.externalTimer = timer;
105+
return PNone.NONE;
106+
}
107+
}
108+
109+
@Builtin(name = "enable", minNumOfPositionalArgs = 1, parameterNames = {"$self", "subcalls", "builtins"})
110+
@GenerateNodeFactory
111+
abstract static class Enable extends PythonBuiltinNode {
112+
@Specialization
113+
PNone doit(Profiler self, long subcalls, long builtins) {
114+
self.subcalls = subcalls > 0;
115+
self.builtins = builtins > 0;
116+
// TODO: deal with any arguments
117+
self.time = System.currentTimeMillis();
118+
self.sampler.setCollecting(true);
119+
return PNone.NONE;
120+
}
121+
122+
@Specialization
123+
PNone doit(Profiler self, long subcalls, @SuppressWarnings("unused") PNone builtins) {
124+
return doit(self, subcalls, self.builtins ? 1 : 0);
125+
}
126+
127+
@Specialization
128+
PNone doit(Profiler self, @SuppressWarnings("unused") PNone subcalls, long builtins) {
129+
return doit(self, self.subcalls ? 1 : 0, builtins);
130+
}
131+
132+
@Specialization
133+
PNone doit(Profiler self, @SuppressWarnings("unused") PNone subcalls, @SuppressWarnings("unused") PNone builtins) {
134+
return doit(self, self.subcalls ? 1 : 0, self.builtins ? 1 : 0);
135+
}
136+
}
137+
138+
@Builtin(name = "disable", minNumOfPositionalArgs = 1)
139+
@GenerateNodeFactory
140+
abstract static class Disable extends PythonBuiltinNode {
141+
@Specialization
142+
PNone doit(Profiler self) {
143+
self.sampler.setCollecting(false);
144+
self.time = (System.currentTimeMillis() - self.time) / 1000D;
145+
return PNone.NONE;
146+
}
147+
}
148+
149+
@Builtin(name = "clear", minNumOfPositionalArgs = 1)
150+
@GenerateNodeFactory
151+
abstract static class Clear extends PythonBuiltinNode {
152+
@Specialization
153+
PNone doit(Profiler self) {
154+
self.sampler.clearData();
155+
return PNone.NONE;
156+
}
157+
}
158+
159+
@Builtin(name = "getstats", minNumOfPositionalArgs = 1, doc = "" +
160+
"getstats() -> list of profiler_entry objects\n" +
161+
"\n" +
162+
"Return all information collected by the profiler.\n" +
163+
"Each profiler_entry is a tuple-like object with the\n" +
164+
"following attributes:\n" +
165+
"\n" +
166+
" code code object or functionname\n" +
167+
" callcount how many times this was called\n" +
168+
" reccallcount how many times called recursively\n" +
169+
" totaltime total time in this entry\n" +
170+
" inlinetime inline time in this entry (not in subcalls)\n" +
171+
" calls details of the calls\n" +
172+
"\n" +
173+
"The calls attribute is either None or a list of\n" +
174+
"profiler_subentry objects:\n" +
175+
"\n" +
176+
" code called code object\n" +
177+
" callcount how many times this is called\n" +
178+
" reccallcount how many times this is called recursively\n" +
179+
" totaltime total time spent in this call\n" +
180+
" inlinetime inline time (not in further subcalls)\n")
181+
@GenerateNodeFactory
182+
abstract static class GetStats extends PythonBuiltinNode {
183+
@Specialization
184+
PList doit(Profiler self) {
185+
double avgSampleSeconds = self.sampler.getPeriod() / 1000D;
186+
List<PList> entries = new ArrayList<>();
187+
for (ProfilerNode<Payload> node : self.sampler.getRootNodes()) {
188+
countNode(entries, node, avgSampleSeconds);
189+
}
190+
self.sampler.close();
191+
return factory().createList(entries.toArray());
192+
}
193+
194+
private void countNode(List<PList> entries, ProfilerNode<Payload> node, double avgSampleTime) {
195+
Collection<ProfilerNode<Payload>> children = node.getChildren();
196+
Object[] profilerEntry = getProfilerEntry(node, avgSampleTime);
197+
Object[] calls = new Object[children.size()];
198+
int callIdx = 0;
199+
for (ProfilerNode<Payload> childNode : children) {
200+
countNode(entries, childNode, avgSampleTime);
201+
calls[callIdx++] = factory().createList(getProfilerEntry(childNode, avgSampleTime));
202+
}
203+
assert callIdx == calls.length;
204+
profilerEntry[profilerEntry.length - 1] = factory().createList(calls);
205+
entries.add(factory().createList(profilerEntry));
206+
}
207+
208+
private static Object[] getProfilerEntry(ProfilerNode<Payload> node, double avgSampleTime) {
209+
SourceSection sec = node.getSourceSection();
210+
String rootName;
211+
if (sec == null) {
212+
rootName = node.getRootName();
213+
} else {
214+
rootName = sec.getSource().getName() + ":" + sec.getStartLine() + "(" + node.getRootName() + ")";
215+
}
216+
if (rootName == null) {
217+
rootName = "<unknown root>";
218+
}
219+
int otherHitCount = node.getPayload().getHitCount();
220+
int selfHitCount = node.getPayload().getSelfHitCount();
221+
long hitCount = (long) otherHitCount + selfHitCount;
222+
Object[] profilerEntry = new Object[] {
223+
rootName,
224+
hitCount,
225+
0,
226+
otherHitCount * avgSampleTime,
227+
selfHitCount * avgSampleTime,
228+
PNone.NONE
229+
};
230+
return profilerEntry;
231+
}
232+
}
233+
}

‎graalpython/lib-graalpython/_lsprof.py

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
from _descriptor import make_named_tuple_class
2+
3+
profiler_entry = make_named_tuple_class(
4+
"profiler_entry",
5+
["code",
6+
"callcount",
7+
"reccallcount",
8+
"totaltime",
9+
"inlinetime",
10+
"calls"]
11+
)
12+
13+
14+
profiler_subentry = make_named_tuple_class(
15+
"profiler_subentry",
16+
["code",
17+
"callcount",
18+
"reccallcount",
19+
"totaltime",
20+
"inlinetime"]
21+
)
22+
23+
24+
def make_wrapped_getstats(original):
25+
def getstats(self):
26+
stats = original(self)
27+
for idx, s in enumerate(stats):
28+
calls = s[-1]
29+
if calls:
30+
s[-1] = [profiler_subentry(c) for c in calls]
31+
stats[idx] = profiler_entry(s)
32+
return stats
33+
return getstats
34+
35+
36+
Profiler.getstats = make_wrapped_getstats(Profiler.getstats)
37+
38+
39+
# cleanup
40+
del make_wrapped_getstats
41+
del make_named_tuple_class

‎graalpython/lib-graalpython/marshal.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
DEFAULT_VERSION = object()
2+
3+
4+
def dump(value, file, version=DEFAULT_VERSION):
5+
if version is DEFAULT_VERSION:
6+
file.write(dumps(value))
7+
else:
8+
file.write(dumps(value, version))
9+
10+
11+
def load(file):
12+
return loads(file.read())

‎mx.graalpython/suite.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@
199199
"dependencies": [
200200
"truffle:TRUFFLE_API",
201201
"tools:TRUFFLE_COVERAGE",
202+
"tools:TRUFFLE_PROFILER",
202203
"sdk:GRAAL_SDK",
203204
"truffle:ANTLR4",
204205
"sulong:SULONG_API",
@@ -342,6 +343,7 @@
342343
"GRAALPYTHON-LAUNCHER",
343344
"truffle:TRUFFLE_API",
344345
"tools:TRUFFLE_COVERAGE",
346+
"tools:TRUFFLE_PROFILER",
345347
"regex:TREGEX",
346348
"sdk:GRAAL_SDK",
347349
"truffle:ANTLR4",

0 commit comments

Comments
 (0)