Skip to content

Commit caa821e

Browse files
timfeltimfel
committed
rework internal _trace module
1 parent c562776 commit caa821e

File tree

5 files changed

+243
-14
lines changed

5 files changed

+243
-14
lines changed

‎docs/user/TOOLS.md

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,17 @@ doesn't currently work).
2020

2121
### Coverage
2222

23-
We are going to support the `trace` module API, but using the Truffle *coverage*
24-
tool.
23+
GraalVM comes with a coverage instrument that can be used with `--coverage`. See
24+
the commandline help for more options on how to use it. In order to work better
25+
with existing Python code, we also partially support the standard library
26+
`trace` module with this low-overhead GraalVM coverage instrument. So you can do
27+
this:
28+
29+
$ graalpython -m trace -m -c -s my_script.py
30+
31+
And this will work similarly to how it will run on CPython. The programmatic API
32+
also works, with some limitations. For example, we do not currently track calls,
33+
only line counts and called functions.
2534

2635
### Profiling
2736

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,6 @@ private static final PythonBuiltins[] initializeBuiltins() {
377377
new MultiprocessingModuleBuiltins(),
378378
new SemLockBuiltins(),
379379
new TraceModuleBuiltins(),
380-
TraceModuleBuiltins.newTraceBuiltins(),
381380
new GraalPythonModuleBuiltins()));
382381
if (!TruffleOptions.AOT) {
383382
ServiceLoader<PythonBuiltins> providers = ServiceLoader.load(PythonBuiltins.class, Python3Core.class.getClassLoader());

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,7 @@ public enum PythonBuiltinClassType implements LazyPythonClass {
192192
RuntimeWarning("RuntimeWarning", BuiltinNames.BUILTINS),
193193
SyntaxWarning("SyntaxWarning", BuiltinNames.BUILTINS),
194194
UnicodeWarning("UnicodeWarning", BuiltinNames.BUILTINS),
195-
UserWarning("UserWarning", BuiltinNames.BUILTINS),
196-
197-
// extras
198-
PTrace("Trace", "_trace");
195+
UserWarning("UserWarning", BuiltinNames.BUILTINS);
199196

200197
private final String name;
201198
private final Shape instanceShape;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TraceModuleBuiltins.java

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
package com.oracle.graal.python.builtins.modules;
2+
3+
import java.io.IOException;
4+
import java.util.List;
5+
import java.util.Map;
6+
7+
import com.oracle.graal.python.PythonLanguage;
8+
import com.oracle.graal.python.builtins.Builtin;
9+
import com.oracle.graal.python.builtins.CoreFunctions;
10+
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
11+
import com.oracle.graal.python.builtins.PythonBuiltins;
12+
import com.oracle.graal.python.builtins.objects.PNone;
13+
import com.oracle.graal.python.builtins.objects.common.HashingCollectionNodes;
14+
import com.oracle.graal.python.builtins.objects.common.SequenceNodes;
15+
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
16+
import com.oracle.graal.python.builtins.objects.dict.PDict;
17+
import com.oracle.graal.python.builtins.objects.function.PArguments;
18+
import com.oracle.graal.python.builtins.objects.module.PythonModule;
19+
import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
20+
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
21+
import com.oracle.graal.python.nodes.attributes.ReadAttributeFromObjectNode;
22+
import com.oracle.graal.python.nodes.attributes.WriteAttributeToObjectNode;
23+
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
24+
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
25+
import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
26+
import com.oracle.graal.python.nodes.util.CastToJavaStringNode;
27+
import com.oracle.graal.python.runtime.PythonContext;
28+
import com.oracle.graal.python.runtime.sequence.PSequence;
29+
import com.oracle.truffle.api.InstrumentInfo;
30+
import com.oracle.truffle.api.TruffleFile;
31+
import com.oracle.truffle.api.TruffleLanguage.Env;
32+
import com.oracle.truffle.api.dsl.Cached;
33+
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
34+
import com.oracle.truffle.api.dsl.NodeFactory;
35+
import com.oracle.truffle.api.dsl.Specialization;
36+
import com.oracle.truffle.api.frame.VirtualFrame;
37+
import com.oracle.truffle.api.instrumentation.SourceSectionFilter;
38+
import com.oracle.truffle.api.library.CachedLibrary;
39+
import com.oracle.truffle.api.object.HiddenKey;
40+
import com.oracle.truffle.tools.coverage.CoverageTracker;
41+
import com.oracle.truffle.tools.coverage.RootCoverage;
42+
import com.oracle.truffle.tools.coverage.SectionCoverage;
43+
import com.oracle.truffle.tools.coverage.SourceCoverage;
44+
import com.oracle.truffle.tools.coverage.impl.CoverageInstrument;
45+
46+
@CoreFunctions(defineModule = "_trace")
47+
public class TraceModuleBuiltins extends PythonBuiltins {
48+
@Override
49+
protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
50+
return TraceModuleBuiltinsFactory.getFactories();
51+
}
52+
53+
private static final HiddenKey TRACKER = new HiddenKey("coverageTracker");
54+
private static final HiddenKey TRACK_FUNCS = new HiddenKey("trackCalledFuncs");
55+
56+
@Builtin(name = "start", parameterNames = {"mod", "count", "countfuncs", "ignoremods", "ignoredirs"}, minNumOfPositionalArgs = 1, declaresExplicitSelf = true)
57+
@GenerateNodeFactory
58+
abstract static class TraceNew extends PythonBuiltinNode {
59+
@Specialization(limit = "getCallSiteInlineCacheMaxDepth()")
60+
PNone doit(VirtualFrame frame, PythonModule mod, Object count, Object countfuncs, PSequence ignoremods, PSequence ignoredirs,
61+
@Cached SequenceNodes.GetSequenceStorageNode getStore,
62+
@Cached SequenceStorageNodes.ToArrayNode toArray,
63+
@Cached CastToJavaStringNode castStr,
64+
@CachedLibrary("count") PythonObjectLibrary lib,
65+
@Cached ReadAttributeFromObjectNode readNode,
66+
@Cached WriteAttributeToObjectNode writeNode) {
67+
Object currentTracker = readNode.execute(mod, TRACKER);
68+
CoverageTracker tracker = null;
69+
70+
SourceSectionFilter.Builder filter = SourceSectionFilter.newBuilder();
71+
filter.includeInternal(false).mimeTypeIs(PythonLanguage.MIME_TYPE);
72+
PythonContext context = getContext();
73+
String stdLibHome = context.getStdlibHome();
74+
filter.sourceIs((src) -> {
75+
String path = src.getPath();
76+
return path != null && !path.contains(stdLibHome);
77+
});
78+
79+
Object[] ignoreMods = toArray.execute(getStore.execute(ignoremods));
80+
Env env = context.getEnv();
81+
for (Object moduleName : ignoreMods) {
82+
String modStr = castStr.execute(moduleName);
83+
if (modStr == null) {
84+
continue;
85+
}
86+
String fileNameSeparator = env.getFileNameSeparator();
87+
modStr = modStr.replace(".", fileNameSeparator);
88+
String modFile = modStr + ".py";
89+
String modInit = modStr + fileNameSeparator + "__init__.py";
90+
filter.sourceIs((src) -> {
91+
String path = src.getPath();
92+
return path != null && !(path.endsWith(modFile) || path.endsWith(modInit));
93+
});
94+
}
95+
96+
Object[] ignoreDirs = toArray.execute(getStore.execute(ignoredirs));
97+
for (Object dir : ignoreDirs) {
98+
String dirStr = castStr.execute(dir);
99+
if (dirStr == null) {
100+
continue;
101+
}
102+
String absDir;
103+
try {
104+
absDir = env.getPublicTruffleFile(dirStr).getCanonicalFile().getPath();
105+
} catch (SecurityException | IOException e) {
106+
continue;
107+
}
108+
filter.sourceIs((src) -> {
109+
String path = src.getPath();
110+
return path != null && !path.equals(absDir);
111+
});
112+
}
113+
114+
if (currentTracker instanceof CoverageTracker) {
115+
tracker = (CoverageTracker) currentTracker;
116+
} else {
117+
Map<String, InstrumentInfo> instruments = env.getInstruments();
118+
InstrumentInfo instrumentInfo = instruments.get(CoverageInstrument.ID);
119+
if (instrumentInfo != null) {
120+
tracker = env.lookup(instrumentInfo, CoverageTracker.class);
121+
if (tracker != null) {
122+
writeNode.execute(mod, TRACKER, tracker);
123+
}
124+
}
125+
}
126+
if (tracker == null) {
127+
throw raise(PythonBuiltinClassType.NotImplementedError, "coverage tracker not available");
128+
}
129+
writeNode.execute(mod, TRACK_FUNCS, countfuncs);
130+
131+
tracker.start(new CoverageTracker.Config(filter.build(), lib.isTrueWithState(count, PArguments.getThreadState(frame))));
132+
133+
return PNone.NONE;
134+
}
135+
}
136+
137+
@Builtin(name = "stop", minNumOfPositionalArgs = 1, declaresExplicitSelf = true)
138+
@GenerateNodeFactory
139+
abstract static class Stop extends PythonUnaryBuiltinNode {
140+
@Specialization
141+
PNone start(PythonModule mod,
142+
@Cached ReadAttributeFromObjectNode readNode) {
143+
Object currentTracker = readNode.execute(mod, TRACKER);
144+
if (currentTracker instanceof CoverageTracker) {
145+
((CoverageTracker) currentTracker).close();
146+
return PNone.NONE;
147+
} else {
148+
throw raise(PythonBuiltinClassType.TypeError, "coverage tracker not running");
149+
}
150+
}
151+
}
152+
153+
@Builtin(name = "results", minNumOfPositionalArgs = 1, declaresExplicitSelf = true)
154+
@GenerateNodeFactory
155+
abstract static class Results extends PythonUnaryBuiltinNode {
156+
@Specialization
157+
PTuple start(VirtualFrame frame, PythonModule mod,
158+
@CachedLibrary(limit = "getCallSiteInlineCacheMaxDepth()") PythonObjectLibrary lib,
159+
@Cached ReadAttributeFromObjectNode readNode,
160+
@Cached HashingCollectionNodes.SetItemNode setItemNode) {
161+
Object currentTracker = readNode.execute(mod, TRACKER);
162+
boolean countFuncs = lib.isTrue(readNode.execute(mod, TRACK_FUNCS));
163+
164+
CoverageTracker tracker;
165+
if (currentTracker instanceof CoverageTracker) {
166+
tracker = (CoverageTracker) currentTracker;
167+
} else {
168+
throw raise(PythonBuiltinClassType.TypeError, "coverage tracker not running");
169+
}
170+
SourceCoverage[] coverage = tracker.getCoverage();
171+
// callers -> not supported
172+
// calledfuncs -> {(filename, modulename, funcname) => 1}
173+
// counts -> {(filename, lineno) => count}
174+
PDict calledFuncs = factory().createDict();
175+
PDict counts = factory().createDict();
176+
for (SourceCoverage c : coverage) {
177+
String filename = c.getSource().getPath();
178+
if (filename == null) {
179+
continue;
180+
}
181+
182+
String modName;
183+
TruffleFile file = getContext().getEnv().getPublicTruffleFile(filename);
184+
String baseName = file.getName();
185+
if (baseName != null) {
186+
if (baseName.endsWith("__init__.py")) {
187+
modName = file.getParent().getName();
188+
} else {
189+
modName = baseName.replaceFirst("\\.py$", "");
190+
}
191+
} else {
192+
continue;
193+
}
194+
195+
for (RootCoverage r : c.getRoots()) {
196+
String name = r.getName();
197+
if (name == null) {
198+
continue;
199+
}
200+
if (countFuncs) {
201+
PTuple tp = factory().createTuple(new Object[] {filename, modName, name});
202+
setItemNode.execute(frame, calledFuncs, tp, 1);
203+
}
204+
for (SectionCoverage s : r.getSectionCoverage()) {
205+
if (s.getSourceSection().hasLines()) {
206+
int startLine = s.getSourceSection().getStartLine();
207+
int endLine = startLine; // s.getSourceSection().getEndLine();
208+
long cnt = s.getCount();
209+
if (cnt < 0) {
210+
cnt = 1;
211+
}
212+
for (int i = startLine; i <= endLine; i++) {
213+
PTuple ctp = factory().createTuple(new Object[] {filename, i});
214+
setItemNode.execute(frame, counts, ctp, cnt);
215+
}
216+
}
217+
}
218+
}
219+
}
220+
return factory().createTuple(new Object[] { counts, calledFuncs });
221+
}
222+
}
223+
}

‎graalpython/lib-python/3/trace.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,6 @@ def __init__(self, count=1, trace=1, countfuncs=0, countcallers=0,
405405
@param outfile file in which to write the results
406406
@param timing true iff timing information be displayed
407407
"""
408-
self._trace = _trace.Trace()
409408
self.infile = infile
410409
self.outfile = outfile
411410
self.ignore = _Ignore(ignoremods, ignoredirs)
@@ -436,6 +435,8 @@ def __init__(self, count=1, trace=1, countfuncs=0, countcallers=0,
436435
# Ahem -- do nothing? Okay.
437436
self.donothing = 1
438437

438+
self._graal_start_args = (count, countfuncs, ignoremods, ignoredirs)
439+
439440
def run(self, cmd):
440441
import __main__
441442
dict = __main__.__dict__
@@ -445,13 +446,13 @@ def runctx(self, cmd, globals=None, locals=None):
445446
if globals is None: globals = {}
446447
if locals is None: locals = {}
447448
if not self.donothing:
448-
self._trace.__start__()
449+
_trace.start(*self._graal_start_args)
449450
try:
450451
exec(cmd, globals, locals)
451452
finally:
452453
if not self.donothing:
453-
self._trace.__stop__()
454-
self.counts, self._calledfuncs = self._trace.results()
454+
_trace.stop()
455+
self.counts, self._calledfuncs = _trace.results()
455456

456457
def runfunc(*args, **kw):
457458
if len(args) >= 2:
@@ -471,13 +472,13 @@ def runfunc(*args, **kw):
471472

472473
result = None
473474
if not self.donothing:
474-
self._trace.__start__()
475+
_trace.start(*self._graal_start_args())
475476
try:
476477
result = func(*args, **kw)
477478
finally:
478479
if not self.donothing:
479-
self._trace.__stop__()
480-
self.counts, self._calledfuncs = self._trace.results()
480+
_trace.stop()
481+
self.counts, self._calledfuncs = _trace.results()
481482
return result
482483
runfunc.__text_signature__ = '($self, func, /, *args, **kw)'
483484

0 commit comments

Comments
 (0)