Skip to content

Commit 280a9b4

Browse files
yoffyoff
committed
Python: Support Model Editor
1 parent db76896 commit 280a9b4

File tree

9 files changed

+420
-0
lines changed

9 files changed

+420
-0
lines changed

‎python/ql/lib/modeling/ModelEditor.qll

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
/** Provides classes and predicates related to handling APIs for the VS Code extension. */
2+
3+
private import python
4+
private import semmle.python.frameworks.data.ModelsAsData
5+
private import semmle.python.frameworks.data.internal.ApiGraphModelsExtensions
6+
private import semmle.python.dataflow.new.internal.DataFlowDispatch as DP
7+
private import Util as Util
8+
9+
/**
10+
* An string describing the kind of source code element being modeled.
11+
*
12+
* See `EndPoint`.
13+
*/
14+
class EndpointKind extends string {
15+
EndpointKind() {
16+
this in ["Function", "InstanceMethod", "ClassMethod", "StaticMethod", "InitMethod", "Class"]
17+
}
18+
}
19+
20+
/**
21+
* An element of the source code to be modeled.
22+
*
23+
* See `EndPointKind` for the possible kinds of elements.
24+
*/
25+
abstract class Endpoint instanceof Scope {
26+
string namespace;
27+
string type;
28+
string name;
29+
30+
Endpoint() {
31+
this.isPublic() and
32+
this.getLocation().getFile() instanceof Util::RelevantFile and
33+
exists(string scopePath, string path, int pathIndex |
34+
scopePath = Util::computeScopePath(this) and
35+
pathIndex = scopePath.indexOf(".", 0, 0)
36+
|
37+
namespace = scopePath.prefix(pathIndex) and
38+
path = scopePath.suffix(pathIndex + 1) and
39+
(
40+
exists(int nameIndex | nameIndex = max(path.indexOf(".")) |
41+
type = path.prefix(nameIndex) and
42+
name = path.suffix(nameIndex + 1)
43+
)
44+
or
45+
not exists(path.indexOf(".")) and
46+
type = "" and
47+
name = path
48+
)
49+
)
50+
}
51+
52+
/** Gets the namespace for this endpoint. This will typically be the package in which it is found. */
53+
string getNamespace() { result = namespace }
54+
55+
/** Gets hte basename of the file where this endpoint is found. */
56+
string getFileName() { result = super.getLocation().getFile().getBaseName() }
57+
58+
/** Gets a string representation of this endpoint. */
59+
string toString() { result = super.toString() }
60+
61+
/** Gets the location of this endpoint. */
62+
Location getLocation() { result = super.getLocation() }
63+
64+
/** Gets the name of the class in which this endpoint is found, or the empty string if it is not found inside a class. */
65+
string getType() { result = type }
66+
67+
/**
68+
* Gets the name of the endpoint if it is not a class, or the empty string if it is a class
69+
*
70+
* If this endpoint is a class, the class name can be obtained via `getType`.
71+
*/
72+
string getName() { result = name }
73+
74+
/**
75+
* Gets a string representation of the parameters of this endpoint.
76+
*
77+
* The string follows a specific format:
78+
* - Positional parameters are listed in order, separated by commas.
79+
* - Keyword parameters are listed in order, separated by commas, each followed by a colon.
80+
* - In the future, positional-only parameters will be listed in order, separated by commas, each followed by a slash.
81+
*/
82+
abstract string getParameters();
83+
84+
/**
85+
* Gets a boolean that is true iff this endpoint is supported by existing modeling.
86+
*
87+
* The check only takes Models ss Data extension models into account.
88+
*/
89+
abstract boolean getSupportedStatus();
90+
91+
/**
92+
* Gets a string that describes the type of support detected this endpoint.
93+
*
94+
* The string can be one of the following:
95+
* - "source" if this endpoint is a known source.
96+
* - "sink" if this endpoint is a known sink.
97+
* - "summary" if this endpoint has a flow summary.
98+
* - "neutral" if this endpoint is a known neutral.
99+
* - "" if this endpoint is not detected as supported.
100+
*/
101+
abstract string getSupportedType();
102+
103+
/** Gets the kind of this endpoint. See `EndPointKind`. */
104+
abstract EndpointKind getKind();
105+
}
106+
107+
private predicate sourceModelPath(string type, string path) { sourceModel(type, path, _, _) }
108+
109+
module FindSourceModel = Util::FindModel<sourceModelPath/2>;
110+
111+
private predicate sinkModelPath(string type, string path) { sinkModel(type, path, _, _) }
112+
113+
module FindSinkModel = Util::FindModel<sinkModelPath/2>;
114+
115+
private predicate summaryModelPath(string type, string path) {
116+
summaryModel(type, path, _, _, _, _)
117+
}
118+
119+
module FindSummaryModel = Util::FindModel<summaryModelPath/2>;
120+
121+
private predicate neutralModelPath(string type, string path) { neutralModel(type, path, _) }
122+
123+
module FindNeutralModel = Util::FindModel<neutralModelPath/2>;
124+
125+
/**
126+
* A callable function or method from source code.
127+
*/
128+
class FunctionEndpoint extends Endpoint instanceof Function {
129+
/**
130+
* Gets the parameter types of this endpoint.
131+
*/
132+
override string getParameters() {
133+
// For now, return the names of positional and keyword parameters. We don't always have type information, so we can't return type names.
134+
// We don't yet handle splat params or dict splat params.
135+
//
136+
// In Python, there are three types of parameters:
137+
// 1. Positional-only parameters: These are parameters that can only be passed by position and not by keyword.
138+
// 2. Positional-or-keyword parameters: These are parameters that can be passed by position or by keyword.
139+
// 3. Keyword-only parameters: These are parameters that can only be passed by keyword.
140+
//
141+
// The syntax for defining these parameters is as follows:
142+
// ```python
143+
// def f(a, /, b, *, c):
144+
// pass
145+
// ```
146+
// In this example, `a` is a positional-only parameter, `b` is a positional-or-keyword parameter, and `c` is a keyword-only parameter.
147+
//
148+
// We handle positional-only parameters by adding a "/" to the parameter name, reminiscient of the syntax above.
149+
// Note that we don't yet have information about positional-only parameters.
150+
// We handle keyword-only parameters by adding a ":" to the parameter name, to be consistent with the MaD syntax and the other languages.
151+
exists(int nrPosOnly, Function f |
152+
f = this and
153+
nrPosOnly = f.getPositionalParameterCount()
154+
|
155+
result =
156+
"(" +
157+
concat(string key, string value |
158+
// TODO: Once we have information about positional-only parameters:
159+
// Handle positional-only parameters by adding a "/"
160+
value = any(int i | i.toString() = key | f.getArgName(i))
161+
or
162+
exists(Name param | param = f.getAKeywordOnlyArg() |
163+
param.getId() = key and
164+
value = key + ":"
165+
)
166+
|
167+
value, "," order by key
168+
) + ")"
169+
)
170+
}
171+
172+
/** Holds if this API has a supported summary. */
173+
pragma[nomagic]
174+
predicate hasSummary() { FindSummaryModel::hasModel(this) }
175+
176+
/** Holds if this API is a known source. */
177+
pragma[nomagic]
178+
predicate isSource() { FindSourceModel::hasModel(this) }
179+
180+
/** Holds if this API is a known sink. */
181+
pragma[nomagic]
182+
predicate isSink() { FindSinkModel::hasModel(this) }
183+
184+
/** Holds if this API is a known neutral. */
185+
pragma[nomagic]
186+
predicate isNeutral() { FindNeutralModel::hasModel(this) }
187+
188+
/**
189+
* Holds if this API is supported by existing CodeQL libraries, that is, it is either a
190+
* recognized source, sink or neutral or it has a flow summary.
191+
*/
192+
predicate isSupported() {
193+
this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
194+
}
195+
196+
override boolean getSupportedStatus() {
197+
if this.isSupported() then result = true else result = false
198+
}
199+
200+
override string getSupportedType() {
201+
this.isSink() and result = "sink"
202+
or
203+
this.isSource() and result = "source"
204+
or
205+
this.hasSummary() and result = "summary"
206+
or
207+
this.isNeutral() and result = "neutral"
208+
or
209+
not this.isSupported() and result = ""
210+
}
211+
212+
override EndpointKind getKind() {
213+
if this.(Function).isMethod()
214+
then
215+
result = this.methodKind()
216+
or
217+
not exists(this.methodKind()) and result = "InstanceMethod"
218+
else result = "Function"
219+
}
220+
221+
private EndpointKind methodKind() {
222+
this.(Function).isMethod() and
223+
(
224+
DP::isClassmethod(this) and result = "ClassMethod"
225+
or
226+
DP::isStaticmethod(this) and result = "StaticMethod"
227+
or
228+
this.(Function).isInitMethod() and result = "InitMethod"
229+
)
230+
}
231+
}
232+
233+
/**
234+
* A class from source code.
235+
*/
236+
class ClassEndpoint extends Endpoint instanceof Class {
237+
override string getType() { result = type + "." + name }
238+
239+
override string getName() { result = "" }
240+
241+
override string getParameters() { result = "" }
242+
243+
override boolean getSupportedStatus() { result = false }
244+
245+
override string getSupportedType() { result = "" }
246+
247+
override EndpointKind getKind() { result = "Class" }
248+
}

‎python/ql/lib/modeling/Util.qll

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/**
2+
* Contains utility methods and classes to assist with generating data extensions models.
3+
*/
4+
5+
private import python
6+
private import semmle.python.ApiGraphs
7+
8+
/**
9+
* A file that probably contains tests.
10+
*/
11+
class TestFile extends File {
12+
TestFile() {
13+
this.getRelativePath().regexpMatch(".*(test|spec|examples).+") and
14+
not this.getAbsolutePath().matches("%/ql/test/%") // allows our test cases to work
15+
}
16+
}
17+
18+
/**
19+
* A file that is relevant in the context of library modeling.
20+
*
21+
* In practice, this means a file that is not part of test code.
22+
*/
23+
class RelevantFile extends File {
24+
RelevantFile() { not this instanceof TestFile and not this.inStdlib() }
25+
}
26+
27+
/**
28+
* Gets the dotted path of a scope.
29+
*/
30+
string computeScopePath(Scope scope) {
31+
// base case
32+
if scope instanceof Module
33+
then
34+
scope.(Module).isPackageInit() and
35+
result = scope.(Module).getPackageName()
36+
or
37+
not scope.(Module).isPackageInit() and
38+
result = scope.(Module).getName()
39+
else
40+
//recursive cases
41+
if scope instanceof Class
42+
then
43+
result = computeScopePath(scope.(Class).getEnclosingScope()) + "." + scope.(Class).getName()
44+
else
45+
if scope instanceof Function
46+
then
47+
result =
48+
computeScopePath(scope.(Function).getEnclosingScope()) + "." + scope.(Function).getName()
49+
else result = "unknown: " + scope.toString()
50+
}
51+
52+
signature predicate modelSig(string type, string path);
53+
54+
/**
55+
* A utility module for finding models of endpoints.
56+
*
57+
* Chiefly the `hasModel` predicate is used to determine if a scope has a model.
58+
*/
59+
module FindModel<modelSig/2 model> {
60+
/**
61+
* Holds if the given scope has a model as identified by the provided predicate `model`.
62+
*/
63+
predicate hasModel(Scope scope) {
64+
exists(string type, string path, string searchPath | model(type, path) |
65+
searchPath = possibleMemberPathPrefix(path, scope.getName()) and
66+
pathToScope(scope, type, searchPath)
67+
)
68+
}
69+
70+
/**
71+
* returns the prefix of `path` that might be a path to `member`
72+
*/
73+
bindingset[path, member]
74+
string possibleMemberPathPrefix(string path, string member) {
75+
// functionName must be a substring of path
76+
exists(int index | index = path.indexOf(["Member", "Method"] + "[" + member + "]") |
77+
result = path.prefix(index)
78+
)
79+
}
80+
81+
/**
82+
* Holds if `(type,path)` evaluates to the given entity, when evalauted from a client of the current library.
83+
*/
84+
bindingset[type, path]
85+
predicate pathToScope(Scope scope, string type, string path) {
86+
scope.getLocation().getFile() instanceof RelevantFile and
87+
scope.isPublic() and // only public methods are modeled
88+
computeScopePath(scope) =
89+
type.replaceAll("!", "") + "." +
90+
path.replaceAll("Member[", "").replaceAll("]", "").replaceAll("Instance.", "") +
91+
scope.getName()
92+
}
93+
}

‎python/ql/src/utils/modeleditor/FrameworkModeEndpoints.ql

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/**
2+
* @name Fetch endpoints for use in the model editor (framework mode)
3+
* @description A list of endpoints accessible (methods and attributes) for consumers of the library. Excludes test and generated code.
4+
* @kind table
5+
* @id py/utils/modeleditor/framework-mode-endpoints
6+
* @tags modeleditor endpoints framework-mode
7+
*/
8+
9+
import modeling.ModelEditor
10+
11+
from Endpoint endpoint
12+
select endpoint, endpoint.getNamespace(), endpoint.getType(), endpoint.getName(),
13+
endpoint.getParameters(), endpoint.getSupportedStatus(), endpoint.getFileName(),
14+
endpoint.getSupportedType(), endpoint.getKind()

‎python/ql/test/modelling/FrameworkModeEndpoints.expected

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
| MyPackage/Foo.py:1:1:1:9 | Class C1 | MyPackage | Foo.C1 | | | false | Foo.py | | Class |
2+
| MyPackage/Foo.py:2:5:2:17 | Function m1 | MyPackage | Foo.C1 | m1 | (self) | true | Foo.py | source | InstanceMethod |
3+
| MyPackage/Foo.py:5:5:5:20 | Function m2 | MyPackage | Foo.C1 | m2 | (self,x) | true | Foo.py | source | InstanceMethod |
4+
| MyPackage/Foo.py:9:5:9:14 | Function m3 | MyPackage | Foo.C1 | m3 | (x) | true | Foo.py | summary | StaticMethod |
5+
| MyPackage/Foo.py:13:5:13:19 | Function m4 | MyPackage | Foo.C1 | m4 | (cls,x) | true | Foo.py | summary | ClassMethod |
6+
| MyPackage/Foo.py:16:1:16:13 | Class C2 | MyPackage | Foo.C2 | | | false | Foo.py | | Class |
7+
| MyPackage/Foo.py:17:5:17:17 | Function m1 | MyPackage | Foo.C2 | m1 | (self) | false | Foo.py | | InstanceMethod |
8+
| MyPackage/Foo.py:20:5:20:27 | Function c2only_m1 | MyPackage | Foo.C2 | c2only_m1 | (self,x) | false | Foo.py | | InstanceMethod |
9+
| MyPackage/Foo.py:23:1:23:9 | Class C3 | MyPackage | Foo.C3 | | | false | Foo.py | | Class |
10+
| MyPackage/Foo.py:24:5:24:26 | Function get_C2_instance | MyPackage | Foo.C3 | get_C2_instance | () | false | Foo.py | | InstanceMethod |
11+
| TopLevel.py:3:1:3:38 | Function top_level_funciton | TopLevel | | top_level_funciton | (x,y,z:) | false | TopLevel.py | | Function |

‎python/ql/test/modelling/FrameworkModeEndpoints.ext.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/python-all
4+
extensible: sourceModel
5+
data:
6+
- ["MyPackage.Foo.C1","Member[m1].ReturnValue","remote"]
7+
- ["MyPackage","Member[Foo].Member[C1].Instance.Member[m2].ReturnValue","remote"]
8+
9+
- addsTo:
10+
pack: codeql/python-all
11+
extensible: summaryModel
12+
data:
13+
- ["MyPackage.Foo.C1!","Member[m3]","Argument[0]","ReturnValue","value"]
14+
- ["MyPackage","Member[Foo].Member[C1].Member[m4]","Argument[0]","ReturnValue","value"]
15+
16+
- addsTo:
17+
pack: codeql/python-all
18+
extensible: typeModel
19+
data:
20+
- ["MyPackage.Foo.C2!","MyPackage","Member[Foo].Member[C3].Member[get_C2_instance].ReturnValue"]

‎python/ql/test/modelling/FrameworkModeEndpoints.qlref

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
utils/modeleditor/FrameworkModeEndpoints.ql

0 commit comments

Comments
 (0)