Skip to content

Commit ce177c3

Browse files
RasmusWLRasmusWL
authored
Merge pull request github#15655 from yoff/python/support-model-editor
Python: Support model editor
2 parents 8e8100f + dc33f0d commit ce177c3

18 files changed

+468
-1
lines changed

‎python/ql/lib/modeling/ModelEditor.qll

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
/** Provides classes and predicates related to handling APIs for the VS Code extension. */
2+
3+
private import python
4+
private import semmle.python.frameworks.data.ModelsAsData
5+
private import semmle.python.frameworks.data.internal.ApiGraphModelsExtensions
6+
private import semmle.python.dataflow.new.internal.DataFlowDispatch as DP
7+
private import Util as Util
8+
9+
/**
10+
* An string describing the kind of source code element being modeled.
11+
*
12+
* See `EndPoint`.
13+
*/
14+
class EndpointKind extends string {
15+
EndpointKind() {
16+
this in ["Function", "InstanceMethod", "ClassMethod", "StaticMethod", "InitMethod", "Class"]
17+
}
18+
}
19+
20+
/**
21+
* An element of the source code to be modeled.
22+
*
23+
* See `EndPointKind` for the possible kinds of elements.
24+
*/
25+
abstract class Endpoint instanceof Util::RelevantScope {
26+
string namespace;
27+
string type;
28+
string name;
29+
30+
Endpoint() {
31+
exists(string scopePath, string path, int pathIndex |
32+
scopePath = Util::computeScopePath(this) and
33+
pathIndex = scopePath.indexOf(".", 0, 0)
34+
|
35+
namespace = scopePath.prefix(pathIndex) and
36+
path = scopePath.suffix(pathIndex + 1) and
37+
(
38+
exists(int nameIndex | nameIndex = max(path.indexOf(".")) |
39+
type = path.prefix(nameIndex) and
40+
name = path.suffix(nameIndex + 1)
41+
)
42+
or
43+
not exists(path.indexOf(".")) and
44+
type = "" and
45+
name = path
46+
)
47+
)
48+
}
49+
50+
/** Gets the namespace for this endpoint. This will typically be the package in which it is found. */
51+
string getNamespace() { result = namespace }
52+
53+
/** Gets hte basename of the file where this endpoint is found. */
54+
string getFileName() { result = super.getLocation().getFile().getBaseName() }
55+
56+
/** Gets a string representation of this endpoint. */
57+
string toString() { result = super.toString() }
58+
59+
/** Gets the location of this endpoint. */
60+
Location getLocation() { result = super.getLocation() }
61+
62+
/** Gets the name of the class in which this endpoint is found, or the empty string if it is not found inside a class. */
63+
string getClass() { result = type }
64+
65+
/**
66+
* Gets the name of the endpoint if it is not a class, or the empty string if it is a class
67+
*
68+
* If this endpoint is a class, the class name can be obtained via `getType`.
69+
*/
70+
string getFunctionName() { result = name }
71+
72+
/**
73+
* Gets a string representation of the parameters of this endpoint.
74+
*
75+
* The string follows a specific format:
76+
* - Normal parameters(where arguments can be passed as either positional or keyword) are listed in order, separated by commas.
77+
* - Keyword-only parameters are listed in order, separated by commas, each followed by a colon.
78+
* - In the future, positional-only parameters will be listed in order, separated by commas, each followed by a slash.
79+
*/
80+
abstract string getParameters();
81+
82+
/**
83+
* Gets a boolean that is true iff this endpoint is supported by existing modeling.
84+
*
85+
* The check only takes Models as Data extension models into account.
86+
*/
87+
abstract boolean getSupportedStatus();
88+
89+
/**
90+
* Gets a string that describes the type of support detected this endpoint.
91+
*
92+
* The string can be one of the following:
93+
* - "source" if this endpoint is a known source.
94+
* - "sink" if this endpoint is a known sink.
95+
* - "summary" if this endpoint has a flow summary.
96+
* - "neutral" if this endpoint is a known neutral.
97+
* - "" if this endpoint is not detected as supported.
98+
*/
99+
abstract string getSupportedType();
100+
101+
/** Gets the kind of this endpoint. See `EndPointKind`. */
102+
abstract EndpointKind getKind();
103+
}
104+
105+
private predicate sourceModelPath(string type, string path) { sourceModel(type, path, _, _) }
106+
107+
module FindSourceModel = Util::FindModel<sourceModelPath/2>;
108+
109+
private predicate sinkModelPath(string type, string path) { sinkModel(type, path, _, _) }
110+
111+
module FindSinkModel = Util::FindModel<sinkModelPath/2>;
112+
113+
private predicate summaryModelPath(string type, string path) {
114+
summaryModel(type, path, _, _, _, _)
115+
}
116+
117+
module FindSummaryModel = Util::FindModel<summaryModelPath/2>;
118+
119+
private predicate neutralModelPath(string type, string path) { neutralModel(type, path, _) }
120+
121+
module FindNeutralModel = Util::FindModel<neutralModelPath/2>;
122+
123+
/**
124+
* A callable function or method from source code.
125+
*/
126+
class FunctionEndpoint extends Endpoint instanceof Function {
127+
/**
128+
* Gets the parameter types of this endpoint.
129+
*/
130+
override string getParameters() {
131+
// For now, return the names of positional and keyword parameters. We don't always have type information, so we can't return type names.
132+
// We don't yet handle splat params or dict splat params.
133+
//
134+
// In Python, there are three types of parameters:
135+
// 1. Positional-only parameters: These are parameters that can only be passed by position and not by keyword.
136+
// 2. Positional-or-keyword parameters: These are parameters that can be passed by position or by keyword.
137+
// 3. Keyword-only parameters: These are parameters that can only be passed by keyword.
138+
//
139+
// The syntax for defining these parameters is as follows:
140+
// ```python
141+
// def f(a, /, b, *, c):
142+
// pass
143+
// ```
144+
// In this example, `a` is a positional-only parameter, `b` is a positional-or-keyword parameter, and `c` is a keyword-only parameter.
145+
//
146+
// We handle positional-only parameters by adding a "/" to the parameter name, reminiscient of the syntax above.
147+
// Note that we don't yet have information about positional-only parameters.
148+
// We handle keyword-only parameters by adding a ":" to the parameter name, to be consistent with the MaD syntax and the other languages.
149+
exists(int nrPosOnly, Function f |
150+
f = this and
151+
nrPosOnly = f.getPositionalParameterCount()
152+
|
153+
result =
154+
"(" +
155+
concat(string key, string value |
156+
// TODO: Once we have information about positional-only parameters:
157+
// Handle positional-only parameters by adding a "/"
158+
value = any(int i | i.toString() = key | f.getArgName(i))
159+
or
160+
exists(Name param | param = f.getAKeywordOnlyArg() |
161+
param.getId() = key and
162+
value = key + ":"
163+
)
164+
|
165+
value, "," order by key
166+
) + ")"
167+
)
168+
}
169+
170+
/** Holds if this API has a supported summary. */
171+
pragma[nomagic]
172+
predicate hasSummary() { FindSummaryModel::hasModel(this) }
173+
174+
/** Holds if this API is a known source. */
175+
pragma[nomagic]
176+
predicate isSource() { FindSourceModel::hasModel(this) }
177+
178+
/** Holds if this API is a known sink. */
179+
pragma[nomagic]
180+
predicate isSink() { FindSinkModel::hasModel(this) }
181+
182+
/** Holds if this API is a known neutral. */
183+
pragma[nomagic]
184+
predicate isNeutral() { FindNeutralModel::hasModel(this) }
185+
186+
/**
187+
* Holds if this API is supported by existing CodeQL libraries, that is, it is either a
188+
* recognized source, sink or neutral or it has a flow summary.
189+
*/
190+
predicate isSupported() {
191+
this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
192+
}
193+
194+
override boolean getSupportedStatus() {
195+
if this.isSupported() then result = true else result = false
196+
}
197+
198+
override string getSupportedType() {
199+
this.isSink() and result = "sink"
200+
or
201+
this.isSource() and result = "source"
202+
or
203+
this.hasSummary() and result = "summary"
204+
or
205+
this.isNeutral() and result = "neutral"
206+
or
207+
not this.isSupported() and result = ""
208+
}
209+
210+
override EndpointKind getKind() {
211+
if this.(Function).isMethod()
212+
then
213+
result = this.methodKind()
214+
or
215+
not exists(this.methodKind()) and result = "InstanceMethod"
216+
else result = "Function"
217+
}
218+
219+
private EndpointKind methodKind() {
220+
this.(Function).isMethod() and
221+
(
222+
DP::isClassmethod(this) and result = "ClassMethod"
223+
or
224+
DP::isStaticmethod(this) and result = "StaticMethod"
225+
or
226+
this.(Function).isInitMethod() and result = "InitMethod"
227+
)
228+
}
229+
}
230+
231+
/**
232+
* A class from source code.
233+
*/
234+
class ClassEndpoint extends Endpoint instanceof Class {
235+
override string getClass() { result = type + "." + name }
236+
237+
override string getFunctionName() { result = "" }
238+
239+
override string getParameters() { result = "" }
240+
241+
override boolean getSupportedStatus() { result = false }
242+
243+
override string getSupportedType() { result = "" }
244+
245+
override EndpointKind getKind() { result = "Class" }
246+
}

‎python/ql/lib/modeling/Util.qll

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/**
2+
* Contains utility methods and classes to assist with generating data extensions models.
3+
*/
4+
5+
private import python
6+
private import semmle.python.ApiGraphs
7+
private import semmle.python.filters.Tests
8+
9+
/** A class to represent scopes that the user might want to model. */
10+
class RelevantScope extends Scope {
11+
RelevantScope() {
12+
this.isPublic() and
13+
not this instanceof TestScope and
14+
exists(this.getLocation().getFile().getRelativePath())
15+
}
16+
}
17+
18+
/**
19+
* Gets the dotted path of a scope.
20+
*/
21+
string computeScopePath(RelevantScope scope) {
22+
// base case
23+
if scope instanceof Module
24+
then
25+
scope.(Module).isPackageInit() and
26+
result = scope.(Module).getPackageName()
27+
or
28+
not scope.(Module).isPackageInit() and
29+
result = scope.(Module).getName()
30+
else
31+
//recursive cases
32+
if scope instanceof Class or scope instanceof Function
33+
then result = computeScopePath(scope.getEnclosingScope()) + "." + scope.getName()
34+
else result = "unknown: " + scope.toString()
35+
}
36+
37+
signature predicate modelSig(string type, string path);
38+
39+
/**
40+
* A utility module for finding models of endpoints.
41+
*
42+
* Chiefly the `hasModel` predicate is used to determine if a scope has a model.
43+
*/
44+
module FindModel<modelSig/2 model> {
45+
/**
46+
* Holds if the given scope has a model as identified by the provided predicate `model`.
47+
*/
48+
predicate hasModel(RelevantScope scope) {
49+
exists(string type, string path, string searchPath | model(type, path) |
50+
searchPath = possibleMemberPathPrefix(path, scope.getName()) and
51+
pathToScope(scope, type, searchPath)
52+
)
53+
}
54+
55+
/**
56+
* returns the prefix of `path` that might be a path to `member`
57+
*/
58+
bindingset[path, member]
59+
string possibleMemberPathPrefix(string path, string member) {
60+
exists(int index | index = path.indexOf(["Member", "Method"] + "[" + member + "]") |
61+
result = path.prefix(index)
62+
)
63+
}
64+
65+
/**
66+
* Holds if `(type,path)` identifies `scope`.
67+
*/
68+
bindingset[type, path]
69+
predicate pathToScope(RelevantScope scope, string type, string path) {
70+
computeScopePath(scope) =
71+
type.replaceAll("!", "") + "." +
72+
path.replaceAll("Member[", "").replaceAll("]", "").replaceAll("Instance.", "") +
73+
scope.getName()
74+
}
75+
}

‎python/ql/lib/semmle/python/Scope.qll

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,10 @@ class Scope extends Scope_ {
8585
this instanceof Module
8686
or
8787
exists(Module m | m = this.getEnclosingScope() and m.isPublic() |
88-
/* If the module has an __all__, is this in it */
88+
// The module is implicitly exported
8989
not exists(getAModuleExport(m))
9090
or
91+
// The module is explicitly exported
9192
getAModuleExport(m) = this.getName()
9293
)
9394
or

‎python/ql/src/utils/modeleditor/FrameworkModeEndpoints.ql

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/**
2+
* @name Fetch endpoints for use in the model editor (framework mode)
3+
* @description A list of endpoints accessible (methods and attributes) for consumers of the library. Excludes test and generated code.
4+
* @kind table
5+
* @id py/utils/modeleditor/framework-mode-endpoints
6+
* @tags modeleditor endpoints framework-mode
7+
*/
8+
9+
import modeling.ModelEditor
10+
11+
from Endpoint endpoint
12+
select endpoint, endpoint.getNamespace(), endpoint.getClass(), endpoint.getFunctionName(),
13+
endpoint.getParameters(), endpoint.getSupportedStatus(), endpoint.getFileName(),
14+
endpoint.getSupportedType(), endpoint.getKind()

‎python/ql/test/modelling/FrameworkModeEndpoints.expected

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
| MyPackage/Foo.py:1:1:1:9 | Class C1 | MyPackage | Foo.C1 | | | false | Foo.py | | Class |
2+
| MyPackage/Foo.py:2:5:2:17 | Function m1 | MyPackage | Foo.C1 | m1 | (self) | true | Foo.py | source | InstanceMethod |
3+
| MyPackage/Foo.py:5:5:5:20 | Function m2 | MyPackage | Foo.C1 | m2 | (self,x) | true | Foo.py | source | InstanceMethod |
4+
| MyPackage/Foo.py:9:5:9:14 | Function m3 | MyPackage | Foo.C1 | m3 | (x) | true | Foo.py | summary | StaticMethod |
5+
| MyPackage/Foo.py:13:5:13:19 | Function m4 | MyPackage | Foo.C1 | m4 | (cls,x) | true | Foo.py | summary | ClassMethod |
6+
| MyPackage/Foo.py:16:1:16:13 | Class C2 | MyPackage | Foo.C2 | | | false | Foo.py | | Class |
7+
| MyPackage/Foo.py:17:5:17:17 | Function m1 | MyPackage | Foo.C2 | m1 | (self) | false | Foo.py | | InstanceMethod |
8+
| MyPackage/Foo.py:20:5:20:27 | Function c2only_m1 | MyPackage | Foo.C2 | c2only_m1 | (self,x) | false | Foo.py | | InstanceMethod |
9+
| MyPackage/Foo.py:23:1:23:9 | Class C3 | MyPackage | Foo.C3 | | | false | Foo.py | | Class |
10+
| MyPackage/Foo.py:24:5:24:26 | Function get_C2_instance | MyPackage | Foo.C3 | get_C2_instance | () | false | Foo.py | | InstanceMethod |
11+
| MyPackage/Foo.py:31:1:31:38 | Function top_level_function | MyPackage | Foo | top_level_function | (x,y,z:) | false | Foo.py | | Function |
12+
| MyPackage/Foo.py:34:1:34:42 | Function func_with_fancy_args | MyPackage | Foo | func_with_fancy_args | () | false | Foo.py | | Function |
13+
| MyPackage/ModuleWithAll.py:2:1:2:10 | Class Foo | MyPackage | ModuleWithAll.Foo | | | false | ModuleWithAll.py | | Class |
14+
| MyPackage/ModuleWithAll.py:3:1:3:10 | Class Bar | MyPackage | ModuleWithAll.Bar | | | false | ModuleWithAll.py | | Class |
15+
| NotPackage/not_in_pacakge_lib.py:1:1:1:34 | Function not_in_pacakge_lib_func | NotPackage | | not_in_pacakge_lib_func | (x,y) | false | not_in_pacakge_lib.py | | Function |
16+
| NotPackage/not_in_pacakge_lib.py:1:1:1:34 | Function not_in_pacakge_lib_func | NotPackage | not_in_pacakge_lib | not_in_pacakge_lib_func | (x,y) | false | not_in_pacakge_lib.py | | Function |
17+
| NotPackage/not_in_pacakge_lib.py:1:1:1:34 | Function not_in_pacakge_lib_func | not_in_pacakge_lib | | not_in_pacakge_lib_func | (x,y) | false | not_in_pacakge_lib.py | | Function |
18+
| NotPackage/not_in_pacakge_lib.py:1:1:1:34 | Function not_in_pacakge_lib_func | not_in_pacakge_lib | not_in_pacakge_lib | not_in_pacakge_lib_func | (x,y) | false | not_in_pacakge_lib.py | | Function |
19+
| NotPackage/not_in_package_script.py:5:1:5:37 | Function not_in_package_script_func | NotPackage | not_in_package_script | not_in_package_script_func | (x,y) | false | not_in_package_script.py | | Function |
20+
| NotPackage/possibly_lib.py:4:1:4:28 | Function possibly_lib_func | NotPackage | possibly_lib | possibly_lib_func | (x,y) | false | possibly_lib.py | | Function |
21+
| TopLevel.py:3:1:3:38 | Function top_level_function | TopLevel | | top_level_function | (x,y,z:) | false | TopLevel.py | | Function |
22+
| not-valid-package/not_in_pacakge_lib_copy.py:1:1:1:34 | Function not_in_pacakge_lib_func | not_in_pacakge_lib_copy | | not_in_pacakge_lib_func | (x,y) | false | not_in_pacakge_lib_copy.py | | Function |

‎python/ql/test/modelling/FrameworkModeEndpoints.ext.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/python-all
4+
extensible: sourceModel
5+
data:
6+
# Test short form of type column
7+
- ["MyPackage.Foo.C1","Member[m1].ReturnValue","remote"]
8+
# Test long form of type column
9+
- ["MyPackage","Member[Foo].Member[C1].Instance.Member[m2].ReturnValue","remote"]
10+
11+
- addsTo:
12+
pack: codeql/python-all
13+
extensible: summaryModel
14+
data:
15+
# Test short form of type column
16+
- ["MyPackage.Foo.C1!","Member[m3]","Argument[0]","ReturnValue","value"]
17+
# Test long form of type column
18+
- ["MyPackage","Member[Foo].Member[C1].Member[m4]","Argument[0]","ReturnValue","value"]
19+
20+
- addsTo:
21+
pack: codeql/python-all
22+
extensible: typeModel
23+
data:
24+
- ["MyPackage.Foo.C2","MyPackage","Member[Foo].Member[C3].Member[get_C2_instance].ReturnValue"]

0 commit comments

Comments
 (0)