Skip to content

Commit 6e05246

Browse files
yoffyoff
authored
Merge pull request github#13935 from yoff/python/mad-on-externals
Python: MaD on externals
2 parents 826b8e6 + ad49ead commit 6e05246

File tree

10 files changed

+117
-126
lines changed

10 files changed

+117
-126
lines changed

‎config/identical-files.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -556,5 +556,9 @@
556556
"EncryptionKeySizes Python/Java": [
557557
"python/ql/lib/semmle/python/security/internal/EncryptionKeySizes.qll",
558558
"java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
559+
],
560+
"Python model summaries test extension": [
561+
"python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
562+
"python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
559563
]
560564
}

‎python/ql/lib/qlpack.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,5 @@ dependencies:
1313
codeql/util: ${workspace}
1414
codeql/yaml: ${workspace}
1515
dataExtensions:
16-
- semmle/python/frameworks/**/model.yml
16+
- semmle/python/frameworks/**/*.model.yml
1717
warnOnImplicitThis: true

‎python/ql/lib/semmle/python/frameworks/data/internal/model.yml renamed to ‎python/ql/lib/semmle/python/frameworks/data/internal/empty.model.yml

File renamed without changes.

‎python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/python-all
4+
extensible: summaryModel
5+
data:
6+
- ["foo", "Member[MS_identity]", "Argument[0]", "ReturnValue", "value"]
7+
- ["foo", "Member[MS_apply_lambda]", "Argument[1]", "Argument[0].Parameter[0]", "value"]
8+
- ["foo", "Member[MS_apply_lambda]", "Argument[0].ReturnValue", "ReturnValue", "value"]
9+
- ["foo", "Member[MS_reversed]", "Argument[0].ListElement", "ReturnValue.ListElement", "value"]
10+
- ["foo", "Member[MS_reversed]", "Argument[0]", "ReturnValue", "taint"]
11+
- ["foo", "Member[MS_list_map]", "Argument[1].ListElement", "Argument[0].Parameter[0]", "value"]
12+
- ["foo", "Member[MS_list_map]", "Argument[0].ReturnValue", "ReturnValue.ListElement", "value"]
13+
- ["foo", "Member[MS_list_map]", "Argument[1]", "ReturnValue", "taint"]
14+
- ["foo", "Member[MS_append_to_list]", "Argument[0].ListElement", "ReturnValue.ListElement", "value"]
15+
- ["foo", "Member[MS_append_to_list]", "Argument[1]", "ReturnValue.ListElement", "value"]
16+
- ["foo", "Member[MS_append_to_list]", "Argument[0]", "ReturnValue", "taint"]
17+
- ["foo", "Member[MS_append_to_list]", "Argument[1]", "ReturnValue", "taint"]
18+
- ["json", "Member[MS_loads]", "Argument[0]", "ReturnValue", "taint"]

‎python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ql

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
import python
2-
private import TestSummaries
32
import experimental.meta.InlineTaintTest
43
import MakeInlineTaintTest<TestTaintTrackingConfig>

‎python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
extensions:
2+
- addsTo:
3+
pack: codeql/python-all
4+
extensible: summaryModel
5+
data:
6+
- ["foo", "Member[MS_identity]", "Argument[0]", "ReturnValue", "value"]
7+
- ["foo", "Member[MS_apply_lambda]", "Argument[1]", "Argument[0].Parameter[0]", "value"]
8+
- ["foo", "Member[MS_apply_lambda]", "Argument[0].ReturnValue", "ReturnValue", "value"]
9+
- ["foo", "Member[MS_reversed]", "Argument[0].ListElement", "ReturnValue.ListElement", "value"]
10+
- ["foo", "Member[MS_reversed]", "Argument[0]", "ReturnValue", "taint"]
11+
- ["foo", "Member[MS_list_map]", "Argument[1].ListElement", "Argument[0].Parameter[0]", "value"]
12+
- ["foo", "Member[MS_list_map]", "Argument[0].ReturnValue", "ReturnValue.ListElement", "value"]
13+
- ["foo", "Member[MS_list_map]", "Argument[1]", "ReturnValue", "taint"]
14+
- ["foo", "Member[MS_append_to_list]", "Argument[0].ListElement", "ReturnValue.ListElement", "value"]
15+
- ["foo", "Member[MS_append_to_list]", "Argument[1]", "ReturnValue.ListElement", "value"]
16+
- ["foo", "Member[MS_append_to_list]", "Argument[0]", "ReturnValue", "taint"]
17+
- ["foo", "Member[MS_append_to_list]", "Argument[1]", "ReturnValue", "taint"]
18+
- ["json", "Member[MS_loads]", "Argument[0]", "ReturnValue", "taint"]

‎python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
11
import python
2-
private import TestSummaries
32
import experimental.dataflow.TestUtil.NormalDataflowTest

‎python/ql/test/experimental/dataflow/model-summaries/TestSummaries.qll

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎python/ql/test/library-tests/frameworks/data/test.ext.yml

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
extensions:
2+
# Contribute empty data sets to avoid errors about an undefined extensionals
3+
- addsTo:
4+
pack: codeql/python-all
5+
extensible: sourceModel
6+
data:
7+
- ["testlib", "Member[getSource].ReturnValue", "test-source"]
8+
- ["testlib.Alias", "", "test-source"]
9+
# testing parameter syntax
10+
- ["testlib", "Member[Callbacks].Member[first].Argument[0].Parameter[0]", "test-source"]
11+
- ["testlib", "Member[Callbacks].Member[param1to3].Argument[0].Parameter[1..3]", "test-source"]
12+
- ["testlib", "Member[Callbacks].Member[nonFirst].Argument[0].Parameter[1..]", "test-source"]
13+
# Common tokens.
14+
- ["testlib", "Member[CommonTokens].Member[makePromise].ReturnValue.Awaited", "test-source"]
15+
- ["testlib", "Member[CommonTokens].Member[Class].Instance", "test-source"]
16+
- ["testlib", "Member[CommonTokens].Member[Super].Subclass.Instance", "test-source"]
17+
# method
18+
- ["testlib", "Member[CommonTokens].Member[Class].Instance.Method[foo]", "test-source"]
19+
# testing non-positional arguments
20+
- ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[foo].Parameter[self]", "test-source"]
21+
- ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[foo].Parameter[named:]", "test-source"]
22+
- ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[secondAndAfter].Parameter[1..]", "test-source"]
23+
- ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[otherSelfTest].Parameter[0]", "test-source"]
24+
- ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[anyParam].Parameter[any]", "test-source"]
25+
- ["testlib", "Member[ArgPos].Member[MyClass].Subclass.Member[anyNamed].Parameter[any-named]", "test-source"]
26+
27+
- addsTo:
28+
pack: codeql/python-all
29+
extensible: sinkModel
30+
data:
31+
- ["testlib", "Member[mySink].Argument[0,sinkName:]", "test-sink"]
32+
# testing argument syntax
33+
- ["testlib", "Member[Args].Member[arg0].Argument[0]", "test-sink"]
34+
- ["testlib", "Member[Args].Member[arg1to3].Argument[1..3]", "test-sink"]
35+
- ["testlib", "Member[Args].Member[lastarg].Argument[N-1]", "test-sink"]
36+
- ["testlib", "Member[Args].Member[nonFist].Argument[1..]", "test-sink"]
37+
# callsite filter.
38+
- ["testlib", "Member[CallFilter].Member[arityOne].WithArity[1].Argument[any]", "test-sink"]
39+
- ["testlib", "Member[CallFilter].Member[twoOrMore].WithArity[2..].Argument[0..]", "test-sink"]
40+
# testing non-positional arguments
41+
- ["testlib", "Member[ArgPos].Instance.Member[self_thing].Argument[self]", "test-sink"]
42+
# any argument
43+
- ["testlib", "Member[ArgPos].Member[anyParam].Argument[any]", "test-sink"]
44+
- ["testlib", "Member[ArgPos].Member[anyNamed].Argument[any-named]", "test-sink"]
45+
# testing package syntax
46+
- ["foo1.bar", "Member[baz1].Argument[any]", "test-sink"]
47+
- ["foo2", "Member[bar].Member[baz2].Argument[any]", "test-sink"]
48+
# testing fuzzy
49+
- ["testlib", "Fuzzy.Member[fuzzyCall].Argument[0]", "test-sink"]
50+
# testing syntax errors
51+
- ["testlib", "Member[foo],Member[bar]", "test-sink"]
52+
- ["testlib", "Member[foo] Member[bar]", "test-sink"]
53+
- ["testlib", "Member[foo]. Member[bar]", "test-sink"]
54+
- ["testlib", "Member[foo], Member[bar]", "test-sink"]
55+
- ["testlib", "Member[foo]..Member[bar]", "test-sink"]
56+
- ["testlib", "Member[foo] .Member[bar]", "test-sink"]
57+
- ["testlib", "Member[foo]Member[bar]", "test-sink"]
58+
- ["testlib", "Member[foo", "test-sink"]
59+
- ["testlib", "Member[foo]]", "test-sink"]
60+
- ["testlib", "Member[foo]].Member[bar]", "test-sink"]
61+
62+
- addsTo:
63+
pack: codeql/python-all
64+
extensible: summaryModel
65+
data:
66+
- ["testlib", "Member[Steps].Member[preserveTaint].Call", "Argument[0]", "ReturnValue", "taint"]
67+
- ["testlib", "Member[Steps].Member[taintIntoCallback]", "Argument[0]", "Argument[1..2].Parameter[0]", "taint"]
68+
- ["testlib", "Member[Steps].Member[preserveArgZeroAndTwo]", "Argument[0,2]", "ReturnValue", "taint"]
69+
- ["testlib", "Member[Steps].Member[preserveAllButFirstArgument].Call", "Argument[1..]", "ReturnValue", "taint"]
70+
71+
- addsTo:
72+
pack: codeql/python-all
73+
extensible: typeModel
74+
data:
75+
- ["testlib.Alias", "testlib", "Member[alias].ReturnValue"]
76+
- ["testlib.Alias", "testlib.Alias", "Member[chain].ReturnValue"]

‎python/ql/test/library-tests/frameworks/data/test.ql

Lines changed: 0 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -5,86 +5,6 @@ import semmle.python.dataflow.new.TaintTracking
55
import semmle.python.dataflow.new.DataFlow
66
private import semmle.python.ApiGraphs
77

8-
class Steps extends ModelInput::SummaryModelCsv {
9-
override predicate row(string row) {
10-
// type;path;input;output;kind
11-
row =
12-
[
13-
"testlib;Member[Steps].Member[preserveTaint].Call;Argument[0];ReturnValue;taint",
14-
"testlib;Member[Steps].Member[taintIntoCallback];Argument[0];Argument[1..2].Parameter[0];taint",
15-
"testlib;Member[Steps].Member[preserveArgZeroAndTwo];Argument[0,2];ReturnValue;taint",
16-
"testlib;Member[Steps].Member[preserveAllButFirstArgument].Call;Argument[1..];ReturnValue;taint",
17-
]
18-
}
19-
}
20-
21-
class Types extends ModelInput::TypeModelCsv {
22-
override predicate row(string row) {
23-
// type1;type2;path
24-
row =
25-
[
26-
"testlib.Alias;testlib;Member[alias].ReturnValue",
27-
"testlib.Alias;testlib.Alias;Member[chain].ReturnValue",
28-
]
29-
}
30-
}
31-
32-
class Sinks extends ModelInput::SinkModelCsv {
33-
override predicate row(string row) {
34-
// type;path;kind
35-
row =
36-
[
37-
"testlib;Member[mySink].Argument[0,sinkName:];test-sink",
38-
// testing argument syntax
39-
"testlib;Member[Args].Member[arg0].Argument[0];test-sink", //
40-
"testlib;Member[Args].Member[arg1to3].Argument[1..3];test-sink", //
41-
"testlib;Member[Args].Member[lastarg].Argument[N-1];test-sink", //
42-
"testlib;Member[Args].Member[nonFist].Argument[1..];test-sink", //
43-
// callsite filter.
44-
"testlib;Member[CallFilter].Member[arityOne].WithArity[1].Argument[any];test-sink", //
45-
"testlib;Member[CallFilter].Member[twoOrMore].WithArity[2..].Argument[0..];test-sink", //
46-
// testing non-positional arguments
47-
"testlib;Member[ArgPos].Instance.Member[self_thing].Argument[self];test-sink", //
48-
// any argument
49-
"testlib;Member[ArgPos].Member[anyParam].Argument[any];test-sink", //
50-
"testlib;Member[ArgPos].Member[anyNamed].Argument[any-named];test-sink", //
51-
// testing package syntax
52-
"foo1.bar;Member[baz1].Argument[any];test-sink", //
53-
"foo2;Member[bar].Member[baz2].Argument[any];test-sink", //
54-
// testing fuzzy
55-
"testlib;Fuzzy.Member[fuzzyCall].Argument[0];test-sink", //
56-
]
57-
}
58-
}
59-
60-
class Sources extends ModelInput::SourceModelCsv {
61-
// type;path;kind
62-
override predicate row(string row) {
63-
row =
64-
[
65-
"testlib;Member[getSource].ReturnValue;test-source", //
66-
"testlib.Alias;;test-source",
67-
// testing parameter syntax
68-
"testlib;Member[Callbacks].Member[first].Argument[0].Parameter[0];test-source", //
69-
"testlib;Member[Callbacks].Member[param1to3].Argument[0].Parameter[1..3];test-source", //
70-
"testlib;Member[Callbacks].Member[nonFirst].Argument[0].Parameter[1..];test-source", //
71-
// Common tokens.
72-
"testlib;Member[CommonTokens].Member[makePromise].ReturnValue.Awaited;test-source", //
73-
"testlib;Member[CommonTokens].Member[Class].Instance;test-source", //
74-
"testlib;Member[CommonTokens].Member[Super].Subclass.Instance;test-source", //
75-
// method
76-
"testlib;Member[CommonTokens].Member[Class].Instance.Method[foo];test-source", //
77-
// testing non-positional arguments
78-
"testlib;Member[ArgPos].Member[MyClass].Subclass.Member[foo].Parameter[self];test-source", //
79-
"testlib;Member[ArgPos].Member[MyClass].Subclass.Member[foo].Parameter[named:];test-source", //
80-
"testlib;Member[ArgPos].Member[MyClass].Subclass.Member[secondAndAfter].Parameter[1..];test-source", //
81-
"testlib;Member[ArgPos].Member[MyClass].Subclass.Member[otherSelfTest].Parameter[0];test-source", //
82-
"testlib;Member[ArgPos].Member[MyClass].Subclass.Member[anyParam].Parameter[any];test-source", //
83-
"testlib;Member[ArgPos].Member[MyClass].Subclass.Member[anyNamed].Parameter[any-named];test-source", //
84-
]
85-
}
86-
}
87-
888
class BasicTaintTracking extends TaintTracking::Configuration {
899
BasicTaintTracking() { this = "BasicTaintTracking" }
9010

@@ -109,24 +29,6 @@ query predicate isSource(DataFlow::Node node, string kind) {
10929
node = ModelOutput::getASourceNode(kind).asSource()
11030
}
11131

112-
class SyntaxErrorTest extends ModelInput::SinkModelCsv {
113-
override predicate row(string row) {
114-
row =
115-
[
116-
"testlib;Member[foo],Member[bar];test-sink", //
117-
"testlib;Member[foo] Member[bar];test-sink", //
118-
"testlib;Member[foo]. Member[bar];test-sink", //
119-
"testlib;Member[foo], Member[bar];test-sink", //
120-
"testlib;Member[foo]..Member[bar];test-sink", //
121-
"testlib;Member[foo] .Member[bar];test-sink", //
122-
"testlib;Member[foo]Member[bar];test-sink", //
123-
"testlib;Member[foo;test-sink", //
124-
"testlib;Member[foo]];test-sink", //
125-
"testlib;Member[foo]].Member[bar];test-sink", //
126-
]
127-
}
128-
}
129-
13032
query predicate syntaxErrors(AccessPathSyntax::AccessPath path) { path.hasSyntaxError() }
13133

13234
query predicate warning = ModelOutput::getAWarning/0;

0 commit comments

Comments
 (0)