Skip to content

Commit dffb120

Browse files
MathiasVPMathiasVP
authored
Merge pull request github#12271 from github/tausbn/ql-fix-json-extraction
QL: Fix JSON extraction
2 parents b28f1a5 + a3a099c commit dffb120

File tree

7 files changed

+152
-34
lines changed

7 files changed

+152
-34
lines changed

‎ql/Cargo.lock

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎ql/extractor/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", re
1414
tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
1515
tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
1616
tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
17-
tree-sitter-json = {git = "https://github.com/tausbn/tree-sitter-json.git", rev = "471ceac44d127e609afa349cf0a59370791fe8b3"}
17+
tree-sitter-json = {git = "https://github.com/tausbn/tree-sitter-json.git", rev = "ea1f655604c32c2f76aad2abed2498a56d81f3a9"}
1818
clap = "2.33"
1919
tracing = "0.1"
2020
tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }

‎ql/extractor/src/main.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,9 @@ fn main() -> std::io::Result<()> {
108108
&& !line.ends_with(".dbscheme")
109109
&& !line.ends_with("qlpack.yml")
110110
&& !line.ends_with(".blame")
111+
&& !line.ends_with(".json")
112+
&& !line.ends_with(".jsonl")
113+
&& !line.ends_with(".jsonc")
111114
{
112115
return Ok(());
113116
}

‎ql/generator/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", re
1515
tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
1616
tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
1717
tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
18-
tree-sitter-json = { git = "https://github.com/tausbn/tree-sitter-json.git", rev = "471ceac44d127e609afa349cf0a59370791fe8b3"}
18+
tree-sitter-json = { git = "https://github.com/tausbn/tree-sitter-json.git", rev = "ea1f655604c32c2f76aad2abed2498a56d81f3a9"}

‎ql/ql/src/codeql_ql/StructuredLogs.qll

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
private import ql
2+
private import codeql_ql.ast.internal.TreeSitter
3+
4+
class Object extends JSON::Object {
5+
JSON::Value getValue(string key) {
6+
exists(JSON::Pair p |
7+
p = this.getChild(_) and p.getKey().(JSON::String).getChild().getValue() = key
8+
|
9+
result = p.getValue()
10+
)
11+
}
12+
13+
string getString(string key) { result = this.getValue(key).(JSON::String).getChild().getValue() }
14+
15+
int getNumber(string key) { result = this.getValue(key).(JSON::Number).getValue().toInt() }
16+
17+
Array getArray(string key) { result = this.getValue(key) }
18+
19+
Object getObject(string key) { result = this.getValue(key) }
20+
21+
string getType() { result = this.getString("type") }
22+
23+
int getEventId() { result = this.getNumber("event_id") }
24+
25+
string getTime() { result = this.getString("time") }
26+
}
27+
28+
class Array extends JSON::Array {
29+
Object getObject(int i) { result = this.getChild(i) }
30+
31+
string getString(int i) { result = this.getChild(i).(JSON::String).getChild().getValue() }
32+
33+
int getNumber(int i) { result = this.getChild(i).(JSON::Number).getValue().toInt() }
34+
35+
Array getArray(int i) { result = this.getChild(i) }
36+
}
37+
38+
abstract class LogEntry extends Object { }
39+
40+
class LogHeader extends LogEntry {
41+
LogHeader() { this.getType() = "LOG_HEADER" }
42+
43+
string getCodeQLVersion() { result = this.getString("codeqlVersion") }
44+
45+
string getLogVersion() { result = this.getString("logVersion") }
46+
}
47+
48+
class QueryStarted extends LogEntry {
49+
QueryStarted() { this.getType() = "QUERY_STARTED" }
50+
51+
string getQueryName() { result = this.getString("queryName") }
52+
53+
int getStage(int i) { result = this.getArray("stage").getNumber(i) }
54+
}
55+
56+
class PredicateStarted extends LogEntry {
57+
PredicateStarted() { this.getType() = "PREDICATE_STARTED" }
58+
59+
string getPredicateName() { result = this.getString("predicateName") }
60+
61+
string getPosition() { result = this.getString("position") }
62+
63+
string getPredicateType() { result = this.getString("predicateType") }
64+
65+
int getQueryCausingWork() { result = this.getNumber("queryCausingWork") }
66+
67+
string getRAHash() { result = this.getString("raHash") }
68+
69+
Object getRA() { result = this.getValue("ra") }
70+
71+
string getDependency(string key) { result = this.getObject("dependencies").getString(key) }
72+
}
73+
74+
class PipelineStarted extends LogEntry {
75+
PipelineStarted() { this.getType() = "PIPELINE_STARTED" }
76+
77+
int getPredicateStartEvent() { result = this.getNumber("predicateStartEvent") }
78+
79+
string getRAReference() { result = this.getString("raReference") }
80+
}
81+
82+
class PipelineCompleted extends LogEntry {
83+
PipelineCompleted() { this.getType() = "PIPELINE_COMPLETED" }
84+
85+
int getStartEvent() { result = this.getNumber("startEvent") }
86+
87+
string getRAReference() { result = this.getString("raReference") }
88+
89+
int getCount(int i) { result = this.getArray("counts").getNumber(i) }
90+
91+
int getDuplicationPercentage(int i) {
92+
result = this.getArray("duplicationPercentages").getNumber(i)
93+
}
94+
95+
int getResultSize() { result = this.getNumber("resultSize") }
96+
}
97+
98+
class PredicateCompleted extends LogEntry {
99+
PredicateCompleted() { this.getType() = "PREDICATE_COMPLETED" }
100+
101+
int getStartEvent() { result = this.getNumber("startEvent") }
102+
103+
int getResultSize() { result = this.getNumber("resultSize") }
104+
}
105+
106+
class QueryCompleted extends LogEntry {
107+
QueryCompleted() { this.getType() = "QUERY_COMPLETED" }
108+
109+
int getStartEvent() { result = this.getNumber("startEvent") }
110+
111+
string getTerminationType() { result = this.getString("terminationType") }
112+
}
113+
114+
class LogFooter extends LogEntry {
115+
LogFooter() { this.getType() = "LOG_FOOTER" }
116+
}
117+
118+
class CacheLookup extends LogEntry {
119+
CacheLookup() { this.getType() = "CACHE_LOOKUP" }
120+
121+
int getRelationSize() { result = this.getNumber("relationSize") }
122+
}
123+
124+
class SentinelEmpty extends LogEntry {
125+
SentinelEmpty() { this.getType() = "SENTINEL_EMPTY" }
126+
}
127+
128+
// Stuff to test whether we've covered all event types
129+
private File logFile() { result = any(LogHeader h).getLocation().getFile() }
130+
131+
private Object missing() {
132+
result =
133+
any(Object o |
134+
o.getLocation().getFile() = logFile() and
135+
not o instanceof LogEntry and
136+
not exists(o.getParent().getParent()) // don't count nested objects
137+
)
138+
}

‎ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1930,12 +1930,6 @@ module JSON {
19301930
final override AstNode getAFieldOrChild() { json_document_child(this, _, result) }
19311931
}
19321932

1933-
/** A class representing `escape_sequence` tokens. */
1934-
class EscapeSequence extends @json_token_escape_sequence, Token {
1935-
/** Gets the name of the primary QL class for this element. */
1936-
final override string getAPrimaryQlClass() { result = "EscapeSequence" }
1937-
}
1938-
19391933
/** A class representing `false` tokens. */
19401934
class False extends @json_token_false, Token {
19411935
/** Gets the name of the primary QL class for this element. */
@@ -1995,16 +1989,10 @@ module JSON {
19951989
final override AstNode getAFieldOrChild() { json_string_child(this, result) }
19961990
}
19971991

1998-
/** A class representing `string_content` nodes. */
1999-
class StringContent extends @json_string_content, AstNode {
1992+
/** A class representing `string_content` tokens. */
1993+
class StringContent extends @json_token_string_content, Token {
20001994
/** Gets the name of the primary QL class for this element. */
20011995
final override string getAPrimaryQlClass() { result = "StringContent" }
2002-
2003-
/** Gets the `i`th child of this node. */
2004-
final EscapeSequence getChild(int i) { json_string_content_child(this, i, result) }
2005-
2006-
/** Gets a field or child node of this node. */
2007-
final override AstNode getAFieldOrChild() { json_string_content_child(this, _, result) }
20081996
}
20091997

20101998
/** A class representing `true` tokens. */

‎ql/ql/src/ql.dbscheme

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1292,24 +1292,13 @@ json_pair_def(
12921292

12931293
json_string_child(
12941294
unique int json_string__: @json_string__ ref,
1295-
unique int child: @json_string_content ref
1295+
unique int child: @json_token_string_content ref
12961296
);
12971297

12981298
json_string_def(
12991299
unique int id: @json_string__
13001300
);
13011301

1302-
#keyset[json_string_content, index]
1303-
json_string_content_child(
1304-
int json_string_content: @json_string_content ref,
1305-
int index: int ref,
1306-
unique int child: @json_token_escape_sequence ref
1307-
);
1308-
1309-
json_string_content_def(
1310-
unique int id: @json_string_content
1311-
);
1312-
13131302
@json_value = @json_array | @json_object | @json_string__ | @json_token_false | @json_token_null | @json_token_number | @json_token_true
13141303

13151304
json_tokeninfo(
@@ -1321,15 +1310,15 @@ json_tokeninfo(
13211310
case @json_token.kind of
13221311
0 = @json_reserved_word
13231312
| 1 = @json_token_comment
1324-
| 2 = @json_token_escape_sequence
1325-
| 3 = @json_token_false
1326-
| 4 = @json_token_null
1327-
| 5 = @json_token_number
1313+
| 2 = @json_token_false
1314+
| 3 = @json_token_null
1315+
| 4 = @json_token_number
1316+
| 5 = @json_token_string_content
13281317
| 6 = @json_token_true
13291318
;
13301319

13311320

1332-
@json_ast_node = @json_array | @json_document | @json_object | @json_pair | @json_string__ | @json_string_content | @json_token
1321+
@json_ast_node = @json_array | @json_document | @json_object | @json_pair | @json_string__ | @json_token
13331322

13341323
@json_ast_node_parent = @file | @json_ast_node
13351324

0 commit comments

Comments
 (0)