Skip to content

Commit 01241cf

Browse files
bradlarsenGlyphack
bradlarsen
authored and
Glyphack
committed
Add a fuzzer for Py_CompileStringExFlags (python#111721)
1 parent 6178924 commit 01241cf

File tree

10 files changed

+265
-0
lines changed

10 files changed

+265
-0
lines changed

‎Modules/_xxtestfuzz/dictionaries/fuzz_pycompile.dict‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
# bits of syntax
2+
"( "
3+
") "
4+
"[ "
5+
"] "
6+
": "
7+
", "
8+
"; "
9+
"{ "
10+
"} "
11+
12+
# operators
13+
"+ "
14+
"- "
15+
"* "
16+
"** "
17+
"/ "
18+
"// "
19+
"| "
20+
"& "
21+
"< "
22+
"> "
23+
"= "
24+
". "
25+
"% "
26+
"` "
27+
"^ "
28+
"~ "
29+
"@ "
30+
"== "
31+
"!= "
32+
"<> "
33+
"<< "
34+
"<= "
35+
">= "
36+
">> "
37+
"+= "
38+
"-= "
39+
"*= "
40+
"** "
41+
"/= "
42+
"//= "
43+
"|= "
44+
"%= "
45+
"&= "
46+
"^= "
47+
"<<= "
48+
">>= "
49+
"**= "
50+
":= "
51+
"@= "
52+
53+
# whitespace
54+
" "
55+
":\\n "
56+
57+
# type signatures and functions
58+
"-> "
59+
": List[int]"
60+
": Dict[int, str]"
61+
62+
"# type:"
63+
"# type: List[int]"
64+
"# type: Dict[int, str]"
65+
66+
", *"
67+
", /"
68+
", *args"
69+
", **kwargs"
70+
", x=42"
71+
72+
73+
# literals
74+
"0x0a"
75+
"0b0000"
76+
"42"
77+
"0o70"
78+
"42j"
79+
"42.01"
80+
"-5"
81+
"+42e-3"
82+
"0_0_0"
83+
"1e1_0"
84+
".1_4"
85+
86+
"{}"
87+
88+
# variable names
89+
"x"
90+
"y"
91+
92+
# strings
93+
"r'x'"
94+
95+
"b'x'"
96+
97+
"rb\"x\""
98+
99+
"br\"x\""
100+
101+
"f'{x + 5}'"
102+
"f\"{x + 5}\""
103+
104+
"'''"
105+
"\"\"\""
106+
107+
"\\u"
108+
"\\x"
109+
110+
# keywords
111+
"def "
112+
"del "
113+
"pass "
114+
"break "
115+
"continue "
116+
"return "
117+
"raise "
118+
"from "
119+
"import "
120+
".. "
121+
"... "
122+
"__future__ "
123+
"as "
124+
"global "
125+
"nonlocal "
126+
"assert "
127+
"print "
128+
"if "
129+
"elif "
130+
"else: "
131+
"while "
132+
"try: "
133+
"except "
134+
"finally: "
135+
"with "
136+
"lambda "
137+
"or "
138+
"and "
139+
"not "
140+
"None "
141+
"__peg_parser__"
142+
"True "
143+
"False "
144+
"yield "
145+
"async "
146+
"await "
147+
"for "
148+
"in "
149+
"is "
150+
"class "
151+
152+
# shebangs and encodings
153+
"#!"
154+
"# coding:"
155+
"# coding="
156+
"# coding: latin-1"
157+
"# coding=latin-1"
158+
"# coding: utf-8"
159+
"# coding=utf-8"
160+
"# coding: ascii"
161+
"# coding=ascii"
162+
"# coding: cp860"
163+
"# coding=cp860"
164+
"# coding: gbk"
165+
"# coding=gbk"

‎Modules/_xxtestfuzz/fuzz_pycompile_corpus/input1.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from __future__ import annotations
2+
3+
def test() -> None:
4+
x: list[int] = []
5+
x: dict[int, str] = {}
6+
x: set[bytes] = {}
7+
print(5 + 42 * 3, x)

‎Modules/_xxtestfuzz/fuzz_pycompile_corpus/input2.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
class Foo(metaclass=42):
2+
__slots__ = ['x']
3+
pass
4+
5+
foo = Foo()

‎Modules/_xxtestfuzz/fuzz_pycompile_corpus/input3.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
def evens():
2+
i = 0
3+
while True:
4+
i += 1
5+
if i % 2 == 0:
6+
yield i

‎Modules/_xxtestfuzz/fuzz_pycompile_corpus/input4.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
async def hello(name: str):
2+
await name
3+
print(name)

‎Modules/_xxtestfuzz/fuzz_pycompile_corpus/input5.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
try:
2+
eval('importer exporter... really long matches')
3+
except SyntaxError:
4+
print("nothing to see here")
5+
finally:
6+
print("all done here")
7+
raise

‎Modules/_xxtestfuzz/fuzz_pycompile_corpus/input6.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
"""Some module docstring"""
2+
import sys
3+
4+
def main():
5+
print("Hello world!", file=sys.stderr)
6+
7+
if __name__ == '__main__':
8+
main()

‎Modules/_xxtestfuzz/fuzz_tests.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ fuzz_csv_reader
88
fuzz_struct_unpack
99
fuzz_ast_literal_eval
1010
fuzz_elementtree_parsewhole
11+
fuzz_pycompile

‎Modules/_xxtestfuzz/fuzzer.c‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -501,6 +501,63 @@ static int fuzz_elementtree_parsewhole(const char* data, size_t size) {
501501
return 0;
502502
}
503503

504+
#define MAX_PYCOMPILE_TEST_SIZE 16384
505+
static char pycompile_scratch[MAX_PYCOMPILE_TEST_SIZE];
506+
507+
static const int start_vals[] = {Py_eval_input, Py_single_input, Py_file_input};
508+
const size_t NUM_START_VALS = sizeof(start_vals) / sizeof(start_vals[0]);
509+
510+
static const int optimize_vals[] = {-1, 0, 1, 2};
511+
const size_t NUM_OPTIMIZE_VALS = sizeof(optimize_vals) / sizeof(optimize_vals[0]);
512+
513+
/* Fuzz `PyCompileStringExFlags` using a variety of input parameters.
514+
* That function is essentially behind the `compile` builtin */
515+
static int fuzz_pycompile(const char* data, size_t size) {
516+
// Ignore overly-large inputs, and account for a NUL terminator
517+
if (size > MAX_PYCOMPILE_TEST_SIZE - 1) {
518+
return 0;
519+
}
520+
521+
// Need 2 bytes for parameter selection
522+
if (size < 2) {
523+
return 0;
524+
}
525+
526+
// Use first byte to determine element of `start_vals` to use
527+
unsigned char start_idx = (unsigned char) data[0];
528+
int start = start_vals[start_idx % NUM_START_VALS];
529+
530+
// Use second byte to determine element of `optimize_vals` to use
531+
unsigned char optimize_idx = (unsigned char) data[1];
532+
int optimize = optimize_vals[optimize_idx % NUM_OPTIMIZE_VALS];
533+
534+
// Create a NUL-terminated C string from the remaining input
535+
memcpy(pycompile_scratch, data + 2, size - 2);
536+
// Put a NUL terminator just after the copied data. (Space was reserved already.)
537+
pycompile_scratch[size - 2] = '\0';
538+
539+
// XXX: instead of always using NULL for the `flags` value to
540+
// `Py_CompileStringExFlags`, there are many flags that conditionally
541+
// change parser behavior:
542+
//
543+
// #define PyCF_TYPE_COMMENTS 0x1000
544+
// #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
545+
// #define PyCF_ONLY_AST 0x0400
546+
//
547+
// It would be good to test various combinations of these, too.
548+
PyCompilerFlags *flags = NULL;
549+
550+
PyObject *result = Py_CompileStringExFlags(pycompile_scratch, "<fuzz input>", start, flags, optimize);
551+
if (result == NULL) {
552+
/* compilation failed, most likely from a syntax error */
553+
PyErr_Clear();
554+
} else {
555+
Py_DECREF(result);
556+
}
557+
558+
return 0;
559+
}
560+
504561
/* Run fuzzer and abort on failure. */
505562
static int _run_fuzz(const uint8_t *data, size_t size, int(*fuzzer)(const char* , size_t)) {
506563
int rv = fuzzer((const char*) data, size);
@@ -642,6 +699,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
642699
}
643700

644701
rv |= _run_fuzz(data, size, fuzz_elementtree_parsewhole);
702+
#endif
703+
#if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_pycompile)
704+
rv |= _run_fuzz(data, size, fuzz_pycompile);
645705
#endif
646706
return rv;
647707
}

‎Tools/c-analyzer/cpython/ignored.tsv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,6 +599,9 @@ Modules/_xxtestfuzz/fuzzer.c - re_error_exception -
599599
Modules/_xxtestfuzz/fuzzer.c - struct_error -
600600
Modules/_xxtestfuzz/fuzzer.c - struct_unpack_method -
601601
Modules/_xxtestfuzz/fuzzer.c - xmlparser_type -
602+
Modules/_xxtestfuzz/fuzzer.c - pycompile_scratch -
603+
Modules/_xxtestfuzz/fuzzer.c - start_vals -
604+
Modules/_xxtestfuzz/fuzzer.c - optimize_vals -
602605
Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput CSV_READER_INITIALIZED -
603606
Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput JSON_LOADS_INITIALIZED -
604607
Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput SRE_COMPILE_INITIALIZED -

0 commit comments

Comments
 (0)