Skip to content

Commit 454fac4

Browse files
rockyrocky
committed
Handle 3.3 MAKE_FUNCTION annotation args properly
1 parent 147155e commit 454fac4

File tree

6 files changed

+123
-73
lines changed

6 files changed

+123
-73
lines changed

‎uncompyle6/bin/uncompile.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,10 +159,10 @@ def main_bin(
159159
"""
160160

161161
version_tuple = sys.version_info[0:2]
162-
if version_tuple < (3, 7):
162+
if version_tuple < (3, 6):
163163
print(
164-
f"Error: This version of the {program} runs from Python 3.7 or greater."
165-
f"You need another branch of this code for Python before 3.7."
164+
f"Error: This version of the {program} runs from Python 3.6 or greater."
165+
f"You need another branch of this code for Python before 3.6."
166166
f""" \n\tYou have version: {version_tuple_to_str()}."""
167167
)
168168
sys.exit(-1)

‎uncompyle6/main.py

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,12 @@
1313
# You should have received a copy of the GNU General Public License
1414
# along with this program. If not, see <http://www.gnu.org/licenses/>.
1515

16+
import ast
1617
import datetime
1718
import os
1819
import os.path as osp
1920
import py_compile
21+
import subprocess
2022
import sys
2123
import tempfile
2224
from typing import Any, Optional, TextIO, Tuple
@@ -51,6 +53,17 @@ def _get_outstream(outfile: str) -> Any:
5153
return open(outfile, mode="w", encoding="utf-8")
5254

5355

56+
def syntax_check(filename: str) -> bool:
57+
with open(filename) as f:
58+
source = f.read()
59+
valid = True
60+
try:
61+
ast.parse(source)
62+
except SyntaxError:
63+
valid = False
64+
return valid
65+
66+
5467
def decompile(
5568
co,
5669
bytecode_version: Tuple[int] = PYTHON_VERSION_TRIPLE,
@@ -368,15 +381,22 @@ def main(
368381
check_type = "syntax check"
369382
if do_verify == "run":
370383
check_type = "run"
371-
result = subprocess.run(
372-
[sys.executable, deparsed_object.f.name],
373-
capture_output=True,
374-
)
375-
valid = result.returncode == 0
376-
output = result.stdout.decode()
377-
if output:
378-
print(output)
379-
pass
384+
if PYTHON_VERSION_TRIPLE >= (3, 7):
385+
result = subprocess.run(
386+
[sys.executable, deparsed_object.f.name],
387+
capture_output=True,
388+
)
389+
valid = result.returncode == 0
390+
output = result.stdout.decode()
391+
if output:
392+
print(output)
393+
pass
394+
else:
395+
result = subprocess.run(
396+
[sys.executable, deparsed_object.f.name],
397+
)
398+
valid = result.returncode == 0
399+
pass
380400
if not valid:
381401
print(result.stderr.decode())
382402

‎uncompyle6/parsers/parse3.py

Lines changed: 60 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -27,22 +27,24 @@
2727
"""
2828

2929
import re
30-
from uncompyle6.scanners.tok import Token
30+
31+
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
32+
3133
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
3234
from uncompyle6.parsers.reducecheck import (
3335
and_invalid,
3436
except_handler_else,
3537
ifelsestmt,
36-
ifstmt,
3738
iflaststmt,
39+
ifstmt,
3840
or_check,
3941
testtrue,
4042
tryelsestmtl3,
4143
tryexcept,
42-
while1stmt
44+
while1stmt,
4345
)
4446
from uncompyle6.parsers.treenode import SyntaxTree
45-
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
47+
from uncompyle6.scanners.tok import Token
4648

4749

4850
class Python3Parser(PythonParser):
@@ -98,7 +100,7 @@ def p_comprehension3(self, args):
98100
"""
99101

100102
def p_dict_comp3(self, args):
101-
""""
103+
""" "
102104
expr ::= dict_comp
103105
stmt ::= dict_comp_func
104106
dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store
@@ -519,7 +521,7 @@ def custom_build_class_rule(self, opname, i, token, tokens, customize, is_pypy):
519521
expr
520522
call
521523
CALL_FUNCTION_3
522-
"""
524+
"""
523525
# FIXME: I bet this can be simplified
524526
# look for next MAKE_FUNCTION
525527
for i in range(i + 1, len(tokens)):
@@ -625,7 +627,11 @@ def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy):
625627
self.add_unique_rule(rule, token.kind, uniq_param, customize)
626628

627629
if "LOAD_BUILD_CLASS" in self.seen_ops:
628-
if next_token == "CALL_FUNCTION" and next_token.attr == 1 and pos_args_count > 1:
630+
if (
631+
next_token == "CALL_FUNCTION"
632+
and next_token.attr == 1
633+
and pos_args_count > 1
634+
):
629635
rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % (
630636
("expr " * (pos_args_count - 1)),
631637
opname,
@@ -764,18 +770,24 @@ def customize_grammar_rules(self, tokens, customize):
764770

765771
elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
766772
if opname == "BUILD_CONST_DICT":
767-
rule = """
773+
rule = (
774+
"""
768775
add_consts ::= ADD_VALUE*
769776
const_list ::= COLLECTION_START add_consts %s
770777
dict ::= const_list
771778
expr ::= dict
772-
""" % opname
779+
"""
780+
% opname
781+
)
773782
else:
774-
rule = """
783+
rule = (
784+
"""
775785
add_consts ::= ADD_VALUE*
776786
const_list ::= COLLECTION_START add_consts %s
777787
expr ::= const_list
778-
""" % opname
788+
"""
789+
% opname
790+
)
779791
self.addRule(rule, nop_func)
780792

781793
elif opname.startswith("BUILD_DICT_OLDER"):
@@ -854,18 +866,24 @@ def customize_grammar_rules(self, tokens, customize):
854866

855867
elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
856868
if opname == "BUILD_CONST_DICT":
857-
rule = """
869+
rule = (
870+
"""
858871
add_consts ::= ADD_VALUE*
859872
const_list ::= COLLECTION_START add_consts %s
860873
dict ::= const_list
861874
expr ::= dict
862-
""" % opname
875+
"""
876+
% opname
877+
)
863878
else:
864-
rule = """
879+
rule = (
880+
"""
865881
add_consts ::= ADD_VALUE*
866882
const_list ::= COLLECTION_START add_consts %s
867883
expr ::= const_list
868-
""" % opname
884+
"""
885+
% opname
886+
)
869887
self.addRule(rule, nop_func)
870888

871889
elif opname_base in (
@@ -946,7 +964,6 @@ def customize_grammar_rules(self, tokens, customize):
946964
"CALL_FUNCTION_VAR_KW",
947965
)
948966
) or opname.startswith("CALL_FUNCTION_KW"):
949-
950967
if opname == "CALL_FUNCTION" and token.attr == 1:
951968
rule = """
952969
dict_comp ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr
@@ -1122,7 +1139,8 @@ def customize_grammar_rules(self, tokens, customize):
11221139
if has_get_iter_call_function1:
11231140
rule_pat = (
11241141
"generator_exp ::= %sload_closure load_genexpr %%s%s expr "
1125-
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
1142+
"GET_ITER CALL_FUNCTION_1"
1143+
% ("pos_arg " * pos_args_count, opname)
11261144
)
11271145
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
11281146

@@ -1190,6 +1208,8 @@ def customize_grammar_rules(self, tokens, customize):
11901208
self.add_unique_rule(rule, opname, token.attr, customize)
11911209

11921210
elif (3, 3) <= self.version < (3, 6):
1211+
# FIXME move this into version-specific custom rules.
1212+
# In fact, some of this has been done for 3.3.
11931213
if annotate_args > 0:
11941214
rule = (
11951215
"mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s"
@@ -1208,7 +1228,6 @@ def customize_grammar_rules(self, tokens, customize):
12081228
)
12091229
self.add_unique_rule(rule, opname, token.attr, customize)
12101230

1211-
12121231
if self.version >= (3, 4):
12131232
if not self.is_pypy:
12141233
load_op = "LOAD_STR"
@@ -1292,14 +1311,16 @@ def customize_grammar_rules(self, tokens, customize):
12921311
if has_get_iter_call_function1:
12931312
rule_pat = (
12941313
"generator_exp ::= %sload_genexpr %%s%s expr "
1295-
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
1314+
"GET_ITER CALL_FUNCTION_1"
1315+
% ("pos_arg " * pos_args_count, opname)
12961316
)
12971317
self.add_make_function_rule(
12981318
rule_pat, opname, token.attr, customize
12991319
)
13001320
rule_pat = (
13011321
"generator_exp ::= %sload_closure load_genexpr %%s%s expr "
1302-
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
1322+
"GET_ITER CALL_FUNCTION_1"
1323+
% ("pos_arg " * pos_args_count, opname)
13031324
)
13041325
self.add_make_function_rule(
13051326
rule_pat, opname, token.attr, customize
@@ -1351,7 +1372,8 @@ def customize_grammar_rules(self, tokens, customize):
13511372
if has_get_iter_call_function1:
13521373
rule_pat = (
13531374
"generator_exp ::= %sload_genexpr %%s%s expr "
1354-
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
1375+
"GET_ITER CALL_FUNCTION_1"
1376+
% ("pos_arg " * pos_args_count, opname)
13551377
)
13561378
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
13571379

@@ -1363,7 +1385,8 @@ def customize_grammar_rules(self, tokens, customize):
13631385
# Todo: For Pypy we need to modify this slightly
13641386
rule_pat = (
13651387
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
1366-
"GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname)
1388+
"GET_ITER CALL_FUNCTION_1"
1389+
% ("expr " * pos_args_count, opname)
13671390
)
13681391
self.add_make_function_rule(
13691392
rule_pat, opname, token.attr, customize
@@ -1450,9 +1473,6 @@ def customize_grammar_rules(self, tokens, customize):
14501473
)
14511474
)
14521475
if self.version >= (3, 3):
1453-
# Normally we remove EXTENDED_ARG from the opcodes, but in the case of
1454-
# annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function.
1455-
# Yes this is a little hacky
14561476
if self.version == (3, 3):
14571477
# 3.3 puts kwargs before pos_arg
14581478
pos_kw_tuple = (
@@ -1466,17 +1486,17 @@ def customize_grammar_rules(self, tokens, customize):
14661486
("kwargs " * kw_args_count),
14671487
)
14681488
rule = (
1469-
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s"
1489+
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
14701490
% (
14711491
pos_kw_tuple[0],
14721492
pos_kw_tuple[1],
1473-
("call " * annotate_args),
1493+
("annotate_arg " * annotate_args),
14741494
opname,
14751495
)
14761496
)
14771497
self.add_unique_rule(rule, opname, token.attr, customize)
14781498
rule = (
1479-
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s"
1499+
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
14801500
% (
14811501
pos_kw_tuple[0],
14821502
pos_kw_tuple[1],
@@ -1485,9 +1505,8 @@ def customize_grammar_rules(self, tokens, customize):
14851505
)
14861506
)
14871507
else:
1488-
# See above comment about use of EXTENDED_ARG
14891508
rule = (
1490-
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s"
1509+
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
14911510
% (
14921511
("kwargs " * kw_args_count),
14931512
("pos_arg " * (pos_args_count)),
@@ -1497,7 +1516,7 @@ def customize_grammar_rules(self, tokens, customize):
14971516
)
14981517
self.add_unique_rule(rule, opname, token.attr, customize)
14991518
rule = (
1500-
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s"
1519+
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
15011520
% (
15021521
("kwargs " * kw_args_count),
15031522
("pos_arg " * pos_args_count),
@@ -1594,7 +1613,7 @@ def customize_grammar_rules(self, tokens, customize):
15941613
}
15951614

15961615
if self.version == (3, 6):
1597-
self.reduce_check_table["and"] = and_invalid
1616+
self.reduce_check_table["and"] = and_invalid
15981617
self.check_reduce["and"] = "AST"
15991618

16001619
self.check_reduce["annotate_tuple"] = "noAST"
@@ -1624,7 +1643,7 @@ def customize_grammar_rules(self, tokens, customize):
16241643
def reduce_is_invalid(self, rule, ast, tokens, first, last):
16251644
lhs = rule[0]
16261645
n = len(tokens)
1627-
last = min(last, n-1)
1646+
last = min(last, n - 1)
16281647
fn = self.reduce_check_table.get(lhs, None)
16291648
if fn:
16301649
if fn(self, lhs, n, rule, ast, tokens, first, last):
@@ -1650,13 +1669,18 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last):
16501669
condition_jump2 = tokens[min(last - 1, len(tokens) - 1)]
16511670
# If there are two *distinct* condition jumps, they should not jump to the
16521671
# same place. Otherwise we have some sort of "and"/"or".
1653-
if condition_jump2.kind.startswith("POP_JUMP_IF") and condition_jump != condition_jump2:
1672+
if (
1673+
condition_jump2.kind.startswith("POP_JUMP_IF")
1674+
and condition_jump != condition_jump2
1675+
):
16541676
return condition_jump.attr == condition_jump2.attr
16551677

1656-
if tokens[last] == "COME_FROM" and tokens[last].off2int() != condition_jump.attr:
1678+
if (
1679+
tokens[last] == "COME_FROM"
1680+
and tokens[last].off2int() != condition_jump.attr
1681+
):
16571682
return False
16581683

1659-
16601684
# if condition_jump.attr < condition_jump2.off2int():
16611685
# print("XXX", first, last)
16621686
# for t in range(first, last): print(tokens[t])
@@ -1678,7 +1702,6 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last):
16781702
< tokens[last].off2int()
16791703
)
16801704
elif lhs == "while1stmt":
1681-
16821705
if while1stmt(self, lhs, n, rule, ast, tokens, first, last):
16831706
return True
16841707

@@ -1700,7 +1723,6 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last):
17001723
return True
17011724
return False
17021725
elif lhs == "while1elsestmt":
1703-
17041726
n = len(tokens)
17051727
if last == n:
17061728
# Adjust for fuzziness in parsing

0 commit comments

Comments
 (0)