Skip to content

Commit 08009f9

Browse files
rockyrocky
committed
improve list comprehensions
1 parent d3ed646 commit 08009f9

File tree

7 files changed

+63
-36
lines changed

7 files changed

+63
-36
lines changed

‎test/stdlib/3.3-exclude.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ SKIP_TESTS=(
1010
# tgt.append(elem)
1111
[test_itertools.py]=1
1212

13-
[test_buffer.py]=1 # FIXME: Works on c90ff51
13+
[test_buffer.py]=pytest # FIXME: Works on c90ff51
1414
[test_cmath.py]=pytest
1515

1616
[test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines

‎test/stdlib/3.8-exclude.sh

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,12 @@ SKIP_TESTS=(
2828

2929
# These and the above may be due to new code generation or tests
3030
# between 3.8.3 and 3.8.5 ?
31-
[test_decorators.py]=1 #
31+
[test_decorators.py]=1 # parse error
3232

33-
[test_dtrace.py]=1 #
34-
[test_exceptions.py]=1 #
33+
[test_dtrace.py]=1 # parse error
34+
[test_exceptions.py]=1 # parse error
3535
[test_ftplib.py]=1 #
36-
[test_gc.py]=1 #
36+
[test_gc.py]=1 # FIXME: return return strip_python_stderr(stderr)
3737
[test_gzip.py]=1 #
3838
[test_hashlib.py]=1 #
3939
[test_iter.py]=1 #
@@ -51,7 +51,6 @@ SKIP_TESTS=(
5151
[test_audioop.py]=1 # test failure
5252
[test_audit.py]=1 # parse error
5353

54-
[test_base64.py]=1 # parse error
5554
[test_baseexception.py]=1 #
5655
[test_bigaddrspace.py]=1 # parse error
5756
[test_bigmem.py]=1 # parse error

‎uncompyle6/parsers/parse2.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2015-2021 Rocky Bernstein
1+
# Copyright (c) 2015-2021, 2024 Rocky Bernstein
22
# Copyright (c) 2000-2002 by hartmut Goebel <[email protected]>
33
#
44
# Copyright (c) 1999 John Aycock
@@ -27,10 +27,11 @@
2727

2828
from __future__ import print_function
2929

30-
from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
30+
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
31+
3132
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
33+
from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
3234
from uncompyle6.parsers.treenode import SyntaxTree
33-
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
3435

3536

3637
class Python2Parser(PythonParser):
@@ -405,7 +406,6 @@ def customize_grammar_rules(self, tokens, customize):
405406
"CALL_FUNCTION_VAR_KW",
406407
"CALL_FUNCTION_KW",
407408
):
408-
409409
args_pos, args_kw = self.get_pos_kw(token)
410410

411411
# number of apply equiv arguments:
@@ -526,7 +526,7 @@ def customize_grammar_rules(self, tokens, customize):
526526
custom_seen_ops.add(opname)
527527
continue
528528
elif opname == "LOAD_LISTCOMP":
529-
self.addRule("expr ::= listcomp", nop_func)
529+
self.addRule("expr ::= list_comp", nop_func)
530530
custom_seen_ops.add(opname)
531531
continue
532532
elif opname == "LOAD_SETCOMP":

‎uncompyle6/parsers/parse3.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1085,7 +1085,9 @@ def customize_grammar_rules(self, tokens, customize):
10851085
)
10861086
custom_ops_processed.add(opname)
10871087
elif opname == "LOAD_LISTCOMP":
1088-
self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
1088+
self.add_unique_rule(
1089+
"expr ::= list_comp", opname, token.attr, customize
1090+
)
10891091
custom_ops_processed.add(opname)
10901092
elif opname == "LOAD_SETCOMP":
10911093
# Should this be generalized and put under MAKE_FUNCTION?
@@ -1154,7 +1156,7 @@ def customize_grammar_rules(self, tokens, customize):
11541156
# and have GET_ITER CALL_FUNCTION_1
11551157
# Todo: For Pypy we need to modify this slightly
11561158
rule_pat = (
1157-
"listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
1159+
"list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
11581160
"GET_ITER CALL_FUNCTION_1"
11591161
% ("pos_arg " * pos_args_count, opname)
11601162
)
@@ -1348,14 +1350,14 @@ def customize_grammar_rules(self, tokens, customize):
13481350
# 'exprs' in the rule above into a
13491351
# tuple.
13501352
rule_pat = (
1351-
"listcomp ::= load_closure LOAD_LISTCOMP %%s%s "
1353+
"list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
13521354
"expr GET_ITER CALL_FUNCTION_1" % (opname,)
13531355
)
13541356
self.add_make_function_rule(
13551357
rule_pat, opname, token.attr, customize
13561358
)
13571359
rule_pat = (
1358-
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
1360+
"list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
13591361
"GET_ITER CALL_FUNCTION_1"
13601362
% ("expr " * pos_args_count, opname)
13611363
)
@@ -1399,7 +1401,7 @@ def customize_grammar_rules(self, tokens, customize):
13991401
# and have GET_ITER CALL_FUNCTION_1
14001402
# Todo: For Pypy we need to modify this slightly
14011403
rule_pat = (
1402-
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
1404+
"list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
14031405
"GET_ITER CALL_FUNCTION_1"
14041406
% ("expr " * pos_args_count, opname)
14051407
)

‎uncompyle6/parsers/parse37base.py

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,10 @@
22
"""
33
Python 3.7 base code. We keep non-custom-generated grammar rules out of this file.
44
"""
5-
from uncompyle6.parser import ParserError, PythonParser, nop_func
6-
from uncompyle6.parsers.treenode import SyntaxTree
75
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
86
from spark_parser.spark import rule2str
97

8+
from uncompyle6.parser import ParserError, PythonParser, nop_func
109
from uncompyle6.parsers.reducecheck import (
1110
and_invalid,
1211
ifelsestmt,
@@ -16,9 +15,10 @@
1615
or_check,
1716
testtrue,
1817
tryelsestmtl3,
19-
while1stmt,
2018
while1elsestmt,
19+
while1stmt,
2120
)
21+
from uncompyle6.parsers.treenode import SyntaxTree
2222

2323

2424
class Python37BaseParser(PythonParser):
@@ -54,7 +54,7 @@ def custom_build_class_rule(self, opname, i, token, tokens, customize):
5454
expr
5555
call
5656
CALL_FUNCTION_3
57-
"""
57+
"""
5858
# FIXME: I bet this can be simplified
5959
# look for next MAKE_FUNCTION
6060
for i in range(i + 1, len(tokens)):
@@ -104,7 +104,6 @@ def custom_build_class_rule(self, opname, i, token, tokens, customize):
104104
# organization for this. For example, arrange organize by opcode base?
105105

106106
def customize_grammar_rules(self, tokens, customize):
107-
108107
is_pypy = False
109108

110109
# For a rough break out on the first word. This may
@@ -348,7 +347,6 @@ def customize_grammar_rules(self, tokens, customize):
348347
self.addRule(rule, nop_func)
349348

350349
elif opname_base in ("BUILD_MAP", "BUILD_MAP_UNPACK"):
351-
352350
if opname == "BUILD_MAP_UNPACK":
353351
self.addRule(
354352
"""
@@ -525,7 +523,6 @@ def customize_grammar_rules(self, tokens, customize):
525523
"CALL_FUNCTION_VAR_KW",
526524
)
527525
) or opname.startswith("CALL_FUNCTION_KW"):
528-
529526
if opname == "CALL_FUNCTION" and token.attr == 1:
530527
rule = """
531528
expr ::= dict_comp
@@ -720,7 +717,9 @@ def customize_grammar_rules(self, tokens, customize):
720717
)
721718
custom_ops_processed.add(opname)
722719
elif opname == "LOAD_LISTCOMP":
723-
self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
720+
self.add_unique_rule(
721+
"expr ::= list_comp", opname, token.attr, customize
722+
)
724723
custom_ops_processed.add(opname)
725724
elif opname == "LOAD_NAME":
726725
if (
@@ -799,7 +798,7 @@ def customize_grammar_rules(self, tokens, customize):
799798
# and have GET_ITER CALL_FUNCTION_1
800799
# Todo: For Pypy we need to modify this slightly
801800
rule_pat = (
802-
"listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
801+
"list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
803802
"GET_ITER CALL_FUNCTION_1"
804803
% ("pos_arg " * args_pos, opname)
805804
)
@@ -897,14 +896,14 @@ def customize_grammar_rules(self, tokens, customize):
897896
# 'exprs' in the rule above into a
898897
# tuple.
899898
rule_pat = (
900-
"listcomp ::= load_closure LOAD_LISTCOMP %%s%s "
899+
"list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
901900
"expr GET_ITER CALL_FUNCTION_1" % (opname,)
902901
)
903902
self.add_make_function_rule(
904903
rule_pat, opname, token.attr, customize
905904
)
906905
rule_pat = (
907-
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
906+
"list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
908907
"GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
909908
)
910909
self.add_make_function_rule(
@@ -938,7 +937,7 @@ def customize_grammar_rules(self, tokens, customize):
938937
# and have GET_ITER CALL_FUNCTION_1
939938
# Todo: For Pypy we need to modify this slightly
940939
rule_pat = (
941-
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
940+
"list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
942941
"GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
943942
)
944943
self.add_make_function_rule(
@@ -1259,7 +1258,8 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last):
12591258
if fn:
12601259
return fn(self, lhs, n, rule, ast, tokens, first, last)
12611260
except Exception:
1262-
import sys, traceback
1261+
import sys
1262+
import traceback
12631263

12641264
print(
12651265
f"Exception in {fn.__name__} {sys.exc_info()[1]}\n"

‎uncompyle6/semantics/gencomp.py

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323

2424
from uncompyle6.parser import get_python_parser
2525
from uncompyle6.scanner import Code
26+
from uncompyle6.scanners.tok import Token
2627
from uncompyle6.semantics.consts import PRECEDENCE
2728
from uncompyle6.semantics.helper import is_lambda_mode
28-
from uncompyle6.scanners.tok import Token
2929

3030

3131
class ComprehensionMixin:
@@ -174,7 +174,10 @@ def comprehension_walk(
174174
tree = tree[1]
175175
pass
176176

177-
if tree in ("genexpr_func", "genexpr_func_async",):
177+
if tree in (
178+
"genexpr_func",
179+
"genexpr_func_async",
180+
):
178181
for i in range(3, 5):
179182
if tree[i] == "comp_iter":
180183
iter_index = i
@@ -332,8 +335,19 @@ def comprehension_walk_newer(
332335
assert store == "store"
333336
n = set_iter_async[2]
334337
elif node == "list_comp" and tree[0] == "expr":
335-
tree = tree[0][0]
336-
n = tree[iter_index]
338+
list_iter = None
339+
for list_iter_try in tree:
340+
if list_iter_try == "list_iter":
341+
list_iter = list_iter_try
342+
break
343+
if not list_iter_try:
344+
tree = tree[0][0]
345+
n = tree[iter_index]
346+
else:
347+
n = list_iter
348+
pass
349+
pass
350+
pass
337351
else:
338352
n = tree[iter_index]
339353

@@ -407,6 +421,9 @@ def comprehension_walk_newer(
407421
n = n[0]
408422

409423
if n in ("list_for", "comp_for"):
424+
if n == "list_for" and not comp_for and n[0] == "expr":
425+
comp_for = n[0]
426+
410427
n_index = 3
411428
if (
412429
(n[2] == "store")
@@ -496,11 +513,21 @@ def comprehension_walk_newer(
496513
if comp_for:
497514
self.preorder(comp_for)
498515
else:
516+
try:
517+
node[in_node_index]
518+
except:
519+
from trepan.api import debug
520+
521+
debug()
499522
self.preorder(node[in_node_index])
500523

501524
# Here is where we handle nested list iterations.
502525
if tree == "list_comp" and self.version != (3, 0):
503-
list_iter = tree[1]
526+
list_iter = None
527+
for list_iter_try in tree:
528+
if list_iter_try == "list_iter":
529+
list_iter = list_iter_try
530+
break
504531
assert list_iter == "list_iter"
505532
if list_iter[0] == "list_for":
506533
self.preorder(list_iter[0][3])
@@ -639,7 +666,6 @@ def listcomp_closure3(self, node):
639666
# Find the list comprehension body. It is the inner-most
640667
# node that is not list_.. .
641668
while n == "list_iter":
642-
643669
# recurse one step
644670
n = n[0]
645671

‎uncompyle6/semantics/n_actions.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1036,7 +1036,7 @@ def n_list_comp_pypy27(self, node):
10361036
self.prec = p
10371037
self.prune() # stop recursing
10381038

1039-
def n_listcomp(self, node):
1039+
def n_list_comp(self, node):
10401040
self.write("[")
10411041
if node[0].kind == "load_closure":
10421042
assert self.version >= (3, 0)

0 commit comments

Comments
 (0)