Bugs found in 3.0 decomplation...

parsers/parse30.py; fix set comprehension grammar bug
uncompyle6/semantics/n_actions.py: evidence of the evils of modifying node data (via node.pop)

Author: rocky

uncompyle6/parsers/parse30.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2017, 2022-2023 Rocky Bernstein
+#  Copyright (c) 2016-2017, 2022-2024 Rocky Bernstein
 """
 spark grammar differences over Python 3.1 for Python 3.0.
 """
@@ -7,8 +7,8 @@
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse31 import Python31Parser
 
-class Python30Parser(Python31Parser):
 
+class Python30Parser(Python31Parser):
     def p_30(self, args):
         """
 
@@ -78,7 +78,6 @@ def p_30(self, args):
         set_comp_func ::= set_comp_header
                           LOAD_ARG FOR_ITER store comp_iter
                           JUMP_BACK ending_return
-                          RETURN_VALUE RETURN_LAST
 
         list_comp_header ::= BUILD_LIST_0 DUP_TOP STORE_FAST
         list_comp        ::= list_comp_header
@@ -118,7 +117,7 @@ def p_30(self, args):
         # From Python 2.6
 
 
-	lc_body     ::= LOAD_FAST expr LIST_APPEND
+        lc_body     ::= LOAD_FAST expr LIST_APPEND
         lc_body     ::= LOAD_NAME expr LIST_APPEND
         list_if     ::= expr jmp_false_then list_iter
         list_if_not ::= expr jmp_true list_iter JUMP_BACK come_froms POP_TOP
@@ -216,9 +215,9 @@ def p_30(self, args):
         compare_chained_right ::= expr COMPARE_OP RETURN_END_IF
         """
 
-
     def remove_rules_30(self):
-        self.remove_rules("""
+        self.remove_rules(
+            """
 
         # The were found using grammar coverage
         while1stmt     ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
@@ -286,37 +285,41 @@ def remove_rules_30(self):
         or               ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
         and              ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
         and              ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
-        """)
+        """
+        )
 
     def customize_grammar_rules(self, tokens, customize):
         super(Python30Parser, self).customize_grammar_rules(tokens, customize)
         self.remove_rules_30()
 
         self.check_reduce["iflaststmtl"] = "AST"
-        self.check_reduce['ifstmt'] = "AST"
+        self.check_reduce["ifstmt"] = "AST"
         self.check_reduce["ifelsestmtc"] = "AST"
         self.check_reduce["ifelsestmt"] = "AST"
         # self.check_reduce["and"] = "stmt"
         return
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
-        invalid = super(Python30Parser,
-                        self).reduce_is_invalid(rule, ast,
-                                                tokens, first, last)
+        invalid = super(Python30Parser, self).reduce_is_invalid(
+            rule, ast, tokens, first, last
+        )
         if invalid:
             return invalid
         lhs = rule[0]
         if (
-            lhs in ("iflaststmtl", "ifstmt",
-                        "ifelsestmt", "ifelsestmtc") and ast[0] == "testexpr"
+            lhs in ("iflaststmtl", "ifstmt", "ifelsestmt", "ifelsestmtc")
+            and ast[0] == "testexpr"
         ):
             testexpr = ast[0]
             if testexpr[0] == "testfalse":
                 testfalse = testexpr[0]
                 if lhs == "ifelsestmtc" and ast[2] == "jump_absolute_else":
                     jump_absolute_else = ast[2]
                     come_from = jump_absolute_else[2]
-                    return come_from == "COME_FROM" and come_from.attr < tokens[first].offset
+                    return (
+                        come_from == "COME_FROM"
+                        and come_from.attr < tokens[first].offset
+                    )
                     pass
                 elif lhs in ("ifelsestmt", "ifelsestmtc") and ast[2] == "jump_cf_pop":
                     jump_cf_pop = ast[2]
@@ -339,20 +342,21 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last):
                     jmp_false = testfalse[1]
                     if last == len(tokens):
                         last -= 1
-                    while (isinstance(tokens[first].offset, str) and first < last):
+                    while isinstance(tokens[first].offset, str) and first < last:
                         first += 1
                     if first == last:
                         return True
-                    while (first < last and isinstance(tokens[last].offset, str)):
+                    while first < last and isinstance(tokens[last].offset, str):
                         last -= 1
                     if rule[0] == "iflaststmtl":
                         return not (jmp_false[0].attr <= tokens[last].offset)
                     else:
                         jmp_false_target = jmp_false[0].attr
                         if tokens[first].offset > jmp_false_target:
                             return True
-                        return (
-                            (jmp_false_target > tokens[last].offset) and tokens[last] != "JUMP_FORWARD")
+                        return (jmp_false_target > tokens[last].offset) and tokens[
+                            last
+                        ] != "JUMP_FORWARD"
                     pass
                 pass
             pass
@@ -361,33 +365,43 @@ def reduce_is_invalid(self, rule, ast, tokens, first, last):
 
     pass
 
+
 class Python30ParserSingle(Python30Parser, PythonParserSingle):
     pass
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     # Check grammar
     p = Python30Parser()
     p.remove_rules_30()
     p.check_grammar()
-    from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+    from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
+
     if PYTHON_VERSION_TRIPLE[:2] == (3, 0):
         lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
+
         s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
-        opcode_set = set(s.opc.opname).union(set(
-            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+        opcode_set = set(s.opc.opname).union(
+            set(
+                """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
                LAMBDA_MARKER RETURN_LAST
-            """.split()))
+            """.split()
+            )
+        )
         ## FIXME: try this
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub(r'_\d+$', '',  t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         import sys
+
         if len(sys.argv) > 1:
             from spark_parser.spark import rule2str
+
             for rule in sorted(p.rule2name.items()):
                 print(rule2str(rule[0]))

uncompyle6/semantics/n_actions.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2022-2023 by Rocky Bernstein
+#  Copyright (c) 2022-2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -16,22 +16,12 @@
 Custom Nonterminal action functions. See NonterminalActions docstring.
 """
 
-from uncompyle6.semantics.consts import (
-    INDENT_PER_LEVEL,
-    NONE,
-    PRECEDENCE,
-    minint,
-)
-
 from uncompyle6.parsers.treenode import SyntaxTree
 from uncompyle6.scanners.tok import Token
+from uncompyle6.semantics.consts import INDENT_PER_LEVEL, NONE, PRECEDENCE, minint
+from uncompyle6.semantics.helper import find_code_node, flatten_list
 from uncompyle6.util import better_repr, get_code_name
 
-from uncompyle6.semantics.helper import (
-    find_code_node,
-    flatten_list,
-)
-
 
 class NonterminalActions:
     """
@@ -826,7 +816,8 @@ def n_list(self, node):
 
         p = self.prec
         self.prec = PRECEDENCE["yield"] - 1
-        lastnode = node.pop()
+        lastnode = node[-1]
+        node = node[:-1]
         lastnodetype = lastnode.kind
 
         # If this build list is inside a CALL_FUNCTION_VAR,