Merge pull request #483 from rocky/xdis-fix-woes

Xdis fix woes

Author: rocky

uncompyle6/bin/uncompile.py

@@ -15,8 +15,8 @@
 import click
 from xdis.version_info import version_tuple_to_str
 
-from uncompyle6 import verify
 from uncompyle6.main import main, status_msg
+from uncompyle6.verify import VerifyCmpError
 from uncompyle6.version import __version__
 
 program = "uncompyle6"
@@ -245,7 +245,7 @@ def main_bin(
             sys.exit(2)
         except KeyboardInterrupt:
             pass
-        except verify.VerifyCmpError:
+        except VerifyCmpError:
             raise
     else:
         from multiprocessing import Process, Queue
@@ -266,18 +266,18 @@ def main_bin(
         tot_files = okay_files = failed_files = verify_failed_files = 0
 
         def process_func():
+            (tot_files, okay_files, failed_files, verify_failed_files) = (
+                0,
+                0,
+                0,
+                0,
+            )
             try:
-                (tot_files, okay_files, failed_files, verify_failed_files) = (
-                    0,
-                    0,
-                    0,
-                    0,
-                )
                 while 1:
                     f = fqueue.get()
                     if f is None:
                         break
-                    (t, o, f, v) = main(src_base, out_base, [f], [], outfile, **options)
+                    (t, o, f, v) = main(src_base, out_base, [f], [], outfile)
                     tot_files += t
                     okay_files += o
                     failed_files += f

uncompyle6/parsers/parse3.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2023 Rocky Bernstein
+#  Copyright (c) 2015-2024 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <[email protected]>
 #  Copyright (c) 2000-2002 by hartmut Goebel <[email protected]>
 #  Copyright (c) 1999 John Aycock
@@ -1221,9 +1221,24 @@ def customize_grammar_rules(self, tokens, customize):
                             )
                         )
                     else:
-                        rule = "mkfunc ::= %s%sload_closure LOAD_CODE LOAD_STR %s" % (
-                            kwargs_str,
-                            "pos_arg " * pos_args_count,
+                        if self.version == (3, 3):
+                            # 3.3 puts kwargs before pos_arg
+                            pos_kw_tuple = (
+                                ("kwargs " * kw_args_count),
+                                ("pos_arg " * pos_args_count),
+                            )
+                        else:
+                            # 3.4 and 3.5 puts pos_arg before kwargs
+                            pos_kw_tuple = (
+                                "pos_arg " * (pos_args_count),
+                                ("kwargs " * kw_args_count),
+                            )
+                        rule = (
+                            "mkfunc ::= %s%s%s " "load_closure LOAD_CODE LOAD_STR %s"
+                        ) % (
+                            pos_kw_tuple[0],
+                            pos_kw_tuple[1],
+                            "annotate_pair " * (annotate_args),
                             opname,
                         )
                     self.add_unique_rule(rule, opname, token.attr, customize)
@@ -1465,12 +1480,12 @@ def customize_grammar_rules(self, tokens, customize):
                         )
                         self.add_unique_rule(rule, opname, token.attr, customize)
                         rule = (
-                            "mkfunc_annotate ::= %s%sannotate_tuple LOAD_CODE LOAD_STR %s"
-                            % (
-                                ("pos_arg " * pos_args_count),
-                                ("annotate_arg " * annotate_args),
-                                opname,
-                            )
+                            "mkfunc_annotate ::= %s%sannotate_tuple LOAD_CODE "
+                            "LOAD_STR %s"
+                        ) % (
+                            ("pos_arg " * pos_args_count),
+                            ("annotate_arg " * annotate_args),
+                            opname,
                         )
                     if self.version >= (3, 3):
                         if self.version == (3, 3):
@@ -1480,29 +1495,19 @@ def customize_grammar_rules(self, tokens, customize):
                                 ("pos_arg " * pos_args_count),
                             )
                         else:
-                            # 3.4 and 3.5puts pos_arg before kwargs
+                            # 3.4 and 3.5 puts pos_arg before kwargs
                             pos_kw_tuple = (
                                 "pos_arg " * (pos_args_count),
                                 ("kwargs " * kw_args_count),
                             )
                         rule = (
-                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
-                            % (
-                                pos_kw_tuple[0],
-                                pos_kw_tuple[1],
-                                ("annotate_arg " * annotate_args),
-                                opname,
-                            )
-                        )
-                        self.add_unique_rule(rule, opname, token.attr, customize)
-                        rule = (
-                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
-                            % (
-                                pos_kw_tuple[0],
-                                pos_kw_tuple[1],
-                                ("annotate_arg " * annotate_args),
-                                opname,
-                            )
+                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE "
+                            "LOAD_STR %s"
+                        ) % (
+                            pos_kw_tuple[0],
+                            pos_kw_tuple[1],
+                            ("annotate_arg " * annotate_args),
+                            opname,
                         )
                     else:
                         rule = (

uncompyle6/scanners/scanner3.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2019, 2021-2023 by Rocky Bernstein
+#  Copyright (c) 2015-2019, 2021-2024 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <[email protected]>
 #  Copyright (c) 2000-2002 by hartmut Goebel <[email protected]>
 #
@@ -482,7 +482,6 @@ def ingest(
 
         last_op_was_break = False
         new_tokens = []
-        operand_value = 0
 
         for i, inst in enumerate(self.insts):
             opname = inst.opname
@@ -534,11 +533,9 @@ def ingest(
             op = inst.opcode
 
             if opname == "EXTENDED_ARG":
-                if i + 1 < n:
-                    operand_value = argval << 16
-                    continue
-                else:
-                    operand_value = 0
+                # EXTEND_ARG adjustments to the operand value should have
+                # already been accounted for in xdis instruction creation.
+                continue
 
             if inst.offset in jump_targets:
                 jump_idx = 0
@@ -645,7 +642,7 @@ def ingest(
                     attr = attr[:4]  # remove last value: attr[5] == False
                 else:
                     pos_args, name_pair_args, annotate_args = parse_fn_counts_30_35(
-                        inst.argval + operand_value
+                        inst.argval
                     )
 
                     pattr = f"{pos_args} positional, {name_pair_args} keyword only, {annotate_args} annotated"

uncompyle6/scanners/scanner35.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017, 2021-2022 by Rocky Bernstein
+#  Copyright (c) 2017, 2021-2022, 2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -22,21 +22,22 @@
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_35 as opc
 
 from uncompyle6.scanners.scanner3 import Scanner3
 
-# bytecode verification, verify(), uses JUMP_OPs from here
-from xdis.opcodes import opcode_35 as opc
 JUMP_OPS = opc.JUMP_OPS
 
-class Scanner35(Scanner3):
 
+class Scanner35(Scanner3):
     def __init__(self, show_asm=None, is_pypy=False):
         Scanner3.__init__(self, (3, 5), show_asm, is_pypy)
         return
+
     pass
 
+
 if __name__ == "__main__":
     from xdis.version_info import PYTHON_VERSION_TRIPLE, version_tuple_to_str
 

uncompyle6/scanners/scanner37base.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2020, 2022-2023 by Rocky Bernstein
+#  Copyright (c) 2015-2020, 2022-2024 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <[email protected]>
 #  Copyright (c) 2000-2002 by hartmut Goebel <[email protected]>
 #
@@ -15,7 +15,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-Python 37 bytecode scanner/deparser base.
+Python 3.7 bytecode scanner/deparser base.
 
 Also we *modify* the instruction sequence to assist deparsing code.
 For example:
@@ -29,20 +29,17 @@
 Finally we save token information.
 """
 
+import sys
 from typing import Any, Dict, List, Set, Tuple
 
-from xdis import iscode, instruction_size, Instruction
-from xdis.bytecode import _get_const_info
-
-from uncompyle6.scanner import Token
 import xdis
 
 # Get all the opcodes into globals
 import xdis.opcodes.opcode_37 as op3
+from xdis import Instruction, instruction_size, iscode
+from xdis.bytecode import _get_const_info
 
-from uncompyle6.scanner import Scanner
-
-import sys
+from uncompyle6.scanner import Scanner, Token
 
 globals().update(op3.opmap)
 
@@ -51,7 +48,9 @@
 
 
 class Scanner37Base(Scanner):
-    def __init__(self, version: Tuple[int, int], show_asm=None, debug="", is_pypy=False):
+    def __init__(
+        self, version: Tuple[int, int], show_asm=None, debug="", is_pypy=False
+    ):
         super(Scanner37Base, self).__init__(version, show_asm, is_pypy)
         self.offset2tok_index = None
         self.debug = debug
@@ -254,7 +253,6 @@ def tokens_append(j, token):
 
         n = len(self.insts)
         for i, inst in enumerate(self.insts):
-
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
@@ -284,7 +282,6 @@ def tokens_append(j, token):
         # To simplify things we want to untangle this. We also
         # do this loop before we compute jump targets.
         for i, inst in enumerate(self.insts):
-
             # One artifact of the "too-small" operand problem, is that
             # some backward jumps, are turned into forward jumps to another
             # "extended arg" backward jump to the same location.
@@ -321,16 +318,9 @@ def tokens_append(j, token):
 
         j = 0
         for i, inst in enumerate(self.insts):
-
             argval = inst.argval
             op = inst.opcode
 
-            if inst.opname == "EXTENDED_ARG":
-                # FIXME: The EXTENDED_ARG is used to signal annotation
-                # parameters
-                if i + 1 < n and self.insts[i + 1].opcode != self.opc.MAKE_FUNCTION:
-                    continue
-
             if inst.offset in jump_targets:
                 jump_idx = 0
                 # We want to process COME_FROMs to the same offset to be in *descending*

uncompyle6/semantics/make_function3.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2021 by Rocky Bernstein
+#  Copyright (c) 2015-2021, 2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -16,18 +16,18 @@
 All the crazy things we have to do to handle Python functions in 3.0-3.5 or so.
 The saga of changes before and after is in other files.
 """
-from xdis import iscode, code_has_star_arg, code_has_star_star_arg, CO_GENERATOR
-from uncompyle6.scanner import Code
-from uncompyle6.parsers.treenode import SyntaxTree
-from uncompyle6.semantics.parser_error import ParserError
+from xdis import CO_GENERATOR, code_has_star_arg, code_has_star_star_arg, iscode
+
 from uncompyle6.parser import ParserError as ParserError2
+from uncompyle6.parsers.treenode import SyntaxTree
+from uncompyle6.scanner import Code
 from uncompyle6.semantics.helper import (
-    print_docstring,
     find_all_globals,
     find_globals_and_nonlocals,
     find_none,
+    print_docstring,
 )
-
+from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.show import maybe_show_tree_param_default
 
 # FIXME: DRY the below code...
@@ -42,8 +42,8 @@ def make_function3_annotate(
 
     def build_param(ast, name, default):
         """build parameters:
-            - handle defaults
-            - handle format tuple parameters
+        - handle defaults
+        - handle format tuple parameters
         """
         if default:
             value = self.traverse(default, indent="")
@@ -300,7 +300,7 @@ def build_param(ast, name, default):
 
 def make_function3(self, node, is_lambda, nested=1, code_node=None):
     """Dump function definition, doc string, and function body in
-      Python version 3.0 and above
+    Python version 3.0 and above
     """
 
     # For Python 3.3, the evaluation stack in MAKE_FUNCTION is:
@@ -333,8 +333,8 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
 
     def build_param(ast, name, default, annotation=None):
         """build parameters:
-            - handle defaults
-            - handle format tuple parameters
+        - handle defaults
+        - handle format tuple parameters
         """
         value = self.traverse(default, indent="")
         maybe_show_tree_param_default(self.showast, name, value)
@@ -419,7 +419,6 @@ def build_param(ast, name, default, annotation=None):
             pass
 
         if len(node) > 2 and (have_kwargs or node[lc_index].kind != "load_closure"):
-
             # Find the index in "node" where the first default
             # parameter value is located. Note this is in contrast to
             # key-word arguments, pairs of (name, value), which appear after "*".
@@ -492,8 +491,6 @@ def build_param(ast, name, default, annotation=None):
             self.ERROR = p
         return
 
-    kw_pairs = 0
-
     i = len(paramnames) - len(defparams)
 
     # build parameters