Towards getting 1.5 unmarshaling correct

Author: rocky

xdis/bytecode.py

@@ -612,8 +612,13 @@ def dis(
         if isinstance(filename, UnicodeForPython3):
             filename = str(filename)
 
+        if isinstance(co.co_code, str):
+            co_code = co.co_code.encode('latin-1')
+        else:
+            co_code = co.co_code
+
         self.disassemble_bytes(
-            co.co_code,
+            co_code,
             varnames=co.co_varnames,
             names=co.co_names,
             constants=co.co_consts,

xdis/codetype/code15.py

@@ -1,4 +1,4 @@
-# (C) Copyright 2020, 2023 by Rocky Bernstein
+# (C) Copyright 2020, 2023, 2025 by Rocky Bernstein
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU General Public License
@@ -72,7 +72,7 @@ def __init__(
         # This messes up decompilers somehow.
         # self.decode_lineno_tab()
         self.fieldtypes = Code15FieldTypes
-        if type(self) == Code15:
+        if type(self) is Code15:
             self.check()
         return
 

xdis/load.py

@@ -410,7 +410,15 @@ def write_bytecode_file(
     if isinstance(code_obj, types.CodeType):
         fp.write(marshal.dumps(code_obj))
     else:
-        fp.write(xdis.marsh.dumps(code_obj, python_version=version_tuple))
+        code_sequence = xdis.marsh.dumps(code_obj, python_version=version_tuple)
+        if isinstance(code_sequence, str):
+            # Python 1.x uses code strings, not bytes. To get this into bytes needed by
+            # fp.write, encode the string using 'latin-1' and 'unicode_escape' to convert escape sequences
+            # into the raw byte values. 'latin-1' is a single-byte encoding that works well for this.
+            code_bytes = code_sequence.encode('latin-1').decode('unicode_escape').encode('latin-1')
+        else:
+            code_bytes = code_sequence
+        fp.write(code_bytes)
     fp.close()
 
 

xdis/marsh.py

@@ -30,7 +30,7 @@
 from types import CodeType, EllipsisType
 from typing import Optional
 
-from xdis.codetype import Code2, Code3
+from xdis.codetype import Code2, Code3, Code15
 from xdis.unmarshal import long
 from xdis.version_info import PYTHON3, PYTHON_VERSION_TRIPLE, version_tuple_to_str
 
@@ -92,7 +92,7 @@ def Ord(c):
 
 class _Marshaller:
     """Python marshalling routine that runs in Python 2 and Python 3.
-    We also extend to allow for xdis Code2 and Code3 types and instances.
+    We also extend to allow for xdis Code15, Code2, and Code3 types and instances.
     """
 
     dispatch = {}
@@ -122,6 +122,9 @@ def dump(self, x) -> None:
             elif isinstance(x, Code2):
                 self.dispatch[Code2](self, x)
                 return
+            elif isinstance(x, Code15):
+                self.dispatch[Code15](self, x)
+                return
             else:
                 for tp in type(x).mro():
                     func = self.dispatch.get(tp)
@@ -260,16 +263,12 @@ def dump_string(self, x) -> None:
         self.w_long(len(x))
         self._write(x)
 
-    if PYTHON_VERSION_TRIPLE > (2, 5):
-        dispatch[bytes] = dump_string
-        dispatch[bytearray] = dump_string
+    dispatch[bytes] = dump_string
+    dispatch[bytearray] = dump_string
 
-    def dump_unicode(self, x) -> None:
-        self._write(TYPE_UNICODE)
-        if not PYTHON3 and self.python_version < (3, 0):
-            s = x.encode("utf8")
-        else:
-            s = x
+    def dump_unicode(self, s) -> None:
+        type_code = TYPE_STRING if self.python_version < (2, 0) else TYPE_UNICODE
+        self._write(type_code)
         self.w_long(len(s))
         self._write(s)
 
@@ -315,6 +314,43 @@ def dump_dict(self, x) -> None:
 
     dispatch[dict] = dump_dict
 
+    def dump_code15(self, x) -> None:
+        # Careful here: many Python 2 code objects are strings,
+        # but Python 3 marshaling, by default, will dump strings as
+        # unicode. Force marsaling this type as string.
+
+        self._write(TYPE_CODE)
+        self.w_short(x.co_argcount)
+        self.w_short(x.co_nlocals)
+        self.w_short(x.co_stacksize)
+        self.w_short(x.co_flags)
+        self.dump_string(x.co_code)
+
+        # If running in a Python3 interpreter, some constants will get
+        # converted from string to unicode. For now, let's see if
+        # that's okay.
+        self.dump(x.co_consts)
+
+        # The tuple "names" in Python 1.x must have string entries
+        self._write(TYPE_TUPLE)
+        self.w_long(len(x.co_names))
+        for name in x.co_names:
+            self.dump_string(name)
+
+        # The tuple "varnames" in Python 1.x also must have string entries
+        self._write(TYPE_TUPLE)
+        self.w_long(len(x.co_varnames))
+        for name in x.co_varnames:
+            self.dump_string(name)
+
+        self.dump_string(x.co_filename)
+        self.dump_string(x.co_name)
+        self.w_long(x.co_firstlineno)
+        self.dump_string(x.co_lnotab)
+        return
+
+    dispatch[Code15] = dump_code15
+
     def dump_code2(self, x) -> None:
         # Careful here: many Python 2 code objects are strings,
         # but Python 3 marshaling, by default, will dump strings as
@@ -1103,11 +1139,15 @@ def dumps(
         buf = []
         for b in buffer:
             if isinstance(b, str) and PYTHON3:
-                try:
-                    s2b = bytes(ord(b[j]) for j in range(len(b)))
-                except ValueError:
-                    s2b = b.encode("utf-8")
-                buf.append(s2b)
+                if python_version < (2, 0):
+                    # Python 1.x has no notion of Unicode. It uses strings.
+                    buf.append(b)
+                else:
+                    try:
+                        s2b = bytes(ord(b[j]) for j in range(len(b)))
+                    except ValueError:
+                        s2b = b.encode("utf-8")
+                    buf.append(s2b)
             elif isinstance(b, bytearray):
                 buf.append(str(b))
             else:

xdis/opcodes/opcode_14.py

@@ -1,4 +1,4 @@
-# (C) Copyright 2018-2021, 2023 by Rocky Bernstein
+# (C) Copyright 2018-2021, 2023, 2025 by Rocky Bernstein
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU General Public License
@@ -68,6 +68,7 @@ def findlinestarts(co, dup_lines: bool=False):
             lineno = code[offset] + code[offset + 1] * 256
             yield (offset + 2, lineno)
             pass
+
         if op >= loc["HAVE_ARGUMENT"]:
             offset += 2
             pass

xdis/unmarshal.py

@@ -99,12 +99,7 @@ def compat_str(s: Union[str, bytes]) -> Union[str, bytes]:
     This handles working with strings between Python2 and Python3.
     """
     if isinstance(s, bytes):
-        try:
-            return s.decode("utf-8")
-        except UnicodeDecodeError:
-            # If not Unicode, return bytes,
-            # and it will get converted to str when needed.
-            return s
+        return s.decode("utf-8", errors="ignore")
     elif not isinstance(s, str):
         return str(s)
     else:
@@ -509,7 +504,9 @@ def t_code(self, save_ref, bytes_for_s: bool = False):
         # of the string.
         co_code_offset_in_file = self.fp.tell() + 5
 
-        co_code = self.r_object(bytes_for_s=True)
+        # bytes_for_code = self.version_tuple >= (2, 0)
+        bytes_for_code = True
+        co_code = self.r_object(bytes_for_s=bytes_for_code)
 
         # FIXME: Check/verify that is true:
         bytes_for_s = self.version_tuple > (3, 0)