Work on Python 3.4+ marshal. This uses ref objects

Author: rocky

test/Makefile

@@ -27,7 +27,7 @@ check: check-short check-roundtrip
                   --bytecode-pypy35 --bytecode-pypy36 --bytecode-pypy37 \
                   --bytecode-pypy38 --bytecode-pypy39 --bytecode-pypy310 $(COMPILE)
 
-check-roundtrip: check-roundtrip-3.0 check-roundtrip-3.1 check-roundtrip-3.3
+check-roundtrip: check-roundtrip-3.0 check-roundtrip-3.1 check-roundtrip-3.3 check-roundtrip-3.4
 
 check-roundtrip-3.0:
 	./test_pyc_roundtrip.py --bytecode-3.0
@@ -38,6 +38,9 @@ check-roundtrip-3.1:
 check-roundtrip-3.3:
 	./test_pyc_roundtrip.py --bytecode-3.3
 
+check-roundtrip-3.4:
+	./test_pyc_roundtrip.py --bytecode-3.4
+
 check-short:
 	$(PYTHON) test_pyenvlib.py --simple
 

test/bytecode_3.4/01_extended_arg.pyc

@@ -16,7 +16,7 @@
 
 import types
 from collections import namedtuple
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import Any, Dict, Optional, Set, Tuple, Union
 
 from xdis.codetype.base import CodeBase
 from xdis.codetype.code13 import Code13
@@ -63,7 +63,8 @@ def codeType2Portable(code, version_tuple=PYTHON_VERSION_TRIPLE):
 
                 # THINK ABOUT: If collection_order isn't defined, i.e. native code
                 # type, should we try to extract it?
-                code.collection_order if hasattr(code, "collection_order") else {}
+                code.collection_order if hasattr(code, "collection_order") else {},
+                code.reference_objects if hasattr(code, "reference_objects") else set(),
             )
         elif version_tuple < (3, 10):
             return Code38(
@@ -207,7 +208,7 @@ def portableCodeType(version_tuple=PYTHON_VERSION_TRIPLE):
 # In contrast to Code3, Code2, etc. you can use CodeTypeUnint for building
 # an incomplete code type, which might be converted to another code type
 # later.
-CodeTypeUnionFields = tuple(Code311FieldNames.split() + ["collection_order"])
+CodeTypeUnionFields = tuple(Code311FieldNames.split() + ["collection_order", "reference_objects"])
 CodeTypeUnion = namedtuple("CodeTypeUnion", CodeTypeUnionFields)
 
 
@@ -235,6 +236,7 @@ def to_portable(
     co_exceptiontable=None,  # 3.11+
     version_triple=PYTHON_VERSION_TRIPLE,
     collection_order: Dict[Union[set, frozenset, dict], Tuple[Any]] = {},
+    reference_objects: Set[Any] = set(),
 ):
     code = CodeTypeUnion(
         co_argcount=co_argcount,
@@ -256,6 +258,7 @@ def to_portable(
         co_cellvars=co_cellvars,
         co_exceptiontable=co_exceptiontable,
         collection_order=collection_order,
+        reference_objects=reference_objects,
     )
     return codeType2Portable(code, version_triple)
 

xdis/codetype/__init__.py

@@ -17,7 +17,7 @@
 import types
 from copy import deepcopy
 from types import CodeType
-from typing import Any, Dict, Tuple, Union
+from typing import Any, Dict, Set, Tuple, Union
 
 from xdis.codetype.code20 import Code2, Code2FieldTypes
 from xdis.version_info import PYTHON_VERSION_TRIPLE, version_tuple_to_str
@@ -61,7 +61,8 @@ def __init__(
         co_lnotab,
         co_freevars,
         co_cellvars,
-        collection_order: Dict[Union[set, frozenset, dict], Tuple[Any]] = {}
+        collection_order: Dict[Union[set, frozenset, dict], Tuple[Any]] = {},
+        reference_objects: Set[Any] = set(),
     ) -> None:
         # Keyword argument parameters in the call below is more robust.
         # Since things change around, robustness is good.
@@ -84,10 +85,19 @@ def __init__(
         self.co_kwonlyargcount = co_kwonlyargcount
         self.fieldtypes = Code3FieldTypes
 
-        # It is helpful to save the order in sets, frozensets and dictionary keys,
-        # so that on writing a bytecode file we can duplicate this order.
+        # The following fields are mostly useful in marshaling a code object.
+        # Keeping marshal order exactly the same is useful in round-trip marshal
+        # testing; but it may also have other benefits.
+
+        # By saving the order in sets, frozensets, and dictionary keys,
+        # these collections can be written in the same order that appeared
+        # in unmarshalling (if that's how the code object was created).
         self.collection_order = collection_order
 
+        # Keeping track of which objects were referenced, allows
+        self.reference_objects = reference_objects
+
+
         if type(self) is Code3:
             self.check()
         return