Word on magics numbers and routiens

set xdis.magic.minor_release_names list release names. Generally these will
be supported.

dict xdis.magic.version2magicint[] gives a way to go from version name to magic
number.

frozenset xdis.INTERIM_MAGIC_INTS gives a (partial) list of used magic
numbers that are interim.

Author: rocky

pytest/test_magic.py

@@ -1,8 +1,16 @@
 """
 Unit test for xdis.magics
 """
+
 import pytest
-from xdis.magics import MAGIC, sysinfo2magic
+from xdis.magics import (
+    INTERIM_MAGIC_INTS,
+    MAGIC,
+    magicint2version,
+    minor_release_names,
+    sysinfo2magic,
+    version2magicint,
+)
 from xdis.version_info import PYTHON_VERSION_TRIPLE
 
 
@@ -12,3 +20,15 @@
 )
 def test_magic():
     assert sysinfo2magic() == MAGIC, (sysinfo2magic(), MAGIC)
+
+    # Check that our interim version numbers are not used as release numbers.
+    interim_version_names = {
+        magicint2version[magic_int] for magic_int in INTERIM_MAGIC_INTS
+    }
+    incorrect_interim_names = interim_version_names.intersection(minor_release_names)
+    if interim_version_names:
+        for incorrect_name in incorrect_interim_names:
+            print(f"Remove {incorrect_name} {version2magicint[incorrect_name]}")
+    assert (
+        not incorrect_interim_names
+    ), f"Remove magic numbers for {incorrect_interim_names}"

xdis/load.py

@@ -29,6 +29,7 @@
 from xdis.dropbox.decrypt25 import fix_dropbox_pyc
 from xdis.magics import (
     GRAAL3_MAGICS,
+    INTERIM_MAGIC_INTS,
     JYTHON_MAGICS,
     PYPY3_MAGICS,
     PYTHON_MAGIC_INT,
@@ -251,34 +252,7 @@ def load_module_from_file_object(
             version = magicint2version.get(magic_int, "")
             raise ImportError(f"Magic int {magic_int} ({version}) is not supported.")
 
-        if magic_int in (
-            3010,
-            3020,
-            3030,
-            3040,
-            3050,
-            3060,
-            3061,
-            3071,
-            3361,
-            3091,
-            3101,
-            3103,
-            3141,
-            3270,
-            3280,
-            3290,
-            3300,
-            3320,
-            3330,
-            3371,
-            62071,
-            62071,
-            62081,
-            62091,
-            62092,
-            62111,
-        ):
+        if magic_int in INTERIM_MAGIC_INTS:
             raise ImportError(
                 "%s is interim Python %s (%d) bytecode which is "
                 "not supported.\nFinal released versions are "

xdis/magics.py

@@ -35,6 +35,7 @@
 import re
 import struct
 import sys
+from collections import defaultdict
 from importlib.util import MAGIC_NUMBER as MAGIC
 from typing import Dict, Set, Tuple
 
@@ -54,11 +55,44 @@
     24881,  # RustPython 3.13
 )
 
+# A list of interim Python version magic numbers used, but were not
+# the *final* major/minor for that release. For example, number 3430
+# to 3435 (3.10a1 to 3.10a7) are in this list, but 3438 (the *last* of
+# the 3.10b magic number changes), is not. Magic number 3438 used in
+# 3.10b was the final release before major/minor release 3.10; it
+# is the magic number used in 3.10 releases 3.10.0 to 3.10.19.
+
+# FIXME: complete the below table.
+# fmt: off
+
+# The following should be interim and this list.
+# The they all seems to be used in Python up to 3.6.15!
+# 3378 3.6b  Used up to 3.6.15
+# 3372 is also 3.6?
+# 3379 is 3.7?
+# 3230 3.3a4
+# 3131 3.0a5 is 3.0.1?
+# 3393 3.7?
+# 3401 3.8.0a3+?
+INTERIM_MAGIC_INTS = frozenset([
+     3010,  3020,  3030,  3040,  3050,  3060,  3061,  3071,  3081,  3091,
+     3101,  3103,  3111,  3141,  3160,  3170,  3190,  3200,  3210, 3220,
+     3250,  3260,  3270,  3280,  3290,  3300,  3320,  3340,
+     3361,  3371,  3373,  3375,  3376,  3377,  3390,  3391,
+     3392,  3400,  3410,  3411,  3412,  3420,  3421,  3422, 3423,
+     3424,  3430,  3431,  3432,  3433,  3435,  3436,  3437,  3438,  3450, 3451,
+    # ...
+    62041, 62051, 62071, 62081, 62071, 62091, 62081, 62091, 62092, 62111,
+    62121, 62121, 62151, 62171, 62181, 62191, 62201,
+])
+# fmt: ofn
 
 def add_magic_from_int(magic_int: int, version: str) -> None:
     magicint2version[magic_int] = version
     versions[int2magic(magic_int)] = version
+    version2magicint[version].append(magic_int)
 
+version2magicint = defaultdict(list)
 
 def int2magic(magic_int: int) -> bytes:
     """Given a magic int like 62211, compute the corresponding magic byte string
@@ -154,11 +188,9 @@ def __by_version(magic_versions: Dict[bytes, str]) -> dict:
 add_magic_from_int(62041, "2.4a0")
 add_magic_from_int(62051, "2.4a3")
 add_magic_from_int(62061, "2.4b1")
-add_magic_from_int(62071, "2.5a0")
-
-# ast-branch
-add_magic_from_int(62081, "2.5a0")
 
+add_magic_from_int(62071, "2.5a0")
+add_magic_from_int(62081, "2.5a0")  # ast-branch
 add_magic_from_int(62091, "2.5a0")  # with
 add_magic_from_int(62092, "2.5a0")  # changed WITH_CLEANUP opcode
 add_magic_from_int(62101, "2.5b3")  # fix wrong code: for x, in ...
@@ -416,7 +448,11 @@ def __by_version(magic_versions: Dict[bytes, str]) -> dict:
 
 # Change CALL_xxx opcodes
 add_magic_from_int(3466, "3.11a4c")
+
+# Add SEND opcode
 add_magic_from_int(3467, "3.11a4d")
+
+# bpo-45711: remove type, traceback from exc_info
 add_magic_from_int(3468, "3.11a4e")
 add_magic_from_int(3469, "3.11a4f")
 add_magic_from_int(3470, "3.11a4g")
@@ -591,19 +627,19 @@ def __by_version(magic_versions: Dict[bytes, str]) -> dict:
 # Add oparg to END_ASYNC_FOR
 add_magic_from_int(3618, "3.14a6b")
 
-# Renumber RESUME opcode from 149 to 128
+# Renumber RESUME opcode from 149 to 128.
 add_magic_from_int(3619, "3.14a6c")
 
-# Optimize bytecode for all/any/tuple called on a genexp
+# Optimize bytecode for all/any/tuple called on a genexp.
 add_magic_from_int(3620, "3.14a7a")
 
-# Optimize LOAD_FAST opcodes into LOAD_FAST_BORROW
+# Optimize LOAD_FAST opcodes into LOAD_FAST_BORROW.
 add_magic_from_int(3621, "3.14a7b")
 
-# Store annotations in different class dict keys
+# Store annotations in different class dict keys.
 add_magic_from_int(3622, "3.14a7c")
 
-# Add BUILD_INTERPOLATION & BUILD_TEMPLATE opcodes
+# Add BUILD_INTERPOLATION & BUILD_TEMPLATE opcodes.
 add_magic_from_int(3623, "3.14a7d")
 
 # Don't optimize LOAD_FAST when local is killed by DELETE_FAST
@@ -612,11 +648,34 @@ def __by_version(magic_versions: Dict[bytes, str]) -> dict:
 # Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST
 add_magic_from_int(3625, "3.14b3")
 
+# Fix miscompilation of some module-level annotations.
+add_magic_from_int(3626, "3.14rc2")
+
+# Fix miscompilation of some module-level annotations.
 add_magic_from_int(3627, "3.14rc3")
 
-# add_magic_from_int(3655, "3.15.0a0")
+# Initial 3.15 version
+add_magic_from_int(3650, "3.15a0")
+
+# Simplify LOAD_CONST.
+add_magic_from_int(3651, "3.15a1a")
+
+# Add Virtual iterators.
+add_magic_from_int(3652, "3.15a1b")
+
+# Fix handling of opcodes that may leave operands on the stack when
+# optimizing LOAD_FAST.
+add_magic_from_int(3653, "3.15a1c")
+
+# Fix missing exception handlers in logical expression.
+add_magic_from_int(3654, "3.15a1d")
+
+# Fix miscompilation of some module-level annotations.
+# add_magic_from_int(3655, "3.15a1e")
 # NOTE: this will change on release!
-add_magic_from_int(3655, "3.15.0")
+add_magic_from_int(3655, "3.15-dev")
+
+# Python 3.16 will start with 3700
 
 # Weird ones
 # WTF? Python 3.2.5 and PyPy have weird magic numbers
@@ -703,7 +762,7 @@ def add_canonic_versions(release_versions: str, canonic: str) -> None:
 add_canonic_versions(
     "3.6b2 3.6 3.6.0 3.6.1 3.6.2 3.6.3 3.6.4 3.6.5 3.6.6 3.6.7 3.6.8 "
     "3.6.9 3.6.10 3.6.11 3.6.12 3.6.13 3.6.14 3.6.15",
-    "3.6rc1",
+    "3.6rc1", # ?? 3.6b2 is not 3.6rc1!
 )
 
 add_canonic_versions("3.7b1", "3.7.0beta3")
@@ -774,8 +833,8 @@ def add_canonic_versions(release_versions: str, canonic: str) -> None:
 add_canonic_versions("3.14 3.14.0", "3.14rc3")
 
 add_canonic_versions(
-    "3.15 3.15.0a1 3.15-dev 3.15.0a0",
-    "3.15.0",
+    "3.15 3.15.0 3.15.0a1 3.15.0a0",
+    "3.15-dev",
 )
 
 # The canonic version for a canonic version is itself
@@ -784,6 +843,12 @@ def add_canonic_versions(release_versions: str, canonic: str) -> None:
 # A set of all Python versions we know about
 python_versions = set(canonic_python_version.keys())
 
+# Python major, minor version names, e.g. 3.6, 3.11pypy, etc.
+# These are not considered interim version number.
+minor_release_names = {
+    python_version for python_version in python_versions if re.match("^[1-3][.][0-9]+(?:pypy|Graal)?$", python_version)
+}
+
 
 def __show(text, magic) -> None:
     print(text, struct.unpack("BBBB", magic), struct.unpack("<HBB", magic))
@@ -869,6 +934,12 @@ def test() -> None:
     print(type(magic_20), len(magic_20), repr(magic_20))
     assert sysinfo2magic() == MAGIC, (sysinfo2magic(), MAGIC)
 
+    # Check that our interim version numbers are not used as release numbers.
+    interim_version_names = {magicint2version[magic_int] for magic_int in INTERIM_MAGIC_INTS}
+    incorrect_interim_names = interim_version_names.intersection(minor_release_names)
+    if interim_version_names:
+        for incorrect_name in incorrect_interim_names:
+            print(f"Remove {incorrect_name} {version2magicint[incorrect_name]}")
 
 if __name__ == "__main__":
     test()