Fix compatibility with Python 3.11 (#467)

Code objects have some new attributes. Those are related
to the enhanced exceptions with code highlighting.
CI job for Python 3.11 is no longer optional.

Author: frenzymadness

.github/workflows/testing.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python_version: [3.6, 3.7, 3.8, 3.9, "3.10-dev", "pypy3"]
+        python_version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev", "pypy3"]
         exclude:
           # Do not test all minor versions on all platforms, especially if they
           # are not the oldest/newest supported versions
@@ -94,48 +94,6 @@ jobs:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: ./coverage.xml
 
-  python-nightly:
-    runs-on: ubuntu-18.04
-    # This entry is made optional for now, see https://github.com/cloudpipe/cloudpickle/pull/420
-    if: "contains(github.event.pull_request.labels.*.name, 'ci python-nightly')"
-    steps:
-    - uses: actions/checkout@v1
-    - name: Install Python from ppa:deadsnakes/nightly
-      run: |
-        sudo add-apt-repository ppa:deadsnakes/nightly
-        sudo apt update
-        sudo apt install python3.11 python3.11-venv python3.11-dev
-        python3.11 -m venv nightly-venv
-        echo "$PWD/nightly-venv/bin" >> $GITHUB_PATH
-    - name: Display Python version
-      run: python -c "import sys; print(sys.version)"
-    - name: Install project and dependencies
-      run: |
-        set -e
-        python -m pip install --upgrade pip
-        python -m pip install -r dev-requirements.txt
-        python -m pip install -e .
-        python ci/install_coverage_subprocess_pth.py
-    - name: Generate old pickles (backward compat)
-      shell: bash
-      run: |
-        git_head=$(git rev-parse HEAD)
-        cp tests/generate_old_pickles.py tests/_generate_old_pickles.py
-        git checkout v1.4.1
-        python tests/_generate_old_pickles.py
-        git checkout ${git_head}
-    - name: Test with pytest
-      run: |
-        COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \
-          PYTHONPATH='.:tests' python -m pytest -r s
-        coverage combine --append
-        coverage xml -i
-    - name: Publish coverage results
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: ./coverage.xml
-
   distributed-downstream-build:
     runs-on: ubuntu-latest
     if: "contains(github.event.pull_request.labels.*.name, 'ci distributed') || contains(github.event.pull_request.labels.*.name, 'ci downstream')"

CHANGES.md

@@ -8,6 +8,9 @@
 - Support for pickling subclasses of generic classes.
   ([PR #448](https://github.com/cloudpipe/cloudpickle/pull/448))
 
+- Support and CI configuration for Python 3.11.
+  ([PR #467](https://github.com/cloudpipe/cloudpickle/pull/467))
+
 2.0.0
 =====
 

cloudpickle/cloudpickle_fast.py

@@ -244,7 +244,23 @@ def _enum_getstate(obj):
 
 def _code_reduce(obj):
     """codeobject reducer"""
-    if hasattr(obj, "co_linetable"):  # pragma: no branch
+    # If you are not sure about the order of arguments, take a look at help
+    # of the specific type from types, for example:
+    # >>> from types import CodeType
+    # >>> help(CodeType)
+    if hasattr(obj, "co_columntable"):  # pragma: no branch
+        # Python 3.11 and later: there are some new attributes
+        # related to the enhanced exceptions.
+        args = (
+            obj.co_argcount, obj.co_posonlyargcount,
+            obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
+            obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
+            obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
+            obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable,
+            obj.co_columntable, obj.co_exceptiontable, obj.co_freevars,
+            obj.co_cellvars,
+        )
+    elif hasattr(obj, "co_linetable"):  # pragma: no branch
         # Python 3.10 and later: obj.co_lnotab is deprecated and constructor
         # expects obj.co_linetable instead.
         args = (

tests/cloudpickle_test.py

@@ -2386,29 +2386,13 @@ def method(self, arg: type_) -> type_:
 
                 def check_annotations(obj, expected_type, expected_type_str):
                     assert obj.__annotations__["attribute"] == expected_type
-                    if sys.version_info >= (3, 11):
-                        # In Python 3.11, type annotations are stored as strings.
-                        # See PEP 563 for more details:
-                        # https://www.python.org/dev/peps/pep-0563/
-                        # Originaly scheduled for 3.10, then postponed.
-                        # See this for more details:
-                        # https://mail.python.org/archives/list/[email protected]/thread/CLVXXPQ2T2LQ5MP2Y53VVQFCXYWQJHKZ/
-                        assert (
-                            obj.method.__annotations__["arg"]
-                            == expected_type_str
-                        )
-                        assert (
-                            obj.method.__annotations__["return"]
-                            == expected_type_str
-                        )
-                    else:
-                        assert (
-                            obj.method.__annotations__["arg"] == expected_type
-                        )
-                        assert (
-                            obj.method.__annotations__["return"]
-                            == expected_type
-                        )
+                    assert (
+                        obj.method.__annotations__["arg"] == expected_type
+                    )
+                    assert (
+                        obj.method.__annotations__["return"]
+                        == expected_type
+                    )
                     return "ok"
 
                 obj = MyClass()

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35, py36, py37, py38, py39, py310, pypy3
+envlist = py35, py36, py37, py38, py39, py310, py311, pypy3
 
 [testenv]
 deps = -rdev-requirements.txt