diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6df3128d..aad97585 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -10,9 +10,9 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: 3.11 - name: Install flake8 @@ -32,20 +32,15 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - # TODO: add "3.12-dev" to the list - python_version: [3.7, 3.8, 3.9, "3.10", "3.11", "pypy-3.9"] + python_version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.9"] exclude: # Do not test all minor versions on all platforms, especially if they # are not the oldest/newest supported versions - - os: windows-latest - python_version: 3.7 - os: windows-latest python_version: 3.8 # as of 4/02/2020, psutil won't build under PyPy + Windows - os: windows-latest python_version: "pypy-3.9" - - os: macos-latest - python_version: 3.7 - os: macos-latest python_version: 3.8 - os: macos-latest @@ -56,11 +51,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python_version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python_version }} + allow-prereleases: true - name: Install project and dependencies shell: bash run: | @@ -85,7 +81,7 @@ jobs: coverage combine --append coverage xml -i - name: Publish coverage results - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml @@ -101,9 +97,9 @@ jobs: matrix: python_version: ["3.10"] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python_version }} - name: Install project and dependencies @@ -133,9 +129,9 @@ jobs: matrix: python_version: ["3.10"] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python_version }} - name: Install project and dependencies @@ -161,9 +157,9 @@ jobs: matrix: python_version: ["3.10"] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python_version }} - name: Install downstream project and dependencies @@ -186,9 +182,9 @@ jobs: matrix: python_version: ["3.10"] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python_version }} - name: Install project and dependencies diff --git a/README.md b/README.md index 701b3d1d..a2101d05 100644 --- a/README.md +++ b/README.md @@ -130,14 +130,14 @@ Running the tests or alternatively for a specific environment: - tox -e py37 + tox -e py312 -- With `py.test` to only run the tests for your current version of +- With `pytest` to only run the tests for your current version of Python: pip install -r dev-requirements.txt - PYTHONPATH='.:tests' py.test + PYTHONPATH='.:tests' pytest History ------- diff --git a/ci/install_coverage_subprocess_pth.py b/ci/install_coverage_subprocess_pth.py index 6a273e4c..927820b9 100644 --- a/ci/install_coverage_subprocess_pth.py +++ b/ci/install_coverage_subprocess_pth.py @@ -5,7 +5,7 @@ import os.path as op from sysconfig import get_path -FILE_CONTENT = u"""\ +FILE_CONTENT = """\ import coverage; coverage.process_startup() """ diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 317be691..73c93f61 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -55,26 +55,18 @@ from .compat import pickle from collections import OrderedDict +from types import CellType from typing import ClassVar, Generic, Union, Tuple, Callable from pickle import _getattribute from importlib._bootstrap import _find_spec try: # pragma: no branch import typing_extensions as _typing_extensions - from typing_extensions import Literal, Final + from typing_extensions import Literal + from typing import Final except ImportError: _typing_extensions = Literal = Final = None -if sys.version_info >= (3, 8): - from types import CellType -else: - def f(): - a = 1 - - def g(): - return a - return g - CellType = type(f().__closure__[0]) # cloudpickle is meant for inter process communication: we expect all @@ -201,20 +193,7 @@ def _whichmodule(obj, name): - Errors arising during module introspection are ignored, as those errors are considered unwanted side effects. """ - if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa - # Workaround bug in old Python versions: prior to Python 3.7, - # T.__module__ would always be set to "typing" even when the TypeVar T - # would be defined in a different module. - if name is not None and getattr(typing, name, None) is obj: - # Built-in TypeVar defined in typing such as AnyStr - return 'typing' - else: - # User defined or third-party TypeVar: __module__ attribute is - # irrelevant, thus trigger a exhaustive search for obj in all - # modules. - module_name = None - else: - module_name = getattr(obj, '__module__', None) + module_name = getattr(obj, '__module__', None) if module_name is not None: return module_name @@ -322,7 +301,7 @@ def _extract_code_globals(co): out_names = _extract_code_globals_cache.get(co) if out_names is None: # We use a dict with None values instead of a set to get a - # deterministic order (assuming Python 3.6+) and avoid introducing + # deterministic order and avoid introducing # non-deterministic pickle bytes as a results. out_names = {name: None for name in _walk_global_ops(co)} @@ -399,7 +378,7 @@ def cell_set(cell, value): ``f = types.FunctionType(code, globals, name, argdefs, closure)``, closure will not be able to contain the yet-to-be-created f. - In Python3.7, cell_contents is writeable, so setting the contents of a cell + cell_contents is writeable, so setting the contents of a cell can be done simply using >>> cell.cell_contents = value @@ -446,12 +425,7 @@ def cell_set(cell, value): test and checker libraries decide to parse the whole file. """ - if sys.version_info[:2] >= (3, 7): # pragma: no branch - cell.cell_contents = value - else: - _cell_set = types.FunctionType( - _cell_set_template_code, {}, '_cell_set', (), (cell,),) - _cell_set(value) + cell.cell_contents = value def _make_cell_set_template_code(): @@ -481,9 +455,6 @@ def _cell_set_factory(value): return _cell_set_template_code -if sys.version_info[:2] < (3, 7): - _cell_set_template_code = _make_cell_set_template_code() - # relevant opcodes STORE_GLOBAL = opcode.opmap['STORE_GLOBAL'] DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL'] @@ -540,51 +511,6 @@ def _extract_class_dict(cls): return clsdict -if sys.version_info[:2] < (3, 7): # pragma: no branch - def _is_parametrized_type_hint(obj): - # This is very cheap but might generate false positives. So try to - # narrow it down is good as possible. - type_module = getattr(type(obj), '__module__', None) - from_typing_extensions = type_module == 'typing_extensions' - from_typing = type_module == 'typing' - - # general typing Constructs - is_typing = getattr(obj, '__origin__', None) is not None - - # typing_extensions.Literal - is_literal = ( - (getattr(obj, '__values__', None) is not None) - and from_typing_extensions - ) - - # typing_extensions.Final - is_final = ( - (getattr(obj, '__type__', None) is not None) - and from_typing_extensions - ) - - # typing.ClassVar - is_classvar = ( - (getattr(obj, '__type__', None) is not None) and from_typing - ) - - # typing.Union/Tuple for old Python 3.5 - is_union = getattr(obj, '__union_params__', None) is not None - is_tuple = getattr(obj, '__tuple_params__', None) is not None - is_callable = ( - getattr(obj, '__result__', None) is not None and - getattr(obj, '__args__', None) is not None - ) - return any((is_typing, is_literal, is_final, is_classvar, is_union, - is_tuple, is_callable)) - - def _create_parametrized_type_hint(origin, args): - return origin[args] -else: - _is_parametrized_type_hint = None - _create_parametrized_type_hint = None - - def parametrized_type_hint_getinitargs(obj): # The distorted type check sematic for typing construct becomes: # ``type(obj) is type(TypeHint)``, which means "obj is a diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index ee1f4b8e..4375045b 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -32,8 +32,8 @@ _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, - _is_parametrized_type_hint, PYPY, cell_set, - parametrized_type_hint_getinitargs, _create_parametrized_type_hint, + PYPY, cell_set, + parametrized_type_hint_getinitargs, builtin_code_type, _make_dict_keys, _make_dict_values, _make_dict_items, _make_function, ) @@ -188,7 +188,7 @@ def _class_getstate(obj): clsdict.pop('_abc_negative_cache_version', None) registry = clsdict.pop('_abc_registry', None) if registry is None: - # in Python3.7+, the abc caches and registered subclasses of a + # The abc caches and registered subclasses of a # class are bundled into the single _abc_impl attribute clsdict.pop('_abc_impl', None) (registry, _, _, _) = abc._get_dump(obj) @@ -355,7 +355,7 @@ def _file_reduce(obj): obj.seek(0) contents = obj.read() obj.seek(curloc) - except IOError as e: + except OSError as e: raise pickle.PicklingError( "Cannot pickle file %s as it cannot be read" % name ) from e @@ -720,11 +720,6 @@ def reducer_override(self, obj): reducers, such as Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248 """ - if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch - return ( - _create_parametrized_type_hint, - parametrized_type_hint_getinitargs(obj) - ) t = type(obj) try: is_anyclass = issubclass(t, type) @@ -785,18 +780,7 @@ def save_global(self, obj, name=None, pack=struct.pack): return self.save_reduce( _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) - if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch - # Parametrized typing constructs in Python < 3.7 are not - # compatible with type checks and ``isinstance`` semantics. For - # this reason, it is easier to detect them using a - # duck-typing-based check (``_is_parametrized_type_hint``) than - # to populate the Pickler's dispatch with type-specific savers. - self.save_reduce( - _create_parametrized_type_hint, - parametrized_type_hint_getinitargs(obj), - obj=obj - ) - elif name is not None: + if name is not None: Pickler.save_global(self, obj, name=name) elif not _should_pickle_by_reference(obj, name=name): self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index 5e9b5277..7a2eb295 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -1,18 +1,5 @@ -import sys +import pickle # noqa: F401 - -if sys.version_info < (3, 8): - try: - import pickle5 as pickle # noqa: F401 - from pickle5 import Pickler # noqa: F401 - except ImportError: - import pickle # noqa: F401 - - # Use the Python pickler for old CPython versions - from pickle import _Pickler as Pickler # noqa: F401 -else: - import pickle # noqa: F401 - - # Pickler will the C implementation in CPython and the Python - # implementation in PyPy - from pickle import Pickler # noqa: F401 +# Pickler will the C implementation in CPython and the Python +# implementation in PyPy +from pickle import Pickler # noqa: F401 diff --git a/dev-requirements.txt b/dev-requirements.txt index 53e56a1c..cae73150 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,10 +1,8 @@ -# Dependencies for running the tests with py.test +# Dependencies for running the tests with pytest flake8 pytest pytest-cov psutil -# To test on older Python versions -pickle5 >=0.0.11 ; python_version == '3.7' and python_implementation == 'CPython' # To be able to test tornado coroutines tornado # To be able to test numpy specific things diff --git a/setup.py b/setup.py index a96140d6..8b1021ae 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import os import re @@ -12,7 +11,7 @@ # Function to parse __version__ in `cloudpickle/__init__.py` def find_version(): here = os.path.abspath(os.path.dirname(__file__)) - with open(os.path.join(here, 'cloudpickle', '__init__.py'), 'r') as fp: + with open(os.path.join(here, 'cloudpickle', '__init__.py')) as fp: version_file = fp.read() version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) @@ -21,7 +20,7 @@ def find_version(): raise RuntimeError("Unable to find version string.") -dist = setup( +setup( name='cloudpickle', version=find_version(), description='Extended pickling support for Python objects', @@ -39,10 +38,11 @@ def find_version(): 'Operating System :: POSIX', 'Operating System :: Microsoft :: Windows', 'Operating System :: MacOS :: MacOS X', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Libraries :: Python Modules', @@ -50,5 +50,5 @@ def find_version(): 'Topic :: System :: Distributed Computing', ], test_suite='tests', - python_requires='>=3.6', + python_requires='>=3.8', ) diff --git a/tests/cloudpickle_file_test.py b/tests/cloudpickle_file_test.py index 25fd9844..d875da1b 100644 --- a/tests/cloudpickle_file_test.py +++ b/tests/cloudpickle_file_test.py @@ -25,7 +25,7 @@ def tearDown(self): def test_empty_file(self): # Empty file open(self.tmpfilepath, 'w').close() - with open(self.tmpfilepath, 'r') as f: + with open(self.tmpfilepath) as f: self.assertEqual('', pickle.loads(cloudpickle.dumps(f)).read()) os.remove(self.tmpfilepath) @@ -43,7 +43,7 @@ def test_r_mode(self): with open(self.tmpfilepath, 'w') as f: f.write(self.teststring) # Open for reading - with open(self.tmpfilepath, 'r') as f: + with open(self.tmpfilepath) as f: new_f = pickle.loads(cloudpickle.dumps(f)) self.assertEqual(self.teststring, new_f.read()) os.remove(self.tmpfilepath) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 2bd1f257..4ab3f50e 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -132,7 +132,7 @@ def tearDown(self): @pytest.mark.skipif( platform.python_implementation() != "CPython" or - (sys.version_info >= (3, 8, 0) and sys.version_info < (3, 8, 2)), + sys.version_info < (3, 8, 2), reason="Underlying bug fixed upstream starting Python 3.8.2") def test_reducer_override_reference_cycle(self): # Early versions of Python 3.8 introduced a reference cycle between a @@ -475,8 +475,7 @@ def test_load_namespace(self): def test_generator(self): def some_generator(cnt): - for i in range(cnt): - yield i + yield from range(cnt) gen2 = pickle_depickle(some_generator, protocol=self.protocol) @@ -766,8 +765,6 @@ def test_module_importability(self): # Check for similar behavior for a module that cannot be imported by # attribute lookup. from _cloudpickle_testpkg.mod import dynamic_submodule_two as m2 - # Note: import _cloudpickle_testpkg.mod.dynamic_submodule_two as m2 - # works only for Python 3.7+ assert _should_pickle_by_reference(m2) assert pickle_depickle(m2, protocol=self.protocol) is m2 @@ -2744,10 +2741,6 @@ def _call_from_registry(k): if "_cloudpickle_testpkg" in list_registry_pickle_by_value(): unregister_pickle_by_value(_cloudpickle_testpkg) - @pytest.mark.skipif( - sys.version_info < (3, 7), - reason="Determinism can only be guaranteed for Python 3.7+" - ) def test_deterministic_pickle_bytes_for_function(self): # Ensure that functions with references to several global names are # pickled to fixed bytes that do not depend on the PYTHONHASHSEED of diff --git a/tests/cloudpickle_testpkg/setup.py b/tests/cloudpickle_testpkg/setup.py index a503b8d3..5cb49f90 100644 --- a/tests/cloudpickle_testpkg/setup.py +++ b/tests/cloudpickle_testpkg/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup -dist = setup( +setup( name='cloudpickle_testpkg', version='0.0.0', description='Package used only for cloudpickle testing purposes', @@ -12,5 +12,5 @@ author_email='cloudpipe@googlegroups.com', license='BSD 3-Clause License', packages=['_cloudpickle_testpkg'], - python_requires='>=3.5', + python_requires='>=3.8', ) diff --git a/tests/test_backward_compat.py b/tests/test_backward_compat.py index b66937b5..d4d57dd3 100644 --- a/tests/test_backward_compat.py +++ b/tests/test_backward_compat.py @@ -19,9 +19,7 @@ def load_obj(filename, check_deprecation_warning='auto'): if check_deprecation_warning == 'auto': - # pickles files generated with cloudpickle_fast.py on old versions of - # coudpickle with Python < 3.8 use non-deprecated reconstructors. - check_deprecation_warning = (sys.version_info < (3, 8)) + check_deprecation_warning = False pickle_filepath = PICKLE_DIRECTORY / filename if not pickle_filepath.exists(): pytest.skip(f"Could not find {str(pickle_filepath)}") diff --git a/tox.ini b/tox.ini index f7d28f0c..5985d534 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,12 @@ [tox] -envlist = py35, py36, py37, py38, py39, py310, py311, pypy3 +envlist = py{38, 39, 310, 311, 312, py3} [testenv] deps = -rdev-requirements.txt setenv = PYTHONPATH = {toxinidir}:{toxinidir}/tests commands = - py.test {posargs:-lv --maxfail=5} + pytest {posargs:-lv --maxfail=5} [pytest] addopts = -s