diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index cb2e909f..00000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[bumpversion] -current_version = 2.6.0 -commit = True -tag = True -tag_name = {new_version} - -[bumpversion:file:papermill/version.py] diff --git a/DEVELOPMENT_GUIDE.md b/DEVELOPMENT_GUIDE.md index 368cad8a..ba63ef3a 100644 --- a/DEVELOPMENT_GUIDE.md +++ b/DEVELOPMENT_GUIDE.md @@ -73,6 +73,6 @@ git push upstream && git push upstream --tags ```bash rm -rf dist/* rm -rf build/* -python setup.py sdist bdist_wheel +python -m build twine upload dist/* ``` diff --git a/MANIFEST.in b/MANIFEST.in index 94affefe..d32a2282 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,19 +6,14 @@ recursive-include papermill *.yml recursive-include papermill *.keep recursive-include papermill *.txt -include setup.py include requirements.txt recursive-include requirements *.txt -include tox.ini -include pytest.ini include README.md include LICENSE include MANIFEST.in include *.md include *.toml -include .bumpversion.cfg - # Documentation prune docs diff --git a/docs/extending-entry-points.rst b/docs/extending-entry-points.rst index bd986082..9da81170 100644 --- a/docs/extending-entry-points.rst +++ b/docs/extending-entry-points.rst @@ -51,18 +51,15 @@ Ensuring your handler is found by papermill ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Once you have developed a new handler, you need to declare papermill entry -points in your ``setup.py`` file. +points in your ``pyproject.toml`` file. -This is done by including the ``entry_points`` key-word argument to ``setup`` -in your setup.py file: +This is done by including the ``[project.entry-points."papermill.io"]`` section +in your pyproject.toml file: -.. code-block:: python +.. code-block:: toml - from setuptools import setup, find_packages - setup( - # all the normal setup.py arguments... - entry_points={"papermill.io": ["sftp://=papermill_sftp:SFTPHandler"]}, - ) + [project.entry-points."papermill.io"] + "sftp://" = "papermill_sftp:SFTPHandler" This indicates to papermill that when a file path begins with ``sftp://``, it should use the class ``papermill_sftp.SFTPHandler`` to handle reading or writing @@ -84,7 +81,7 @@ from an sftp server and writes back to it, so we could do the following:: Our project structure will look like this:: papermill_sftp - |- setup.py + |- pyproject.toml |- src |- papermill_sftp |- __init__.py @@ -153,24 +150,32 @@ implement a listdir option for now. raise NotImplementedError -The ``setup.py`` file contains the following code: +The ``pyproject.toml`` file contains the following code: -.. code-block:: python +.. code-block:: toml + + [build-system] + requires = ["setuptools>=61.0", "wheel"] + build-backend = "setuptools.build_meta" + + [project] + name = "papermill_sftp" + version = "0.1" + description = "An SFTP I/O handler for papermill." + authors = [ + {name = "My Name", email = "my.email@gmail.com"} + ] + dependencies = ["pysftp"] + + [project.urls] + Repository = "https://github.com/my_username/papermill_sftp.git" + + [project.entry-points."papermill.io"] + "sftp://" = "papermill_sftp:SFTPHandler" - from setuptools import setup, find_packages - - setup( - name="papermill_sftp", - version="0.1", - url="https://github.com/my_username/papermill_sftp.git", - author="My Name", - author_email="my.email@gmail.com", - description="An SFTP I/O handler for papermill.", - packages=find_packages("./src"), - package_dir={"": "src"}, - install_requires=["pysftp"], - entry_points={"papermill.io": ["sftp://=papermill_sftp:SFTPHandler"]}, - ) + [tool.setuptools] + packages = ["papermill_sftp"] + package-dir = {"" = "src"} When executing, papermill will check if the input or output path begin with ``sftp://``, and if so, use the SFTPHandler from the papermill_sftp project. @@ -214,7 +219,7 @@ time it took to execute each cell as additional output after every code cell. The project structure is:: papermill_timing - |- setup.py + |- pyproject.toml |- src |- papermill_timing |- __init__.py @@ -249,7 +254,7 @@ library to create a `notebook node object`_. cell.outputs = [output_node] + cell.outputs Once this is in place, we need to add our engine as an entry point to our -``setup.py`` script - for this, see the following section. +``pyproject.toml`` file - for this, see the following section. Ensuring your engine is found by papermill ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -257,25 +262,33 @@ Ensuring your engine is found by papermill Custom engines can be specified as `entry points`_, under the ``papermill.engine`` prefix. The entry point needs to reference the class that we have just implemented. For example, if you write an engine called -TimingEngine in a package called papermill_timing, then in the ``setup.py`` +TimingEngine in a package called papermill_timing, then in the ``pyproject.toml`` file, you should specify: -.. code-block:: python +.. code-block:: toml + + [build-system] + requires = ["setuptools>=61.0", "wheel"] + build-backend = "setuptools.build_meta" + + [project] + name = "papermill_timing" + version = "0.1" + description = "A papermill engine that logs additional timing information about code." + authors = [ + {name = "My Name", email = "my.email@gmail.com"} + ] + dependencies = ["papermill", "nbformat"] + + [project.urls] + Repository = "https://github.com/my_username/papermill_timing.git" + + [project.entry-points."papermill.engine"] + timer_engine = "papermill_timing:CustomEngine" - from setuptools import setup, find_packages - - setup( - name="papermill_timing", - version="0.1", - url="https://github.com/my_username/papermill_timing.git", - author="My Name", - author_email="my.email@gmail.com", - description="A papermill engine that logs additional timing information about code.", - packages=find_packages("./src"), - package_dir={"": "src"}, - install_requires=["papermill", "nbformat"], - entry_points={"papermill.engine": ["timer_engine=papermill_timing:CustomEngine"]}, - ) + [tool.setuptools] + packages = ["papermill_timing"] + package-dir = {"" = "src"} This allows users to specify the engine from ``papermill_timing`` by passing the command line argument ``--engine timer_engine``. diff --git a/pyproject.toml b/pyproject.toml index edd2e6a8..57cb957c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,163 @@ -# Migration to pyproject.toml is in progress +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "papermill" +dynamic = ["version"] +description = "Parameterize and run Jupyter and nteract Notebooks" +readme = "README.md" +license = {text = "BSD"} +authors = [ + {name = "nteract contributors", email = "nteract@googlegroups.com"} +] +keywords = ["jupyter", "mapreduce", "nteract", "pipeline", "notebook"] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +requires-python = ">=3.10" +dependencies = [ + "click", + "pyyaml", + "nbformat>=5.2.0", + "nbclient>=0.2.0", + "tqdm>=4.32.2", + "requests", + "entrypoints", + "tenacity>=5.0.2", + "aiohttp>=3.9.0; python_version=='3.12'", +] + +[project.urls] +Documentation = "https://papermill.readthedocs.io" +Funding = "https://nteract.io" +Source = "https://github.com/nteract/papermill/" +Tracker = "https://github.com/nteract/papermill/issues" + +[project.scripts] +papermill = "papermill.__main__:papermill" + +[project.optional-dependencies] +s3 = ["boto3"] +azure = [ + "azure-datalake-store>=0.0.30,<1.0.0a0", + "azure-storage-blob>=12.1.0", + "requests>=2.21.0", + "azure-identity>=1.3.1", +] +gcs = ["gcsfs>=0.2.0"] +hdfs = ["pyarrow>=2.0"] +github = ["PyGithub>=1.55"] +black = ["black>=19.3b0"] +all = [ + "boto3", + "azure-datalake-store>=0.0.30,<1.0.0a0", + "azure-storage-blob>=12.1.0", + "requests>=2.21.0", + "azure-identity>=1.3.1", + "gcsfs>=0.2.0", + "pyarrow>=2.0", + "PyGithub>=1.55", + "black>=19.3b0", +] +dev = [ + "boto3", + "botocore", + "codecov", + "coverage", + "google_compute_engine", + "ipython>=5.0,<9.0.0", + "ipywidgets", + "notebook", + "moto>=5.0.0,<5.1.0", + "pytest>=4.1", + "pytest-cov>=2.6.1", + "pytest-mock>=1.10", + "pytest-env>=0.6.2", + "requests>=2.21.0", + "check-manifest", + "attrs>=17.4.0", + "pre-commit", + "tox", + "bumpversion", + "recommonmark", + "pip>=18.1", + "wheel>=0.31.0", + "setuptools>=38.6.0", + "twine>=1.11.0", + "azure-datalake-store>=0.0.30,<1.0.0a0", + "azure-storage-blob>=12.1.0", + "azure-identity>=1.3.1", + "gcsfs>=0.2.0", + "pyarrow>=2.0", + "PyGithub>=1.55", + "black>=19.3b0", +] +test = [ + "boto3", + "botocore", + "codecov", + "coverage", + "google_compute_engine", + "ipython>=5.0,<9.0.0", + "ipywidgets", + "notebook", + "moto>=5.0.0,<5.1.0", + "pytest>=4.1", + "pytest-cov>=2.6.1", + "pytest-mock>=1.10", + "pytest-env>=0.6.2", + "requests>=2.21.0", + "check-manifest", + "attrs>=17.4.0", + "pre-commit", + "tox", + "bumpversion", + "recommonmark", + "pip>=18.1", + "wheel>=0.31.0", + "setuptools>=38.6.0", + "twine>=1.11.0", + "azure-datalake-store>=0.0.30,<1.0.0a0", + "azure-storage-blob>=12.1.0", + "azure-identity>=1.3.1", + "gcsfs>=0.2.0", + "pyarrow>=2.0", + "PyGithub>=1.55", + "black>=19.3b0", +] +docs = [ + "boto3", + "azure-datalake-store>=0.0.30,<1.0.0a0", + "azure-storage-blob>=12.1.0", + "requests>=2.21.0", + "azure-identity>=1.3.1", + "gcsfs>=0.2.0", + "pyarrow>=2.0", + "PyGithub>=1.55", + "black>=19.3b0", + "Sphinx>=7.2.6", + "furo>=2023.9.10", + "myst-parser>=2.0.0", + "moto>=4.2.8", + "sphinx-copybutton>=0.5.2", +] + +[tool.setuptools] +packages = ["papermill"] +include-package-data = true + +[tool.setuptools.dynamic] +version = {attr = "papermill.version.version"} # Example configuration for Black. [tool.black] @@ -58,3 +217,90 @@ format.quote-style = "preserve" [tool.ruff.lint.pydocstyle] # Use Google-style docstrings. convention = "google" + +[tool.pytest.ini_options] +env = [ + "AWS_SECRET_ACCESS_KEY=foobar_secret", + "AWS_ACCESS_KEY_ID=foobar_key", +] +filterwarnings = [ + "ignore:.*imp module is deprecated.*:DeprecationWarning", +] + +[tool.bumpversion] +current_version = "2.6.0" +commit = true +tag = true +tag_name = "{new_version}" + +[[tool.bumpversion.files]] +filename = "papermill/version.py" + +[tool.tox] +legacy_tox_ini = """ +[tox] +skipsdist = true +envlist = py{310,311,312,313}, dist, manifest, docs, binder + +[gh-actions] +python = + 3.10: py310 + 3.11: py311 + 3.12: py312 + 3.13: py313, docs, dist + +# Manifest +[testenv:manifest] +skip_install = true +deps = check-manifest +commands = check-manifest +ignore = + .readthedocs.yaml + +# Docs +[testenv:docs] +description = invoke sphinx-build to build the HTML docs +deps = + .[docs] +extras = docs +commands = + sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W -bhtml {posargs} + python -c 'import pathlib; print("documentation available under file://\\{0\\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' + +# Binder +[testenv:binder] +description = ensure /binder/*ipynb are runnable +deps = + -r binder/requirements.txt +commands = python -c "import glob; import papermill as pm; [pm.execute_notebook(input, '{toxworkdir}/out.ipynb', parameters=\\{'binder_dir':'binder'\\}) for input in glob.glob('binder/**/*.ipynb')]" + +# Distro +[testenv:dist] +skip_install = true +deps = build +commands = + python -m build --outdir {distdir} + /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.tar.gz' + /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.whl' + +[testenv] +# disable Python's hash randomization for tests that stringify dicts, etc +setenv = + PYTHONHASHSEED = 0 + AWS_ACCESS_KEY_ID=foobar_key + AWS_SECRET_ACCESS_KEY=foobar_secret +passenv = * +basepython = + py310: python3.10 + py311: python3.11 + py312: python3.12 + py313: python3.13 + manifest: python3.13 + dist: python3.13 + docs: python3.13 + binder: python3.13 +deps = .[dev] +# Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line... +allowlist_externals = /bin/bash +commands = pytest -v --maxfail=2 --cov=papermill -W always {posargs} +""" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index a9ea21ea..00000000 --- a/pytest.ini +++ /dev/null @@ -1,6 +0,0 @@ -[pytest] -env = - AWS_SECRET_ACCESS_KEY=foobar_secret - AWS_ACCESS_KEY_ID=foobar_key -filterwarnings = - ignore:.*imp module is deprecated.*:DeprecationWarning diff --git a/setup.py b/setup.py deleted file mode 100644 index 9e55a883..00000000 --- a/setup.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python -""" -setup.py - -See: -https://packaging.python.org/tutorials/packaging-projects/ -https://packaging.python.org/en/latest/distributing.html -https://github.com/pypa/sampleproject - -""" - -import os - -from setuptools import setup - -local_path = os.path.dirname(__file__) -# Fix for tox which manipulates execution pathing -if not local_path: - local_path = '.' -here = os.path.abspath(local_path) - - -def version(): - with open(f"{here}/papermill/version.py") as ver: - for line in ver.readlines(): - if line.startswith('version ='): - return line.split(' = ')[-1].strip()[1:-1] - raise ValueError('No version found in papermill/version.py') - - -def read(fname): - with open(fname) as fhandle: - return fhandle.read() - - -def read_reqs(fname, folder=None): - path_dir = os.path.join(here, folder) if folder else here - req_path = os.path.join(path_dir, fname) - return [req.strip() for req in read(req_path).splitlines() if req.strip()] - - -s3_reqs = read_reqs('s3.txt', folder='requirements') -azure_reqs = read_reqs('azure.txt', folder='requirements') -gcs_reqs = read_reqs('gcs.txt', folder='requirements') -hdfs_reqs = read_reqs('hdfs.txt', folder='requirements') -github_reqs = read_reqs('github.txt', folder='requirements') -docs_only_reqs = read_reqs('docs.txt', folder='requirements') -black_reqs = ['black >= 19.3b0'] -all_reqs = s3_reqs + azure_reqs + gcs_reqs + hdfs_reqs + github_reqs + black_reqs -docs_reqs = all_reqs + docs_only_reqs -dev_reqs = read_reqs('dev.txt', folder='requirements') + all_reqs -extras_require = { - "test": dev_reqs, - "dev": dev_reqs, - "all": all_reqs, - "s3": s3_reqs, - "azure": azure_reqs, - "gcs": gcs_reqs, - "hdfs": hdfs_reqs, - "github": github_reqs, - "black": black_reqs, - "docs": docs_reqs, -} - -# Get the long description from the README file -with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: - long_description = f.read() - -setup( - name='papermill', - version=version(), - description='Parameterize and run Jupyter and nteract Notebooks', - author='nteract contributors', - author_email='nteract@googlegroups.com', - license='BSD', - # Note that this is a string of words separated by whitespace, not a list. - keywords='jupyter mapreduce nteract pipeline notebook', - long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/nteract/papermill', - packages=['papermill'], - python_requires='>=3.10', - install_requires=read_reqs('requirements.txt'), - extras_require=extras_require, - entry_points={'console_scripts': ['papermill = papermill.__main__:papermill']}, - project_urls={ - 'Documentation': 'https://papermill.readthedocs.io', - 'Funding': 'https://nteract.io', - 'Source': 'https://github.com/nteract/papermill/', - 'Tracker': 'https://github.com/nteract/papermill/issues', - }, - classifiers=[ - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - ], -) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 0ba4ed7a..00000000 --- a/tox.ini +++ /dev/null @@ -1,64 +0,0 @@ -[tox] -skipsdist = true -envlist = py{310,311,312,313}, dist, manifest, docs, binder - -[gh-actions] -python = - 3.10: py310 - 3.11: py311 - 3.12: py312 - 3.13: py313, docs, dist - -# Manifest -[testenv:manifest] -skip_install = true -deps = check-manifest -commands = check-manifest -ignore = - .readthedocs.yaml - -# Docs -[testenv:docs] -description = invoke sphinx-build to build the HTML docs -deps = - .[docs] -extras = docs -commands = - sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W -bhtml {posargs} - python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' - -# Binder -[testenv:binder] -description = ensure /binder/*ipynb are runnable -deps = - -r binder/requirements.txt -commands = python -c "import glob; import papermill as pm; [pm.execute_notebook(input, '{toxworkdir}/out.ipynb', parameters=\{'binder_dir':'binder'\}) for input in glob.glob('binder/**/*.ipynb')]" - -# Distro -[testenv:dist] -skip_install = true -commands = - python setup.py sdist --dist-dir={distdir} bdist_wheel --dist-dir={distdir} - /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.tar.gz' - /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.whl' - -[testenv] -# disable Python's hash randomization for tests that stringify dicts, etc -setenv = - PYTHONHASHSEED = 0 - AWS_ACCESS_KEY_ID=foobar_key - AWS_SECRET_ACCESS_KEY=foobar_secret -passenv = * -basepython = - py310: python3.10 - py311: python3.11 - py312: python3.12 - py313: python3.13 - manifest: python3.13 - dist: python3.13 - docs: python3.13 - binder: python3.13 -deps = .[dev] -# Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line... -allowlist_externals = /bin/bash -commands = pytest -v --maxfail=2 --cov=papermill -W always {posargs}