Switch project setup to pyproject.toml (#123)

Since we're hiring a new Python engineer hopefully soon, I thought it's
time to get rid of some tech debt.

This rewrites the project setup to use `pyproject.toml` instead of
`setup.py`, as `pyproject.toml` is the modern, recommended way to
specify project metadata, and `setup.py` is being deprecated.

I've also updated the publication to PyPI to use the official GitHub
action, and to use OIDC for authentication instead of an API token.

Author: fnesveda

.flake8

@@ -3,8 +3,7 @@ filename =
     ./docs/*.py,
     ./scripts/*.py,
     ./src/*.py,
-    ./tests/*.py,
-    ./setup.py
+    ./tests/*.py
 per-file-ignores =
     docs/*: D
     scripts/*: D

.github/workflows/release.yaml

@@ -41,10 +41,16 @@ jobs:
     name: Check whether the documentation is up to date
     uses: ./.github/workflows/check_docs.yaml
 
-  deploy:
+  publish_to_pypi:
     name: Publish to PyPI
     needs: [lint_and_type_checks, unit_tests, integration_tests, check_docs]
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/apify-client
 
     steps:
     - name: Checkout repository
@@ -56,9 +62,7 @@ jobs:
         python-version: 3.8
 
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install --upgrade setuptools twine wheel
+      run: make install-dev
 
     - # Determine if this is a prerelease or latest release
       name: Determine release type
@@ -86,7 +90,7 @@ jobs:
     - # Check whether the released version is listed in CHANGELOG.md
       name: Check whether the released version is listed in the changelog
       if: steps.get-release-type.outputs.release_type != 'alpha'
-      run: python ./scripts/check_version_in_changelog.py
+      run: make check-changelog-entry
 
     - # Check version consistency and increment pre-release version number for prereleases (must be the last step before build)
       name: Bump pre-release version
@@ -95,18 +99,15 @@ jobs:
 
     - # Build a source distribution and a python3-only wheel
       name: Build distribution files
-      run: python setup.py sdist bdist_wheel
+      run: make build
 
     - # Check whether the package description will render correctly on PyPI
       name: Check package rendering on PyPI
-      run: python -m twine check dist/*
+      run: make twine-check
 
     - # Publish package to PyPI using their official GitHub action
       name: Publish package to PyPI
-      run: python -m twine upload --non-interactive --disable-progress-bar dist/*
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      uses: pypa/gh-action-pypi-publish@release/v1
 
     - # Tag the current commit with the version tag if this is not made from the release event (releases are tagged with the release process)
       name: Tag Version

CHANGELOG.md

@@ -7,6 +7,10 @@ Changelog
 
 - added support for `is_status_message_terminal` flag in Actor run status message update
 
+### Internal changes
+
+- switched from `setup.py` to `pyproject.toml` for specifying project setup
+
 [1.0.0](../../releases/tag/v1.0.0) - 2023-03-13
 -----------------------------------------------
 

Makefile

@@ -8,10 +8,18 @@ clean:
 
 install-dev:
 	python -m pip install --upgrade pip
-	pip install --upgrade setuptools wheel
 	pip install --no-cache-dir -e ".[dev]"
 	pre-commit install
 
+build:
+	python -m build
+
+publish:
+	python -m twine upload dist/*
+
+twine-check:
+	python -m twine check dist/*
+
 lint:
 	python3 -m flake8
 

README.md

@@ -61,12 +61,12 @@ To generate the documentation, just run `make docs`.
 
 Publishing new versions to [PyPI](https://pypi.org/project/apify-client) happens automatically through GitHub Actions.
 
-On each commit to the `master` branch, a new beta release is published, taking the version number from `src/apify_client/_version.py`
+On each commit to the `master` branch, a new beta release is published, taking the version number from `pyproject.toml`
 and automatically incrementing the beta version suffix by 1 from the last beta release published to PyPI.
 
-A stable version is published when a new release is created using GitHub Releases, again taking the version number from `src/apify_client/_version.py`. The built package assets are automatically uploaded to the GitHub release.
+A stable version is published when a new release is created using GitHub Releases, again taking the version number from `pyproject.toml`. The built package assets are automatically uploaded to the GitHub release.
 
-If there is already a stable version with the same version number as in `src/apify_client/_version.py` published to PyPI, the publish process fails,
+If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish process fails,
 so don't forget to update the version number before releasing a new version.
 The release process also fails when the released version is not described in `CHANGELOG.md`,
 so don't forget to describe the changes in the new version there.

mypy.ini

@@ -4,8 +4,7 @@ files =
     docs,
     scripts,
     src,
-    tests,
-    setup.py
+    tests
 check_untyped_defs = True
 disallow_incomplete_defs = True
 disallow_untyped_calls = True

pyproject.toml

@@ -0,0 +1,77 @@
+[project]
+name = "apify_client"
+version = "1.1.0"
+description = "Apify API client for Python"
+readme = "README.md"
+license = {text = "Apache Software License"}
+authors = [
+    {name = "Apify Technologies s.r.o.", email = "[email protected]" }
+]
+keywords = ["apify", "api", "client", "scraping", "automation"]
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Software Development :: Libraries",
+]
+
+requires-python = ">=3.8"
+dependencies = [
+    "httpx ~= 0.24.0"
+]
+
+[project.optional-dependencies]
+dev = [
+    "autopep8 ~= 2.0.2",
+    "build ~= 0.10.0",
+    "flake8 ~= 6.0.0",
+    "flake8-bugbear ~= 23.3.23",
+    "flake8-commas ~= 2.1.0",
+    "flake8-comprehensions ~= 3.12.0",
+    "flake8-datetimez ~= 20.10.0",
+    "flake8-docstrings ~= 1.7.0",
+    "flake8-isort ~= 6.0.0",
+    "flake8-noqa ~= 1.3.1",
+    "flake8-pytest-style ~= 1.7.2",
+    "flake8-quotes ~= 3.3.2",
+    "flake8-unused-arguments ~= 0.0.13",
+    "isort ~= 5.12.0",
+    "mypy ~= 1.2.0",
+    "pep8-naming ~= 0.13.3",
+    "pre-commit ~= 3.3.1",
+    "pytest ~= 7.3.1",
+    "pytest-asyncio ~= 0.21.0",
+    "pytest-only ~= 2.0.0",
+    "pytest-randomly ~= 3.12.0",
+    "pytest-timeout ~= 2.1.0",
+    "pytest-xdist ~= 3.2.1",
+    "redbaron ~= 0.9.2",
+    "sphinx ~= 6.1.3",
+    "sphinx-autodoc-typehints ~= 1.22",
+    "sphinx-markdown-builder == 0.5.4",
+    "twine ~= 4.0.2",
+]
+
+[project.urls]
+"Homepage" = "https://docs.apify.com/api/client/python/"
+"Documentation" = "https://docs.apify.com/api/client/python/"
+"Source" = "https://github.com/apify/apify-client-python"
+"Issue tracker" = "https://github.com/apify/apify-client-python/issues"
+"Apify Homepage" = "https://apify.com"
+
+[build-system]
+requires = ["setuptools>=64.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["apify_client"]
+
+[tool.setuptools.package-data]
+apify_client = ["py.typed"]

scripts/print_current_package_version.py

@@ -2,6 +2,6 @@
 
 from utils import get_current_package_version
 
-# Print the current package version from the src/package_name/_version.py file to stdout
+# Print the current package version from the pyproject.toml file to stdout
 if __name__ == '__main__':
     print(get_current_package_version(), end='')

scripts/update_version_for_prerelease.py

@@ -8,7 +8,7 @@
 from utils import PACKAGE_NAME, get_current_package_version, set_current_package_version
 
 # Checks whether the current package version number was not already used in a published release,
-# and if not, modifies the package version number in src/package_name/_version.py
+# and if not, modifies the package version number in pyproject.toml
 # from a stable release version (X.Y.Z) to a prerelease version (X.Y.ZbN or X.Y.Z.aN or X.Y.Z.rcN)
 if __name__ == '__main__':
     if len(sys.argv) != 2:
@@ -55,6 +55,6 @@
             if prerelease_version > latest_prerelease:
                 latest_prerelease = prerelease_version
 
-    # Write the latest prerelease version number to src/package_name/_version.py
+    # Write the latest prerelease version number to pyproject.toml
     new_prerelease_version_number = f'{current_version}{prerelease_prefix}{latest_prerelease + 1}'
     set_current_package_version(new_prerelease_version_number)

scripts/utils.py

@@ -3,40 +3,40 @@
 
 PACKAGE_NAME = 'apify_client'
 REPO_ROOT = pathlib.Path(__file__).parent.resolve() / '..'
-VERSION_FILE_PATH = REPO_ROOT / f'src/{PACKAGE_NAME}/_version.py'
+PYPROJECT_TOML_FILE_PATH = REPO_ROOT / 'pyproject.toml'
 
 
-# Load the current version number from src/package_name/_version.py
-# It is on a line in the format __version__ = 1.2.3
+# Load the current version number from pyproject.toml
+# It is on a line in the format `version = "1.2.3"`
 def get_current_package_version() -> str:
-    with open(VERSION_FILE_PATH, 'r') as version_file:
-        for line in version_file:
-            if line.startswith('__version__'):
+    with open(PYPROJECT_TOML_FILE_PATH, 'r') as pyproject_toml_file:
+        for line in pyproject_toml_file:
+            if line.startswith('version = '):
                 delim = '"' if '"' in line else "'"
                 version = line.split(delim)[1]
                 return version
         else:
             raise RuntimeError('Unable to find version string.')
 
 
-# Write the given version number from src/package_name/_version.py
-# It replaces the version number on the line with the format __version__ = 1.2.3
+# Write the given version number from pyproject.toml
+# It replaces the version number on the line with the format `version = "1.2.3"`
 def set_current_package_version(version: str) -> None:
-    with open(VERSION_FILE_PATH, 'r+') as version_file:
-        updated_version_file_lines = []
+    with open(PYPROJECT_TOML_FILE_PATH, 'r+') as pyproject_toml_file:
+        updated_pyproject_toml_file_lines = []
         version_string_found = False
-        for line in version_file:
-            if line.startswith('__version__'):
+        for line in pyproject_toml_file:
+            if line.startswith('version = '):
                 version_string_found = True
-                line = f"__version__ = '{version}'"
-            updated_version_file_lines.append(line)
+                line = f'version = "{version}"'
+            updated_pyproject_toml_file_lines.append(line)
 
         if not version_string_found:
             raise RuntimeError('Unable to find version string.')
 
-        version_file.seek(0)
-        version_file.write('\n'.join(updated_version_file_lines))
-        version_file.truncate()
+        pyproject_toml_file.seek(0)
+        pyproject_toml_file.write('\n'.join(updated_pyproject_toml_file_lines))
+        pyproject_toml_file.truncate()
 
 
 # Generate convert a docstring from a sync resource client method