[Internal] Make databricks-sdk-py compliant with PEP 621 and PEP 625 (#895)

## What changes are proposed in this pull request?

We've received messages from PyPI that the current SDK library is not
compliant with PEP 625. With help from the administrators at PyPI, I've
converted our `setup.py` to a `pyproject.toml` and, with help from the
setuptools `build` frontend, we can produce PEP 625-compliant artifacts.

Additionally, we can run `check-manifest` to verify that our source
distribution and wheel contain the same set of files. I noticed in
particular that all of our examples and docs are included in both our
wheel and sdist, polluting customer's python environments with
unnecessary packages.

## How is this tested?

Ran `python -m build` and saw that the desired build artifacts were
produced.

The `check-manifest` job should be passing.

Tested uploading the resulting artifact to Test PyPI:
https://github.com/databricks/databricks-sdk-py/actions/runs/13438801138/job/37547674236

NO_CHANGELOG=true

Author: mgyucht

.github/workflows/push.yml

@@ -39,3 +39,19 @@ jobs:
 
       - name: Fail on differences
         run: git diff --exit-code
+
+  check-manifest:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.13
+
+      - name: Check MANIFEST.in
+        run: |
+          pip install check-manifest
+          check-manifest .

.github/workflows/release-test.yml

@@ -17,8 +17,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.13
+
+      - name: Install build
+        run: pip install build
+
       - name: Build wheel
-        run: make dev install dist
+        run: python -m build
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/release.yml

@@ -20,8 +20,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.13
+
+      - name: Install build
+        run: pip install build
+
       - name: Build wheel
-        run: make dev install dist
+        run: python -m build
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -30,8 +37,8 @@ jobs:
         with:
           draft: true
           files: |
-            dist/databricks-*.whl
-            dist/databricks-*.tar.gz
+            dist/databricks_*.whl
+            dist/databricks_*.tar.gz
 
       - uses: pypa/gh-action-pypi-publish@release/v1
         name: Publish package distributions to PyPI

MANIFEST.in

@@ -0,0 +1,23 @@
+include README.md
+include LICENSE
+include NOTICE
+include DCO
+include Makefile
+include SECURITY.md
+include CHANGELOG.md
+include CONTRIBUTING.md
+include pyproject.toml
+include databricks/sdk/py.typed
+
+exclude .codegen.json
+exclude .readthedocs.yaml
+exclude docs-requirements.txt
+exclude NEXT_CHANGELOG.md
+exclude tagging.py
+exclude .package.json
+
+recursive-exclude docs *
+recursive-exclude examples *
+recursive-exclude tests *
+recursive-exclude .codegen *
+recursive-exclude .vscode *

Makefile

@@ -36,8 +36,5 @@ benchmark:
 coverage: test
 	open htmlcov/index.html
 
-dist:
-	python3 setup.py bdist_wheel sdist
-
 clean:
 	rm -fr dist *.egg-info .pytest_cache build htmlcov

pyproject.toml

@@ -0,0 +1,74 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "databricks-sdk"
+dynamic = ["version"]
+description = "Databricks SDK for Python (Beta)"
+readme = "README.md"
+requires-python = ">=3.7"
+keywords = ["databricks", "sdk"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Operating System :: OS Independent"
+]
+dependencies = [
+    "requests>=2.28.1,<3",
+    "google-auth~=2.0"
+]
+
+[project.urls]
+Documentation = "https://databricks-sdk-py.readthedocs.io"
+
+[project.optional-dependencies]
+dev = [
+    "pytest",
+    "pytest-cov",
+    "pytest-xdist",
+    "pytest-mock",
+    "yapf",
+    "pycodestyle",
+    "autoflake",
+    "isort",
+    "wheel",
+    "ipython",
+    "ipywidgets",
+    "requests-mock",
+    "pyfakefs",
+    "databricks-connect",
+    "pytest-rerunfailures",
+    "openai",
+    'langchain-openai; python_version > "3.7"',
+    "httpx",
+    "build", # some integration tests depend on the databricks-sdk-py wheel
+]
+notebook = [
+    "ipython>=8,<9",
+    "ipywidgets>=8,<9"
+]
+openai = [
+    "openai",
+    'langchain-openai; python_version > "3.7"',
+    "httpx"
+]
+
+[tool.setuptools.dynamic]
+version = { attr = "databricks.sdk.version.__version__" }
+
+[tool.setuptools.packages.find]
+include = ["databricks", "databricks.*"]
+
+[tool.setuptools.package-data]
+"databricks.sdk" = ["py.typed"]

setup.py

@@ -25,7 +25,7 @@ def fresh_wheel_file(tmp_path) -> Path:
     build_root = tmp_path / 'databricks-sdk-py'
     shutil.copytree(project_root, build_root)
     try:
-        completed_process = subprocess.run([sys.executable, 'setup.py', 'bdist_wheel'],
+        completed_process = subprocess.run([sys.executable, '-m', 'build'],
                                            capture_output=True,
                                            cwd=build_root)
         if completed_process.returncode != 0: