From 33c99f6af97e4053c5f56b44105353b2dc15400c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 1 Aug 2025 15:50:24 +0200 Subject: [PATCH 1/2] document recommend ci additions for pypi/test-pypi uploads closes #311 --- CHANGELOG.md | 2 + docs/integrations.md | 295 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 296 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca2177d9..26e25b2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ - fix #1059: add `SETUPTOOLS_SCM_PRETEND_METADATA` environment variable to override individual ScmVersion fields - add `scm` parameter support to `get_version()` function for nested SCM configuration - fix #987: expand documentation on git archival files and add cli tools for good defaults +- fix #311: document github/gitlab ci pipelines that enable auto-upload to test-pypi/pypi + ### Changed diff --git a/docs/integrations.md b/docs/integrations.md index ba097fe6..5c6fa8d3 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -47,4 +47,297 @@ build: - export SETUPTOOLS_SCM_OVERRIDES_FOR_${READTHEDOCS_PROJECT//-/_}='{scm.git.pre_parse="fail_on_shallow"}' ``` -This configuration uses the `SETUPTOOLS_SCM_OVERRIDES_FOR_${NORMALIZED_DIST_NAME}` environment variable to override the `scm.git.pre_parse` setting specifically for your project when building on ReadTheDocs, forcing setuptools-scm to fail with a clear error if the repository is shallow. \ No newline at end of file +This configuration uses the `SETUPTOOLS_SCM_OVERRIDES_FOR_${NORMALIZED_DIST_NAME}` environment variable to override the `scm.git.pre_parse` setting specifically for your project when building on ReadTheDocs, forcing setuptools-scm to fail with a clear error if the repository is shallow. + +## CI/CD and Package Publishing + +### Publishing to PyPI from CI/CD + +When publishing packages to PyPI or test-PyPI from CI/CD pipelines, you often need to remove local version components that are not allowed on public package indexes according to [PEP 440](https://peps.python.org/pep-0440/#local-version-identifiers). + +setuptools-scm provides the `no-local-version` local scheme and environment variable overrides to handle this scenario cleanly. + +#### The Problem + +By default, setuptools-scm generates version numbers like: +- `1.2.3.dev4+g1a2b3c4d5` (development version with git hash) +- `1.2.3+dirty` (dirty working directory) + +These local version components (`+g1a2b3c4d5`, `+dirty`) prevent uploading to PyPI. + +#### The Solution + +Use the `SETUPTOOLS_SCM_OVERRIDES_FOR_${NORMALIZED_DIST_NAME}` environment variable to override the `local_scheme` to `no-local-version` when building for upload to PyPI. + +### GitHub Actions Example + +Here's a complete GitHub Actions workflow that: +- Runs tests on all branches +- Uploads development versions to test-PyPI from feature branches +- Uploads development versions to PyPI from the main branch (with no-local-version) +- Uploads tagged releases to PyPI (using exact tag versions) + +```yaml title=".github/workflows/ci.yml" +name: CI/CD + +on: + push: + branches: ["main", "develop"] + pull_request: + branches: ["main", "develop"] + release: + types: [published] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + with: + # Fetch full history for setuptools-scm + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build pytest + pip install -e . + + - name: Run tests + run: pytest + + publish-test-pypi: + needs: test + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref != 'refs/heads/main' + env: + # Replace MYPACKAGE with your actual package name (normalized) + # For package "my-awesome.package", use "MY_AWESOME_PACKAGE" + SETUPTOOLS_SCM_OVERRIDES_FOR_MYPACKAGE: '{"local_scheme": "no-local-version"}' + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Upload to test-PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + + publish-pypi: + needs: test + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + # Replace MYPACKAGE with your actual package name (normalized) + SETUPTOOLS_SCM_OVERRIDES_FOR_MYPACKAGE: '{"local_scheme": "no-local-version"}' + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + + publish-release: + needs: test + runs-on: ubuntu-latest + if: github.event_name == 'release' + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} +``` + +### GitLab CI Example + +Here's an equivalent GitLab CI configuration: + +```yaml title=".gitlab-ci.yml" +stages: + - test + - publish + +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + +cache: + paths: + - .cache/pip/ + +before_script: + - python -m pip install --upgrade pip + +test: + stage: test + image: python:3.11 + script: + - pip install build pytest + - pip install -e . + - pytest + parallel: + matrix: + - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12"] + image: python:${PYTHON_VERSION} + +publish-test-pypi: + stage: publish + image: python:3.11 + variables: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: $TEST_PYPI_API_TOKEN + # Replace MYPACKAGE with your actual package name (normalized) + SETUPTOOLS_SCM_OVERRIDES_FOR_MYPACKAGE: '{"local_scheme": "no-local-version"}' + script: + - pip install build twine + - python -m build + - twine upload --repository testpypi dist/* + rules: + - if: $CI_COMMIT_BRANCH != "main" && $CI_PIPELINE_SOURCE == "push" + +publish-pypi: + stage: publish + image: python:3.11 + variables: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: $PYPI_API_TOKEN + # Replace MYPACKAGE with your actual package name (normalized) + SETUPTOOLS_SCM_OVERRIDES_FOR_MYPACKAGE: '{"local_scheme": "no-local-version"}' + script: + - pip install build twine + - python -m build + - twine upload dist/* + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + +publish-release: + stage: publish + image: python:3.11 + variables: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: $PYPI_API_TOKEN + script: + - pip install build twine + - python -m build + - twine upload dist/* + rules: + - if: $CI_COMMIT_TAG +``` + +### Configuration Details + +#### Environment Variable Format + +The environment variable `SETUPTOOLS_SCM_OVERRIDES_FOR_${NORMALIZED_DIST_NAME}` must be set where: + +1. **`${NORMALIZED_DIST_NAME}`** is your package name normalized according to PEP 503: + - Convert to uppercase + - Replace hyphens and dots with underscores + - Examples: `my-package` → `MY_PACKAGE`, `my.package` → `MY_PACKAGE` + +2. **Value** must be a valid TOML inline table format: + ```bash + SETUPTOOLS_SCM_OVERRIDES_FOR_MYPACKAGE='{"local_scheme": "no-local-version"}' + ``` + +#### Alternative: pyproject.toml Configuration + +Instead of environment variables, you can configure this in your `pyproject.toml`: + +```toml title="pyproject.toml" +[tool.setuptools_scm] +# Use no-local-version by default for CI builds +local_scheme = "no-local-version" +``` + +However, the environment variable approach is preferred for CI/CD as it allows different schemes for local development vs. CI builds. + +#### Version Examples + +**Development versions from main branch** (with `local_scheme = "no-local-version"`): +- Development commit: `1.2.3.dev4+g1a2b3c4d5` → `1.2.3.dev4` ✅ (uploadable to PyPI) +- Dirty working directory: `1.2.3+dirty` → `1.2.3` ✅ (uploadable to PyPI) + +**Tagged releases** (without overrides, using default local scheme): +- Tagged commit: `1.2.3` → `1.2.3` ✅ (uploadable to PyPI) +- Tagged release on dirty workdir: `1.2.3+dirty` → `1.2.3+dirty` ❌ (should not happen in CI) + +### Security Notes + +- Store PyPI API tokens as repository secrets +- Use separate tokens for test-PyPI and production PyPI +- Consider using [Trusted Publishers](https://docs.pypi.org/trusted-publishers/) for enhanced security + +### Troubleshooting + +**Package name normalization**: If your override isn't working, verify the package name normalization: + +```python +import re +dist_name = "my-awesome.package" +normalized = re.sub(r"[-_.]+", "-", dist_name) +env_var_name = normalized.replace("-", "_").upper() +print(f"SETUPTOOLS_SCM_OVERRIDES_FOR_{env_var_name}") +# Output: SETUPTOOLS_SCM_OVERRIDES_FOR_MY_AWESOME_PACKAGE +``` + +**Fetch depth**: Always use `fetch-depth: 0` in GitHub Actions to ensure setuptools-scm has access to the full git history for proper version calculation. \ No newline at end of file From 538251c9a12d70a9ade1cbdf1395ef0c23f33682 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 1 Aug 2025 16:02:10 +0200 Subject: [PATCH 2/2] add test-pypi upload for main --- .github/workflows/python-tests.yml | 4 ++++ _own_version_helper.py | 11 ++++++++++- docs/integrations.md | 4 +++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 5aad6dda..77dd1129 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -22,6 +22,9 @@ jobs: package: name: Build & inspect our package. runs-on: ubuntu-latest + env: + # Use no-local-version for package builds to ensure clean versions for PyPI uploads + SETUPTOOLS_SCM_NO_LOCAL: "1" steps: - uses: actions/checkout@v4 @@ -117,6 +120,7 @@ jobs: test-pypi-upload: runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: [test] permissions: id-token: write diff --git a/_own_version_helper.py b/_own_version_helper.py index d0d7433c..12ffeb07 100644 --- a/_own_version_helper.py +++ b/_own_version_helper.py @@ -9,6 +9,7 @@ from __future__ import annotations import logging +import os from typing import Callable @@ -22,6 +23,7 @@ from setuptools_scm.fallbacks import parse_pkginfo from setuptools_scm.version import ScmVersion from setuptools_scm.version import get_local_node_and_date +from setuptools_scm.version import get_no_local_node from setuptools_scm.version import guess_next_dev_version log = logging.getLogger("setuptools_scm") @@ -48,11 +50,18 @@ def parse(root: str, config: Configuration) -> ScmVersion | None: def scm_version() -> str: + # Use no-local-version if SETUPTOOLS_SCM_NO_LOCAL is set (for CI uploads) + local_scheme = ( + get_no_local_node + if os.environ.get("SETUPTOOLS_SCM_NO_LOCAL") + else get_local_node_and_date + ) + return get_version( relative_to=__file__, parse=parse, version_scheme=guess_next_dev_version, - local_scheme=get_local_node_and_date, + local_scheme=local_scheme, ) diff --git a/docs/integrations.md b/docs/integrations.md index 5c6fa8d3..e94a6ea0 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -299,7 +299,9 @@ The environment variable `SETUPTOOLS_SCM_OVERRIDES_FOR_${NORMALIZED_DIST_NAME}` SETUPTOOLS_SCM_OVERRIDES_FOR_MYPACKAGE='{"local_scheme": "no-local-version"}' ``` -#### Alternative: pyproject.toml Configuration +#### Alternative Approaches + +**Option 1: pyproject.toml Configuration** Instead of environment variables, you can configure this in your `pyproject.toml`: