Fix PyPI publish: use twine with explicit file patterns

oberstet · oberstet · commit ebdc991b0321 · 2025-12-13T00:37:02.000+01:00
Workflow (.github/workflows/release.yml):
- Replace pypa/gh-action-pypi-publish with twine upload
- Add PyPI version check before upload (skip if exists)
- Add final validation step with twine check on all packages
- Use explicit patterns (dist/*.whl dist/*.tar.gz) to avoid
  uploading metadata files (build-info.txt, CHECKSUMS.sha256)
- Remove keep-metadata from check-release-fileset for PyPI
- Match autobahn-python release workflow pattern

Justfile (publish-pypi, publish-rtd, publish):
- Sync publish-pypi recipe with autobahn-python pattern
- Download to temp dir, use explicit patterns for twine upload
- Add tag format validation (vX.Y.Z)
- Add wheel/sdist count verification
- Sync publish-rtd recipe with autobahn-python (RTD API trigger)
- Add publish meta-recipe summary output

This fixes the "InvalidDistribution: Unknown distribution format:
'build-info.txt'" error during PyPI publish.

Note: This work was completed with AI assistance (Claude Code).
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -804,6 +804,7 @@ jobs:
         with:
           distdir: dist
           mode: strict
+          # keep-metadata: false (default - removes CHECKSUMS, build-info.txt etc for PyPI)
           targets: |
             cpy311-linux-x86_64-manylinux_2_28
             cpy312-linux-x86_64-manylinux_2_28
@@ -826,13 +827,148 @@ jobs:
             cpy314-win-amd64
             source
 
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          # Use OIDC trusted publishing (no password needed)
-          # Automatically generates and uploads attestations
-          packages-dir: dist/
-          verify-metadata: true
-          skip-existing: false
-          print-hash: true
-          attestations: true
+      - name: Check if version already exists on PyPI
+        id: pypi_check
+        run: |
+          # Extract version from tag name (v25.12.2 -> 25.12.2)
+          VERSION="${{ needs.check-workflows.outputs.tag_name }}"
+          VERSION="${VERSION#v}"
+          echo "Checking if zlmdb version ${VERSION} exists on PyPI..."
+
+          # Query PyPI JSON API
+          HTTP_CODE=$(curl -s -o /tmp/pypi_response.json -w "%{http_code}" "https://pypi.org/pypi/zlmdb/${VERSION}/json")
+
+          if [ "${HTTP_CODE}" = "200" ]; then
+            echo "⚠️  WARNING: Version ${VERSION} already exists on PyPI!"
+            echo "⚠️  PyPI does not allow re-uploading the same version."
+            echo "⚠️  Skipping PyPI upload to avoid error."
+            echo "exists=true" >> $GITHUB_OUTPUT
+          elif [ "${HTTP_CODE}" = "404" ]; then
+            echo "✅ Version ${VERSION} does not exist on PyPI yet - proceeding with upload"
+            echo "exists=false" >> $GITHUB_OUTPUT
+          else
+            echo "⚠️  Unexpected HTTP code ${HTTP_CODE} from PyPI API"
+            echo "⚠️  Response:"
+            cat /tmp/pypi_response.json || echo "(no response)"
+            echo "⚠️  Proceeding with upload anyway (will fail if version exists)"
+            echo "exists=false" >> $GITHUB_OUTPUT
+          fi
+
+          rm -f /tmp/pypi_response.json
+
+      - name: Final validation before PyPI upload
+        if: steps.pypi_check.outputs.exists == 'false'
+        run: |
+          set -o pipefail
+          echo "======================================================================"
+          echo "==> FINAL PYPI VALIDATION: All Packages"
+          echo "======================================================================"
+          echo ""
+          echo "Last chance to catch corrupted packages before PyPI upload."
+          echo ""
+          # Install both packaging and twine from master for PEP 639 (Core Metadata 2.4) support
+          # Use --break-system-packages for consistency (safe in CI)
+          python3 -m pip install --break-system-packages git+https://github.com/pypa/packaging.git
+          python3 -m pip install --break-system-packages git+https://github.com/pypa/twine.git
+          echo ""
+
+          echo "==> Validation environment:"
+          echo "Python: $(python3 --version)"
+          echo "setuptools: $(python3 -m pip show setuptools | grep '^Version:' || echo 'not installed')"
+          echo "packaging: $(python3 -m pip show packaging | grep '^Version:' || echo 'not installed')"
+          echo "twine: $(twine --version)"
+          echo ""
+
+          HAS_ERRORS=0
+
+          for pkg in dist/*.whl dist/*.tar.gz; do
+            if [ ! -f "$pkg" ]; then
+              continue
+            fi
+
+            PKG_NAME=$(basename "$pkg")
+            echo "==> Validating: $PKG_NAME"
+
+            # For wheels: full integrity check
+            if [[ "$pkg" == *.whl ]]; then
+              if ! unzip -t "$pkg" > /dev/null 2>&1; then
+                echo "  ❌ ZIP test FAIL - CORRUPTED WHEEL!"
+                HAS_ERRORS=1
+              elif ! python3 -m zipfile -t "$pkg" > /dev/null 2>&1; then
+                echo "  ❌ Python zipfile test FAIL - CORRUPTED WHEEL!"
+                HAS_ERRORS=1
+              else
+                # Run twine check and capture output
+                twine check "$pkg" 2>&1 | tee /tmp/twine_pypi_output.txt
+                TWINE_EXIT=${PIPESTATUS[0]}
+
+                # Fail on nonzero exit or any error-like output
+                if [ "$TWINE_EXIT" -eq 0 ] && ! grep -Eqi "ERROR|FAILED|InvalidDistribution" /tmp/twine_pypi_output.txt; then
+                  echo "  ✅ All checks PASS"
+                else
+                  echo "  ❌ Twine check FAIL"
+                  cat /tmp/twine_pypi_output.txt
+                  HAS_ERRORS=1
+                fi
+                rm -f /tmp/twine_pypi_output.txt
+              fi
+            # For source dists: gzip + tar integrity
+            elif [[ "$pkg" == *.tar.gz ]]; then
+              if ! gzip -t "$pkg" 2>/dev/null; then
+                echo "  ❌ Gzip test FAIL - CORRUPTED TARBALL!"
+                HAS_ERRORS=1
+              elif ! tar -tzf "$pkg" > /dev/null 2>&1; then
+                echo "  ❌ Tar test FAIL - CORRUPTED TARBALL!"
+                HAS_ERRORS=1
+              else
+                # Run twine check and capture output
+                twine check "$pkg" 2>&1 | tee /tmp/twine_pypi_output.txt
+                TWINE_EXIT=${PIPESTATUS[0]}
+
+                # Fail on nonzero exit or any error-like output
+                if [ "$TWINE_EXIT" -eq 0 ] && ! grep -Eqi "ERROR|FAILED|InvalidDistribution" /tmp/twine_pypi_output.txt; then
+                  echo "  ✅ All checks PASS"
+                else
+                  echo "  ❌ Twine check FAIL"
+                  cat /tmp/twine_pypi_output.txt
+                  HAS_ERRORS=1
+                fi
+                rm -f /tmp/twine_pypi_output.txt
+              fi
+            fi
+            echo ""
+          done
+
+          if [ $HAS_ERRORS -eq 1 ]; then
+            echo "======================================================================"
+            echo "❌ PYPI VALIDATION FAILED - UPLOAD BLOCKED"
+            echo "======================================================================"
+            echo ""
+            echo "Corrupted packages detected. PyPI upload BLOCKED."
+            echo ""
+            exit 1
+          else
+            echo "======================================================================"
+            echo "✅ ALL PACKAGES VALIDATED - Safe to upload to PyPI"
+            echo "======================================================================"
+          fi
+
+      - name: Publish to PyPI using bleeding-edge twine
+        if: steps.pypi_check.outputs.exists == 'false'
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          echo "==> Publishing to PyPI using twine from master..."
+          # Install bleeding-edge packaging and twine for PEP 639 support
+          # Use --break-system-packages for consistency (safe in CI)
+          python3 -m pip install --break-system-packages git+https://github.com/pypa/packaging.git
+          python3 -m pip install --break-system-packages git+https://github.com/pypa/twine.git
+
+          echo "Upload environment:"
+          echo "twine: $(twine --version)"
+          echo "packaging: $(python3 -m pip show packaging | grep '^Version:')"
+          echo ""
+
+          # Upload to PyPI - explicit patterns to avoid uploading metadata files
+          twine upload dist/*.whl dist/*.tar.gz --verbose
diff --git a/justfile b/justfile
@@ -1388,8 +1388,22 @@ dist venv="": clean-build (build venv) (build-sourcedist venv)
     echo "==> Contents of wheel:"
     unzip -l dist/zlmdb-*-py*.whl || echo "Wheel not found"
 
-# Publish package to PyPI (requires twine setup) - meta-recipe
+# Publish package to PyPI and Read the Docs (meta-recipe)
 publish venv="" tag="": (publish-pypi venv tag) (publish-rtd tag)
+    #!/usr/bin/env bash
+    set -e
+    TAG="{{ tag }}"
+    if [ -z "${TAG}" ]; then
+        TAG=$(git describe --tags --abbrev=0)
+    fi
+    echo ""
+    echo "════════════════════════════════════════════════════════════"
+    echo "✅ Successfully published version ${TAG}"
+    echo "════════════════════════════════════════════════════════════"
+    echo ""
+    echo "📦 PyPI: https://pypi.org/project/zlmdb/${TAG#v}/"
+    echo "📚 RTD:  https://zlmdb.readthedocs.io/en/${TAG}/"
+    echo ""
 
 # Download GitHub release artifacts (usage: `just download-github-release` for nightly, or `just download-github-release stable`)
 download-github-release release_type="nightly":
@@ -1409,45 +1423,150 @@ download-github-release release_type="nightly":
     ls -la dist/
 
 # Download release artifacts from GitHub and publish to PyPI
-publish-pypi venv="" tag="": (install-tools venv)
+publish-pypi venv="" tag="":
     #!/usr/bin/env bash
     set -e
     VENV_NAME="{{ venv }}"
     if [ -z "${VENV_NAME}" ]; then
+        echo "==> No venv name specified. Auto-detecting from system Python..."
         VENV_NAME=$(just --quiet _get-system-venv-name)
+        echo "==> Defaulting to venv: '${VENV_NAME}'"
     fi
-    VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}")
+    VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}"
 
+    # Determine which tag to use
     TAG="{{ tag }}"
     if [ -z "${TAG}" ]; then
-        echo "==> No tag specified, using local build..."
-        just dist ${VENV_NAME}
-    else
-        echo "==> Downloading release artifacts for tag ${TAG}..."
-        mkdir -p dist/
-        gh release download --repo crossbario/zlmdb --pattern "*.whl" --pattern "*.tar.gz" --dir dist/ "${TAG}"
+        echo "==> No tag specified. Using latest git tag..."
+        TAG=$(git describe --tags --abbrev=0)
+        echo "==> Using tag: ${TAG}"
+    fi
+
+    # Verify tag looks like a version tag
+    if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        echo "❌ Error: Tag '${TAG}' doesn't look like a version tag (expected format: vX.Y.Z)"
+        exit 1
+    fi
+
+    # Create temp directory for downloads
+    TEMP_DIR=$(mktemp -d)
+    echo "==> Downloading release artifacts from GitHub release ${TAG}..."
+    echo "    Temp directory: ${TEMP_DIR}"
+
+    # Download all release assets
+    gh release download "${TAG}" --repo crossbario/zlmdb --dir "${TEMP_DIR}"
+
+    echo ""
+    echo "==> Downloaded files:"
+    ls -lh "${TEMP_DIR}"
+    echo ""
+
+    # Count wheels and source distributions
+    WHEEL_COUNT=$(find "${TEMP_DIR}" -name "*.whl" | wc -l)
+    SDIST_COUNT=$(find "${TEMP_DIR}" -name "*.tar.gz" | wc -l)
+
+    echo "Found ${WHEEL_COUNT} wheel(s) and ${SDIST_COUNT} source distribution(s)"
+
+    if [ "${WHEEL_COUNT}" -eq 0 ] || [ "${SDIST_COUNT}" -eq 0 ]; then
+        echo "❌ Error: Expected at least 1 wheel and 1 source distribution"
+        echo "    Wheels found: ${WHEEL_COUNT}"
+        echo "    Source dist found: ${SDIST_COUNT}"
+        rm -rf "${TEMP_DIR}"
+        exit 1
+    fi
+
+    # Ensure twine is installed
+    if [ ! -f "${VENV_PATH}/bin/twine" ]; then
+        echo "==> Installing twine in ${VENV_NAME}..."
+        "${VENV_PATH}/bin/pip" install twine
     fi
 
-    echo "==> Verifying artifacts..."
-    ${VENV_PYTHON} -m twine check dist/*
+    echo "==> Publishing to PyPI using twine..."
+    # Use explicit patterns to avoid uploading metadata files (build-info.txt, CHECKSUMS.sha256, etc.)
+    "${VENV_PATH}/bin/twine" upload "${TEMP_DIR}"/*.whl "${TEMP_DIR}"/*.tar.gz
 
-    echo "==> Publishing to PyPI..."
-    ${VENV_PYTHON} -m twine upload dist/*
+    # Cleanup
+    rm -rf "${TEMP_DIR}"
+    echo "✅ Successfully published ${TAG} to PyPI"
 
 # Trigger Read the Docs build for a specific tag
 publish-rtd tag="":
     #!/usr/bin/env bash
     set -e
+
+    # Determine which tag to use
     TAG="{{ tag }}"
     if [ -z "${TAG}" ]; then
-        echo "==> No tag specified. RTD will build from webhook on push."
-        echo "    To manually trigger: https://readthedocs.org/projects/zlmdb/builds/"
+        echo "==> No tag specified. Using latest git tag..."
+        TAG=$(git describe --tags --abbrev=0)
+        echo "==> Using tag: ${TAG}"
+    fi
+
+    # Verify tag looks like a version tag
+    if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        echo "❌ Error: Tag '${TAG}' doesn't look like a version tag (expected format: vX.Y.Z)"
+        exit 1
+    fi
+
+    # Check if RTD_TOKEN is set
+    if [ -z "${RTD_TOKEN}" ]; then
+        echo "❌ Error: RTD_TOKEN environment variable is not set"
+        echo ""
+        echo "To trigger RTD builds, you need to:"
+        echo "1. Get an API token from https://readthedocs.org/accounts/tokens/"
+        echo "2. Export it: export RTD_TOKEN=your_token_here"
+        echo ""
+        exit 1
+    fi
+
+    echo "==> Triggering Read the Docs build for ${TAG}..."
+    echo ""
+
+    # Trigger build via RTD API
+    # See: https://docs.readthedocs.io/en/stable/api/v3.html#post--api-v3-projects-(string-project_slug)-versions-(string-version_slug)-builds-
+    RTD_PROJECT="zlmdb"
+    RTD_API_URL="https://readthedocs.org/api/v3/projects/${RTD_PROJECT}/versions/${TAG}/builds/"
+
+    echo "==> Calling RTD API..."
+    echo "    Project: ${RTD_PROJECT}"
+    echo "    Version: ${TAG}"
+    echo "    URL: ${RTD_API_URL}"
+    echo ""
+
+    # Trigger the build
+    HTTP_CODE=$(curl -X POST \
+        -H "Authorization: Token ${RTD_TOKEN}" \
+        -w "%{http_code}" \
+        -s -o /tmp/rtd_response.json \
+        "${RTD_API_URL}")
+
+    echo "==> API Response (HTTP ${HTTP_CODE}):"
+    cat /tmp/rtd_response.json | python3 -m json.tool 2>/dev/null || cat /tmp/rtd_response.json
+    echo ""
+
+    if [ "${HTTP_CODE}" = "202" ] || [ "${HTTP_CODE}" = "201" ]; then
+        echo "✅ Read the Docs build triggered successfully!"
+        echo ""
+        echo "Check build status at:"
+        echo "  https://readthedocs.org/projects/${RTD_PROJECT}/builds/"
+        echo ""
+        echo "Documentation will be available at:"
+        echo "  https://${RTD_PROJECT}.readthedocs.io/en/${TAG}/"
+        echo "  https://${RTD_PROJECT}.readthedocs.io/en/stable/ (if marked as stable)"
+        echo ""
     else
-        echo "==> RTD build triggered by GitHub webhook on tag push."
-        echo "    Monitor build at: https://readthedocs.org/projects/zlmdb/builds/"
-        echo "    Documentation will be available at: https://zlmdb.readthedocs.io/en/${TAG}/"
+        echo "❌ Error: Failed to trigger RTD build (HTTP ${HTTP_CODE})"
+        echo ""
+        echo "Common issues:"
+        echo "- Invalid RTD_TOKEN"
+        echo "- Version/tag doesn't exist in RTD project"
+        echo "- Network/API connectivity problems"
+        echo ""
+        exit 1
     fi
 
+    rm -f /tmp/rtd_response.json
+
 # -----------------------------------------------------------------------------
 # -- Utilities
 # -----------------------------------------------------------------------------
