G-Research · jgiannuzzi · Oct 22, 2025 · Oct 22, 2025
diff --git a/.github/scripts/generate_index.py b/.github/scripts/generate_index.py
@@ -0,0 +1,170 @@
+import os
+import sys
+import re
+import itertools
+import requests
+import hashlib
+
+from urllib.parse import quote
+from pathlib import Path
+from github import Github
+from typing import List, Dict, Set
+
+# Define yanked versions - modify this dictionary as needed
+yanked_versions = {
+         "confluent-kafka": {
+             "2.11.0+gr",
+             "2.11.0+gr.1",
+         },
+    }
+
+HTML_TEMPLATE = """<!DOCTYPE html>
+ <html>
+ <head>
+     <title>{package_name}</title>
+ </head>
+ <body>
+     <h1>{package_name}</h1>
+     {package_links}
+ </body>
+ </html>
+"""
+
+def normalize(name):
+    """Normalize package name according to PEP 503."""
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+def calculate_sha256(file_path):
+    with open(file_path, "rb") as f:
+        digest = hashlib.file_digest(f, "sha256")
+
+    return digest.hexdigest()
+
+def extract_version_from_filename(filename: str) -> str:
+    """Extract version from wheel or sdist filename."""
+    # Remove extension
+    name = filename.replace('.tar.gz', '').replace('.whl', '')
+
+    # For wheels: package-version-python-abi-platform
+    # For sdist: package-version
+    parts = name.split('-')
+    if len(parts) >= 2:
+        return parts[1]
+    return ""
+
+class PackageIndexBuilder:
+    def __init__(self, token: str, repo_name: str, output_dir: str, yanked_versions: Dict[str, Set[str]] = None):
+        self.github = Github(token)
+        self.repo = self.github.get_repo(repo_name)
+        self.output_dir = Path(output_dir)
+        self.packages: Dict[str, List[Dict]] = {}
+        self.yanked_versions = yanked_versions or {}
+
+        # Set up authenticated session
+        self.session = requests.Session()
+        self.session.headers.update({
+            "Authorization": f"token {token}",
+            "Accept": "application/octet-stream",
+        })
+
+    def is_version_yanked(self, package_name: str, version: str) -> bool:
+        """Check if a specific version of a package is yanked."""
+        normalized_package = normalize(package_name)
+        return normalized_package in self.yanked_versions and version in self.yanked_versions[normalized_package]
+
+    def collect_packages(self):
+        print("Query release assets")
+
+        for release in self.repo.get_releases():
+            for asset in release.get_assets():
+                if asset.name.endswith(('.whl', '.tar.gz')):
+                    package_name = normalize(asset.name.split('-')[0])
+                    if package_name not in self.packages:
+                        self.packages[package_name] = []
+
+                    version = extract_version_from_filename(asset.name)
+                    self.packages[package_name].append({
+                        'filename': asset.name,
+                        'url': asset.url,
+                        'size': asset.size,
+                        'upload_time': asset.created_at.strftime('%Y-%m-%d %H:%M:%S'),
+                        'version': version,
+                    })
+
+    def generate_index_html(self):
+        # Generate main index
+        package_list = self.packages.keys()
+        main_index = HTML_TEMPLATE.format(
+            package_name="Simple Package Index",
+            package_links="\n".join([f'<a href="{x}/">{x}</a><br/>' for x in package_list])
+        )
+
+        with open(self.output_dir / "index.html", "w") as f:
+            f.write(main_index)
+
+        for package, assets in self.packages.items():
+
+            package_dir = self.output_dir / package
+            package_dir.mkdir(exist_ok=True)
+
+            # Generate package-specific index.html
+            file_links = []
+            assets = sorted(assets, key=lambda x: x["filename"])
+            for filename, items in itertools.groupby(assets, key=lambda x: x["filename"]):
+                asset_info = next(items)
+                url = asset_info['url']
+                version = asset_info['version']
+
+                # Download the file
+                with open(package_dir / filename, 'wb') as f:
+                    print (f"Downloading '{filename}' from '{url}'")
+                    response = self.session.get(url, stream=True)
+                    response.raise_for_status()
+                    for chunk in response.iter_content(chunk_size=8192):
+                        if chunk:
+                            f.write(chunk)
+
+                sha256_hash = calculate_sha256(package_dir / filename)
+
+                # Check if this version is yanked
+                yanked_attr = ""
+                if self.is_version_yanked(package, version):
+                    yanked_attr = ' data-yanked="true"'
+
+                file_links.append(
+                    f'<a href="{quote(filename)}#sha256={sha256_hash}"{yanked_attr}>{filename}</a><br/>'
+                )
+
+            package_index = HTML_TEMPLATE.format(
+                package_name=f"Links for {package}",
+                package_links="\n".join(file_links)
+            )
+
+            with open(package_dir / "index.html", "w") as f:
+                f.write(package_index)
+
+    def build(self):
+        # Create output directory
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Collect and generate
+        self.collect_packages()
+        self.generate_index_html()
+
+
+def main():
+    # Get environment variables
+    token = os.environ.get("GITHUB_TOKEN")
+    repo = os.environ.get("GITHUB_REPOSITORY")
+    print (repo)
+    output_dir = os.environ.get("OUTPUT_DIR", "dist")
+
+    if not all([token, repo]):
+        print ("Missing required environment variables")
+        sys.exit(1)
+
+    builder = PackageIndexBuilder(token, repo, output_dir, yanked_versions)
+    builder.build()
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
@@ -0,0 +1,134 @@
+# .github/workflows/build-wheels.yml
+name: Build and Package Wheels
+
+on:
+  pull_request:
+  push:
+
+env:
+  LIBRDKAFKA_VERSION: v2.12.1.1
+
+jobs:
+
+  build-linux:
+    name: Build wheels for Linux ${{ matrix.arch }}
+    strategy:
+      matrix:
+        include:
+          - arch: x64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
+    env:
+      OS_NAME: linux
+      ARCH: ${{ matrix.arch }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+      - name: Build wheels
+        run: |
+          ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ env.OS_NAME }}-${{ env.ARCH }}
+          path: wheelhouse/confluent_kafka*.whl
+
+  build-windows:
+    name: Build wheels for Windows
+    runs-on: windows-latest
+    env:
+      OS_NAME: windows
+      ARCH: x64
+      CHERE_INVOKING: yes
+      MSYSTEM: UCRT64
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+      - name: Setup MSYS2
+        uses: msys2/setup-msys2@v2
+      - name: Build wheels
+        shell: bash
+        run: |
+          ./tools/mingw-w64/msys2-dependencies.sh
+          bash tools/mingw-w64/semaphore_commands.sh
+          bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
+          tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ env.OS_NAME }}-${{ env.ARCH }}
+          path: wheelhouse/confluent_kafka*.whl
+
+  create_release_artifacts:
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    name: Create release artifacts
+    needs: [build-linux, build-windows]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - name: Check version consistency
+        run: |
+          tag_version=${GITHUB_REF/refs\/tags\/v/}
+          proj_version=$(uv run --no-project --with poetry poetry version -s)
+          if [ "$tag_version" != "$proj_version" ]; then
+            echo "::error title=Version mismatch::Tag version \"$tag_version\" does not match project version \"$proj_version\"."
+            exit 1
+          fi
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          pattern: wheels-*
+          merge-multiple: true
+
+      - name: Create release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            artifacts/confluent_kafka*
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  publish_simple_package_index:
+    name: Build&publish a PyPI-compatible package index
+    runs-on: ubuntu-latest
+    needs: [create_release_artifacts]
+    if: always() && !failure() && !cancelled() && (github.event_name == 'push')
+    concurrency:
+      group: simple_package_index
+      cancel-in-progress: true
+    permissions:
+      contents: write
+      actions: read
+      packages: read
+      pages: write
+      id-token: write
+    steps:    
+      - uses: actions/checkout@v2
+      - name: Generate Package Index
+        run: |        
+          python -m pip install --upgrade pip
+          pip install PyGithub
+          python .github/scripts/generate_index.py
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OUTPUT_DIR: dist
+
+      - name: Upload Site Artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: 'dist'
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
+
+      - name: Display GitHub Pages URL
+        run: |
+          echo "Package Index URL: ${{ steps.deployment.outputs.page_url }}"
diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "confluent-kafka"
-version = "2.12.1"
+version = "2.12.1.1+gr"
 description = "Confluent's Python client for Apache Kafka"
 classifiers = [
     "Development Status :: 5 - Production/Stable",

diff --git a/tools/wheels/install-librdkafka.sh b/tools/wheels/install-librdkafka.sh
@@ -19,7 +19,19 @@ echo "$0: Installing librdkafka $VER to $DEST"
 [[ -d "$DEST" ]] || mkdir -p "$DEST"
 pushd "$DEST"
 
-curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
+# Check if variable exists
+if [ -z "${GITHUB_TOKEN}" ]; then
+    echo "Error: GITHUB_TOKEN is not set"
+    exit 1
+fi
+
+curl -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+ -H "Accept: application/vnd.github.v3+json" \
+ -L \
+ -o lrk$VER.zip \
+https://nuget.pkg.github.com/G-Research/download/librdkafka.redist/$VER/librdkafka.redist.$VER.nupkg
+
+#curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
 
 unzip lrk$VER.zip
 
@@ -28,12 +40,7 @@ ARCH=${ARCH:-x64}
 if [[ $OSTYPE == linux* ]]; then
     # Linux
 
-    # Copy the librdkafka build with least dependencies to librdkafka.so.1
-    if [[ $ARCH == arm64* ]]; then
-        cp -v runtimes/linux-$ARCH/native/{librdkafka.so,librdkafka.so.1}
-    else
-        cp -v runtimes/linux-$ARCH/native/{centos8-librdkafka.so,librdkafka.so.1}
-    fi
+    cp -v runtimes/linux-$ARCH/native/{librdkafka.so,librdkafka.so.1}
     ldd runtimes/linux-$ARCH/native/librdkafka.so.1
 
 elif [[ $OSTYPE == darwin* ]]; then