Skip to content

Commit 2dbfd66

Browse files
jgiannuzzijgiannuzzi
authored
CI/CD script for v2.11.0-gr (#6)
* CI/CD script for v2.11.0-gr * Do not include libsasl2 and its dependencies in the resulting wheels * Another attempt at excluding libsasl2 from the wheels * Escaping is hard * Create a universal build for linux-x64 using the system-provided sasl2 library * Attempt to fix universal build
1 parent 0d81d29 commit 2dbfd66

File tree

10 files changed

+312
-14
lines changed

10 files changed

+312
-14
lines changed

‎.github/scripts/generate_index.py‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import os
2+
import sys
3+
import re
4+
import itertools
5+
import requests
6+
import hashlib
7+
8+
from urllib.parse import quote
9+
from pathlib import Path
10+
from github import Github
11+
from typing import List, Dict
12+
13+
HTML_TEMPLATE = """<!DOCTYPE html>
14+
<html>
15+
<head>
16+
<title>{package_name}</title>
17+
</head>
18+
<body>
19+
<h1>{package_name}</h1>
20+
{package_links}
21+
</body>
22+
</html>
23+
"""
24+
25+
def normalize(name):
26+
"""Normalize package name according to PEP 503."""
27+
return re.sub(r"[-_.]+", "-", name).lower()
28+
29+
def calculate_sha256(file_path):
30+
with open(file_path, "rb") as f:
31+
digest = hashlib.file_digest(f, "sha256")
32+
33+
return digest.hexdigest()
34+
35+
class PackageIndexBuilder:
36+
def __init__(self, token: str, repo_name: str, output_dir: str):
37+
self.github = Github(token)
38+
self.repo = self.github.get_repo(repo_name)
39+
self.output_dir = Path(output_dir)
40+
self.packages: Dict[str, List[Dict]] = {}
41+
42+
# Set up authenticated session
43+
self.session = requests.Session()
44+
self.session.headers.update({
45+
"Authorization": f"token {token}",
46+
"Accept": "application/octet-stream",
47+
})
48+
49+
def collect_packages(self):
50+
51+
print ("Query release assets")
52+
53+
for release in self.repo.get_releases():
54+
for asset in release.get_assets():
55+
if asset.name.endswith(('.whl', '.tar.gz')):
56+
package_name = normalize(asset.name.split('-')[0])
57+
if package_name not in self.packages:
58+
self.packages[package_name] = []
59+
60+
self.packages[package_name].append({
61+
'filename': asset.name,
62+
'url': asset.url,
63+
'size': asset.size,
64+
'upload_time': asset.created_at.strftime('%Y-%m-%d %H:%M:%S'),
65+
})
66+
67+
def generate_index_html(self):
68+
# Generate main index
69+
package_list = self.packages.keys()
70+
main_index = HTML_TEMPLATE.format(
71+
package_name="Simple Package Index",
72+
package_links="\n".join([f'<a href="{x}/">{x}</a><br/>' for x in package_list])
73+
)
74+
75+
with open(self.output_dir / "index.html", "w") as f:
76+
f.write(main_index)
77+
78+
for package, assets in self.packages.items():
79+
80+
package_dir = self.output_dir / package
81+
package_dir.mkdir(exist_ok=True)
82+
83+
# Generate package-specific index.html
84+
file_links = []
85+
assets = sorted(assets, key=lambda x: x["filename"])
86+
for filename, items in itertools.groupby(assets, key=lambda x: x["filename"]):
87+
url = next(items)['url']
88+
89+
# Download the file
90+
with open(package_dir / filename, 'wb') as f:
91+
print (f"Downloading '{filename}' from '{url}'")
92+
response = self.session.get(url, stream=True)
93+
response.raise_for_status()
94+
for chunk in response.iter_content(chunk_size=8192):
95+
if chunk:
96+
f.write(chunk)
97+
98+
sha256_hash = calculate_sha256(package_dir / filename)
99+
file_links.append(f'<a href="{quote(filename)}#sha256={sha256_hash}">{filename}</a><br/>')
100+
101+
package_index = HTML_TEMPLATE.format(
102+
package_name=f"Links for {package}",
103+
package_links="\n".join(file_links)
104+
)
105+
106+
with open(package_dir / "index.html", "w") as f:
107+
f.write(package_index)
108+
109+
def build(self):
110+
# Create output directory
111+
self.output_dir.mkdir(parents=True, exist_ok=True)
112+
113+
# Collect and generate
114+
self.collect_packages()
115+
self.generate_index_html()
116+
117+
118+
def main():
119+
# Get environment variables
120+
token = os.environ.get("GITHUB_TOKEN")
121+
repo = os.environ.get("GITHUB_REPOSITORY")
122+
print (repo)
123+
output_dir = os.environ.get("OUTPUT_DIR", "dist")
124+
125+
if not all([token, repo]):
126+
print ("Missing required environment variables")
127+
sys.exit(1)
128+
129+
builder = PackageIndexBuilder(token, repo, output_dir)
130+
builder.build()
131+
132+
if __name__ == "__main__":
133+
main()

‎.github/workflows/package.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# .github/workflows/build-wheels.yml
2+
name: Build and Package Wheels
3+
4+
on:
5+
pull_request:
6+
push:
7+
8+
env:
9+
LIBRDKAFKA_VERSION: v2.11.0-gr
10+
11+
jobs:
12+
13+
build-linux-x64:
14+
name: Build wheels for Linux x64
15+
runs-on: ubuntu-latest
16+
env:
17+
OS_NAME: linux
18+
ARCH: x64
19+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
20+
steps:
21+
- uses: actions/checkout@v4
22+
- name: Build wheels
23+
run: |
24+
./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
25+
- uses: actions/upload-artifact@v4
26+
with:
27+
name: wheels-${{ env.OS_NAME }}-${{ env.ARCH }}
28+
path: wheelhouse/confluent_kafka*.whl
29+
30+
build-windows:
31+
name: Build wheels for Windows
32+
runs-on: windows-latest
33+
env:
34+
OS_NAME: windows
35+
ARCH: x64
36+
CHERE_INVOKING: yes
37+
MSYSTEM: UCRT64
38+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
39+
steps:
40+
- uses: actions/checkout@v4
41+
- name: Setup MSYS2
42+
uses: msys2/setup-msys2@v2
43+
- name: Build wheels
44+
shell: bash
45+
run: |
46+
./tools/mingw-w64/msys2-dependencies.sh
47+
bash tools/mingw-w64/semaphore_commands.sh
48+
bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
49+
tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
50+
- uses: actions/upload-artifact@v4
51+
with:
52+
name: wheels-${{ env.OS_NAME }}-${{ env.ARCH }}
53+
path: wheelhouse/confluent_kafka*.whl
54+
55+
publish_pypi_index:
56+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
57+
name: Build a PyPI-compatible index
58+
needs: [build-linux-x64, build-windows]
59+
runs-on: ubuntu-latest
60+
permissions:
61+
contents: write
62+
actions: read
63+
packages: read
64+
pages: write
65+
id-token: write
66+
steps:
67+
- uses: actions/checkout@v2
68+
- uses: actions/download-artifact@v4
69+
with:
70+
path: artifacts
71+
pattern: wheels-*
72+
merge-multiple: true
73+
74+
- name: Create release
75+
uses: softprops/action-gh-release@v2
76+
with:
77+
files: |
78+
artifacts/confluent_kafka*
79+
env:
80+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
81+
82+
- name: Generate Package Index
83+
run: |
84+
python -m pip install --upgrade pip
85+
pip install PyGithub
86+
python .github/scripts/generate_index.py
87+
env:
88+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
89+
OUTPUT_DIR: dist
90+
91+
- name: Upload Site Artifact
92+
uses: actions/upload-pages-artifact@v3
93+
with:
94+
path: 'dist'
95+
96+
- name: Deploy to GitHub Pages
97+
uses: actions/deploy-pages@v4

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "confluent-kafka"
7-
version = "2.11.0"
7+
version = "2.11.0+gr"
88
description = "Confluent's Python client for Apache Kafka"
99
classifiers = [
1010
"Development Status :: 5 - Production/Stable",

‎setup.py‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/usr/bin/env python
22

3+
import ctypes.util
34
import os
45
from setuptools import setup
56
from setuptools import Extension
@@ -16,13 +17,44 @@
1617
else:
1718
librdkafka_libname = 'rdkafka'
1819

19-
module = Extension('confluent_kafka.cimpl',
20-
libraries=[librdkafka_libname],
21-
sources=[os.path.join(ext_dir, 'confluent_kafka.c'),
22-
os.path.join(ext_dir, 'Producer.c'),
23-
os.path.join(ext_dir, 'Consumer.c'),
24-
os.path.join(ext_dir, 'Metadata.c'),
25-
os.path.join(ext_dir, 'AdminTypes.c'),
26-
os.path.join(ext_dir, 'Admin.c')])
20+
# Define the default module to build, without external dependencies.
21+
module_defs = [
22+
{
23+
'name': 'confluent_kafka.cimpl.nodeps.cimpl',
24+
'libname': librdkafka_libname,
25+
}
26+
]
2727

28-
setup(ext_modules=[module])
28+
# Check for GSSAPI support and add the appropriate module definitions.
29+
if ctypes.util.find_library('rdkafka_sasl2_2'):
30+
module_defs.append(
31+
{
32+
'name': 'confluent_kafka.cimpl.sasl2_2.cimpl',
33+
'libname': 'rdkafka_sasl2_2',
34+
}
35+
)
36+
if ctypes.util.find_library('rdkafka_sasl2_3'):
37+
module_defs.append(
38+
{
39+
'name': 'confluent_kafka.cimpl.sasl2_3.cimpl',
40+
'libname': 'rdkafka_sasl2_3',
41+
}
42+
)
43+
44+
setup(
45+
ext_modules=[
46+
Extension(
47+
mod_def['name'],
48+
libraries=[mod_def['libname']],
49+
sources=[
50+
os.path.join(ext_dir, 'confluent_kafka.c'),
51+
os.path.join(ext_dir, 'Producer.c'),
52+
os.path.join(ext_dir, 'Consumer.c'),
53+
os.path.join(ext_dir, 'Metadata.c'),
54+
os.path.join(ext_dir, 'AdminTypes.c'),
55+
os.path.join(ext_dir, 'Admin.c'),
56+
],
57+
)
58+
for mod_def in module_defs
59+
]
60+
)

‎src/confluent_kafka/cimpl/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
try:
2+
from .sasl2_3.cimpl import *
3+
variant = "sasl2_3"
4+
except ImportError:
5+
try:
6+
from .sasl2_2.cimpl import *
7+
variant = "sasl2_2"
8+
except ImportError:
9+
from .nodeps.cimpl import *
10+
variant = "nodeps"

‎src/confluent_kafka/cimpl/nodeps/__init__.py‎

Whitespace-only changes.

‎src/confluent_kafka/cimpl/sasl2_2/__init__.py‎

Whitespace-only changes.

‎src/confluent_kafka/cimpl/sasl2_3/__init__.py‎

Whitespace-only changes.

‎tools/wheels/build-wheels.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ case $OSTYPE in
3535
# Need to set up env vars (in docker) so that cibuildwheel finds librdkafka.
3636
lib_dir=dest/runtimes/linux-$ARCH/native
3737
export CIBW_ENVIRONMENT="CFLAGS=-I\$PWD/dest/build/native/include LDFLAGS=-L\$PWD/$lib_dir LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$PWD/$lib_dir"
38+
# Do not include libsasl2 and its dependencies in the resulting wheel.
39+
export CIBW_REPAIR_WHEEL_COMMAND="auditwheel repair --exclude libsasl2.so.2 --exclude libsasl2.so.3 -w {dest_dir} {wheel}"
3840
;;
3941
darwin*)
4042
os=macos

‎tools/wheels/install-librdkafka.sh‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,46 @@ echo "$0: Installing librdkafka $VER to $DEST"
1919
[[ -d "$DEST" ]] || mkdir -p "$DEST"
2020
pushd "$DEST"
2121

22-
curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
22+
# Check if variable exists
23+
if [ -z "${GITHUB_TOKEN}" ]; then
24+
echo "Error: GITHUB_TOKEN is not set"
25+
exit 1
26+
fi
27+
28+
curl -H "Authorization: Bearer ${GITHUB_TOKEN}" \
29+
-H "Accept: application/vnd.github.v3+json" \
30+
-L \
31+
-o lrk$VER.zip \
32+
https://nuget.pkg.github.com/G-Research/download/librdkafka.redist/$VER/librdkafka.redist.$VER.nupkg
33+
34+
#curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
2335

2436
unzip lrk$VER.zip
2537

2638
ARCH=${ARCH:-x64}
2739

2840
if [[ $OSTYPE == linux* ]]; then
2941
# Linux
42+
LIBDIR=runtimes/linux-$ARCH/native
3043

3144
# Copy the librdkafka build with least dependencies to librdkafka.so.1
3245
if [[ $ARCH == arm64* ]]; then
33-
cp -v runtimes/linux-$ARCH/native/{librdkafka.so,librdkafka.so.1}
46+
cp -v $LIBDIR/{librdkafka.so,librdkafka.so.1}
3447
else
35-
cp -v runtimes/linux-$ARCH/native/{centos8-librdkafka.so,librdkafka.so.1}
48+
cp -v $LIBDIR/{centos8-librdkafka.so,librdkafka.so.1}
49+
# Copy the librdkafka build with sasl2 support to 2 versions:
50+
# librdkafka_sasl2_2.so.1 for debian-based distros
51+
# librdkafka_sasl2_3.so.1 for rpm-based distros
52+
patchelf --set-soname librdkafka_sasl2_2.so.1 --output $LIBDIR/{librdkafka_sasl2_2.so.1,librdkafka.so}
53+
patchelf --replace-needed libsasl2.so.3 libsasl2.so.2 $LIBDIR/librdkafka_sasl2_2.so.1
54+
ln -s librdkafka_sasl2_2.so.1 $LIBDIR/librdkafka_sasl2_2.so
55+
patchelf --set-soname librdkafka_sasl2_3.so.1 --output $LIBDIR/{librdkafka_sasl2_3.so.1,librdkafka.so}
56+
ln -s librdkafka_sasl2_3.so.1 $LIBDIR/librdkafka_sasl2_3.so
3657
fi
37-
ldd runtimes/linux-$ARCH/native/librdkafka.so.1
58+
for lib in $LIBDIR/librdkafka*.so.1; do
59+
echo $lib
60+
ldd $lib
61+
done
3862

3963
elif [[ $OSTYPE == darwin* ]]; then
4064
# MacOS X

0 commit comments

Comments
 (0)