Skip to content

Commit 34e4303

Browse files
mensfeldmensfeld
authored
build native extensions (#632)
* build native extensions * remarks * platforms setup * move ostruct where it should be * improve wait * execution remarks * remarks * remarks
1 parent 77e896c commit 34e4303

26 files changed

+3172
-161
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 0 additions & 101 deletions
This file was deleted.

‎.github/workflows/ci_linux_x86_64_gnu.yml‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
name: CI Linux x86_64 GNU
2+
3+
concurrency:
4+
group: ${{ github.workflow }}-${{ github.ref }}
5+
cancel-in-progress: true
6+
7+
on:
8+
pull_request:
9+
branches: [ main, master ]
10+
push:
11+
branches: [ main, master ]
12+
schedule:
13+
- cron: '0 1 * * *'
14+
15+
permissions:
16+
contents: read
17+
18+
env:
19+
BUNDLE_RETRY: 6
20+
BUNDLE_JOBS: 4
21+
22+
jobs:
23+
build_install:
24+
timeout-minutes: 30
25+
runs-on: ubuntu-latest
26+
steps:
27+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
28+
with:
29+
fetch-depth: 0
30+
- name: Install package dependencies
31+
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
32+
- name: Set up Ruby
33+
uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
34+
with:
35+
ruby-version: '3.4' # Use one Ruby version for building
36+
bundler-cache: false
37+
- name: Build gem with mini_portile
38+
run: |
39+
set -e
40+
bundle install
41+
cd ext && bundle exec rake
42+
cd ..
43+
- name: Upload built gem and bundle
44+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
45+
with:
46+
name: rdkafka-built-gem
47+
path: |
48+
vendor/bundle/
49+
.bundle/
50+
ext/
51+
lib/
52+
retention-days: 1
53+
54+
specs_install:
55+
timeout-minutes: 30
56+
runs-on: ubuntu-latest
57+
needs: build_install
58+
strategy:
59+
fail-fast: false
60+
matrix:
61+
ruby:
62+
- '3.5.0-preview1'
63+
- '3.4'
64+
- '3.3'
65+
- '3.2'
66+
- '3.1'
67+
- 'jruby-10.0'
68+
include:
69+
- ruby: '3.4'
70+
coverage: 'true'
71+
- ruby: 'jruby-10.0'
72+
continue-on-error: true
73+
steps:
74+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
75+
with:
76+
fetch-depth: 0
77+
- name: Download built gem
78+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
79+
with:
80+
name: rdkafka-built-gem
81+
path: ./
82+
- name: Set up Ruby
83+
uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
84+
with:
85+
ruby-version: ${{matrix.ruby}}
86+
bundler-cache: false
87+
- name: Start Kafka with Docker Compose
88+
run: |
89+
docker compose up -d
90+
echo "Waiting for Kafka to be ready..."
91+
92+
sleep 10
93+
94+
echo "=== Container status ==="
95+
docker compose ps kafka
96+
97+
for i in {1..30}; do
98+
echo "=== Attempt $i/30 ==="
99+
100+
echo "Testing kafka-topics command..."
101+
if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
102+
echo "Kafka topics command succeeded!"
103+
break
104+
else
105+
echo "Kafka topics command failed (exit code: $?)"
106+
fi
107+
108+
echo "Sleeping 2 seconds..."
109+
sleep 2
110+
done
111+
- name: Install remaining dependencies
112+
env:
113+
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
114+
run: |
115+
# Only install gems that aren't Ruby-version specific
116+
bundle install
117+
- name: Run all specs
118+
env:
119+
GITHUB_COVERAGE: ${{matrix.coverage}}
120+
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
121+
continue-on-error: ${{ matrix.continue-on-error || false }}
122+
run: |
123+
bundle exec rspec
124+
125+
build_precompiled:
126+
timeout-minutes: 30
127+
runs-on: ubuntu-latest
128+
steps:
129+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
130+
with:
131+
fetch-depth: 0
132+
- name: Install build dependencies
133+
run: |
134+
sudo apt-get update
135+
sudo apt-get install -y --no-install-recommends \
136+
build-essential \
137+
gcc \
138+
make \
139+
patch \
140+
tar \
141+
wget \
142+
ca-certificates \
143+
libsasl2-dev \
144+
libssl-dev \
145+
zlib1g-dev \
146+
libzstd-dev
147+
- name: Cache build-tmp directory
148+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
149+
with:
150+
path: ext/build-tmp
151+
key: build-tmp-${{ runner.os }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2
152+
- name: Build precompiled librdkafka.so
153+
run: |
154+
cd ext
155+
./build_linux_x86_64_gnu.sh
156+
- name: Upload precompiled library
157+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
158+
with:
159+
name: librdkafka-precompiled-linux
160+
path: ext/
161+
retention-days: 1
162+
163+
specs_precompiled:
164+
timeout-minutes: 30
165+
runs-on: ubuntu-latest
166+
needs: build_precompiled
167+
strategy:
168+
fail-fast: false
169+
matrix:
170+
ruby:
171+
- '3.5.0-preview1'
172+
- '3.4'
173+
- '3.3'
174+
- '3.2'
175+
- '3.1'
176+
include:
177+
- ruby: '3.4'
178+
coverage: 'true'
179+
steps:
180+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
181+
with:
182+
fetch-depth: 0
183+
- name: Download precompiled library
184+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
185+
with:
186+
name: librdkafka-precompiled-linux
187+
path: ext/
188+
- name: Set up Ruby
189+
uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
190+
with:
191+
ruby-version: ${{ matrix.ruby }}
192+
bundler-cache: false
193+
- name: Start Kafka with Docker Compose
194+
run: |
195+
docker compose up -d
196+
echo "Waiting for Kafka to be ready..."
197+
198+
sleep 10
199+
200+
echo "=== Container status ==="
201+
docker compose ps kafka
202+
203+
for i in {1..30}; do
204+
echo "=== Attempt $i/30 ==="
205+
206+
echo "Testing kafka-topics command..."
207+
if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
208+
echo "Kafka topics command succeeded!"
209+
break
210+
else
211+
echo "Kafka topics command failed (exit code: $?)"
212+
fi
213+
214+
echo "Sleeping 2 seconds..."
215+
sleep 2
216+
done
217+
- name: Install bundle with precompiled library
218+
env:
219+
GITHUB_COVERAGE: ${{ matrix.coverage }}
220+
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
221+
run: |
222+
bundle install
223+
echo "Bundle install completed with precompiled library"
224+
- name: Remove build dependencies to test static linking
225+
continue-on-error: true
226+
run: |
227+
echo "Removing build dependencies to verify precompiled library is truly self-contained..."
228+
229+
# Remove packages one by one to avoid dependency conflicts
230+
packages_to_remove="build-essential gcc g++ make patch tar wget libsasl2-dev libssl-dev zlib1g-dev libzstd-dev"
231+
232+
for package in $packages_to_remove; do
233+
if dpkg -l | grep -q "^ii.*$package "; then
234+
echo "Removing $package..."
235+
sudo dpkg --remove --force-depends $package 2>/dev/null || echo "Could not remove $package"
236+
else
237+
echo "$package is not installed"
238+
fi
239+
done
240+
241+
echo "Build dependencies removal completed"
242+
echo "Remaining build tools:"
243+
which gcc g++ make 2>/dev/null || echo "No build tools found in PATH (good!)"
244+
- name: Run specs with precompiled library
245+
env:
246+
GITHUB_COVERAGE: ${{ matrix.coverage }}
247+
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
248+
run: |
249+
bundle exec rspec

0 commit comments

Comments
 (0)