Skip to content

Commit b578b41

Browse files
emasabemasab
authored
Upgrade librdkafka to v2.6.0 (#120)
* Upgrade librdkafka to v2.6.0 * Add install-from-source * Dynamic linking for tests * Speed up librdkafka build
1 parent a4754c7 commit b578b41

File tree

7 files changed

+70
-50
lines changed

7 files changed

+70
-50
lines changed

‎.semaphore/semaphore.yml‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ global_job_config:
2828
- git submodule update --init --recursive
2929
- cd deps/librdkafka
3030
- git fetch origin
31-
- git checkout v2.5.3
31+
- git checkout v2.6.0
3232
- cd ../../
3333
- cache clear
3434

@@ -70,7 +70,8 @@ blocks:
7070
jobs:
7171
- name: "Build from source and test"
7272
commands:
73-
- npm install # this will actually not build anything if we have a release, but rather, fetch things using node-pre-gyp - so change this later.
73+
- export CKJS_LINKING=dynamic
74+
- npm run install-from-source
7475
- make test
7576

7677
- name: 'macOS arm64/m1: Build and test'
@@ -82,7 +83,8 @@ blocks:
8283
jobs:
8384
- name: 'Build from source and test'
8485
commands:
85-
- npm install # this will actually not build anything if we have a release, but rather, fetch things using node-pre-gyp - so change this later.
86+
- export CKJS_LINKING=dynamic
87+
- npm run install-from-source
8688
- make test
8789

8890
- name: "Linux amd64: Build, test, lint"
@@ -93,7 +95,8 @@ blocks:
9395
type: s1-prod-ubuntu20-04-amd64-2
9496
prologue:
9597
commands:
96-
- npm install # this will actually not build anything if we have a release, but rather, fetch things using node-pre-gyp - so change this later.
98+
- export CKJS_LINKING=dynamic
99+
- npm run install-from-source
97100
jobs:
98101
- name: "Test"
99102
commands:
@@ -130,10 +133,7 @@ blocks:
130133
- sudo apt-get install -y librdkafka-dev bc
131134
- export CKJS_LINKING=dynamic
132135
- export BUILD_LIBRDKAFKA=0
133-
- npm install
134-
- npx node-pre-gyp --build-from-source clean
135-
- npx node-pre-gyp --build-from-source configure
136-
- npx node-pre-gyp --build-from-source build
136+
- npm run install-from-source
137137
jobs:
138138
- name: "Performance Test"
139139
commands:
@@ -168,7 +168,7 @@ blocks:
168168
- sem-version node 18.19.0
169169
- export NODE_ABI=108
170170
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
171-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
171+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
172172
- npx node-pre-gyp package
173173
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
174174
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -177,7 +177,7 @@ blocks:
177177
- sem-version node 20.10.0
178178
- export NODE_ABI=115
179179
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
180-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
180+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
181181
- npx node-pre-gyp package
182182
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
183183
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -186,7 +186,7 @@ blocks:
186186
- sem-version node 21.4.0
187187
- export NODE_ABI=120
188188
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
189-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
189+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
190190
- npx node-pre-gyp package
191191
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
192192
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -195,7 +195,7 @@ blocks:
195195
- sem-version node 22.2.0
196196
- export NODE_ABI=127
197197
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
198-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
198+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
199199
- npx node-pre-gyp package
200200
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
201201
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -224,7 +224,7 @@ blocks:
224224
- sem-version node 18.19.0
225225
- export NODE_ABI=108
226226
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
227-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
227+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
228228
- npx node-pre-gyp package
229229
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
230230
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -233,7 +233,7 @@ blocks:
233233
- sem-version node 20.10.0
234234
- export NODE_ABI=115
235235
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
236-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
236+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
237237
- npx node-pre-gyp package
238238
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
239239
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -242,7 +242,7 @@ blocks:
242242
- sem-version node 21.4.0
243243
- export NODE_ABI=120
244244
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
245-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
245+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
246246
- npx node-pre-gyp package
247247
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
248248
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -251,7 +251,7 @@ blocks:
251251
- sem-version node 22.2.0
252252
- export NODE_ABI=127
253253
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
254-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
254+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
255255
- npx node-pre-gyp package
256256
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
257257
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -373,7 +373,7 @@ blocks:
373373
- sem-version node 18.19.0
374374
- export NODE_ABI=108
375375
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
376-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
376+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
377377
- npx node-pre-gyp package
378378
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
379379
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -382,7 +382,7 @@ blocks:
382382
- sem-version node 20.10.0
383383
- export NODE_ABI=115
384384
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
385-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
385+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
386386
- npx node-pre-gyp package
387387
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
388388
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -391,7 +391,7 @@ blocks:
391391
- sem-version node 21.4.0
392392
- export NODE_ABI=120
393393
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
394-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
394+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
395395
- npx node-pre-gyp package
396396
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
397397
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -400,7 +400,7 @@ blocks:
400400
- sem-version node 22.2.0
401401
- export NODE_ABI=127
402402
- export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz"
403-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
403+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
404404
- npx node-pre-gyp package
405405
- ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}
406406
- artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}"
@@ -450,7 +450,7 @@ blocks:
450450
- $env:NODE_ABI = 108
451451
- $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz"
452452
- echo "$env:ARTIFACT_KEY"
453-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
453+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
454454
- npx node-pre-gyp package
455455
- ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}"
456456
- artifact push workflow "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" --destination "releases/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}"
@@ -463,7 +463,7 @@ blocks:
463463
- $env:NODE_ABI = 115
464464
- $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz"
465465
- echo "$env:ARTIFACT_KEY"
466-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
466+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
467467
- npx node-pre-gyp package
468468
- ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}"
469469
- artifact push workflow "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" --destination "releases/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}"
@@ -476,7 +476,7 @@ blocks:
476476
- $env:NODE_ABI = 120
477477
- $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz"
478478
- echo "$env:ARTIFACT_KEY"
479-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
479+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
480480
- npx node-pre-gyp package
481481
- ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}"
482482
- artifact push workflow "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" --destination "releases/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}"
@@ -489,7 +489,7 @@ blocks:
489489
- $env:NODE_ABI = 127
490490
- $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz"
491491
- echo "$env:ARTIFACT_KEY"
492-
- npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet.
492+
- npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet.
493493
- npx node-pre-gyp package
494494
- ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}"
495495
- artifact push workflow "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" --destination "releases/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}"

‎deps/librdkafka‎

Submodule librdkafka updated 39 files

‎deps/librdkafka.gyp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
"deps/librdkafka/src/librdkafka.so",
3333
],
3434
"action": [
35-
"make", "-C", "librdkafka", "libs", "install"
35+
"make", "-j", "-C", "librdkafka", "libs", "install"
3636
],
3737
"conditions": [
3838
[

‎lib/error.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ LibrdKafkaError.wrap = errorWrap;
2727
* @enum {number}
2828
* @constant
2929
*/
30-
// ====== Generated from librdkafka master file src-cpp/rdkafkacpp.h ======
30+
// ====== Generated from librdkafka 2.6.0 file src-cpp/rdkafkacpp.h ======
3131
LibrdKafkaError.codes = {
3232

3333
/* Internal errors to rdkafka: */

‎package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,16 @@
22
"name": "@confluentinc/kafka-javascript",
33
"version": "v0.2.1",
44
"description": "Node.js bindings for librdkafka",
5-
"librdkafka": "2.5.3",
6-
"librdkafka_win": "2.5.3",
5+
"librdkafka": "2.6.0",
6+
"librdkafka_win": "2.6.0",
77
"main": "lib/index.js",
88
"types": "types/index.d.ts",
99
"scripts": {
1010
"configure": "node-gyp configure",
1111
"build": "node-gyp build",
1212
"test": "make test",
1313
"install": "node-pre-gyp install --fallback-to-build",
14+
"install-from-source": "npm ci && node-pre-gyp install --build-from-source=@confluentinc/kafka-javascript --fallback-to-build",
1415
"prepack": "node ./ci/prepublish.js",
1516
"test:types": "tsc -p ."
1617
},

0 commit comments

Comments
 (0)