Update librdkafka from 1.8.2 -> 1.9.2 (#961)

Author: GaryWilber

README.md

@@ -16,7 +16,7 @@ I am looking for *your* help to make this project even better! If you're interes
 
 The `node-rdkafka` library is a high-performance NodeJS client for [Apache Kafka](http://kafka.apache.org/) that wraps the native  [librdkafka](https://github.com/edenhill/librdkafka) library.  All the complexity of balancing writes across partitions and managing (possibly ever-changing) brokers should be encapsulated in the library.
 
-__This library currently uses `librdkafka` version `1.8.2`.__
+__This library currently uses `librdkafka` version `1.9.2`.__
 
 ## Reference Docs
 
@@ -59,7 +59,7 @@ Using Alpine Linux? Check out the [docs](https://github.com/Blizzard/node-rdkafk
 
 ### Windows
 
-Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.1.8.2.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
+Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.1.9.2.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
 
 Requirements:
  * [node-gyp for Windows](https://github.com/nodejs/node-gyp#on-windows)  (the easies way to get it: `npm install --global --production windows-build-tools`, if your node version is 6.x or below, pleasse use `npm install --global --production [email protected]`)
@@ -96,7 +96,7 @@ var Kafka = require('node-rdkafka');
 
 ## Configuration
 
-You can pass many configuration options to `librdkafka`.  A full list can be found in `librdkafka`'s [Configuration.md](https://github.com/edenhill/librdkafka/blob/v1.8.2/CONFIGURATION.md)
+You can pass many configuration options to `librdkafka`.  A full list can be found in `librdkafka`'s [Configuration.md](https://github.com/edenhill/librdkafka/blob/v1.9.2/CONFIGURATION.md)
 
 Configuration keys that have the suffix `_cb` are designated as callbacks. Some
 of these keys are informational and you can choose to opt-in (for example, `dr_cb`). Others are callbacks designed to
@@ -131,7 +131,7 @@ You can also get the version of `librdkafka`
 const Kafka = require('node-rdkafka');
 console.log(Kafka.librdkafkaVersion);
 
-// #=> 1.8.2
+// #=> 1.9.2
 ```
 
 ## Sending Messages
@@ -144,7 +144,7 @@ var producer = new Kafka.Producer({
 });
 ```
 
-A `Producer` requires only `metadata.broker.list` (the Kafka brokers) to be created.  The values in this list are separated by commas.  For other configuration options, see the [Configuration.md](https://github.com/edenhill/librdkafka/blob/v1.8.2/CONFIGURATION.md) file described previously.
+A `Producer` requires only `metadata.broker.list` (the Kafka brokers) to be created.  The values in this list are separated by commas.  For other configuration options, see the [Configuration.md](https://github.com/edenhill/librdkafka/blob/v1.9.2/CONFIGURATION.md) file described previously.
 
 The following example illustrates a list with several `librdkafka` options set.
 

ci/librdkafka-defs-generator.js

@@ -171,7 +171,7 @@ function updateErrorDefinitions(file) {
 
   // validate body
   const emptyCheck = body
-    .replace(/((  \/\*)|(   ?\*)).*/g, '')
+    .replace(/((\s+\/\*)|(   ?\*)).*/g, '')
     .replace(/  ERR_\w+: -?\d+,?\r?\n/g, '')
     .trim()
   if (emptyCheck !== '') {

config.d.ts

@@ -1,12 +1,12 @@
-// ====== Generated from librdkafka 1.8.2 file CONFIGURATION.md ======
+// ====== Generated from librdkafka 1.9.2 file CONFIGURATION.md ======
 // Code that generated this is a derivative work of the code from Nam Nguyen
 // https://gist.github.com/ntgn81/066c2c8ec5b4238f85d1e9168a04e3fb
 
 export interface GlobalConfig {
     /**
      * Indicates the builtin features for this build of librdkafka. An application can either query this value or attempt to set it with its list of required features to check for library support.
      *
-     * @default gzip, snappy, ssl, sasl, regex, lz4, sasl_gssapi, sasl_plain, sasl_scram, plugins, zstd, sasl_oauthbearer
+     * @default gzip, snappy, ssl, sasl, regex, lz4, sasl_gssapi, sasl_plain, sasl_scram, plugins, zstd, sasl_oauthbearer, http, oidc
      */
     "builtin.features"?: string;
 
@@ -177,6 +177,13 @@ export interface GlobalConfig {
      */
     "broker.address.family"?: 'any' | 'v4' | 'v6';
 
+    /**
+     * Maximum time allowed for broker connection setup (TCP connection setup as well SSL and SASL handshake). If the connection to the broker is not fully functional after this the connection will be closed and retried.
+     *
+     * @default 30000
+     */
+    "socket.connection.setup.timeout.ms"?: number;
+
     /**
      * Close broker connections after the specified time of inactivity. Disable with 0. If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info).
      *
@@ -268,7 +275,7 @@ export interface GlobalConfig {
     "enable.random.seed"?: boolean;
 
     /**
-     * Log broker disconnects. It might be useful to turn this off when interacting with 0.9 brokers with an aggressive `connection.max.idle.ms` value.
+     * Log broker disconnects. It might be useful to turn this off when interacting with 0.9 brokers with an aggressive `connections.max.idle.ms` value.
      *
      * @default true
      */
@@ -544,10 +551,42 @@ export interface GlobalConfig {
     "enable.sasl.oauthbearer.unsecure.jwt"?: boolean;
 
     /**
-     * SASL/OAUTHBEARER token refresh callback (set with rd_kafka_conf_set_oauthbearer_token_refresh_cb(), triggered by rd_kafka_poll(), et.al. This callback will be triggered when it is time to refresh the client's OAUTHBEARER token.
+     * SASL/OAUTHBEARER token refresh callback (set with rd_kafka_conf_set_oauthbearer_token_refresh_cb(), triggered by rd_kafka_poll(), et.al. This callback will be triggered when it is time to refresh the client's OAUTHBEARER token. Also see `rd_kafka_conf_enable_sasl_queue()`.
      */
     "oauthbearer_token_refresh_cb"?: any;
 
+    /**
+     * Set to "default" or "oidc" to control which login method to be used. If set to "oidc", the following properties must also be be specified: `sasl.oauthbearer.client.id`, `sasl.oauthbearer.client.secret`, and `sasl.oauthbearer.token.endpoint.url`.
+     *
+     * @default default
+     */
+    "sasl.oauthbearer.method"?: 'default' | 'oidc';
+
+    /**
+     * Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when `sasl.oauthbearer.method` is set to "oidc".
+     */
+    "sasl.oauthbearer.client.id"?: string;
+
+    /**
+     * Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when `sasl.oauthbearer.method` is set to "oidc".
+     */
+    "sasl.oauthbearer.client.secret"?: string;
+
+    /**
+     * Client use this to specify the scope of the access request to the broker. Only used when `sasl.oauthbearer.method` is set to "oidc".
+     */
+    "sasl.oauthbearer.scope"?: string;
+
+    /**
+     * Allow additional information to be provided to the broker. Comma-separated list of key=value pairs. E.g., "supportFeatureX=true,organizationId=sales-emea".Only used when `sasl.oauthbearer.method` is set to "oidc".
+     */
+    "sasl.oauthbearer.extensions"?: string;
+
+    /**
+     * OAuth/OIDC issuer token endpoint HTTP(S) URI used to retrieve token. Only used when `sasl.oauthbearer.method` is set to "oidc".
+     */
+    "sasl.oauthbearer.token.endpoint.url"?: string;
+
     /**
      * List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically.
      */

deps/librdkafka

@@ -18,7 +18,7 @@
 outputDir = 'librdkafka.redist'
 outputFile = outputDir + '.zip'
 dllPath = outputDir + '/runtimes/win{}-x64/native'.format(librdkafkaWinSufix)
-libPath = outputDir + '/build/native/lib/win{}/x64/win{}-x64-Release/v140'.format(librdkafkaWinSufix, librdkafkaWinSufix)
+libPath = outputDir + '/build/native/lib/win{}/x64/win{}-x64-Release/v142'.format(librdkafkaWinSufix, librdkafkaWinSufix)
 includePath = outputDir + '/build/native/include/librdkafka'
 
 # download librdkafka from nuget
@@ -63,6 +63,7 @@ def createdir(dir):
 shutil.copy2(includePath + '/rdkafkacpp.h', depsIncludeDir)
 
 shutil.copy2(dllPath + '/libcrypto-1_1-x64.dll', buildReleaseDir)
+shutil.copy2(dllPath + '/libcurl.dll', buildReleaseDir)
 shutil.copy2(dllPath + '/librdkafka.dll', buildReleaseDir)
 shutil.copy2(dllPath + '/librdkafkacpp.dll', buildReleaseDir)
 shutil.copy2(dllPath + '/libssl-1_1-x64.dll', buildReleaseDir)

deps/windows-install.py

@@ -1,4 +1,4 @@
-// ====== Generated from librdkafka 1.8.2 file src-cpp/rdkafkacpp.h ======
+// ====== Generated from librdkafka 1.9.2 file src-cpp/rdkafkacpp.h ======
 export const CODES: { ERRORS: {
   /* Internal errors to rdkafka: */
   /** Begin internal error codes (**-200**) */
@@ -20,15 +20,15 @@ export const CODES: { ERRORS: {
   /** Produced message timed out (**-192**) */
   ERR__MSG_TIMED_OUT: number,
   /** Reached the end of the topic+partition queue on
-   *  the broker. Not really an error.
-   *  This event is disabled by default,
-   *  see the `enable.partition.eof` configuration property (**-191**) */
+  *  the broker. Not really an error.
+  *  This event is disabled by default,
+  *  see the `enable.partition.eof` configuration property (**-191**) */
   ERR__PARTITION_EOF: number,
   /** Permanent: Partition does not exist in cluster (**-190**) */
   ERR__UNKNOWN_PARTITION: number,
   /** File or filesystem error (**-189**) */
   ERR__FS: number,
-   /** Permanent: Topic does not exist in cluster (**-188**) */
+  /** Permanent: Topic does not exist in cluster (**-188**) */
   ERR__UNKNOWN_TOPIC: number,
   /** All broker connections are down (**-187**) */
   ERR__ALL_BROKERS_DOWN: number,
@@ -50,9 +50,9 @@ export const CODES: { ERRORS: {
   ERR__UNKNOWN_GROUP: number,
   /** Operation in progress (**-178**) */
   ERR__IN_PROGRESS: number,
-   /** Previous operation in progress, wait for it to finish (**-177**) */
+  /** Previous operation in progress, wait for it to finish (**-177**) */
   ERR__PREV_IN_PROGRESS: number,
-   /** This operation would interfere with an existing subscription (**-176**) */
+  /** This operation would interfere with an existing subscription (**-176**) */
   ERR__EXISTING_SUBSCRIPTION: number,
   /** Assigned partitions (rebalance_cb) (**-175**) */
   ERR__ASSIGN_PARTITIONS: number,
@@ -163,15 +163,15 @@ export const CODES: { ERRORS: {
   ERR_NETWORK_EXCEPTION: number,
   /** Coordinator load in progress (**14**) */
   ERR_COORDINATOR_LOAD_IN_PROGRESS: number,
-  /** Group coordinator load in progress (**14**) */
+/** Group coordinator load in progress (**14**) */
   ERR_GROUP_LOAD_IN_PROGRESS: number,
   /** Coordinator not available (**15**) */
   ERR_COORDINATOR_NOT_AVAILABLE: number,
-  /** Group coordinator not available (**15**) */
+/** Group coordinator not available (**15**) */
   ERR_GROUP_COORDINATOR_NOT_AVAILABLE: number,
   /** Not coordinator (**16**) */
   ERR_NOT_COORDINATOR: number,
-  /** Not coordinator for group (**16**) */
+/** Not coordinator for group (**16**) */
   ERR_NOT_COORDINATOR_FOR_GROUP: number,
   /** Invalid topic (**17**) */
   ERR_TOPIC_EXCEPTION: number,

errors.d.ts

@@ -27,7 +27,7 @@ LibrdKafkaError.wrap = errorWrap;
  * @enum {number}
  * @constant
  */
-// ====== Generated from librdkafka 1.8.2 file src-cpp/rdkafkacpp.h ======
+// ====== Generated from librdkafka 1.9.2 file src-cpp/rdkafkacpp.h ======
 LibrdKafkaError.codes = {
 
   /* Internal errors to rdkafka: */
@@ -50,15 +50,15 @@ LibrdKafkaError.codes = {
   /** Produced message timed out*/
   ERR__MSG_TIMED_OUT: -192,
   /** Reached the end of the topic+partition queue on
-   *  the broker. Not really an error.
-   *  This event is disabled by default,
-   *  see the `enable.partition.eof` configuration property. */
+  *  the broker. Not really an error.
+  *  This event is disabled by default,
+  *  see the `enable.partition.eof` configuration property. */
   ERR__PARTITION_EOF: -191,
   /** Permanent: Partition does not exist in cluster. */
   ERR__UNKNOWN_PARTITION: -190,
   /** File or filesystem error */
   ERR__FS: -189,
-   /** Permanent: Topic does not exist in cluster. */
+  /** Permanent: Topic does not exist in cluster. */
   ERR__UNKNOWN_TOPIC: -188,
   /** All broker connections are down. */
   ERR__ALL_BROKERS_DOWN: -187,
@@ -80,9 +80,9 @@ LibrdKafkaError.codes = {
   ERR__UNKNOWN_GROUP: -179,
   /** Operation in progress */
   ERR__IN_PROGRESS: -178,
-   /** Previous operation in progress, wait for it to finish. */
+  /** Previous operation in progress, wait for it to finish. */
   ERR__PREV_IN_PROGRESS: -177,
-   /** This operation would interfere with an existing subscription */
+  /** This operation would interfere with an existing subscription */
   ERR__EXISTING_SUBSCRIPTION: -176,
   /** Assigned partitions (rebalance_cb) */
   ERR__ASSIGN_PARTITIONS: -175,
@@ -193,15 +193,15 @@ LibrdKafkaError.codes = {
   ERR_NETWORK_EXCEPTION: 13,
   /** Coordinator load in progress */
   ERR_COORDINATOR_LOAD_IN_PROGRESS: 14,
-  /** Group coordinator load in progress */
+/** Group coordinator load in progress */
   ERR_GROUP_LOAD_IN_PROGRESS: 14,
   /** Coordinator not available */
   ERR_COORDINATOR_NOT_AVAILABLE: 15,
-  /** Group coordinator not available */
+/** Group coordinator not available */
   ERR_GROUP_COORDINATOR_NOT_AVAILABLE: 15,
   /** Not coordinator */
   ERR_NOT_COORDINATOR: 16,
-  /** Not coordinator for group */
+/** Not coordinator for group */
   ERR_NOT_COORDINATOR_FOR_GROUP: 16,
   /** Invalid topic */
   ERR_TOPIC_EXCEPTION: 17,

lib/error.js

@@ -2,7 +2,7 @@
   "name": "node-rdkafka",
   "version": "v2.13.0",
   "description": "Node.js bindings for librdkafka",
-  "librdkafka": "1.8.2",
+  "librdkafka": "1.9.2",
   "main": "lib/index.js",
   "scripts": {
     "configure": "node-gyp configure",
@@ -46,4 +46,4 @@
   "engines": {
     "node": ">=6.0.0"
   }
-}
+}