Skip to content

Commit b98a9f8

Browse files
committed
fix(pulsar): Upgrade pulsar 6.5.0 and patch tokio-util
We want to upgrade to pulsar@6.5.0, but it contains an updated dependency for tokio-util which breaks vector. Vector has maintained a fork of `tokio` and patched it, so this same patch was applied to the version that pulsar 6.5.0 uses, which is `tokio-util@0.7.17`. We are testing this version of pulsar as a potential fix for the `SlowDown` error which causes partitions to no longer be consumed. We need consumers to be pulling from all available partitions. Ref: LOG-22655
1 parent b213e4e commit b98a9f8

File tree

3 files changed

+52
-28
lines changed

3 files changed

+52
-28
lines changed

Cargo.lock

Lines changed: 47 additions & 26 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@ openssl-probe = { version = "0.1.6", default-features = false }
386386
ordered-float = { version = "4.6.0", default-features = false, features = ["serde"] }
387387
percent-encoding = { version = "2.3.1", default-features = false }
388388
postgres-openssl = { version = "0.5.1", default-features = false, features = ["runtime"], optional = true }
389-
pulsar = { version = "6.3.1", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true }
389+
pulsar = { version = "6.5.0", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true }
390390
quick-junit = { version = "0.5.1" }
391391
rand.workspace = true
392392
rand_distr.workspace = true
@@ -492,7 +492,7 @@ zstd = { version = "0.13.0", default-features = false }
492492

493493
[patch.crates-io]
494494
# The upgrade for `tokio-util` >= 0.6.9 is blocked on https://github.com/vectordotdev/vector/issues/11257.
495-
tokio-util = { git = "https://github.com/vectordotdev/tokio", branch = "tokio-util-0.7.13-framed-read-continue-on-error" }
495+
tokio-util = { git = "https://github.com/mezmo/tokio", branch = "tokio-util-0.7.17-framed-read-continue-on-error", version = "0.7.17" }
496496
nix = { git = "https://github.com/vectordotdev/nix.git", branch = "memfd/gnu/musl" }
497497
# The `heim` crates depend on `ntapi` 0.3.7 on Windows, but that version has an
498498
# unaligned access bug fixed in the following revision.

src/sinks/pulsar/config.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,9 @@ impl PulsarSinkConfig {
336336
batch_size: self.batch.max_events,
337337
batch_byte_size: self.batch.max_bytes,
338338
compression: None,
339+
batch_timeout: None,
340+
block_queue_if_full: false,
341+
routing_policy: None,
339342
};
340343

341344
match &self.compression {

0 commit comments

Comments
 (0)