Skip to content

Commit 9f61859

Browse files
lhotarilhotari
authored
Use PIP-45 metadata store config to replace deprecated ZK config and make PulsarMetadataBookieDriver configurable in BK (#576)
1 parent a55b1bb commit 9f61859

File tree

6 files changed

+94
-6
lines changed

6 files changed

+94
-6
lines changed

‎.ci/chart_test.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
6262
install_type="upgrade"
6363
echo "Wait 10 seconds"
6464
sleep 10
65+
# check pulsar environment
66+
ci::check_pulsar_environment
6567
# test that we can access the admin api
6668
ci::test_pulsar_admin_api_access
6769
# produce messages with old version of pulsar and consume with new version

‎.ci/configure_ci_runner_for_debugging.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ function k9s() {
2727
# install k9s on the fly
2828
if [ ! -x /usr/local/bin/k9s ]; then
2929
echo "Installing k9s..."
30-
curl -L -s https://github.com/derailed/k9s/releases/download/v0.32.5/k9s_Linux_amd64.tar.gz | sudo tar xz -C /usr/local/bin k9s
30+
curl -L -s https://github.com/derailed/k9s/releases/download/v0.40.5/k9s_Linux_amd64.tar.gz | sudo tar xz -C /usr/local/bin k9s
3131
fi
3232
command k9s "$@"
3333
}

‎.ci/values-common.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,12 @@ bookkeeper:
5555
diskUsageWarnThreshold: "0.999"
5656
PULSAR_PREFIX_diskUsageThreshold: "0.999"
5757
PULSAR_PREFIX_diskUsageWarnThreshold: "0.999"
58+
# minimal memory use for bookkeeper
59+
# https://bookkeeper.apache.org/docs/reference/config#db-ledger-storage-settings
60+
dbStorage_writeCacheMaxSizeMb: "32"
61+
dbStorage_readAheadCacheMaxSizeMb: "32"
62+
dbStorage_rocksDB_writeBufferSizeMB: "8"
63+
dbStorage_rocksDB_blockCacheSize: "8388608"
5864

5965
broker:
6066
replicaCount: 1

‎charts/pulsar/templates/_bookkeeper.tpl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,22 @@ Define bookie common config
9898
*/}}
9999
{{- define "pulsar.bookkeeper.config.common" -}}
100100
{{- if .Values.components.zookeeper }}
101+
{{- if (and (hasKey .Values.pulsar_metadata "bookkeeper") .Values.pulsar_metadata.bookkeeper.usePulsarMetadataBookieDriver) }}
102+
# there's a bug when using PulsarMetadataBookieDriver since it always appends /ledgers to the metadataServiceUri
103+
# Possibly a bug in org.apache.pulsar.metadata.bookkeeper.AbstractMetadataDriver#resolveLedgersRootPath in Pulsar code base
104+
metadataServiceUri: "metadata-store:zk:{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
105+
{{- else }}
101106
zkServers: "{{ template "pulsar.zookeeper.connect" . }}"
102107
zkLedgersRootPath: "{{ .Values.metadataPrefix }}/ledgers"
108+
{{- end }}
103109
{{- else if .Values.components.oxia }}
104110
metadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
105111
{{- end }}
112+
{{- /* metadataStoreSessionTimeoutMillis maps to zkTimeout in bookkeeper.conf for both zookeeper and oxia metadata stores */}}
113+
{{- if (and (hasKey .Values.pulsar_metadata "bookkeeper") (hasKey .Values.pulsar_metadata.bookkeeper "metadataStoreSessionTimeoutMillis")) }}
114+
zkTimeout: "{{ .Values.pulsar_metadata.bookkeeper.metadataStoreSessionTimeoutMillis }}"
115+
{{- end }}
116+
106117
# enable bookkeeper http server
107118
httpServerEnabled: "true"
108119
httpServerPort: "{{ .Values.bookkeeper.ports.http }}"

‎charts/pulsar/templates/broker-configmap.yaml‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,19 @@ metadata:
2929
data:
3030
# Metadata settings
3131
{{- if .Values.components.zookeeper }}
32-
zookeeperServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
32+
metadataStoreUrl: "zk:{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
33+
{{- $configMetadataStoreUrl := "" }}
3334
{{- if .Values.pulsar_metadata.configurationStore }}
34-
configurationStoreServers: "{{ template "pulsar.configurationStore.connect" . }}{{ .Values.pulsar_metadata.configurationStoreMetadataPrefix }}"
35+
{{- $configMetadataStoreUrl = printf "zk:%s%s" (include "pulsar.configurationStore.connect" .) .Values.pulsar_metadata.configurationStoreMetadataPrefix }}
36+
{{- else }}
37+
{{- $configMetadataStoreUrl = printf "zk:%s%s" (include "pulsar.zookeeper.connect" .) .Values.metadataPrefix }}
3538
{{- end }}
36-
{{- if not .Values.pulsar_metadata.configurationStore }}
37-
configurationStoreServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
39+
configurationMetadataStoreUrl: "{{ $configMetadataStoreUrl }}"
40+
# setting bookkeeperMetadataServiceUri causes a NPE in WorkerUtils.initializeDlogNamespace which is a bug in Pulsar
41+
# omit setting bookkeeperMetadataServiceUri until the bug is fixed when functions are enabled.
42+
# bookkeeperMetadataServiceUri will default to configurationMetadataStoreUrl + "/ledgers" in that case
43+
{{- if not .Values.components.functions }}
44+
bookkeeperMetadataServiceUri: "metadata-store:{{ $configMetadataStoreUrl }}/ledgers"
3845
{{- end }}
3946
{{- end }}
4047
{{- if .Values.components.oxia }}
@@ -43,11 +50,35 @@ data:
4350
bookkeeperMetadataServiceUri: "{{ template "pulsar.oxia.metadata.url.bookkeeper" . }}"
4451
{{- end }}
4552

53+
{{- if hasKey .Values.pulsar_metadata "metadataStoreAllowReadOnlyOperations" }}
54+
PULSAR_PREFIX_metadataStoreAllowReadOnlyOperations: "{{ .Values.pulsar_metadata.metadataStoreAllowReadOnlyOperations }}"
55+
{{- end }}
56+
{{- if hasKey .Values.pulsar_metadata "metadataStoreSessionTimeoutMillis" }}
57+
metadataStoreSessionTimeoutMillis: "{{ .Values.pulsar_metadata.metadataStoreSessionTimeoutMillis }}"
58+
{{- end }}
59+
{{- if hasKey .Values.pulsar_metadata "metadataStoreOperationTimeoutSeconds" }}
60+
metadataStoreOperationTimeoutSeconds: "{{ .Values.pulsar_metadata.metadataStoreOperationTimeoutSeconds }}"
61+
{{- end }}
62+
{{- if hasKey .Values.pulsar_metadata "metadataStoreCacheExpirySeconds" }}
63+
metadataStoreCacheExpirySeconds: "{{ .Values.pulsar_metadata.metadataStoreCacheExpirySeconds }}"
64+
{{- end }}
65+
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingEnabled" }}
66+
metadataStoreBatchingEnabled: "{{ .Values.pulsar_metadata.metadataStoreBatchingEnabled }}"
67+
{{- end }}
68+
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxDelayMillis" }}
69+
metadataStoreBatchingMaxDelayMillis: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxDelayMillis }}"
70+
{{- end }}
71+
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxOperations" }}
72+
metadataStoreBatchingMaxOperations: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxOperations }}"
73+
{{- end }}
74+
{{- if hasKey .Values.pulsar_metadata "metadataStoreBatchingMaxSizeKb" }}
75+
metadataStoreBatchingMaxSizeKb: "{{ .Values.pulsar_metadata.metadataStoreBatchingMaxSizeKb }}"
76+
{{- end }}
77+
4678
# Broker settings
4779
clusterName: {{ template "pulsar.cluster.name" . }}
4880
exposeTopicLevelMetricsInPrometheus: "true"
4981
numHttpServerThreads: "8"
50-
zooKeeperSessionTimeoutMillis: "30000"
5182
statusFilePath: "{{ template "pulsar.home" . }}/logs/status"
5283

5384
# Tiered storage settings

‎charts/pulsar/values.yaml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,44 @@ pulsar_metadata:
880880
## Timeout for running metadata initialization
881881
initTimeout: 60
882882

883+
## Allow read-only operations on the metadata store when the metadata store is not available.
884+
## This is useful when you want to continue serving requests even if the metadata store is not fully available with quorum.
885+
metadataStoreAllowReadOnlyOperations: false
886+
887+
## The session timeout for the metadata store in milliseconds.
888+
metadataStoreSessionTimeoutMillis: 30000
889+
890+
## Metadata store operation timeout in seconds.
891+
metadataStoreOperationTimeoutSeconds: 30
892+
893+
## The expiry time for the metadata store cache in seconds.
894+
metadataStoreCacheExpirySeconds: 300
895+
896+
## Whether we should enable metadata operations batching
897+
metadataStoreBatchingEnabled: true
898+
899+
## Maximum delay to impose on batching grouping (in milliseconds)
900+
metadataStoreBatchingMaxDelayMillis: 5
901+
902+
## Maximum number of operations to include in a singular batch
903+
metadataStoreBatchingMaxOperations: 1000
904+
905+
## Maximum size of a batch (in KB)
906+
metadataStoreBatchingMaxSizeKb: 128
907+
908+
## BookKeeper metadata configuration settings with Pulsar Helm Chart deployments
909+
bookkeeper:
910+
## Controls whether to use the PIP-45 metadata driver (PulsarMetadataBookieDriver) for BookKeeper components
911+
## when using ZooKeeper as a metadata store.
912+
## This is a global setting that applies to all BookKeeper components.
913+
## When set to true, BookKeeper components will use the PIP-45 metadata driver (PulsarMetadataBookieDriver).
914+
## When set to false, BookKeeper components will use BookKeeper's default ZooKeeper connection implementation.
915+
usePulsarMetadataBookieDriver: false
916+
917+
## The session timeout for the metadata store in milliseconds. This setting is mapped to `zkTimeout` in `bookkeeper.conf`.
918+
## due to implementation details in the PulsarMetadataBookieDriver, it also applies when Oxia metadata store is enabled.
919+
metadataStoreSessionTimeoutMillis: 30000
920+
883921
# resources for bin/pulsar initialize-cluster-metadata
884922
resources:
885923
# requests:

0 commit comments

Comments
 (0)