Coordinator fails to start #837
Description
After upgrading Oxia from 0.14.4 to 0.15.0 in Apache Pulsar Helmchart, Oxia Coordinator fails to start successfully.
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0xcc9938]
The PR is apache/pulsar-helm-chart#641 . I have added the additional rules to the role:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update"] This is the full logs on the coordinator:
{"level":"info","time":"2025-12-15T12:45:39.49319768Z","event-type":"ADDED","k8s-config-map":"pulsar-ci-oxia-coordinator","k8s-namespace":"pulsar","time":"2025-12-15T12:45:39.493299611Z","message":"Got watch event from K8S"}
{"level":"info","time":"2025-12-15T12:45:39.494053392Z","config":{"InternalServiceAddr":"0.0.0.0:6649","AdminServiceAddr":"0.0.0.0:6651","InternalSecureServiceAddr":"","MetricsServiceAddr":"0.0.0.0:8080","MetadataProviderName":"configmap","K8SMetadataNamespace":"pulsar","K8SMetadataConfigMapName":"pulsar-ci-oxia-coordinator-status","FileMetadataPath":"data/cluster-status.json","RaftBootstrapNodes":null,"RaftAddress":"","RaftDataDir":"data/raft"},"time":"2025-12-15T12:45:39.49411706Z","message":"Starting Oxia coordinator"}
{"level":"info","time":"2025-12-15T12:45:39.494646481Z","component":"coordinator","time":"2025-12-15T12:45:39.494651681Z","message":"Waiting to become leader"}
{"level":"info","time":"2025-12-15T12:45:39.494847046Z","component":"metadata-config-map","sub-component":"k8s-client","time":"2025-12-15T12:45:39.494869679Z","message":"attempting to acquire leader lease pulsar/pulsar-ci-oxia-coordinator-status..."}
{"level":"info","time":"2025-12-15T12:45:39.500623394Z","component":"metadata-config-map","sub-component":"k8s-client","time":"2025-12-15T12:45:39.500632061Z","message":"successfully acquired lease pulsar/pulsar-ci-oxia-coordinator-status"}
{"level":"info","time":"2025-12-15T12:45:39.50222963Z","component":"metadata-config-map","identity":"pulsar-ci-oxia-coordinator-b548d97b6-ggqqm","time":"2025-12-15T12:45:39.502238015Z","message":"Started leading - lease acquired"}
{"level":"info","time":"2025-12-15T12:45:39.50225653Z","component":"coordinator","time":"2025-12-15T12:45:39.502260277Z","message":"This coordinator is now leader"}
{"level":"info","time":"2025-12-15T12:45:39.505951063Z","component":"cluster-config-resource","time":"2025-12-15T12:45:39.505967304Z","message":"Received cluster config change event"}
{"level":"info","time":"2025-12-15T12:45:39.507411416Z","component":"data-server-controller","data-server":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649","time":"2025-12-15T12:45:39.507417938Z","message":"Started data server controller"}
{"level":"info","time":"2025-12-15T12:45:39.507530138Z","component":"data-server-controller","data-server":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649","time":"2025-12-15T12:45:39.507534867Z","message":"Started data server controller"}
{"level":"info","time":"2025-12-15T12:45:39.507658087Z","component":"data-server-controller","data-server":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649","time":"2025-12-15T12:45:39.507663107Z","message":"Started data server controller"}
{"level":"info","time":"2025-12-15T12:45:39.507778723Z","clusterConfig":{"namespaces":[{"name":"default","initialShardCount":3,"replicationFactor":3,"notificationsEnabled":null},{"name":"broker","initialShardCount":3,"replicationFactor":3,"notificationsEnabled":null},{"name":"bookkeeper","initialShardCount":3,"replicationFactor":3,"notificationsEnabled":null}],"servers":[{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"serverMetadata":null,"loadBalancer":null},"component":"coordinator","time":"2025-12-15T12:45:39.508001029Z","message":"Checking cluster config"}
{"level":"info","time":"2025-12-15T12:45:39.50814076Z","component":"shard-controller","namespace":"broker","shard":3,"shard-metadata":{"status":"Unknown","term":-1,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":0,"max":1431655765}},"time":"2025-12-15T12:45:39.50818849Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.508232752Z","component":"shard-controller","namespace":"broker","shard":4,"shard-metadata":{"status":"Unknown","term":-1,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":1431655766,"max":2863311531}},"time":"2025-12-15T12:45:39.508393263Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.508549605Z","component":"shard-controller","namespace":"broker","shard":5,"shard-metadata":{"status":"Unknown","term":-1,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":2863311532,"max":4294967295}},"time":"2025-12-15T12:45:39.508649222Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.508942571Z","component":"shard-controller","namespace":"default","shard":0,"shard-metadata":{"status":"Election","term":1,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":0,"max":1431655765}},"time":"2025-12-15T12:45:39.508955395Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.508997954Z","component":"shard-controller","namespace":"default","shard":1,"shard-metadata":{"status":"Unknown","term":-1,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":1431655766,"max":2863311531}},"time":"2025-12-15T12:45:39.509085077Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.509238805Z","component":"shard-controller","namespace":"default","shard":2,"shard-metadata":{"status":"Election","term":2,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":2863311532,"max":4294967295}},"time":"2025-12-15T12:45:39.509305019Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.509505233Z","component":"shard-controller","namespace":"bookkeeper","shard":6,"shard-metadata":{"status":"Unknown","term":-1,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":0,"max":1431655765}},"time":"2025-12-15T12:45:39.509515563Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.509723973Z","component":"shard-controller","namespace":"bookkeeper","shard":7,"shard-metadata":{"status":"Unknown","term":-1,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":1431655766,"max":2863311531}},"time":"2025-12-15T12:45:39.509859667Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.510061144Z","component":"shard-controller","namespace":"bookkeeper","shard":8,"shard-metadata":{"status":"Election","term":0,"leader":null,"ensemble":[{"name":null,"public":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649"},{"name":null,"public":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648","internal":"pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649"}],"removedNodes":[],"pendingDeleteShardNodes":[],"int32HashRange":{"min":2863311532,"max":4294967295}},"time":"2025-12-15T12:45:39.510081692Z","message":"Started shard controller"}
{"level":"info","time":"2025-12-15T12:45:39.587964468Z","bindAddress":"[::]:6649","grpc-server":"coordinator","time":"2025-12-15T12:45:39.587978885Z","message":"Started Grpc server"}
{"level":"info","time":"2025-12-15T12:45:39.588139405Z","bindAddress":"[::]:6651","grpc-server":"admin","time":"2025-12-15T12:45:39.588145206Z","message":"Started Grpc server"}
{"level":"info","time":"2025-12-15T12:45:39.588247137Z","time":"2025-12-15T12:45:39.588249952Z","message":"Serving Prometheus metrics at http://localhost:8080/metrics"}
{"level":"info","time":"2025-12-15T12:45:39.60118221Z","component":"cluster-config-resource","time":"2025-12-15T12:45:39.601198561Z","message":"No cluster config changes detected"}
{"level":"info","time":"2025-12-15T12:45:39.601330538Z","component":"shard-controller","namespace":"bookkeeper","shard":8,"time":"2025-12-15T12:45:39.601339745Z","message":"Starting a new election"}
{"level":"info","time":"2025-12-15T12:45:39.602504564Z","component":"shard-controller","namespace":"broker","shard":3,"time":"2025-12-15T12:45:39.60251295Z","message":"Starting a new election"}
{"level":"info","time":"2025-12-15T12:45:39.6028706Z","component":"shard-controller","namespace":"broker","shard":4,"time":"2025-12-15T12:45:39.602878684Z","message":"Starting a new election"}
{"level":"info","time":"2025-12-15T12:45:39.602993449Z","component":"shard-controller","namespace":"broker","shard":5,"time":"2025-12-15T12:45:39.603002797Z","message":"Starting a new election"}
{"level":"info","time":"2025-12-15T12:45:39.603103495Z","component":"shard-controller","namespace":"default","shard":0,"time":"2025-12-15T12:45:39.603109917Z","message":"Starting a new election"}
{"level":"info","time":"2025-12-15T12:45:39.603220073Z","component":"shard-controller","namespace":"default","shard":1,"time":"2025-12-15T12:45:39.603227297Z","message":"Starting a new election"}
{"level":"info","time":"2025-12-15T12:45:39.603327514Z","component":"shard-controller","namespace":"default","shard":2,"time":"2025-12-15T12:45:39.603831938Z","message":"Starting a new election"}
{"level":"info","time":"2025-12-15T12:45:39.60397701Z","component":"shard-controller","namespace":"bookkeeper","shard":6,"time":"2025-12-15T12:45:39.603984974Z","message":"Starting a new election"}
{"level":"info","time":"2025-12-15T12:45:39.604153019Z","component":"shard-controller","namespace":"bookkeeper","shard":7,"time":"2025-12-15T12:45:39.604162286Z","message":"Starting a new election"}
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0xcc9938]
goroutine 72 [running]:
github.com/pingcap/errors.IsNotFound({0x0?, 0x0?})
/go/pkg/mod/github.com/pingcap/errors@v0.11.4/juju_adaptor.go:112 +0x18
github.com/oxia-db/oxia/oxiad/coordinator/metadata.(*clientImpl[...]).Upsert(0x2dce600?, {0x7ffcc39dc60d, 0x41fd25?}, {0x7ffcc39dc629?, 0x21}, 0xc00028e7e0)
/src/oxia/oxiad/coordinator/metadata/k8s_client.go:109 +0x1c5
github.com/oxia-db/oxia/oxiad/coordinator/metadata.(*metadataProviderConfigMap).Store(0xc0002cd220, 0xc00051ba88, {0xc00019c9d8, 0x4})
/src/oxia/oxiad/coordinator/metadata/metadata_configmap.go:143 +0x2d8
github.com/oxia-db/oxia/oxiad/coordinator/resource.(*status).UpdateShardMetadata.func1()
/src/oxia/oxiad/coordinator/resource/status_resource.go:152 +0x3a
github.com/cenkalti/backoff/v4.RetryNotifyWithTimer.Operation.withEmptyData.func1()
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:18 +0x13
github.com/cenkalti/backoff/v4.doRetryNotify[...](0xc0004bcf78?, {0x2da3960, 0xc00003fda0}, 0xc0004bd0e8, {0x0, 0x0?})
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:88 +0x11c
github.com/cenkalti/backoff/v4.RetryNotifyWithTimer(0x0?, {0x2da3960?, 0xc00003fda0?}, 0x526fd8?, {0x0?, 0x0?})
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:61 +0x56
github.com/cenkalti/backoff/v4.RetryNotify(...)
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:49
github.com/oxia-db/oxia/oxiad/coordinator/resource.(*status).UpdateShardMetadata(0xc0003b7040, {0xc00019ca50, 0xa}, 0x8, {0x2, 0x1, 0x0, {0xc00020d100, 0x3, 0x3}, ...})
/src/oxia/oxiad/coordinator/resource/status_resource.go:151 +0x3e9
github.com/oxia-db/oxia/oxiad/coordinator/controller.(*ShardElection).start(0xc00026f400)
/src/oxia/oxiad/coordinator/controller/shard_controller_election.go:411 +0x179
github.com/oxia-db/oxia/oxiad/coordinator/controller.(*ShardElection).Start.func1()
/src/oxia/oxiad/coordinator/controller/shard_controller_election.go:504 +0x29
github.com/cenkalti/backoff/v4.doRetryNotify[...](0xc0004bd858?, {0x7f2829499e00, 0xc00009bac0}, 0xc0004bd848, {0x0, 0x0?})
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:88 +0x155
github.com/cenkalti/backoff/v4.RetryNotifyWithData[...](...)
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:54
github.com/oxia-db/oxia/oxiad/coordinator/controller.(*ShardElection).Start(0x2d99ba0?)
/src/oxia/oxiad/coordinator/controller/shard_controller_election.go:503 +0x1fe
github.com/oxia-db/oxia/oxiad/coordinator/controller.(*shardController).onElectLeader(0xc000126b60, 0x0)
/src/oxia/oxiad/coordinator/controller/shard_controller.go:326 +0x378
github.com/oxia-db/oxia/oxiad/coordinator/controller.(*shardController).run(0xc000126b60, 0xc0001f2f50)
/src/oxia/oxiad/coordinator/controller/shard_controller.go:196 +0x15f
github.com/oxia-db/oxia/oxiad/coordinator/controller.NewShardController.func2.1()
/src/oxia/oxiad/coordinator/controller/shard_controller.go:174 +0x1b
github.com/oxia-db/oxia/common/process.DoWithLabels.func1({0x2dc2468?, 0xc00002ecc0?})
/src/oxia/common/process/pprof.go:46 +0x13
runtime/pprof.Do({0x2dc24a0?, 0xc0002e6960?}, {{0xc00003e4e0?, 0x2e06e40?, 0x64?}}, 0xc000531dd8)
/usr/local/go/src/runtime/pprof/runtime.go:51 +0x8c
github.com/oxia-db/oxia/common/process.DoWithLabels({0x2dc24a0, 0xc0002e6960}, 0xc000531f38?, 0xc000531f68)
/src/oxia/common/process/pprof.go:42 +0x295
github.com/oxia-db/oxia/oxiad/coordinator/controller.NewShardController.func2()
/src/oxia/oxiad/coordinator/controller/shard_controller.go:167 +0x257
sync.(*WaitGroup).Go.func1()
/usr/local/go/src/sync/waitgroup.go:239 +0x4a
created by sync.(*WaitGroup).Go in goroutine 1
/usr/local/go/src/sync/waitgroup.go:237 +0x73
configmap/pulsar-ci-oxia-coordinator:
apiVersion: v1
data:
config.yaml: |
namespaces:
- name: default
initialShardCount: 3
replicationFactor: 3
- name: broker
initialShardCount: 3
replicationFactor: 3
- name: bookkeeper
initialShardCount: 3
replicationFactor: 3
servers:
- public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
kind: ConfigMap
metadata:
annotations:
meta.helm.sh/release-name: pulsar-ci
meta.helm.sh/release-namespace: pulsar
creationTimestamp: "2025-12-15T15:38:18Z"
labels:
app: pulsar
app.kubernetes.io/managed-by: Helm
chart: pulsar-4.4.0
cluster: pulsar-ci
component: oxia-coordinator
heritage: Helm
release: pulsar-ci
name: pulsar-ci-oxia-coordinator
namespace: pulsar
resourceVersion: "981"
uid: 2d3d33cd-43ab-40b7-be11-ece54a7e4aabconfigmap/pulsar-ci-oxia-coordinator-status
apiVersion: v1
data:
status: |
namespaces:
bookkeeper:
replicationFactor: 3
shards:
6:
status: 2
term: 0
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 0
max: 1431655765
7:
status: 0
term: -1
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 1431655766
max: 2863311531
8:
status: 2
term: 0
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 2863311532
max: 4294967295
broker:
replicationFactor: 3
shards:
3:
status: 0
term: -1
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 0
max: 1431655765
4:
status: 0
term: -1
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 1431655766
max: 2863311531
5:
status: 2
term: 0
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 2863311532
max: 4294967295
default:
replicationFactor: 3
shards:
0:
status: 2
term: 1
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 0
max: 1431655765
1:
status: 2
term: 0
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 1431655766
max: 2863311531
2:
status: 2
term: 4
leader: null
ensemble:
- name: null
public: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-0.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-1.pulsar-ci-oxia-svc:6649
- name: null
public: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc.pulsar.svc.cluster.local:6648
internal: pulsar-ci-oxia-server-2.pulsar-ci-oxia-svc:6649
removedNodes: []
pendingDeleteShardNodes: []
int32HashRange:
min: 2863311532
max: 4294967295
shardIdGenerator: 9
serverIdx: 0
kind: ConfigMap
metadata:
creationTimestamp: "2025-12-15T15:38:56Z"
name: pulsar-ci-oxia-coordinator-status
namespace: pulsar
resourceVersion: "2682"
uid: e2d149bd-bb52-4196-9291-b7bd6fb74421