-
Notifications
You must be signed in to change notification settings - Fork 44
Description
What did you do to encounter the bug?
- deploy
mdbccustom resource with 2spec.statefulSet.spec.template.spec.topologySpreadConstraints - wait for reconciliation
- stateful set has correct number of
topologySpreadConstraints
- stateful set has correct number of
- update
mdbccustom resource to have just 1spec.statefulSet.spec.template.spec.topologySpreadConstraints
What did you expect?
After reducing number of topologySpreadConstraints on mdbc resource from 2 to 1 I would expect that statefulset will have correct number of topologySpreadConstraints - 1
What happened instead?
After reducing topologySpreadConstraints from 2 to 1 reconciliation was not triggered and statefulSet has still 2 topologySpreadConstraints
Even when reconciliation was triggered after some time due to other reason, number of topologySpreadConstraints on statefulSet was still 2 instead of 1
Operator Information
- Operator Version -
mongodb/mongodb-kubernetes:1.6.0 - MongoDB Image used -
MongoDBCommunityCRD withmongodb/mongodb-community-server:8.2.2-ubi8
Kubernetes Cluster Information
- Distribution - AKS
- Version - 1.32.9
- Image Registry location (quay, or an internal registry) - azure container registry
Additional context
% k get pods -n management -l app=management-application-mongodb-headless
NAME READY STATUS RESTARTS AGE
management-application-mongodb-0 2/2 Running 0 6h27m
management-application-mongodb-1 2/2 Running 0 4m29s
management-application-mongodb-2 2/2 Running 0 3m42s
% k get mdbc --all-namespaces
NAMESPACE NAME PHASE VERSION
management management-application-mongodb Running 8.2.2-ubi8
mdbc yaml
apiVersion: mongodbcommunity.mongodb.com/v1
kind: MongoDBCommunity
metadata:
annotations:
meta.helm.sh/release-name: example-application-mongodb-deployment
meta.helm.sh/release-namespace: example
mongodb.com/v1.lastAppliedMongoDBVersion: 8.2.2-ubi8
mongodb.com/v1.lastSuccessfulConfiguration: 'REDACTED'
creationTimestamp: "2025-12-19T10:06:10Z"
generation: 4
labels:
app.kubernetes.io/managed-by: Helm
helm.toolkit.fluxcd.io/name: example-application-mongodb-deployment
helm.toolkit.fluxcd.io/namespace: flux-resources
name: example-application-mongodb
namespace: example
resourceVersion: "REDACTED"
uid: REDACTED
spec:
featureCompatibilityVersion: "8.0"
members: 3
security:
authentication:
ignoreUnknownUsers: true
modes:
- SCRAM
statefulSet:
spec:
minReadySeconds: 30
serviceName: example-application-mongodb-headless
template:
metadata:
labels:
sidecar.istio.io/inject: "false"
sidecar.gw.example.com/inject: "false"
spec:
containers:
- livenessProbe:
exec:
command:
- /bin/sh
- -c
- HOME=/tmp mongosh --quiet --norc --eval "db.adminCommand('ping')"
failureThreshold: 3
periodSeconds: 10
timeoutSeconds: 5
name: mongod
resources:
limits:
memory: 1500Mi
requests:
cpu: 100m
memory: 1500Mi
startupProbe:
exec:
command:
- /bin/sh
- -c
- HOME=/tmp mongosh --quiet --norc --eval "db.adminCommand('ping')"
failureThreshold: 30
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
- name: mongodb-agent
readinessProbe:
failureThreshold: 60
periodSeconds: 20
timeoutSeconds: 10
resources:
limits:
memory: 130Mi
requests:
cpu: 100m
memory: 130Mi
startupProbe:
exec:
command:
- curl
- -sf
- http://localhost:5000/healthz
failureThreshold: 10
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
nodeSelector:
node.example.com/agentpool: worker
priorityClassName: platform-system
tolerations:
- effect: PreferNoSchedule
key: node.example.com/agentpool
operator: Equal
value: worker
- effect: NoSchedule
key: kubernetes.azure.com/scalesetpriority
operator: Equal
value: spot
topologySpreadConstraints:
- labelSelector:
matchLabels:
app: example-application-mongodb-headless
matchLabelKeys:
- controller-revision-hash
maxSkew: 1
nodeTaintsPolicy: Honor
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: data-volume
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10G
storageClassName: custom-retain
volumeMode: Filesystem
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: logs-volume
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2G
storageClassName: custom-retain
volumeMode: Filesystem
type: ReplicaSet
users:
- db: example-cloud
name: REDACTED
passwordSecretRef:
name: REDACTED
roles:
- db: example-cloud
name: dbOwner
scramCredentialsSecretName: example-center
- db: admin
name: REDACTED
passwordSecretRef:
name: REDACTED
roles:
- db: admin
name: backup
- db: admin
name: restore
- db: admin
name: clusterMonitor
scramCredentialsSecretName: mongodb-backup-restore
version: 8.2.2-ubi8
status:
currentMongoDBMembers: 3
currentStatefulSetReplicas: 3
mongoUri: REDACTED
phase: Running
version: 8.2.2-ubi8
stateful set created by operator
apiVersion: apps/v1
kind: StatefulSet
metadata:
creationTimestamp: "2025-12-19T10:06:10Z"
generation: 779
name: example-application-mongodb
namespace: example
ownerReferences:
- apiVersion: mongodbcommunity.mongodb.com/v1
blockOwnerDeletion: true
controller: true
kind: MongoDBCommunity
name: example-application-mongodb
uid: REDACTED
resourceVersion: "REDACTED"
uid: REDACTED
spec:
persistentVolumeClaimRetentionPolicy:
whenDeleted: Retain
whenScaled: Retain
podexamplePolicy: OrderedReady
replicas: 3
revisionHistoryLimit: 10
selector:
matchLabels:
app: example-application-mongodb-headless
serviceName: example-application-mongodb-headless
template:
metadata:
creationTimestamp: null
labels:
app: example-application-mongodb-headless
sidecar.istio.io/inject: "false"
sidecar.gw.example.com/inject: "false"
spec:
containers:
- args:
- ""
command:
- /bin/sh
- -c
- "\nif [ -e \"/hooks/version-upgrade\" ]; then\n\t#run post-start hook to
handle version changes (if exists)\n /hooks/version-upgrade\nfi\n\n#
wait for config and keyfile to be created by the agent\necho \"Waiting for
config and keyfile files to be created by the agent...\"\nwhile ! [ -f /data/automation-mongod.conf
-a -f /var/lib/mongodb-mms-automation/authentication/keyfile ]; do\n\tsleep
3;\n\techo \"Waiting...\"\ndone\n\n# sleep is important after agent issues
shutdown command\n# k8s restarts the mongod container too quickly for the
agent to realize mongod is down\necho \"Sleeping for 15s...\"\nsleep 15\n\n#
start mongod with this configuration\necho \"Starting mongod...\"\nexec
mongod -f /data/automation-mongod.conf\n"
env:
- name: AGENT_STATUS_FILEPATH
value: /healthstatus/agent-health-status.json
image: examplesaas.azurecr.io/mongodb/mongodb-community-server:8.2.2-ubi8
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- /bin/sh
- -c
- HOME=/tmp mongosh --quiet --norc --eval "db.adminCommand('ping')"
failureThreshold: 3
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
name: mongod
resources:
limits:
memory: 1500Mi
requests:
cpu: 100m
memory: 1500Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
startupProbe:
exec:
command:
- /bin/sh
- -c
- HOME=/tmp mongosh --quiet --norc --eval "db.adminCommand('ping')"
failureThreshold: 30
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /data
name: data-volume
- mountPath: /healthstatus
name: healthstatus
- mountPath: /hooks
name: hooks
- mountPath: /var/log/mongodb-mms-automation
name: logs-volume
- mountPath: /var/lib/mongodb-mms-automation/authentication
name: example-application-mongodb-keyfile
- mountPath: /tmp
name: tmp
- command:
- /bin/bash
- -c
- |-
current_uid=$(id -u)
declare -r current_uid
if ! grep -q "${current_uid}" /etc/passwd ; then
sed -e "s/^mongodb:/builder:/" /etc/passwd > /tmp/passwd
echo "mongodb:x:$(id -u):$(id -g):,,,:/:/bin/bash" >> /tmp/passwd
export NSS_WRAPPER_PASSWD=/tmp/passwd
export LD_PRELOAD=libnss_wrapper.so
export NSS_WRAPPER_GROUP=/etc/group
fi
agent/mongodb-agent -healthCheckFilePath=/var/log/mongodb-mms-automation/healthstatus/agent-health-status.json -serveStatusPort=5000 -cluster=/var/lib/automation/config/cluster-config.json -skipMongoStart -noDaemonize -useLocalMongoDbTools -logFile /var/log/mongodb-mms-automation/automation-agent.log -logLevel INFO -maxLogFileDurationHrs 24
env:
- name: AGENT_STATUS_FILEPATH
value: /var/log/mongodb-mms-automation/healthstatus/agent-health-status.json
- name: AUTOMATION_CONFIG_MAP
value: example-application-mongodb-config
- name: HEADLESS_AGENT
value: "true"
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
image: examplesaas.azurecr.io/mongodb/mongodb-agent:108.0.2.8729-1
imagePullPolicy: Always
name: mongodb-agent
readinessProbe:
exec:
command:
- /opt/scripts/readinessprobe
failureThreshold: 60
initialDelaySeconds: 5
periodSeconds: 20
successThreshold: 1
timeoutSeconds: 10
resources:
limits:
memory: 130Mi
requests:
cpu: 100m
memory: 130Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
startupProbe:
exec:
command:
- curl
- -sf
- http://localhost:5000/healthz
failureThreshold: 10
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /opt/scripts
name: agent-scripts
- mountPath: /var/lib/automation/config
name: automation-config
readOnly: true
- mountPath: /data
name: data-volume
- mountPath: /var/log/mongodb-mms-automation/healthstatus
name: healthstatus
- mountPath: /var/log/mongodb-mms-automation
name: logs-volume
- mountPath: /var/lib/mongodb-mms-automation/authentication
name: example-application-mongodb-keyfile
- mountPath: /tmp
name: tmp
dnsPolicy: ClusterFirst
initContainers:
- command:
- cp
- version-upgrade-hook
- /hooks/version-upgrade
image: examplesaas.azurecr.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.10
imagePullPolicy: Always
name: mongod-posthook
resources:
limits:
cpu: "1"
memory: 500M
requests:
cpu: 500m
memory: 400M
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /hooks
name: hooks
- command:
- cp
- /probes/readinessprobe
- /opt/scripts/readinessprobe
image: examplesaas.azurecr.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.23
imagePullPolicy: Always
name: mongodb-agent-readinessprobe
resources:
limits:
cpu: "1"
memory: 500M
requests:
cpu: 500m
memory: 400M
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /opt/scripts
name: agent-scripts
nodeSelector:
node.example.com/agentpool: worker
priorityClassName: platform-system
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
fsGroup: 2000
runAsNonRoot: true
runAsUser: 2000
serviceAccount: mongodb-kubernetes-appdb
serviceAccountName: mongodb-kubernetes-appdb
terminationGracePeriodSeconds: 30
tolerations:
- effect: PreferNoSchedule
key: node.example.com/agentpool
operator: Equal
value: worker
- effect: NoSchedule
key: kubernetes.azure.com/scalesetpriority
operator: Equal
value: spot
topologySpreadConstraints:
- labelSelector:
matchLabels:
app: example-application-mongodb-headless
matchLabelKeys:
- controller-revision-hash
maxSkew: 1
nodeTaintsPolicy: Honor
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: DoNotSchedule
- labelSelector:
matchLabels:
app: example-application-mongodb-headless
matchLabelKeys:
- controller-revision-hash
maxSkew: 1
nodeTaintsPolicy: Honor
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
volumes:
- emptyDir: {}
name: agent-scripts
- name: automation-config
secret:
defaultMode: 416
secretName: example-application-mongodb-config
- emptyDir: {}
name: healthstatus
- emptyDir: {}
name: hooks
- emptyDir: {}
name: example-application-mongodb-keyfile
- emptyDir: {}
name: tmp
updateStrategy:
type: RollingUpdate
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
name: data-volume
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10G
storageClassName: custom-retain
volumeMode: Filesystem
status:
phase: Pending
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
creationTimestamp: null
name: logs-volume
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2G
storageClassName: custom-retain
volumeMode: Filesystem
status:
phase: Pending
status:
availableReplicas: 3
collisionCount: 0
currentReplicas: 3
currentRevision: redacted-6778c449fc
observedGeneration: 779
readyReplicas: 3
replicas: 3
updateRevision: redacted-6778c449fc
updatedReplicas: 3
I can see in the operator logs that reconciliation started later due to different reason, but stateful set topologySpreadConstraints was not updated. Operator logs:
1768290174064 2026-01-13T07:42:54.064Z {"level":"info","ts":1768290174.064393,"caller":"agent/agent_readiness.go:62","msg":"All 3 Agents have reached Goal state","ReplicaSet":"example/example-application-mongodb"}
1768290174054 2026-01-13T07:42:54.054Z {"level":"info","ts":1768290174.0547287,"caller":"controllers/replica_set_controller.go:364","msg":"Creating/Updating AutomationConfig","ReplicaSet":"example/example-application-mongodb"}
1768290174054 2026-01-13T07:42:54.054Z {"level":"info","ts":1768290174.0545614,"caller":"controllers/replica_set_controller.go:472","msg":"Create/Update operation succeeded","ReplicaSet":"example/example-application-mongodb","operation":"updated"}
1768290174035 2026-01-13T07:42:54.035Z {"level":"info","ts":1768290174.0352888,"caller":"controllers/replica_set_controller.go:156","msg":"Reconciling MongoDB","ReplicaSet":"example/example-application-mongodb"}
1768290174021 2026-01-13T07:42:54.021Z {"level":"info","ts":1768290174.021209,"caller":"controllers/mongodb_status_options.go:110","msg":"ReplicaSet is not yet ready, retrying in 10 seconds"}
1768290173983 2026-01-13T07:42:53.983Z {"level":"info","ts":1768290173.983354,"caller":"controllers/replica_set_controller.go:344","msg":"Creating/Updating StatefulSet for Arbiters","ReplicaSet":"example/example-application-mongodb"}
1768290173874 2026-01-13T07:42:53.874Z {"level":"info","ts":1768290173.874106,"caller":"controllers/replica_set_controller.go:339","msg":"Creating/Updating StatefulSet","ReplicaSet":"example/example-application-mongodb"}
1768290173874 2026-01-13T07:42:53.874Z {"level":"info","ts":1768290173.8740652,"caller":"agent/agent_readiness.go:62","msg":"All 3 Agents have reached Goal state","ReplicaSet":"example/example-application-mongodb"}
1768290173853 2026-01-13T07:42:53.853Z {"level":"info","ts":1768290173.8538957,"caller":"controllers/replica_set_controller.go:364","msg":"Creating/Updating AutomationConfig","ReplicaSet":"example/example-application-mongodb"}
1768290173852 2026-01-13T07:42:53.852Z {"level":"info","ts":1768290173.852983,"caller":"controllers/replica_set_controller.go:472","msg":"Create/Update operation succeeded","ReplicaSet":"example/example-application-mongodb","operation":"updated"}
1768290173839 2026-01-13T07:42:53.839Z {"level":"info","ts":1768290173.8394437,"caller":"controllers/replica_set_controller.go:156","msg":"Reconciling MongoDB","ReplicaSet":"example/example-application-mongodb"}