[openstack/nova] Nanny for orphaned migrations

Since we should only have few migrations ongoing, the query should be quite fast.
We trigger it often as it will be a blocker to re-installing kvm hosts,
as they get decomissioned each time, but have to wait for no
migrations being associated with the compute-host.

Author: fwiesel

openstack/nova/templates/bin-configmap.yaml

@@ -17,3 +17,6 @@ data:
 {{ include (include "job_bin_path" (tuple . "nanny-db-purge" "nanny")) . | indent 4 }}
   nanny-db-soft-delete-excessive-instance-faults: |
 {{ include (include "job_bin_path" (tuple . "nanny-db-soft-delete-excessive-instance-faults" "nanny")) . | indent 4 }}
+  nanny-db-error-out-orphaned-migrations: |
+{{ include (include "job_bin_path" (tuple . "nanny-db-error-out-orphaned-migrations" "nanny")) . | indent 4 }}
+

openstack/nova/templates/bin/nanny/_nanny-db-error-out-orphaned-migrations.tpl

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "INFO: Erroring out all orphaned migrations"
+exec nova-manage db error_out_orphaned_migrations

openstack/nova/templates/nanny-db-error-out-orphaned-migrations-cronjob.yaml

@@ -0,0 +1,82 @@
+{{- if .Values.nanny.error_out_orphaned_migrations.enabled }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: nova-nanny-db-error-out-orphaned-migrations
+  labels:
+    system: openstack
+    component: nova
+    type: nanny
+spec:
+  schedule: {{ .Values.nanny.error_out_orphaned_migrations.schedule | quote }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          {{- tuple . "nanny-db-error-out-orphaned-migrations" "nanny" | include "job_metadata" | indent 10 }}
+        spec:
+          restartPolicy: OnFailure
+          podFailurePolicy:
+            rules:
+            - action: Ignore
+              onExitCodes:
+                operator: In
+                values: [1]
+          volumes:
+          - name: container-init
+            configMap:
+              name: nova-bin
+              defaultMode: 0755
+              items:
+              - key: nanny-db-error-out-orphaned-migrations
+                path: nanny-script
+          {{- include "utils.trust_bundle.volumes" . | indent 10 }}
+          - name: nova-etc
+            projected:
+              sources:
+              - configMap:
+                  name: nova-etc
+                  items:
+                  - key:  nova.conf
+                    path: nova.conf
+                  - key:  logging.ini
+                    path: logging.ini
+                  - key:  release
+                    path: release
+              - secret:
+                  name: nova-etc
+                  items:
+                  - key: api-db.conf
+                    path: nova.conf.d/api-db.conf
+          initContainers:
+          {{- tuple . (dict "service" (include "nova.helpers.database_services" .)) | include "utils.snippets.kubernetes_entrypoint_init_container" | indent 10 }}
+          containers:
+          - name: nanny
+            image: {{ tuple . "nanny" | include "container_image_nova" }}
+            imagePullPolicy: IfNotPresent
+            command:
+            - /bin/sh
+            - -c
+            - |
+              /container.init/nanny-script
+              exit_code=$?
+              {{- include "utils.script.job_finished_hook" . | nindent 14 }}
+              exit $exit_code
+            env:
+            {{- if .Values.sentry.enabled }}
+            {{- include "utils.sentry_config" . | nindent 12 }}
+            {{- end }}
+            - name: PYTHONWARNINGS
+              value: {{ .Values.python_warnings | quote }}
+            volumeMounts:
+            - mountPath: /etc/nova
+              name: nova-etc
+            - mountPath: /container.init
+              name: container-init
+            {{- include "utils.trust_bundle.volume_mount" . | indent 12 }}
+            resources:
+              requests:
+                memory: "500Mi"
+                cpu: "100m"
+{{- end }}

openstack/nova/values.yaml

@@ -1036,3 +1036,7 @@ nanny:
     schedule: "30 * * * *"
     max_faults: 10
     max_rows: 25
+  error_out_orphaned_migrations:
+    enabled: true
+    # cron schedule (minute | hour | day of month | month | day of week)
+    schedule: "7,22,37,52 * * * *"