fix: update job hook settings for ArgoCD for OpenStack Helm

The references used here are:
https://argo-cd.readthedocs.io/en/stable/user-guide/helm/#helm-hooks
https://argo-cd.readthedocs.io/en/stable/user-guide/sync-options/#force-sync

There are certain jobs that need to run before the API deployments and
other pods from an OpenStack Helm project can start up successfully but
OpenStack Helm puts all jobs in the post sync phase which doesn't work
in ArgoCD. It works for stock OpenStack Helm because this just orders
how helm applies the manifests. They tolerate the fact that things fail
and restart a bit before coming up. But with ArgoCD it will only apply
post sync resources once the sync resources are up successfully
resulting in things never coming up. To address this we need to move
several jobs to the sync phase to complete with the deployments.

Further the jobs that OpenStack Helm creates are expected to stay around
so that it's init container can check if they exist before starting up.
Many fields in Kubernetes Job's are immutable therefore we need to
ensure that we replace the job when things change. We also don't want
ArgoCD to perform any diff (where it will error) before doing that
replacement.

ref:PUC-1082

Author: cardoe

components/cinder/values.yaml

@@ -136,39 +136,60 @@ manifests:
   service_ingress_api: false
   deployment_backup: false
 
-# We don't want to enable OpenStack Helm's
-# helm.sh/hooks because they set them as
-# post-install,post-upgrade which in ArgoCD
-# maps to PostSync. However the deployments
-# and statefulsets in OpenStack Helm
-# depend on the jobs to complete to become
-# healthy. Which they cannot because they are in
-# the post step and not in the main step.
-# Turning this on results in the keys jobs
-# editing the annotation which deletes the item
-# and wipes our keys.
-helm3_hook: false
-
 annotations:
+  # we need to modify the annotations on OpenStack Helm
+  # jobs because they use helm.sh/hooks: post-install,post-upgrade
+  # which means they will get applied in the post phase which
+  # is after the API deployment. With standard helm this works
+  # out because it just orders how things are applied but with
+  # ArgoCD it will wait until the sync phase is successful.
+  # Unfortunately the API deployments need several jobs to occur
+  # before it will go successful like creating the keystone user,
+  # service, endpoints and syncing the DB. These jobs also have
+  # a helm.sh/hook-weight to order them which is good but by moving
+  # them to the sync phase the weight is now wrong with resources
+  # they depend on like secrets and configmaps so we need to
+  # override them to 0 because there is no way in OpenStack Helm
+  # to set annotations on deployments and daemonssets nicely.
+  # Other jobs might need to be moved as well. We do this by
+  # moving them to the sync phase. Additionally since the jobs
+  # are using fixed names and not generated names for each run
+  # ArgoCD attempts to edit them but they have immutable fields
+  # so we must force the replacement instead of attempting to diff them.
+  # Lastly the hook-delete-policy controls the finalizer which
+  # prevents the deletion of the job. In this case we're saying
+  # the old job needs to be removed before applying the new one
+  # which gets around the immutable case above.
   job:
     cinder_db_sync:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     cinder_ks_service:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     cinder_ks_user:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     cinder_ks_endpoints:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     cinder_image_repo_sync:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
     cinder_clean:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
     cinder_create_internal_tenant:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true

components/glance/values.yaml

@@ -112,39 +112,60 @@ manifests:
   secret_keystone: true
   service_ingress_api: false
 
-# We don't want to enable OpenStack Helm's
-# helm.sh/hooks because they set them as
-# post-install,post-upgrade which in ArgoCD
-# maps to PostSync. However the deployments
-# and statefulsets in OpenStack Helm
-# depend on the jobs to complete to become
-# healthy. Which they cannot because they are in
-# the post step and not in the main step.
-# Turning this on results in the keys jobs
-# editing the annotation which deletes the item
-# and wipes our keys.
-helm3_hook: false
-
 annotations:
+  # we need to modify the annotations on OpenStack Helm
+  # jobs because they use helm.sh/hooks: post-install,post-upgrade
+  # which means they will get applied in the post phase which
+  # is after the API deployment. With standard helm this works
+  # out because it just orders how things are applied but with
+  # ArgoCD it will wait until the sync phase is successful.
+  # Unfortunately the API deployments need several jobs to occur
+  # before it will go successful like creating the keystone user,
+  # service, endpoints and syncing the DB. These jobs also have
+  # a helm.sh/hook-weight to order them which is good but by moving
+  # them to the sync phase the weight is now wrong with resources
+  # they depend on like secrets and configmaps so we need to
+  # override them to 0 because there is no way in OpenStack Helm
+  # to set annotations on deployments and daemonssets nicely.
+  # Other jobs might need to be moved as well. We do this by
+  # moving them to the sync phase. Additionally since the jobs
+  # are using fixed names and not generated names for each run
+  # ArgoCD attempts to edit them but they have immutable fields
+  # so we must force the replacement instead of attempting to diff them.
+  # Lastly the hook-delete-policy controls the finalizer which
+  # prevents the deletion of the job. In this case we're saying
+  # the old job needs to be removed before applying the new one
+  # which gets around the immutable case above.
   job:
     glance_db_sync:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     glance_ks_service:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     glance_ks_user:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     glance_ks_endpoints:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     glance_metadefs_load:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
     glance_storage_init:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
     glance_bootstrap:
-      argocd.argoproj.io/hook: Sync
+      # relies on the services to be up so it can be post
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true

components/horizon/values.yaml

@@ -77,21 +77,33 @@ pod:
         # usually set on per-deployment basis.
         min_available: 0
 
-# We don't want to enable OpenStack Helm's
-# helm.sh/hooks because they set them as
-# post-install,post-upgrade which in ArgoCD
-# maps to PostSync. However the deployments
-# and statefulsets in OpenStack Helm
-# depend on the jobs to complete to become
-# healthy. Which they cannot because they are in
-# the post step and not in the main step.
-# Turning this on results in the keys jobs
-# editing the annotation which deletes the item
-# and wipes our keys.
-helm3_hook: false
-
 annotations:
+  # we need to modify the annotations on OpenStack Helm
+  # jobs because they use helm.sh/hooks: post-install,post-upgrade
+  # which means they will get applied in the post phase which
+  # is after the API deployment. With standard helm this works
+  # out because it just orders how things are applied but with
+  # ArgoCD it will wait until the sync phase is successful.
+  # Unfortunately the API deployments need several jobs to occur
+  # before it will go successful like creating the keystone user,
+  # service, endpoints and syncing the DB. These jobs also have
+  # a helm.sh/hook-weight to order them which is good but by moving
+  # them to the sync phase the weight is now wrong with resources
+  # they depend on like secrets and configmaps so we need to
+  # override them to 0 because there is no way in OpenStack Helm
+  # to set annotations on deployments and daemonssets nicely.
+  # Other jobs might need to be moved as well. We do this by
+  # moving them to the sync phase. Additionally since the jobs
+  # are using fixed names and not generated names for each run
+  # ArgoCD attempts to edit them but they have immutable fields
+  # so we must force the replacement instead of attempting to diff them.
+  # Lastly the hook-delete-policy controls the finalizer which
+  # prevents the deletion of the job. In this case we're saying
+  # the old job needs to be removed before applying the new one
+  # which gets around the immutable case above.
   job:
     horizon_db_sync:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"

components/ironic/values.yaml

@@ -220,30 +220,48 @@ pod:
         # usually set on per-deployment basis.
         min_available: 0
 
-# we don't want to enable OpenStack Helm's
-# helm.sh/hooks because they set them as
-# post-install,post-upgrade which in ArgoCD
-# maps to PostSync. However the deployments
-# and statefulsets in OpenStack Helm
-# depend on the jobs to complete to become
-# healthy. Which they cannot because they are in
-# the post step and not in the main step.
-# Turning this on results in the keys jobs
-# editing the annotation which deletes the item
-# and wipes our keys.
-helm3_hook: false
-
 annotations:
+  # we need to modify the annotations on OpenStack Helm
+  # jobs because they use helm.sh/hooks: post-install,post-upgrade
+  # which means they will get applied in the post phase which
+  # is after the API deployment. With standard helm this works
+  # out because it just orders how things are applied but with
+  # ArgoCD it will wait until the sync phase is successful.
+  # Unfortunately the API deployments need several jobs to occur
+  # before it will go successful like creating the keystone user,
+  # service, endpoints and syncing the DB. These jobs also have
+  # a helm.sh/hook-weight to order them which is good but by moving
+  # them to the sync phase the weight is now wrong with resources
+  # they depend on like secrets and configmaps so we need to
+  # override them to 0 because there is no way in OpenStack Helm
+  # to set annotations on deployments and daemonssets nicely.
+  # Other jobs might need to be moved as well. We do this by
+  # moving them to the sync phase. Additionally since the jobs
+  # are using fixed names and not generated names for each run
+  # ArgoCD attempts to edit them but they have immutable fields
+  # so we must force the replacement instead of attempting to diff them.
+  # Lastly the hook-delete-policy controls the finalizer which
+  # prevents the deletion of the job. In this case we're saying
+  # the old job needs to be removed before applying the new one
+  # which gets around the immutable case above.
   job:
     ironic_db_sync:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     ironic_ks_service:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     ironic_ks_user:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     ironic_ks_endpoints:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"

components/keystone/values.yaml

@@ -289,33 +289,51 @@ manifests:
   secret_keystone: true
   service_ingress_api: false
 
-# we don't want to enable OpenStack Helm's
-# helm.sh/hooks because they set them as
-# post-install,post-upgrade which in ArgoCD
-# maps to PostSync. However the deployments
-# and statefulsets in OpenStack Helm
-# depend on the jobs to complete to become
-# healthy. Which they cannot because they are in
-# the post step and not in the main step.
-# Turning this on results in the keys jobs
-# editing the annotation which deletes the item
-# and wipes our keys.
-helm3_hook: false
-
 annotations:
+  # we need to modify the annotations on OpenStack Helm
+  # jobs because they use helm.sh/hooks: post-install,post-upgrade
+  # which means they will get applied in the post phase which
+  # is after the API deployment. With standard helm this works
+  # out because it just orders how things are applied but with
+  # ArgoCD it will wait until the sync phase is successful.
+  # Unfortunately the API deployments need several jobs to occur
+  # before it will go successful like creating the keystone user,
+  # service, endpoints and syncing the DB. These jobs also have
+  # a helm.sh/hook-weight to order them which is good but by moving
+  # them to the sync phase the weight is now wrong with resources
+  # they depend on like secrets and configmaps so we need to
+  # override them to 0 because there is no way in OpenStack Helm
+  # to set annotations on deployments and daemonssets nicely.
+  # Other jobs might need to be moved as well. We do this by
+  # moving them to the sync phase. Additionally since the jobs
+  # are using fixed names and not generated names for each run
+  # ArgoCD attempts to edit them but they have immutable fields
+  # so we must force the replacement instead of attempting to diff them.
+  # Lastly the hook-delete-policy controls the finalizer which
+  # prevents the deletion of the job. In this case we're saying
+  # the old job needs to be removed before applying the new one
+  # which gets around the immutable case above.
   job:
-    keystone_fernet_setup:
+    keystone_db_sync:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-    keystone_db_sync:
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
+    keystone_fernet_setup:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     keystone_credential_setup:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     keystone_domain_manage:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
     keystone_bootstrap:
-      argocd.argoproj.io/hook: Sync
+      # relies on services to be up so it can remain post
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true

components/neutron/values.yaml

@@ -199,30 +199,48 @@ manifests:
   deployment_ironic_agent: true
   service_ingress_server: false
 
-# We don't want to enable OpenStack Helm's
-# helm.sh/hooks because they set them as
-# post-install,post-upgrade which in ArgoCD
-# maps to PostSync. However the deployments
-# and statefulsets in OpenStack Helm
-# depend on the jobs to complete to become
-# healthy. Which they cannot because they are in
-# the post step and not in the main step.
-# Turning this on results in the keys jobs
-# editing the annotation which deletes the item
-# and wipes our keys.
-helm3_hook: false
-
 annotations:
+  # we need to modify the annotations on OpenStack Helm
+  # jobs because they use helm.sh/hooks: post-install,post-upgrade
+  # which means they will get applied in the post phase which
+  # is after the API deployment. With standard helm this works
+  # out because it just orders how things are applied but with
+  # ArgoCD it will wait until the sync phase is successful.
+  # Unfortunately the API deployments need several jobs to occur
+  # before it will go successful like creating the keystone user,
+  # service, endpoints and syncing the DB. These jobs also have
+  # a helm.sh/hook-weight to order them which is good but by moving
+  # them to the sync phase the weight is now wrong with resources
+  # they depend on like secrets and configmaps so we need to
+  # override them to 0 because there is no way in OpenStack Helm
+  # to set annotations on deployments and daemonssets nicely.
+  # Other jobs might need to be moved as well. We do this by
+  # moving them to the sync phase. Additionally since the jobs
+  # are using fixed names and not generated names for each run
+  # ArgoCD attempts to edit them but they have immutable fields
+  # so we must force the replacement instead of attempting to diff them.
+  # Lastly the hook-delete-policy controls the finalizer which
+  # prevents the deletion of the job. In this case we're saying
+  # the old job needs to be removed before applying the new one
+  # which gets around the immutable case above.
   job:
     neutron_db_sync:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     neutron_ks_service:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     neutron_ks_user:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"
     neutron_ks_endpoints:
       argocd.argoproj.io/hook: Sync
       argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+      argocd.argoproj.io/sync-options: Force=true
+      argocd.argoproj.io/sync-wave: "0"