Default to retrying platform create once

Adds new jobCreateRetries property to cluster types to allow
tooling that is not fully idempotent to skip the retry of create.
Before this change, we only retry on delete.

Author: JohnGarbutt

azimuth_caas_operator/models/v1alpha1/cluster_type.py

@@ -29,6 +29,8 @@ class ClusterTypeSpec(schema.BaseModel):
     # The timeout (in seconds) to apply to the kubernetes job resource
     # which creates, updates and deletes the cluster instances
     jobTimeout: int = pydantic.Field(default=1200)
+    # The number of retries for the kubernetes job resource
+    jobCreateRetries: int = 1
     # Option to add cloud specific details, like the image
     extraVars: schema.Dict[str, schema.Any] = pydantic.Field(default_factory=dict)
     # Option to define cluster-type specific details, like inventory

azimuth_caas_operator/tests/models/test_crds.py

@@ -61,6 +61,9 @@ def test_cluster_type_crd_json(self):
                   "jobTimeout": {
                     "type": "integer"
                   },
+                  "jobCreateRetries": {
+                    "type": "integer"
+                  },
                   "extraVars": {
                     "additionalProperties": {
                       "x-kubernetes-preserve-unknown-fields": true
@@ -301,6 +304,9 @@ def test_cluster_crd_json(self):
                       "jobTimeout": {
                         "type": "integer"
                       },
+                      "jobCreateRetries": {
+                        "type": "integer"
+                      },
                       "extraVars": {
                         "additionalProperties": {
                           "x-kubernetes-preserve-unknown-fields": true

azimuth_caas_operator/tests/utils/test_ansible_runner.py

@@ -174,27 +174,31 @@ def test_get_job_remove(self):
         },
         clear=True,
     )
-    def test_get_job_remove_with_trust_bundle(self):
+    def test_get_job_create_with_trust_bundle(self):
         cluster = cluster_crd.get_fake()
         cluster.spec.leaseName = None
         cluster_type = cluster_type_crd.get_fake()
+        cluster_type.spec.jobCreateRetries = 3
 
         job = ansible_runner.get_job(
-            cluster, cluster_type.spec, "test1-tfstate", "trust-bundle", remove=True
+            cluster,
+            cluster_type.spec,
+            "test1-tfstate",
+            "trust-bundle",
         )
 
         expected = """\
 apiVersion: batch/v1
 kind: Job
 metadata:
-  generateName: test1-remove-
+  generateName: test1-create-
   labels:
-    azimuth-caas-action: remove
+    azimuth-caas-action: create
     azimuth-caas-cluster: test1
   namespace: ns1
 spec:
   activeDeadlineSeconds: 1200
-  backoffLimit: 1
+  backoffLimit: 3
   template:
     spec:
       containers:
@@ -206,8 +210,7 @@ def test_get_job_remove_with_trust_bundle(self):
           \\nif [ -f /runner/project/requirements.yml ]; then\\n  ansible-galaxy install\\
           \\ -r /runner/project/requirements.yml\\nelif [ -f /runner/project/roles/requirements.yml\\
           \\ ]; then\\n  ansible-galaxy install -r /runner/project/roles/requirements.yml\\n\\
-          fi\\nansible-runner run /runner -j\\nopenstack application credential delete\\
-          \\ az-caas-test1 || true\\n"
+          fi\\nansible-runner run /runner -j\\n"
         env:
         - name: RUNNER_PLAYBOOK
           value: sample.yaml
@@ -322,7 +325,7 @@ def test_get_job_remove_with_trust_bundle(self):
       - emptyDir: {}
         name: ansible-home
       - configMap:
-          name: test1-remove
+          name: test1-create
         name: env
       - name: cloudcreds
         secret:
@@ -339,7 +342,6 @@ def test_get_job_remove_with_trust_bundle(self):
       - configMap:
           name: trust-bundle
         name: trust-bundle
-  ttlSecondsAfterFinished: 36000
 """  # noqa
         self.assertEqual(expected, yaml.safe_dump(job))
 

azimuth_caas_operator/utils/ansible_runner.py

@@ -259,6 +259,9 @@ def get_job(
 
     remove_app_cred = remove and (cluster.spec.leaseName is None)
 
+    # default to 1 retry for create, if not specified in the type
+    back_off_limit = 1 if remove else cluster_type_spec.jobCreateRetries
+
     # TODO(johngarbutt): need get secret keyname from somewhere
     job_yaml = f"""apiVersion: batch/v1
 kind: Job
@@ -477,7 +480,7 @@ def get_job(
         if trust_bundle_configmap_name
         else ""
     }
-  backoffLimit: {1 if remove else 0}
+  backoffLimit: {back_off_limit}
   # Set timeout so that jobs don't get stuck in configuring state if something goes wrong
   activeDeadlineSeconds: {cluster_type_spec.jobTimeout}"""  # noqa
     return yaml.safe_load(job_yaml)