Merge pull request #337 from azimuth-cloud/feature/retry-create-by-default

Default to retrying platform create once

Author: sd109

azimuth_caas_operator/models/v1alpha1/cluster_type.py

@@ -29,6 +29,8 @@ class ClusterTypeSpec(schema.BaseModel):
     # The timeout (in seconds) to apply to the kubernetes job resource
     # which creates, updates and deletes the cluster instances
     jobTimeout: int = pydantic.Field(default=1200)
+    # The number of retries for the kubernetes job resource
+    jobCreateRetries: int = 1
     # Option to add cloud specific details, like the image
     extraVars: schema.Dict[str, schema.Any] = pydantic.Field(default_factory=dict)
     # Option to define cluster-type specific details, like inventory

azimuth_caas_operator/tests/models/test_crds.py

@@ -61,6 +61,9 @@ def test_cluster_type_crd_json(self):
                   "jobTimeout": {
                     "type": "integer"
                   },
+                  "jobCreateRetries": {
+                    "type": "integer"
+                  },
                   "extraVars": {
                     "additionalProperties": {
                       "x-kubernetes-preserve-unknown-fields": true
@@ -301,6 +304,9 @@ def test_cluster_crd_json(self):
                       "jobTimeout": {
                         "type": "integer"
                       },
+                      "jobCreateRetries": {
+                        "type": "integer"
+                      },
                       "extraVars": {
                         "additionalProperties": {
                           "x-kubernetes-preserve-unknown-fields": true

azimuth_caas_operator/tests/utils/test_ansible_runner.py

@@ -174,27 +174,31 @@ def test_get_job_remove(self):
         },
         clear=True,
     )
-    def test_get_job_remove_with_trust_bundle(self):
+    def test_get_job_create_with_trust_bundle(self):
         cluster = cluster_crd.get_fake()
         cluster.spec.leaseName = None
         cluster_type = cluster_type_crd.get_fake()
+        cluster_type.spec.jobCreateRetries = 3
 
         job = ansible_runner.get_job(
-            cluster, cluster_type.spec, "test1-tfstate", "trust-bundle", remove=True
+            cluster,
+            cluster_type.spec,
+            "test1-tfstate",
+            "trust-bundle",
         )
 
         expected = """\
 apiVersion: batch/v1
 kind: Job
 metadata:
-  generateName: test1-remove-
+  generateName: test1-create-
   labels:
-    azimuth-caas-action: remove
+    azimuth-caas-action: create
     azimuth-caas-cluster: test1
   namespace: ns1
 spec:
   activeDeadlineSeconds: 1200
-  backoffLimit: 1
+  backoffLimit: 3
   template:
     spec:
       containers:
@@ -206,8 +210,7 @@ def test_get_job_remove_with_trust_bundle(self):
           \\nif [ -f /runner/project/requirements.yml ]; then\\n  ansible-galaxy install\\
           \\ -r /runner/project/requirements.yml\\nelif [ -f /runner/project/roles/requirements.yml\\
           \\ ]; then\\n  ansible-galaxy install -r /runner/project/roles/requirements.yml\\n\\
-          fi\\nansible-runner run /runner -j\\nopenstack application credential delete\\
-          \\ az-caas-test1 || true\\n"
+          fi\\nansible-runner run /runner -j\\n"
         env:
         - name: RUNNER_PLAYBOOK
           value: sample.yaml
@@ -322,7 +325,7 @@ def test_get_job_remove_with_trust_bundle(self):
       - emptyDir: {}
         name: ansible-home
       - configMap:
-          name: test1-remove
+          name: test1-create
         name: env
       - name: cloudcreds
         secret:
@@ -339,7 +342,6 @@ def test_get_job_remove_with_trust_bundle(self):
       - configMap:
           name: trust-bundle
         name: trust-bundle
-  ttlSecondsAfterFinished: 36000
 """  # noqa
         self.assertEqual(expected, yaml.safe_dump(job))
 

azimuth_caas_operator/utils/ansible_runner.py

@@ -259,6 +259,9 @@ def get_job(
 
     remove_app_cred = remove and (cluster.spec.leaseName is None)
 
+    # default to 1 retry for create, if not specified in the type
+    back_off_limit = 1 if remove else cluster_type_spec.jobCreateRetries
+
     # TODO(johngarbutt): need get secret keyname from somewhere
     job_yaml = f"""apiVersion: batch/v1
 kind: Job
@@ -477,7 +480,7 @@ def get_job(
         if trust_bundle_configmap_name
         else ""
     }
-  backoffLimit: {1 if remove else 0}
+  backoffLimit: {back_off_limit}
   # Set timeout so that jobs don't get stuck in configuring state if something goes wrong
   activeDeadlineSeconds: {cluster_type_spec.jobTimeout}"""  # noqa
     return yaml.safe_load(job_yaml)