fix: Work-around the 409 conflict errors

Author: Alan Christie

operator/handlers.py

@@ -142,6 +142,55 @@ def create(spec: Dict[str, Any], name: str, namespace: str, **_: Any) -> Dict[st
     logging.info("Starting create (name=%s namespace=%s)...", name, namespace)
     logging.info("spec=%s (name=%s)", spec, name)
 
+    # All Data-Manager provided material
+    # will be namespaced under the 'imDataManager' property
+    material: Dict[str, Any] = spec.get("imDataManager", {})
+
+    notebook_interface = material.get("notebook", {}).get("interface", "lab")
+
+    image = material.get("image", _DEFAULT_IMAGE)
+    image_parts = image.split(":")
+    image_tag = "latest" if len(image_parts) == 1 else image_parts[1]
+    image_pull_policy = (
+        "Always" if image_tag.lower() in ["latest", "stable"] else "IfNotPresent"
+    )
+
+    service_account = material.get("serviceAccountName", _DEFAULT_SA)
+
+    resources = material.get("resources", {})
+    cpu_limit = resources.get("limits", {}).get("cpu", _DEFAULT_CPU_LIMIT)
+    cpu_request = resources.get("requests", {}).get("cpu", _DEFAULT_CPU_REQUEST)
+    memory_limit = resources.get("limits", {}).get("memory", _DEFAULT_MEM_LIMIT)
+    memory_request = resources.get("requests", {}).get("memory", _DEFAULT_MEM_REQUEST)
+
+    # Data Manager API compliance.
+    #
+    # The user and group IDs we're asked to run as.
+    # The files in the container project volume will be owned
+    # by this user and group. We must run as group 100.
+    # We use the supplied group ID and pass that into the container
+    # as the Kubernetes 'File System Group' (fsGroup).
+    # This should allow us to run and manipulate the files.
+    sc_run_as_user = material.get("securityContext", {}).get(
+        "runAsUser", _DEFAULT_USER_ID
+    )
+    sc_run_as_group = material.get("securityContext", {}).get(
+        "runAsGroup", _DEFAULT_GROUP_ID
+    )
+
+    # Project storage
+    project_claim_name = material.get("project", {}).get("claimName")
+    project_id = material.get("project", {}).get("id")
+
+    ingress_proxy_body_size = material.get(
+        "ingressProxyBodySize", _DEFAULT_INGRESS_PROXY_BODY_SIZE
+    )
+
+    ingress_class = material.get("ingressClass", _DEFAULT_INGRESS_CLASS)
+    ingress_domain = material.get("ingressDomain", _DEFAULT_INGRESS_DOMAIN)
+    ingress_tls_secret = material.get("ingressTlsSecret", _DEFAULT_INGRESS_TLS_SECRET)
+    ingress_path = f"/{name}"
+
     # ConfigMaps
     # ----------
 
@@ -169,11 +218,43 @@ def create(spec: Dict[str, Any], name: str, namespace: str, **_: Any) -> Dict[st
         "data": {"jupyter_notebook_config.json": _NOTEBOOK_CONFIG % config_vars},
     }
 
+    create_response = {
+        "notebook": {
+            "url": f"http://{ingress_domain}{ingress_path}?token={token}",
+            "token": token,
+            "interface": notebook_interface,
+        },
+        "image": image,
+        "serviceAccountName": service_account,
+        "resources": {
+            "requests": {"memory": memory_request},
+            "limits": {"memory": memory_limit},
+        },
+        "project": {"claimName": project_claim_name, "id": project_id},
+    }
+
     kopf.adopt(bp_cm_body)
     kopf.adopt(startup_cm_body)
     kopf.adopt(config_cm_body)
     core_api = kubernetes.client.CoreV1Api()
-    core_api.create_namespaced_config_map(namespace, bp_cm_body)
+
+    # We currently see a number of 409 exceptions with the objects we create
+    # (with the reason 'Conflict'). As each one has a unique name
+    # we have to assume there is a serious underlying problem
+    # in kopf of kubernetes. For now, if the first object we create
+    # already exists let us assume they all do?
+    #
+    # Added as a work-wround for sc-
+    try:
+        core_api.create_namespaced_config_map(namespace, bp_cm_body)
+    except kubernetes.client.exceptions.ApiException as ex:
+        if ex.status != 409 or ex.reason != "Conflict":
+            raise ex
+        # Warn, but ignore and return a valid 'create' response now.
+        logging.warning(
+            "Got ApiException [409/Conflict] creating BP ConfigMap. Ignoring [#10]"
+        )
+        return create_response
     core_api.create_namespaced_config_map(namespace, startup_cm_body)
     core_api.create_namespaced_config_map(namespace, config_cm_body)
 
@@ -184,46 +265,6 @@ def create(spec: Dict[str, Any], name: str, namespace: str, **_: Any) -> Dict[st
 
     logging.info("Creating Deployment %s...", name)
 
-    # All Data-Manager provided material
-    # will be namespaced under the 'imDataManager' property
-    material: Dict[str, Any] = spec.get("imDataManager", {})
-
-    notebook_interface = material.get("notebook", {}).get("interface", "lab")
-
-    image = material.get("image", _DEFAULT_IMAGE)
-    image_parts = image.split(":")
-    image_tag = "latest" if len(image_parts) == 1 else image_parts[1]
-    image_pull_policy = (
-        "Always" if image_tag.lower() in ["latest", "stable"] else "IfNotPresent"
-    )
-
-    service_account = material.get("serviceAccountName", _DEFAULT_SA)
-
-    resources = material.get("resources", {})
-    cpu_limit = resources.get("limits", {}).get("cpu", _DEFAULT_CPU_LIMIT)
-    cpu_request = resources.get("requests", {}).get("cpu", _DEFAULT_CPU_REQUEST)
-    memory_limit = resources.get("limits", {}).get("memory", _DEFAULT_MEM_LIMIT)
-    memory_request = resources.get("requests", {}).get("memory", _DEFAULT_MEM_REQUEST)
-
-    # Data Manager API compliance.
-    #
-    # The user and group IDs we're asked to run as.
-    # The files in the container project volume will be owned
-    # by this user and group. We must run as group 100.
-    # We use the supplied group ID and pass that into the container
-    # as the Kubernetes 'File System Group' (fsGroup).
-    # This should allow us to run and manipulate the files.
-    sc_run_as_user = material.get("securityContext", {}).get(
-        "runAsUser", _DEFAULT_USER_ID
-    )
-    sc_run_as_group = material.get("securityContext", {}).get(
-        "runAsGroup", _DEFAULT_GROUP_ID
-    )
-
-    # Project storage
-    project_claim_name = material.get("project", {}).get("claimName")
-    project_id = material.get("project", {}).get("id")
-
     # Command is simply our custom start script,
     # which is mounted at /usr/local/bin
     command_items = ["bash", "/usr/local/bin/start.sh"]
@@ -368,15 +409,6 @@ def create(spec: Dict[str, Any], name: str, namespace: str, **_: Any) -> Dict[st
 
     logging.info("Creating Ingress %s...", name)
 
-    ingress_proxy_body_size = material.get(
-        "ingressProxyBodySize", _DEFAULT_INGRESS_PROXY_BODY_SIZE
-    )
-
-    ingress_class = material.get("ingressClass", _DEFAULT_INGRESS_CLASS)
-    ingress_domain = material.get("ingressDomain", _DEFAULT_INGRESS_DOMAIN)
-    ingress_tls_secret = material.get("ingressTlsSecret", _DEFAULT_INGRESS_TLS_SECRET)
-    ingress_path = f"/{name}"
-
     ingress_body: Dict[Any, Any] = {
         "kind": "Ingress",
         "apiVersion": "networking.k8s.io/v1",
@@ -424,17 +456,4 @@ def create(spec: Dict[str, Any], name: str, namespace: str, **_: Any) -> Dict[st
     # Done
     # ----
 
-    return {
-        "notebook": {
-            "url": f"http://{ingress_domain}{ingress_path}?token={token}",
-            "token": token,
-            "interface": notebook_interface,
-        },
-        "image": image,
-        "serviceAccountName": service_account,
-        "resources": {
-            "requests": {"memory": memory_request},
-            "limits": {"memory": memory_limit},
-        },
-        "project": {"claimName": project_claim_name, "id": project_id},
-    }
+    return create_response