RHOAIENG-33283: Change ConfigMaps to Secrets

Author: kryanbeane

src/codeflare_sdk/ray/rayjobs/config.py

@@ -24,6 +24,8 @@
 from kubernetes.client import (
     V1ConfigMapVolumeSource,
     V1KeyToPath,
+    V1LocalObjectReference,
+    V1SecretVolumeSource,
     V1Toleration,
     V1Volume,
     V1VolumeMount,
@@ -415,8 +417,6 @@ def _build_pod_spec(self, container: V1Container, is_head: bool) -> V1PodSpec:
 
         # Add image pull secrets if specified
         if hasattr(self, "image_pull_secrets") and self.image_pull_secrets:
-            from kubernetes.client import V1LocalObjectReference
-
             pod_spec.image_pull_secrets = [
                 V1LocalObjectReference(name=secret)
                 for secret in self.image_pull_secrets
@@ -448,12 +448,12 @@ def _build_env_vars(self) -> list:
         """Build environment variables list."""
         return [V1EnvVar(name=key, value=value) for key, value in self.envs.items()]
 
-    def add_file_volumes(self, configmap_name: str, mount_path: str = MOUNT_PATH):
+    def add_file_volumes(self, secret_name: str, mount_path: str = MOUNT_PATH):
         """
         Add file volume and mount references to cluster configuration.
 
         Args:
-            configmap_name: Name of the ConfigMap containing files
+            secret_name: Name of the Secret containing files
             mount_path: Where to mount files in containers (default: /home/ray/scripts)
         """
         # Check if file volume already exists
@@ -478,7 +478,7 @@ def add_file_volumes(self, configmap_name: str, mount_path: str = MOUNT_PATH):
 
         # Add file volume to cluster configuration
         file_volume = V1Volume(
-            name=volume_name, config_map=V1ConfigMapVolumeSource(name=configmap_name)
+            name=volume_name, secret=V1SecretVolumeSource(secret_name=secret_name)
         )
         self.volumes.append(file_volume)
 
@@ -487,36 +487,37 @@ def add_file_volumes(self, configmap_name: str, mount_path: str = MOUNT_PATH):
         self.volume_mounts.append(file_mount)
 
         logger.info(
-            f"Added file volume '{configmap_name}' to cluster config: mount_path={mount_path}"
+            f"Added file volume '{secret_name}' to cluster config: mount_path={mount_path}"
         )
 
-    def validate_configmap_size(self, files: Dict[str, str]) -> None:
+    def validate_secret_size(self, files: Dict[str, str]) -> None:
         total_size = sum(len(content.encode("utf-8")) for content in files.values())
         if total_size > 1024 * 1024:  # 1MB
             raise ValueError(
-                f"ConfigMap size exceeds 1MB limit. Total size: {total_size} bytes"
+                f"Secret size exceeds 1MB limit. Total size: {total_size} bytes"
             )
 
-    def build_file_configmap_spec(
+    def build_file_secret_spec(
         self, job_name: str, namespace: str, files: Dict[str, str]
     ) -> Dict[str, Any]:
         """
-        Build ConfigMap specification for files
+        Build Secret specification for files
 
         Args:
-            job_name: Name of the RayJob (used for ConfigMap naming)
+            job_name: Name of the RayJob (used for Secret naming)
             namespace: Kubernetes namespace
             files: Dictionary of file_name -> file_content
 
         Returns:
-            Dict: ConfigMap specification ready for Kubernetes API
+            Dict: Secret specification ready for Kubernetes API
         """
-        configmap_name = f"{job_name}-files"
+        secret_name = f"{job_name}-files"
         return {
             "apiVersion": "v1",
-            "kind": "ConfigMap",
+            "kind": "Secret",
+            "type": "Opaque",
             "metadata": {
-                "name": configmap_name,
+                "name": secret_name,
                 "namespace": namespace,
                 "labels": {
                     "ray.io/job-name": job_name,
@@ -528,19 +529,19 @@ def build_file_configmap_spec(
         }
 
     def build_file_volume_specs(
-        self, configmap_name: str, mount_path: str = MOUNT_PATH
+        self, secret_name: str, mount_path: str = MOUNT_PATH
     ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
         """
         Build volume and mount specifications for files
 
         Args:
-            configmap_name: Name of the ConfigMap containing files
+            secret_name: Name of the Secret containing files
             mount_path: Where to mount files in containers
 
         Returns:
             Tuple of (volume_spec, mount_spec) as dictionaries
         """
-        volume_spec = {"name": "ray-job-files", "configMap": {"name": configmap_name}}
+        volume_spec = {"name": "ray-job-files", "secret": {"secretName": secret_name}}
 
         mount_spec = {"name": "ray-job-files", "mountPath": mount_path}
 

src/codeflare_sdk/ray/rayjobs/rayjob.py

@@ -31,7 +31,7 @@
 from python_client.kuberay_cluster_api import RayClusterApi
 from codeflare_sdk.ray.rayjobs.config import ManagedClusterConfig
 from codeflare_sdk.ray.rayjobs.runtime_env import (
-    create_file_configmap,
+    create_file_secret,
     extract_all_local_files,
     process_runtime_env,
 )
@@ -169,10 +169,10 @@ def submit(self) -> str:
         # Extract files from entrypoint and runtime_env working_dir
         files = extract_all_local_files(self)
 
-        # Create ConfigMap for files (will be mounted to submitter pod)
-        configmap_name = None
+        # Create Secret for files (will be mounted to submitter pod)
+        secret_name = None
         if files:
-            configmap_name = f"{self.name}-files"
+            secret_name = f"{self.name}-files"
 
         rayjob_cr = self._build_rayjob_cr()
 
@@ -182,9 +182,9 @@ def submit(self) -> str:
         if result:
             logger.info(f"Successfully submitted RayJob {self.name}")
 
-            # Create ConfigMap with owner reference after RayJob exists
+            # Create Secret with owner reference after RayJob exists
             if files:
-                create_file_configmap(self, files, result)
+                create_file_secret(self, files, result)
 
             return self.name
         else:
@@ -285,10 +285,10 @@ def _build_rayjob_cr(self) -> Dict[str, Any]:
 
         # Add submitterPodTemplate if we have files to mount
         if files:
-            configmap_name = f"{self.name}-files"
+            secret_name = f"{self.name}-files"
             rayjob_cr["spec"][
                 "submitterPodTemplate"
-            ] = self._build_submitter_pod_template(files, configmap_name)
+            ] = self._build_submitter_pod_template(files, secret_name)
 
         # Configure cluster: either use existing or create new
         if self._cluster_config is not None:
@@ -311,17 +311,17 @@ def _build_rayjob_cr(self) -> Dict[str, Any]:
         return rayjob_cr
 
     def _build_submitter_pod_template(
-        self, files: Dict[str, str], configmap_name: str
+        self, files: Dict[str, str], secret_name: str
     ) -> Dict[str, Any]:
         """
-        Build submitterPodTemplate with ConfigMap volume mount for local files.
+        Build submitterPodTemplate with Secret volume mount for local files.
 
         If files contain working_dir.zip, an init container will unzip it before
         the main submitter container runs.
 
         Args:
             files: Dict of file_name -> file_content
-            configmap_name: Name of the ConfigMap containing the files
+            secret_name: Name of the Secret containing the files
 
         Returns:
             submitterPodTemplate specification
@@ -337,8 +337,8 @@ def _build_submitter_pod_template(
         ):
             image = self._cluster_config.image
 
-        # Build ConfigMap items for each file
-        config_map_items = []
+        # Build Secret items for each file
+        secret_items = []
         entrypoint_path = files.get(
             "__entrypoint_path__"
         )  # Metadata for single file case
@@ -349,9 +349,9 @@ def _build_submitter_pod_template(
 
             # For single file case, use the preserved path structure
             if entrypoint_path:
-                config_map_items.append({"key": file_name, "path": entrypoint_path})
+                secret_items.append({"key": file_name, "path": entrypoint_path})
             else:
-                config_map_items.append({"key": file_name, "path": file_name})
+                secret_items.append({"key": file_name, "path": file_name})
 
         # Check if we need to unzip working_dir
         has_working_dir_zip = "working_dir.zip" in files
@@ -378,9 +378,9 @@ def _build_submitter_pod_template(
                 "volumes": [
                     {
                         "name": "ray-job-files",
-                        "configMap": {
-                            "name": configmap_name,
-                            "items": config_map_items,
+                        "secret": {
+                            "secretName": secret_name,
+                            "items": secret_items,
                         },
                     }
                 ],

src/codeflare_sdk/ray/rayjobs/runtime_env.py

@@ -39,7 +39,7 @@ def _normalize_runtime_env(
 
 def extract_all_local_files(job: RayJob) -> Optional[Dict[str, str]]:
     """
-    Prepare local files for ConfigMap upload.
+    Prepare local files for Secret upload.
 
     - If runtime_env has local working_dir: zip entire directory into single file
     - If single entrypoint file (no working_dir): extract that file
@@ -76,7 +76,7 @@ def extract_all_local_files(job: RayJob) -> Optional[Dict[str, str]]:
         logger.info(f"Zipping local working_dir: {working_dir}")
         zip_data = _zip_directory(working_dir)
         if zip_data:
-            # Encode zip as base64 for ConfigMap storage
+            # Encode zip as base64 for Secret storage
             zip_base64 = base64.b64encode(zip_data).decode("utf-8")
             return {"working_dir.zip": zip_base64}
 
@@ -128,7 +128,7 @@ def _extract_single_entrypoint_file(job: RayJob) -> Optional[Dict[str, str]]:
     Extract single Python file from entrypoint if no working_dir specified.
 
     Returns a dict with metadata about the file path structure so we can
-    preserve it when mounting via ConfigMap.
+    preserve it when mounting via Secret.
 
     Args:
         job: RayJob instance
@@ -150,8 +150,8 @@ def _extract_single_entrypoint_file(job: RayJob) -> Optional[Dict[str, str]]:
                 with open(file_path, "r") as f:
                     content = f.read()
 
-                # Use basename as key (ConfigMap keys can't have slashes)
-                # But store the full path for later use in ConfigMap item.path
+                # Use basename as key (Secret keys can't have slashes)
+                # But store the full path for later use in Secret item.path
                 filename = os.path.basename(file_path)
                 relative_path = file_path.lstrip("./")
 
@@ -283,23 +283,23 @@ def parse_requirements_file(requirements_path: str) -> Optional[List[str]]:
         return None
 
 
-def create_configmap_from_spec(
-    job: RayJob, configmap_spec: Dict[str, Any], rayjob_result: Dict[str, Any] = None
+def create_secret_from_spec(
+    job: RayJob, secret_spec: Dict[str, Any], rayjob_result: Dict[str, Any] = None
 ) -> str:
     """
-    Create ConfigMap from specification via Kubernetes API.
+    Create Secret from specification via Kubernetes API.
 
     Args:
-        configmap_spec: ConfigMap specification dictionary
+        secret_spec: Secret specification dictionary
         rayjob_result: The result from RayJob creation containing UID
 
     Returns:
-        str: Name of the created ConfigMap
+        str: Name of the created Secret
     """
 
-    configmap_name = configmap_spec["metadata"]["name"]
+    secret_name = secret_spec["metadata"]["name"]
 
-    metadata = client.V1ObjectMeta(**configmap_spec["metadata"])
+    metadata = client.V1ObjectMeta(**secret_spec["metadata"])
 
     # Add owner reference if we have the RayJob result
     if (
@@ -308,7 +308,7 @@ def create_configmap_from_spec(
         and rayjob_result.get("metadata", {}).get("uid")
     ):
         logger.info(
-            f"Adding owner reference to ConfigMap '{configmap_name}' with RayJob UID: {rayjob_result['metadata']['uid']}"
+            f"Adding owner reference to Secret '{secret_name}' with RayJob UID: {rayjob_result['metadata']['uid']}"
         )
         metadata.owner_references = [
             client.V1OwnerReference(
@@ -322,52 +322,55 @@ def create_configmap_from_spec(
         ]
     else:
         logger.warning(
-            f"No valid RayJob result with UID found, ConfigMap '{configmap_name}' will not have owner reference. Result: {rayjob_result}"
+            f"No valid RayJob result with UID found, Secret '{secret_name}' will not have owner reference. Result: {rayjob_result}"
         )
 
-    # Convert dict spec to V1ConfigMap
-    configmap = client.V1ConfigMap(
+    # Convert dict spec to V1Secret
+    # Use stringData instead of data to avoid double base64 encoding
+    # Our zip files are already base64-encoded, so stringData will handle the final encoding
+    secret = client.V1Secret(
         metadata=metadata,
-        data=configmap_spec["data"],
+        type=secret_spec.get("type", "Opaque"),
+        string_data=secret_spec["data"],
     )
 
-    # Create ConfigMap via Kubernetes API
+    # Create Secret via Kubernetes API
     k8s_api = client.CoreV1Api(get_api_client())
     try:
-        k8s_api.create_namespaced_config_map(namespace=job.namespace, body=configmap)
+        k8s_api.create_namespaced_secret(namespace=job.namespace, body=secret)
         logger.info(
-            f"Created ConfigMap '{configmap_name}' with {len(configmap_spec['data'])} files"
+            f"Created Secret '{secret_name}' with {len(secret_spec['data'])} files"
         )
     except client.ApiException as e:
         if e.status == 409:  # Already exists
-            logger.info(f"ConfigMap '{configmap_name}' already exists, updating...")
-            k8s_api.replace_namespaced_config_map(
-                name=configmap_name, namespace=job.namespace, body=configmap
+            logger.info(f"Secret '{secret_name}' already exists, updating...")
+            k8s_api.replace_namespaced_secret(
+                name=secret_name, namespace=job.namespace, body=secret
             )
         else:
-            raise RuntimeError(f"Failed to create ConfigMap '{configmap_name}': {e}")
+            raise RuntimeError(f"Failed to create Secret '{secret_name}': {e}")
 
-    return configmap_name
+    return secret_name
 
 
-def create_file_configmap(
+def create_file_secret(
     job: RayJob, files: Dict[str, str], rayjob_result: Dict[str, Any]
 ):
     """
-    Create ConfigMap with owner reference for local files.
+    Create Secret with owner reference for local files.
     """
-    # Use a basic config builder for ConfigMap creation
+    # Use a basic config builder for Secret creation
     config_builder = ManagedClusterConfig()
 
-    # Filter out metadata keys (like __entrypoint_path__) from ConfigMap data
-    configmap_files = {k: v for k, v in files.items() if not k.startswith("__")}
+    # Filter out metadata keys (like __entrypoint_path__) from Secret data
+    secret_files = {k: v for k, v in files.items() if not k.startswith("__")}
 
-    # Validate and build ConfigMap spec
-    config_builder.validate_configmap_size(configmap_files)
-    configmap_spec = config_builder.build_file_configmap_spec(
-        job_name=job.name, namespace=job.namespace, files=configmap_files
+    # Validate and build Secret spec
+    config_builder.validate_secret_size(secret_files)
+    secret_spec = config_builder.build_file_secret_spec(
+        job_name=job.name, namespace=job.namespace, files=secret_files
     )
 
-    # Create ConfigMap with owner reference
+    # Create Secret with owner reference
     # TODO Error handling
-    create_configmap_from_spec(job, configmap_spec, rayjob_result)
+    create_secret_from_spec(job, secret_spec, rayjob_result)