Add v25.11.0 (#86)

* Prep for v25.10.0

* Switch to Flex LBs and update OKE module version

* Add custom wildcard domain for Grafana

* Add support to use image URI

* Add support for credential-provider for OCIR

* update schema.yaml

* updated oke-cluster.tf

---------

Co-authored-by: Andrei Ilas <[email protected]>

Author: OguzPastirmaci

files/oke-ubuntu-cloud-init.sh

@@ -30,6 +30,38 @@ EOF
     fi
 }
 
+# Install OKE credential provider for OCIR
+download_oke_credential_provider_for_ocir() {
+    ARCH=$(uname -m)
+
+    case "$ARCH" in
+    x86_64)
+        ARCH="amd64"
+        ;;
+    aarch64 | arm64)
+        ARCH="arm64"
+        ;;
+    *)
+        return 1
+        ;;
+    
+    esac
+    
+    wget --tries=5 --waitretry=3 --retry-connrefused -O /usr/local/bin/credential-provider-oke \
+        https://github.com/oracle-devrel/oke-credential-provider-for-ocir/releases/latest/download/oke-credential-provider-for-ocir-linux-$ARCH && \
+        chmod +x /usr/local/bin/credential-provider-oke || true
+    
+    mkdir -p /etc/kubernetes/
+    wget --tries=5 --waitretry=3 --retry-connrefused -P /etc/kubernetes/ \
+        https://github.com/oracle-devrel/oke-credential-provider-for-ocir/releases/latest/download/credential-provider-config.yaml || true
+    
+    if [[ -f /usr/local/bin/credential-provider-oke && -f /etc/kubernetes/credential-provider-config.yaml ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
 # Disable nvidia-imex.service for GB200 and GB300 shapes for Dynamic Resource Allocation (DRA) compatibility
 SHAPE=$(curl -H "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/shape 2>/dev/null) || true
 if [[ -z "$SHAPE" ]]; then
@@ -43,17 +75,28 @@ elif [[ "$SHAPE" == BM.GPU.GB200* ]] || [[ "$SHAPE" == BM.GPU.GB300* ]]; then
     fi
 fi
 
+kubernetes_version="${1-}"
+setup_credential_provider="${2:-false}"
+
+if [[ "$setup_credential_provider" == "true" ]]; then
+    credential_provider_done=$(download_oke_credential_provider_for_ocir)
+else
+    credential_provider_done=1
+fi
+
 case "$ID" in
     ubuntu)
         echo "Detected Ubuntu"
         if command -v oke >/dev/null 2>&1; then
             echo "[Ubuntu] oke binary already present, running bootstrap only"
-            kubernetes_version="${1-}"
             configure_crio_defaults "$kubernetes_version"
-            oke bootstrap
+            if [[ "$credential_provider_done" -eq 0 ]]; then
+                oke bootstrap --kubelet-extra-args "--image-credential-provider-bin-dir=/usr/local/bin/ --image-credential-provider-config=/etc/kubernetes/credential-provider-config.yaml"
+            else
+                oke bootstrap
+            fi
         else
             echo "[Ubuntu] oke binary not found, installing package"
-            kubernetes_version="${1-}"
             oke_package_version="${kubernetes_version:1}"
             oke_package_repo_version="${oke_package_version:0:4}"
             oke_package_name="oci-oke-node-all-$oke_package_version"
@@ -78,26 +121,38 @@ EOF
 
             echo "[Ubuntu] Running bootstrap"
             configure_crio_defaults "$kubernetes_version"
-            oke bootstrap
+            if [[ "$credential_provider_done" -eq 0 ]]; then
+                oke bootstrap --kubelet-extra-args "--image-credential-provider-bin-dir=/usr/local/bin/ --image-credential-provider-config=/etc/kubernetes/credential-provider-config.yaml"
+            else
+                oke bootstrap
+            fi
         fi
         ;;
     ol)
         echo "Detected Oracle Linux"
         if command -v oke >/dev/null 2>&1; then
             echo "[Oracle Linux] oke binary already present, running bootstrap only"
-            kubernetes_version="${1-}"
+            
             configure_crio_defaults "$kubernetes_version"
-            oke bootstrap
+            if [[ "$credential_provider_done" -eq 0 ]]; then
+                oke bootstrap --kubelet-extra-args "--image-credential-provider-bin-dir=/usr/local/bin/ --image-credential-provider-config=/etc/kubernetes/credential-provider-config.yaml"
+            else
+                oke bootstrap
+            fi
         else
             echo "[Oracle Linux] oke binary not found, fetching init script"
             curl --fail -H "Authorization: Bearer Oracle" \
                  -L0 http://169.254.169.254/opc/v2/instance/metadata/oke_init_script \
             | base64 --decode >/var/run/oke-init.sh
 
             echo "[Oracle Linux] Running init script"
-            kubernetes_version="${1-}"
             configure_crio_defaults "$kubernetes_version"
-            bash /var/run/oke-init.sh
+            if [[ "$credential_provider_done" -eq 0 ]]; then
+                bash /var/run/oke-init.sh --kubelet-extra-args "--image-credential-provider-bin-dir=/usr/local/bin/ --image-credential-provider-config=/etc/kubernetes/credential-provider-config.yaml"
+            else
+                bash /var/run/oke-init.sh
+            fi
+            
         fi
         ;;
     *)
@@ -106,4 +161,4 @@ EOF
         ;;
 esac
 
-echo "OKE setup completed successfully."
+echo "OKE setup completed successfully."

terraform/iam.tf

@@ -21,7 +21,7 @@ locals {
   compartment_matches  = format("instance.compartment.id = '%v'", var.compartment_ocid)
   compartment_rule     = format("ANY {%v}", join(", ", [local.compartment_matches]))
 
-  rule_templates = [
+  rule_templates = compact([
     "Allow dynamic-group %v to manage cluster-node-pools in compartment id %v",
     "Allow dynamic-group %v to manage cluster-family in compartment id %v",
     "Allow dynamic-group %v to manage file-family in compartment id %v",
@@ -37,8 +37,9 @@ locals {
     "Allow dynamic-group %v to {CLUSTER_JOIN} in compartment id %v",
     "Allow dynamic-group %v to read metrics in compartment id %v",
     "Allow dynamic-group %v to use metrics in compartment id %v where target.metrics.namespace='gpu_infrastructure_health'",
-    "Allow dynamic-group %v to use metrics in compartment id %v where target.metrics.namespace='rdma_infrastructure_health'"
-  ]
+    "Allow dynamic-group %v to use metrics in compartment id %v where target.metrics.namespace='rdma_infrastructure_health'",
+    var.setup_credential_provider_for_ocir ? "Allow dynamic-group %v to read repos in compartment id %v" : ""
+  ])
 
   wris_template = [
     "request.principal.type = 'workload'",

terraform/image.tf

@@ -0,0 +1,18 @@
+# Copyright (c) 2025 Oracle Corporation and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
+
+locals {
+  unique_image_urls = distinct(compact([var.worker_ops_image_custom_uri, var.worker_cpu_image_custom_uri, var.worker_gpu_image_custom_uri, var.worker_rdma_image_custom_uri]))
+}
+
+resource "oci_core_image" "imported_image" {
+  for_each = toset(local.unique_image_urls)
+
+  compartment_id = var.compartment_ocid
+  display_name   = format("%v-%v", element(split("/", each.value), length(split("/", each.value))-1), local.state_id) 
+
+  image_source_details {
+    source_type = "objectStorageUri"
+    source_uri  = each.value
+  }
+}

terraform/oke-cluster.tf

@@ -152,7 +152,7 @@ locals {
 module "oke" {
   source  = "oracle-terraform-modules/oke/oci"
   version = "5.3.3"
-  
+
   providers = { oci.home = oci.home }
 
   region         = var.region
@@ -196,11 +196,14 @@ module "oke" {
           }
         ]
       }
+    },
+    var.install_monitoring && var.install_node_problem_detector_kube_prometheus_stack ?
+    {
       "KubernetesMetricsServer" = {
         remove_addon_resources_on_delete = true
         override_existing                = true
       }
-    },
+    } : {},
     var.install_monitoring && var.install_node_problem_detector_kube_prometheus_stack && var.preferred_kubernetes_services == "public" ?
     {
       "CertManager" = {

terraform/oke-workers.tf

@@ -7,17 +7,17 @@ locals {
     trimspace(local.ssh_public_key),
   ])
 
-  worker_ops_image_id    = coalesce(var.worker_ops_image_custom_id, "none")
+  worker_ops_image_id    = var.worker_ops_image_use_uri ? lookup(lookup(oci_core_image.imported_image, var.worker_ops_image_custom_uri, {}), "id", null) : coalesce(var.worker_ops_image_custom_id, "none")
   worker_cpu_image_type  = contains(["platform", "custom"], lower(var.worker_cpu_image_type)) ? "custom" : "oke"
-  worker_cpu_image_id    = coalesce(var.worker_cpu_image_custom_id, var.worker_cpu_image_platform_id, "none")
+  worker_cpu_image_id    = var.worker_cpu_image_use_uri ? lookup(lookup(oci_core_image.imported_image, var.worker_cpu_image_custom_uri, {}), "id", null) : coalesce(var.worker_cpu_image_custom_id, var.worker_cpu_image_platform_id, "none")
   worker_gpu_image_type  = contains(["platform", "custom"], lower(var.worker_gpu_image_type)) ? "custom" : "oke"
-  worker_gpu_image_id    = coalesce(var.worker_gpu_image_custom_id, var.worker_gpu_image_platform_id, "none")
+  worker_gpu_image_id    = var.worker_gpu_image_use_uri ? lookup(lookup(oci_core_image.imported_image, var.worker_gpu_image_use_uri, {}), "id", null) : coalesce(var.worker_gpu_image_custom_id, var.worker_gpu_image_platform_id, "none")
   worker_rdma_image_type = contains(["platform", "custom"], lower(var.worker_rdma_image_type)) ? "custom" : "oke"
-  worker_rdma_image_id   = coalesce(var.worker_rdma_image_custom_id, var.worker_rdma_image_platform_id, "none")
+  worker_rdma_image_id   = var.worker_rdma_image_use_uri ? lookup(lookup(oci_core_image.imported_image, var.worker_rdma_image_custom_uri, {}), "id", null) : coalesce(var.worker_rdma_image_custom_id, var.worker_rdma_image_platform_id, "none")
 
   runcmd_bootstrap = local.create_workers ? format(
-    "curl -sL -o /var/run/oke-ubuntu-cloud-init.sh https://raw.githubusercontent.com/oracle-quickstart/oci-hpc-oke/refs/heads/main/files/oke-ubuntu-cloud-init.sh && (bash /var/run/oke-ubuntu-cloud-init.sh '%v' '%v' || echo 'Error bootstrapping OKE' >&2)",
-    var.kubernetes_version, var.override_hostnames,
+    "curl -sL -o /var/run/oke-ubuntu-cloud-init.sh https://raw.githubusercontent.com/oracle-quickstart/oci-hpc-oke/refs/heads/main/files/oke-ubuntu-cloud-init.sh && (bash /var/run/oke-ubuntu-cloud-init.sh '%v' '%v' '%v' || echo 'Error bootstrapping OKE' >&2)",
+    var.kubernetes_version, var.setup_credential_provider_for_ocir, var.override_hostnames
   ) : ""
 
   runcmd_nvme_raid = var.nvme_raid_enabled ? format(
@@ -86,7 +86,6 @@ locals {
       boot_volume_size = var.worker_gpu_boot_volume_size
       image_type       = "custom"
       image_id         = local.worker_gpu_image_id
-      cloud_init       = [{ content_type = "text/cloud-config", content = yamlencode(local.cloud_init) }]
       node_labels      = { "oci.oraclecloud.com/disable-gpu-device-plugin" : var.disable_gpu_device_plugin ? "true" : "false" },
       cloud_init       = [{ content_type = "text/cloud-config", content = yamlencode(local.cloud_init) }]
     }