Update stack to add AMD

OguzPastirmaci · OguzPastirmaci · commit 041549f3dc47 · 2025-04-07T13:33:51.000-07:00
diff --git a/README.md b/README.md
@@ -176,6 +176,11 @@ kubectl apply -f https://raw.githubusercontent.com/oracle-quickstart/oci-hpc-oke
 kubectl apply -f https://raw.githubusercontent.com/oracle-quickstart/oci-hpc-oke/main/manifests/nccl-tests/BM.GPU.B4.8-nccl-test.yaml
 ```
 
+##### BM.GPU.MI300X.8
+```
+kubectl apply -f https://raw.githubusercontent.com/oracle-quickstart/oci-hpc-oke/main/manifests/rccl-tests/BM.GPU.MI300X.8.yaml
+```
+
 The initial pull of the container will take long. Once the master pod `nccl-allreduce-job0-mpimaster-0` starts running, you can check it logs for the NCCL test result.
 
 ```sh
diff --git a/manifests/rccl-tests/BM.GPU.MI300X.8.yaml b/manifests/rccl-tests/BM.GPU.MI300X.8.yaml
@@ -0,0 +1,111 @@
+apiVersion: batch.volcano.sh/v1alpha1
+kind: Job
+metadata:
+  annotations:
+  name: rccl-tests-job0
+spec:
+  minAvailable: 0
+  plugins:
+    ssh: []
+    svc: []
+  queue: default
+  schedulerName: volcano
+  tasks:
+  - name: mpimaster
+    policies:
+    - action: CompleteJob
+      event: TaskCompleted
+    replicas: 1
+    template:
+      metadata:
+      spec:
+        containers:
+        - command:
+          - /bin/bash
+          - -c
+          - |
+            sysctl --system
+            NUM_GPUS=8
+            NUM_HOSTS=$(sed -n '$=' /etc/volcano/mpiworker.host)
+            NP=$(($NUM_HOSTS*$NUM_GPUS))
+            mpirun --allow-run-as-root \
+            -mca plm_rsh_args "-p 2222" \
+            --bind-to numa \
+            --mca oob_tcp_if_exclude docker,lo \
+            --mca btl ^openib \
+            -x NCCL_DEBUG=VERSION \
+            -x NCCL_IB_HCA==mlx5_0,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_7,mlx5_8,mlx5_9 \
+            -x NCCL_SOCKET_IFNAME=eth0 \
+            -x NCCL_IB_TC=41 \
+            -x NCCL_IB_SL=0 \
+            -x NCCL_IB_GID_INDEX=3 \
+            -x NCCL_IB_QPS=2 \
+            -x NCCL_IB_SPLIT_DATA_ON_QPS=4 \
+            -x NCCL_ALGO=Ring \
+            -hostfile /etc/volcano/mpiworker.host \
+            -N 8 -np $NP \
+            /workspace/rccl-tests/build/all_reduce_perf -b 1G -e 16G -f 2 -g 1
+          ports:
+          - { name: mpijob-port, containerPort: 2222, protocol: TCP }
+          image: iad.ocir.io/hpc_limited_availability/oke/rccl-tests:rocm-6.2.1-ofed-5.9-0.5.6.0.127
+          imagePullPolicy: Always
+          name: mpimaster
+          resources:
+            limits:
+              ephemeral-storage: 32Gi
+            requests:
+              cpu: 8
+              ephemeral-storage: 32Gi
+              memory: 2Gi
+          securityContext:
+            privileged: true
+            capabilities:
+              add: [IPC_LOCK, SYS_PTRACE]
+          volumeMounts:
+          - { mountPath: /dev/shm, name: shm }
+          workingDir: /workspace
+        dnsPolicy: ClusterFirstWithHostNet
+        hostNetwork: true
+        restartPolicy: OnFailure
+        terminationGracePeriodSeconds: 2
+        volumes:
+        - { name: shm, emptyDir: { medium: Memory, sizeLimit: 128Gi }}
+  - minAvailable: 0
+    name: mpiworker
+    replicas: 2
+    template:
+      spec:
+        containers:
+        - command:
+          - /bin/bash
+          - -c
+          - sysctl --system; mkdir -p /var/run/sshd; /usr/sbin/sshd -D -p 2222
+          ports:
+          - { name: mpijob-port, containerPort: 2222, protocol: TCP }
+          image: iad.ocir.io/hpc_limited_availability/oke/rccl-tests:rocm-6.2.1-ofed-5.9-0.5.6.0.127
+          imagePullPolicy: Always
+          name: mpiworker
+          resources:
+            limits:
+              ephemeral-storage: 32Gi
+              amd.com/gpu: 8
+            requests:
+              cpu: 200
+              ephemeral-storage: 32Gi
+              memory: 1024Gi
+              amd.com/gpu: 8
+          securityContext:
+            privileged: true
+            capabilities:
+              add: [IPC_LOCK, SYS_PTRACE]
+          volumeMounts:
+          - { mountPath: /dev/shm, name: shm }
+          workingDir: /workspace
+        dnsPolicy: ClusterFirstWithHostNet
+        hostNetwork: true
+        restartPolicy: OnFailure
+        terminationGracePeriodSeconds: 2
+        tolerations:
+        - { key: amd.com/gpu, operator: Exists }
+        volumes:
+        - { name: shm, emptyDir: { medium: Memory, sizeLimit: 128Gi }}
diff --git a/terraform/files/grafana/dashboards/rdma.json b/terraform/files/grafana/dashboards/rdma.json
@@ -316,7 +316,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\",device=~\"mlx5_.+\"}[1m])",
+          "expr": "sum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.H100.8\", device=~\"mlx5_0|mlx5_1|mlx5_3|mlx5_4|mlx5_5|mlx5_6|mlx5_7|mlx5_8|mlx5_9|mlx5_10|mlx5_12|mlx5_13|mlx5_14|mlx5_15|mlx5_16|mlx5_17\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.H200.8\", device=~\"mlx5_0|mlx5_3|mlx5_4|mlx5_5|mlx5_6|mlx5_9|mlx5_10|mlx5_11\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.B4.8\", device=~\"mlx5_1|mlx5_2|mlx5_3|mlx5_4|mlx5_5|mlx5_6|mlx5_7|mlx5_8|mlx5_9|mlx5_10|mlx5_11|mlx5_12|mlx5_14|mlx5_15|mlx5_16|mlx5_17\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.A100-v2.8\", device=~\"mlx5_1|mlx5_2|mlx5_3|mlx5_4|mlx5_5|mlx5_6|mlx5_7|mlx5_8|mlx5_9|mlx5_10|mlx5_11|mlx5_12|mlx5_14|mlx5_15|mlx5_16|mlx5_17\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU4.8\", device=~\"mlx5_0|mlx5_1|mlx5_2|mlx5_3|mlx5_6|mlx5_7|mlx5_8|mlx5_9|mlx5_10|mlx5_11|mlx5_12|mlx5_13|mlx5_14|mlx5_15|mlx5_16|mlx5_17\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.MI300X.8\", device=~\"mlx5_0|mlx5_1|mlx5_2|mlx5_3|mlx5_6|mlx5_7|mlx5_8|mlx5_9\"}[1m]))\n",
           "interval": "",
           "intervalFactor": 1,
           "legendFormat": "{{device}}",
@@ -416,7 +416,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "irate(node_infiniband_port_data_transmitted_bytes_total{instance=~\"$node\",device=~\"mlx5_.+\"}[1m])",
+          "expr": "sum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.H100.8\", device=~\"mlx5_0|mlx5_1|mlx5_3|mlx5_4|mlx5_5|mlx5_6|mlx5_7|mlx5_8|mlx5_9|mlx5_10|mlx5_12|mlx5_13|mlx5_14|mlx5_15|mlx5_16|mlx5_17\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.H200.8\", device=~\"mlx5_0|mlx5_3|mlx5_4|mlx5_5|mlx5_6|mlx5_9|mlx5_10|mlx5_11\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.B4.8\", device=~\"mlx5_1|mlx5_2|mlx5_3|mlx5_4|mlx5_5|mlx5_6|mlx5_7|mlx5_8|mlx5_9|mlx5_10|mlx5_11|mlx5_12|mlx5_14|mlx5_15|mlx5_16|mlx5_17\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.A100-v2.8\", device=~\"mlx5_1|mlx5_2|mlx5_3|mlx5_4|mlx5_5|mlx5_6|mlx5_7|mlx5_8|mlx5_9|mlx5_10|mlx5_11|mlx5_12|mlx5_14|mlx5_15|mlx5_16|mlx5_17\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU4.8\", device=~\"mlx5_0|mlx5_1|mlx5_2|mlx5_3|mlx5_6|mlx5_7|mlx5_8|mlx5_9|mlx5_10|mlx5_11|mlx5_12|mlx5_13|mlx5_14|mlx5_15|mlx5_16|mlx5_17\"}[1m])) \nor \nsum by (device) (irate(node_infiniband_port_data_received_bytes_total{instance=~\"$node\", instance_shape=\"BM.GPU.MI300X.8\", device=~\"mlx5_0|mlx5_1|mlx5_2|mlx5_3|mlx5_6|mlx5_7|mlx5_8|mlx5_9\"}[1m]))\n",
           "interval": "",
           "intervalFactor": 1,
           "legendFormat": "{{device}}",
@@ -458,7 +458,7 @@
         },
         "definition": "",
         "includeAll": false,
-        "label": "Host",
+        "label": "Instance",
         "multi": true,
         "name": "node",
         "options": [],
@@ -478,6 +478,6 @@
   "timezone": "browser",
   "title": "RDMA Received/Transmitted bytes",
   "uid": "rdma-data",
-  "version": 13,
+  "version": 2,
   "weekStart": ""
 }
diff --git a/terraform/grafana.tf b/terraform/grafana.tf
@@ -48,7 +48,10 @@ resource "kubernetes_config_map_v1" "grafana_alerts" {
 
 
 resource "random_password" "grafana_admin_password" {
-  length           = 12
-  special          = true
+  length           = 16
+  min_lower        = 1
+  min_upper        = 1
+  min_numeric      = 1
+  min_special      = 1
   override_special = "!#$%&*()-_=+[]:?"
 }
diff --git a/terraform/kube-prometheus-stack.tf b/terraform/kube-prometheus-stack.tf
@@ -2,11 +2,11 @@
 # # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
 resource "helm_release" "prometheus" {
-  count             = var.install_node_problem_detector_kube_prometheus_stack ? 1 : 0
-  depends_on        = [
+  count = var.install_node_problem_detector_kube_prometheus_stack ? 1 : 0
+  depends_on = [
     module.oke,
     time_sleep.wait_for_lb_termination
-    ]
+  ]
   namespace         = var.monitoring_namespace
   name              = "kube-prometheus-stack"
   chart             = "kube-prometheus-stack"
@@ -26,5 +26,5 @@ resource "helm_release" "prometheus" {
 }
 
 resource "time_sleep" "wait_for_lb_termination" {
-  destroy_duration = "30s"
+  destroy_duration = "60s"
 }
diff --git a/terraform/oke-cluster.tf b/terraform/oke-cluster.tf
@@ -54,7 +54,7 @@ module "oke" {
       "NvidiaGpuPlugin" = {
         remove_addon_resources_on_delete = true
         override_existing                = true
-        configurations                   = [
+        configurations = [
           {
             key   = "isDcgmExporterDisabled"
             value = "true"
@@ -65,27 +65,26 @@ module "oke" {
     anytrue([
       var.worker_rdma_shape == "BM.GPU.MI300X.8",
       var.worker_gpu_shape == "BM.GPU.MI300X.8"
-    ]) ? {
+      ]) ? {
       "AmdGpuPlugin" = {
         remove_addon_resources_on_delete = true
         override_existing                = true
       }
     } : {}
   )
-  cni_type                    = var.cni_type == "VCN-Native Pod Networking" ? "npn" : "flannel"
-  control_plane_allowed_cidrs = flatten(tolist([var.control_plane_allowed_cidrs]))
-  control_plane_is_public     = true
-  create_bastion              = var.create_bastion
-  create_cluster              = true
-  create_iam_defined_tags     = false
-  create_iam_resources        = false
-  create_iam_tag_namespace    = false
-  create_operator             = var.create_operator
-  create_vcn                  = var.create_vcn
-  kubernetes_version          = var.kubernetes_version
-  load_balancers              = "internal"
-  lockdown_default_seclist    = true
-  # TODO input variable + schema for image selection
+  cni_type                           = var.cni_type == "VCN-Native Pod Networking" ? "npn" : "flannel"
+  control_plane_allowed_cidrs        = flatten(tolist([var.control_plane_allowed_cidrs]))
+  control_plane_is_public            = true
+  create_bastion                     = var.create_bastion
+  create_cluster                     = true
+  create_iam_defined_tags            = false
+  create_iam_resources               = false
+  create_iam_tag_namespace           = false
+  create_operator                    = var.create_operator
+  create_vcn                         = var.create_vcn
+  kubernetes_version                 = var.kubernetes_version
+  load_balancers                     = "internal"
+  lockdown_default_seclist           = true
   operator_image_type                = "platform"
   operator_image_os                  = "Canonical Ubuntu" # Ignored when bastion_image_type = "custom"
   operator_image_os_version          = "22.04"
@@ -103,7 +102,7 @@ module "oke" {
     boot_volume_size = var.operator_shape_boot
   }
   output_detail = true
-  pods_cidr     = "10.240.0.0/12" # TODO input var (but keep expanded default)
+  pods_cidr     = "10.240.0.0/12"
   # services_cidr                     = "10.96.0.0/16"
   #preferred_load_balancer           = "internal"
   ssh_public_key                    = trimspace(var.ssh_public_key)
@@ -139,7 +138,7 @@ module "oke" {
   }
   allow_rules_internal_lb = {
     "Allow TCP ingress to internal load balancers from internal VCN/DRG" = {
-      protocol = "all", port = -1, source = "10.0.0.0/8", source_type = "CIDR_BLOCK",
+      protocol = "all", port = -1, source = var.vcn_cidrs, source_type = "CIDR_BLOCK",
     }
   }
 }
diff --git a/terraform/output.tf b/terraform/output.tf
@@ -3,10 +3,7 @@
 
 # Terraform
 output "state_id" { value = module.oke.state_id }
-
-# Identity
-# output "dynamic_group_ids" { value = module.oke.dynamic_group_ids }
-# output "policy_statements" { value = module.oke.policy_statements }
+output "stack_version" { value = "v25.3.1" }
 
 # Network
 output "vcn_id" { value = module.oke.vcn_id }
@@ -35,33 +32,27 @@ output "cluster_id" { value = module.oke.cluster_id }
 output "cluster_name" { value = local.cluster_name }
 output "cluster_public_endpoint" { value = local.cluster_public_endpoint }
 output "cluster_private_endpoint" { value = local.cluster_private_endpoint }
-output "cluster_kubeconfig" { value = module.oke.cluster_kubeconfig }
 output "cluster_ca_cert" { value = base64decode(module.oke.cluster_ca_cert) }
 output "control_plane_subnet_id" { value = module.oke.control_plane_subnet_id }
 output "control_plane_subnet_cidr" { value = module.oke.control_plane_subnet_cidr }
 output "control_plane_nsg_id" { value = module.oke.control_plane_nsg_id }
 output "int_lb_subnet_id" { value = module.oke.int_lb_subnet_id }
 output "int_lb_subnet_cidr" { value = module.oke.int_lb_subnet_cidr }
+output "int_lb_nsg_id" { value = module.oke.int_lb_nsg_id }
 output "pub_lb_subnet_id" { value = module.oke.pub_lb_subnet_id }
 output "pub_lb_subnet_cidr" { value = module.oke.pub_lb_subnet_cidr }
+output "pub_lb_nsg_id" { value = module.oke.pub_lb_nsg_id }
+
 
 # Workers
 output "worker_subnet_id" { value = module.oke.worker_subnet_id }
 output "worker_nsg_id" { value = module.oke.worker_nsg_id }
+output "worker_subnet_cidr" { value = module.oke.worker_subnet_cidr }
 output "worker_ops_pool_id" { value = lookup(module.oke.worker_pool_ids, "oke-ops", null) }
 output "worker_cpu_pool_id" { value = lookup(module.oke.worker_pool_ids, "oke-cpu", null) }
 output "worker_gpu_pool_id" { value = lookup(module.oke.worker_pool_ids, "oke-gpu", null) }
 output "worker_rdma_pool_id" { value = lookup(module.oke.worker_pool_ids, "oke-rdma", null) }
 
-# Storage
-#output "fss_ad" { value = oci_file_storage_file_system.fss.0.availability_domain }
-#output "fss_filesystem_id" { value = oci_file_storage_file_system.fss.0.id }
-#output "fss_volume_name" { value = local.fss_volume_name }
-#output "fss_nsg_id" { value = local.fss_nsg_id }
-#output "fss_subnet_id" { value = local.fss_subnet_id }
-#output "fss_mount_target_id" { value = oci_file_storage_mount_target.fss.0.id }
-#output "fss_export_set_id" { value = oci_file_storage_export_set.fss.0.id }
-
 # Monitoring
 output "grafana_public_ip" {
   value = format("kubectl get svc -n %v -l app.kubernetes.io/instance=kube-prometheus-stack,app.kubernetes.io/name=grafana -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}'", var.monitoring_namespace)
diff --git a/terraform/variables.tf b/terraform/variables.tf
