Merge pull request #133 from oci-hpc/2.10.3_more_counters

Fix For RDMA HW Counters

Author: arnaudfroidmont

bastion.tf

@@ -486,18 +486,34 @@ data "oci_objectstorage_namespace" "compartment_namespace" {
     compartment_id = var.targetCompartment
 }
 
+locals {
+  rdma_nic_metric_bucket_name = "RDMA_NIC_metrics"
+  par_path = ".."
+}
+/*
+saving the PAR into file: ../PAR_file_for_metrics.
+this PAR is used by the scripts to upload NIC metrics to object storage (i.e. script: upload_rdma_nic_metrics.sh)
+*/
+
+data "oci_objectstorage_bucket" "RDMA_NIC_Metrics_bucket_check" {
+  name           = local.rdma_nic_metric_bucket_name
+  namespace      = data.oci_objectstorage_namespace.compartment_namespace.namespace
+}
+
+
 resource "oci_objectstorage_bucket" "RDMA_NIC_metrics_bucket" {
-  count = var.bastion_object_storage_par ? 1 : 0
+  count = (var.bastion_object_storage_par && data.oci_objectstorage_bucket.RDMA_NIC_Metrics_bucket_check.bucket_id == null) ? 1 : 0
   compartment_id = var.targetCompartment
-  name           = "RDMA_NIC_metrics"
+  name           = local.rdma_nic_metric_bucket_name
   namespace      = data.oci_objectstorage_namespace.compartment_namespace.namespace
+  versioning     = "Enabled"
 }
 
 resource "oci_objectstorage_preauthrequest" "RDMA_NIC_metrics_par" {
-  count = var.bastion_object_storage_par ? 1 : 0
+  count = (var.bastion_object_storage_par && data.oci_objectstorage_bucket.RDMA_NIC_Metrics_bucket_check.bucket_id == null) ? 1 : 0
   depends_on  = [oci_objectstorage_bucket.RDMA_NIC_metrics_bucket]
   access_type = "AnyObjectWrite"
-  bucket      = oci_objectstorage_bucket.RDMA_NIC_metrics_bucket[0].name
+  bucket      = local.rdma_nic_metric_bucket_name
   name         = format("%s-%s", "RDMA_NIC_metrics_bucket", var.tenancy_ocid)
   namespace    = data.oci_objectstorage_namespace.compartment_namespace.namespace
   time_expires = "2030-08-01T00:00:00+00:00"
@@ -506,18 +522,12 @@ resource "oci_objectstorage_preauthrequest" "RDMA_NIC_metrics_par" {
 
 output "RDMA_NIC_metrics_url" {
  depends_on = [oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par]
- value = var.bastion_object_storage_par ? "https://objectstorage.${var.region}.oraclecloud.com${oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par[0].access_uri}" : ""
+ value = (var.bastion_object_storage_par && data.oci_objectstorage_bucket.RDMA_NIC_Metrics_bucket_check.bucket_id == null) ? "https://objectstorage.${var.region}.oraclecloud.com${oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par[0].access_uri}" : ""
 }
 
-locals {
-  par_path = "/opt/oci-hpc"
-}
-/*
-saving the PAR into file: /opt/oci-hpc/PAR_file_for_metrics.
-this PAR is used by the scripts to upload NIC metrics to object storage (i.e. script: upload_rdma_nic_metrics.sh)
-*/
+
 resource "local_file" "PAR" {
-    count = var.bastion_object_storage_par ? 1 : 0
+    count = (var.bastion_object_storage_par && data.oci_objectstorage_bucket.RDMA_NIC_Metrics_bucket_check.bucket_id == null) ? 1 : 0
     depends_on = [oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par]
     content     = "https://objectstorage.${var.region}.oraclecloud.com${oci_objectstorage_preauthrequest.RDMA_NIC_metrics_par[0].access_uri}"
     filename = "${local.par_path}/PAR_file_for_metrics"

playbooks/roles/telegraf/tasks/common.yml

@@ -47,6 +47,23 @@
     - influxdb.conf
     - net.conf
     - ethtool_counters.conf
+    - infiniband_mlx5_0_hw_counters.conf
+    - infiniband_mlx5_1_hw_counters.conf
+    - infiniband_mlx5_2_hw_counters.conf
+    - infiniband_mlx5_3_hw_counters.conf
+    - infiniband_mlx5_4_hw_counters.conf
+    - infiniband_mlx5_5_hw_counters.conf
+    - infiniband_mlx5_6_hw_counters.conf
+    - infiniband_mlx5_7_hw_counters.conf
+    - infiniband_mlx5_8_hw_counters.conf
+    - infiniband_mlx5_9_hw_counters.conf
+    - infiniband_mlx5_10_hw_counters.conf
+    - infiniband_mlx5_11_hw_counters.conf
+    - infiniband_mlx5_12_hw_counters.conf
+    - infiniband_mlx5_13_hw_counters.conf
+    - infiniband_mlx5_14_hw_counters.conf
+    - infiniband_mlx5_15_hw_counters.conf
+    - infiniband_mlx5_16_hw_counters.conf
 - name: restart telegraf
   become: true
   service:

playbooks/roles/telegraf/templates/infiniband_mlx5_0_hw_counters.conf.j2

@@ -1,4 +1,4 @@
-  [[inputs.multifile]]
+[[inputs.multifile]]
     name_override = "infiniband_mlx5_0_hw_counters"
     base_dir = "/sys/class/infiniband"
     interval = "300s"
@@ -10,41 +10,40 @@
 
     [[inputs.multifile.file]]
       file = "mlx5_0/ports/1/hw_counters/np_ecn_marked_roce_packets"
-      conversion = “int"
+      conversion = "int"
 
     [[inputs.multifile.file]]
       file = "mlx5_0/ports/1/hw_counters/out_of_sequence"
-      conversion = “int"
+      conversion = "int"
 
     [[inputs.multifile.file]]
       file = "mlx5_0/ports/1/hw_counters/packet_seq_err"
-      conversion = “int"
+      conversion = "int"
 
     [[inputs.multifile.file]]
       file = "mlx5_0/ports/1/hw_counters/local_ack_timeout_err"
-      conversion = “int"
+      conversion = "int"
 
     [[inputs.multifile.file]]
     file = "mlx5_0/ports/1/hw_counters/roce_adp_retrans"
-    conversion = “int"
+    conversion = "int"
 
     [[inputs.multifile.file]]
       file = "mlx5_0/ports/1/hw_counters/np_cnp_sent"
-      conversion = “int"
+      conversion = "int"
 
     [[inputs.multifile.file]]
       file = "mlx5_0/ports/1/hw_counters/rp_cnp_handled"
-      conversion = “int"
+      conversion = "int"
 
     [[inputs.multifile.file]]
       file = "mlx5_0/ports/1/hw_counters/rp_cnp_ignored"
-      conversion = “int"
+      conversion = "int"
 
     [[inputs.multifile.file]]
     file = "mlx5_0/ports/1/hw_counters/rx_icrc_encapsulated"
-    conversion = “int"
+    conversion = "int"
 
     [[inputs.multifile.file]]
     file = "mlx5_0/ports/1/hw_counters/roce_slow_restart"
     conversion = "int"
-

playbooks/roles/telegraf/templates/infiniband_mlx5_10_hw_counters.conf.j2

@@ -1,51 +1,49 @@
-
-
-  [[inputs.multifile]]
-      name_override = "infiniband_mlx5_10_hw_counters"
-      base_dir = "/sys/class/infiniband"
-      interval = "300s"
-
-      [[inputs.multifile.tags]]
-        device="mlx5_10"
-        port="1"
-        type="hw_counters"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_10/ports/1/hw_counters/np_ecn_marked_roce_packets"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_10/ports/1/hw_counters/out_of_sequence"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_10/ports/1/hw_counters/packet_seq_err"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_10/ports/1/hw_counters/local_ack_timeout_err"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-      file = "mlx5_10/ports/1/hw_counters/roce_adp_retrans"
-      conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_10/ports/1/hw_counters/np_cnp_sent"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_10/ports/1/hw_counters/rp_cnp_handled"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_10/ports/1/hw_counters/rp_cnp_ignored"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-      file = "mlx5_10/ports/1/hw_counters/rx_icrc_encapsulated"
-      conversion = “int"
-
-      [[inputs.multifile.file]]
-      file = "mlx5_10/ports/1/hw_counters/roce_slow_restart"
-      conversion = "int"  
+[[inputs.multifile]]
+    name_override = "infiniband_mlx5_10_hw_counters"
+    base_dir = "/sys/class/infiniband"
+    interval = "300s"
+
+    [[inputs.multifile.tags]]
+      device="mlx5_10"
+      port="1"
+      type="hw_counters"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/np_ecn_marked_roce_packets"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/out_of_sequence"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/packet_seq_err"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/local_ack_timeout_err"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_10/ports/1/hw_counters/roce_adp_retrans"
+    conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/np_cnp_sent"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/rp_cnp_handled"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/rp_cnp_ignored"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_10/ports/1/hw_counters/rx_icrc_encapsulated"
+    conversion = "int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_10/ports/1/hw_counters/roce_slow_restart"
+    conversion = "int"

playbooks/roles/telegraf/templates/infiniband_mlx5_11_hw_counters.conf.j2

@@ -1,51 +1,49 @@
-
-
-  [[inputs.multifile]]
-      name_override = "infiniband_mlx5_11_hw_counters"
-      base_dir = "/sys/class/infiniband"
-      interval = "300s"
-
-      [[inputs.multifile.tags]]
-        device="mlx5_11"
-        port="1"
-        type="hw_counters"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_11/ports/1/hw_counters/np_ecn_marked_roce_packets"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_11/ports/1/hw_counters/out_of_sequence"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_11/ports/1/hw_counters/packet_seq_err"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_11/ports/1/hw_counters/local_ack_timeout_err"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-      file = "mlx5_11/ports/1/hw_counters/roce_adp_retrans"
-      conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_11/ports/1/hw_counters/np_cnp_sent"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_11/ports/1/hw_counters/rp_cnp_handled"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-        file = "mlx5_11/ports/1/hw_counters/rp_cnp_ignored"
-        conversion = “int"
-
-      [[inputs.multifile.file]]
-      file = "mlx5_11/ports/1/hw_counters/rx_icrc_encapsulated"
-      conversion = “int"
-
-      [[inputs.multifile.file]]
-      file = "mlx5_11/ports/1/hw_counters/roce_slow_restart"
-      conversion = "int"  
+[[inputs.multifile]]
+    name_override = "infiniband_mlx5_11_hw_counters"
+    base_dir = "/sys/class/infiniband"
+    interval = "60s"
+
+    [[inputs.multifile.tags]]
+      device="mlx5_11"
+      port="1"
+      type="hw_counters"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/np_ecn_marked_roce_packets"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/out_of_sequence"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/packet_seq_err"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/local_ack_timeout_err"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_11/ports/1/hw_counters/roce_adp_retrans"
+    conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/np_cnp_sent"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/rp_cnp_handled"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/rp_cnp_ignored"
+      conversion = "int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_11/ports/1/hw_counters/rx_icrc_encapsulated"
+    conversion = "int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_11/ports/1/hw_counters/roce_slow_restart"
+    conversion = "int"