Merge pull request #131 from oci-hpc/2.10.3_more_counters

Publish NIC metrics to influxDB

Author: arnaudfroidmont

playbooks/roles/telegraf/tasks/common.yml

@@ -46,7 +46,24 @@
     - infiniband.conf
     - influxdb.conf
     - net.conf
-    - infiniband_hw_counters.conf
+    - ethtool_counters.conf
+    - infiniband_mlx5_0_hw_counters.conf
+    - infiniband_mlx5_1_hw_counters.conf
+    - infiniband_mlx5_2_hw_counters.conf
+    - infiniband_mlx5_3_hw_counters.conf
+    - infiniband_mlx5_4_hw_counters.conf
+    - infiniband_mlx5_5_hw_counters.conf
+    - infiniband_mlx5_6_hw_counters.conf
+    - infiniband_mlx5_7_hw_counters.conf
+    - infiniband_mlx5_8_hw_counters.conf
+    - infiniband_mlx5_9_hw_counters.conf
+    - infiniband_mlx5_10_hw_counters.conf
+    - infiniband_mlx5_11_hw_counters.conf
+    - infiniband_mlx5_12_hw_counters.conf
+    - infiniband_mlx5_13_hw_counters.conf
+    - infiniband_mlx5_14_hw_counters.conf
+    - infiniband_mlx5_15_hw_counters.conf
+    - infiniband_mlx5_16_hw_counters.conf
 - name: restart telegraf
   become: true
   service:

playbooks/roles/telegraf/templates/ethtool_counters.conf.j2

@@ -0,0 +1,12 @@
+# Returns ethtool statistics for given interfaces
+ [[inputs.ethtool]]
+
+  interface_include = ["rdma*","enp*"]
+  interval = "300s"
+  fieldpass = ["tx_pci_signal_integrity","rx_steer_missed_packets","rx_vport_multicast_bytes","rx_vport_rdma_unicast_packets",
+  "rx_vport_rdma_unicast_bytes","tx_vport_rdma_unicast_packets","tx_vport_rdma_unicast_bytes","tx_packets_phy","rx_packets_phy",
+  "tx_bytes_phy","rx_bytes_phy", "rx_multicast_phy","rx_65_to_127_bytes_phy","rx_2048_to_4095_bytes_phy", "rx_4096_to_8191_bytes_phy",
+  "rx_crc_errors_phy","rx_symbol_err_phy", "rx_discards_phy","tx_discards_phy","tx_errors_phy","rx_64_bytes_phy","link_down_events_phy",
+  "rx_out_of_buffer","module_bus_stuck","module_high_temp","rx_buffer_passed_thres_phy","tx_pause_storm_warning_events","tx_pause_storm_error_events",
+  "rx_pcs_symbol_err_phy","rx_pci_signal_integrity","tx_pci_signal_integrity","rx_prio0_bytes","rx_prio0_packets","tx_prio0_bytes",
+  "tx_prio0_packets","rx_prio0_buf_discard","rx_prio0_cong_discard","rx_prio0_marked","outbound_pci_buffer_overflow"]

playbooks/roles/telegraf/templates/infiniband_mlx5_0_hw_counters.conf.j2

@@ -0,0 +1,50 @@
+  [[inputs.multifile]]
+    name_override = "infiniband_mlx5_0_hw_counters"
+    base_dir = "/sys/class/infiniband"
+    interval = "300s"
+
+    [[inputs.multifile.tags]]
+      device="mlx5_0"
+      port="1"
+      type="hw_counters"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_0/ports/1/hw_counters/np_ecn_marked_roce_packets"
+      conversion = “int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_0/ports/1/hw_counters/out_of_sequence"
+      conversion = “int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_0/ports/1/hw_counters/packet_seq_err"
+      conversion = “int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_0/ports/1/hw_counters/local_ack_timeout_err"
+      conversion = “int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_0/ports/1/hw_counters/roce_adp_retrans"
+    conversion = “int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_0/ports/1/hw_counters/np_cnp_sent"
+      conversion = “int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_0/ports/1/hw_counters/rp_cnp_handled"
+      conversion = “int"
+
+    [[inputs.multifile.file]]
+      file = "mlx5_0/ports/1/hw_counters/rp_cnp_ignored"
+      conversion = “int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_0/ports/1/hw_counters/rx_icrc_encapsulated"
+    conversion = “int"
+
+    [[inputs.multifile.file]]
+    file = "mlx5_0/ports/1/hw_counters/roce_slow_restart"
+    conversion = "int"
+

playbooks/roles/telegraf/templates/infiniband_mlx5_10_hw_counters.conf.j2

@@ -0,0 +1,51 @@
+
+
+  [[inputs.multifile]]
+      name_override = "infiniband_mlx5_10_hw_counters"
+      base_dir = "/sys/class/infiniband"
+      interval = "300s"
+
+      [[inputs.multifile.tags]]
+        device="mlx5_10"
+        port="1"
+        type="hw_counters"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_10/ports/1/hw_counters/np_ecn_marked_roce_packets"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_10/ports/1/hw_counters/out_of_sequence"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_10/ports/1/hw_counters/packet_seq_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_10/ports/1/hw_counters/local_ack_timeout_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/roce_adp_retrans"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_10/ports/1/hw_counters/np_cnp_sent"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_10/ports/1/hw_counters/rp_cnp_handled"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_10/ports/1/hw_counters/rp_cnp_ignored"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/rx_icrc_encapsulated"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_10/ports/1/hw_counters/roce_slow_restart"
+      conversion = "int"  

playbooks/roles/telegraf/templates/infiniband_mlx5_11_hw_counters.conf.j2

@@ -0,0 +1,51 @@
+
+
+  [[inputs.multifile]]
+      name_override = "infiniband_mlx5_11_hw_counters"
+      base_dir = "/sys/class/infiniband"
+      interval = "300s"
+
+      [[inputs.multifile.tags]]
+        device="mlx5_11"
+        port="1"
+        type="hw_counters"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_11/ports/1/hw_counters/np_ecn_marked_roce_packets"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_11/ports/1/hw_counters/out_of_sequence"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_11/ports/1/hw_counters/packet_seq_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_11/ports/1/hw_counters/local_ack_timeout_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/roce_adp_retrans"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_11/ports/1/hw_counters/np_cnp_sent"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_11/ports/1/hw_counters/rp_cnp_handled"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_11/ports/1/hw_counters/rp_cnp_ignored"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/rx_icrc_encapsulated"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_11/ports/1/hw_counters/roce_slow_restart"
+      conversion = "int"  

playbooks/roles/telegraf/templates/infiniband_mlx5_12_hw_counters.conf.j2

@@ -0,0 +1,51 @@
+
+
+  [[inputs.multifile]]
+      name_override = "infiniband_mlx5_12_hw_counters"
+      base_dir = "/sys/class/infiniband"
+      interval = "300s"
+
+      [[inputs.multifile.tags]]
+        device="mlx5_12"
+        port="1"
+        type="hw_counters"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_12/ports/1/hw_counters/np_ecn_marked_roce_packets"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_12/ports/1/hw_counters/out_of_sequence"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_12/ports/1/hw_counters/packet_seq_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_12/ports/1/hw_counters/local_ack_timeout_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_12/ports/1/hw_counters/roce_adp_retrans"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_12/ports/1/hw_counters/np_cnp_sent"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_12/ports/1/hw_counters/rp_cnp_handled"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_12/ports/1/hw_counters/rp_cnp_ignored"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_12/ports/1/hw_counters/rx_icrc_encapsulated"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_12/ports/1/hw_counters/roce_slow_restart"
+      conversion = "int"  

playbooks/roles/telegraf/templates/infiniband_mlx5_13_hw_counters.conf.j2

@@ -0,0 +1,50 @@
+
+  [[inputs.multifile]]
+      name_override = "infiniband_mlx5_13_hw_counters"
+      base_dir = "/sys/class/infiniband"
+      interval = "300s"
+
+      [[inputs.multifile.tags]]
+        device="mlx5_13"
+        port="1"
+        type="hw_counters"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_13/ports/1/hw_counters/np_ecn_marked_roce_packets"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_13/ports/1/hw_counters/out_of_sequence"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_13/ports/1/hw_counters/packet_seq_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_13/ports/1/hw_counters/local_ack_timeout_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_13/ports/1/hw_counters/roce_adp_retrans"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_13/ports/1/hw_counters/np_cnp_sent"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_13/ports/1/hw_counters/rp_cnp_handled"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_13/ports/1/hw_counters/rp_cnp_ignored"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_13/ports/1/hw_counters/rx_icrc_encapsulated"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_13/ports/1/hw_counters/roce_slow_restart"
+      conversion = "int"  

playbooks/roles/telegraf/templates/infiniband_mlx5_14_hw_counters.conf.j2

@@ -0,0 +1,50 @@
+
+  [[inputs.multifile]]
+      name_override = "infiniband_mlx5_14_hw_counters"
+      base_dir = "/sys/class/infiniband"
+      interval = "300s"
+
+      [[inputs.multifile.tags]]
+        device="mlx5_14"
+        port="1"
+        type="hw_counters"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_14/ports/1/hw_counters/np_ecn_marked_roce_packets"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_14/ports/1/hw_counters/out_of_sequence"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_14/ports/1/hw_counters/packet_seq_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_14/ports/1/hw_counters/local_ack_timeout_err"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_14/ports/1/hw_counters/roce_adp_retrans"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_14/ports/1/hw_counters/np_cnp_sent"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_14/ports/1/hw_counters/rp_cnp_handled"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+        file = "mlx5_14/ports/1/hw_counters/rp_cnp_ignored"
+        conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_14/ports/1/hw_counters/rx_icrc_encapsulated"
+      conversion = “int"
+
+      [[inputs.multifile.file]]
+      file = "mlx5_14/ports/1/hw_counters/roce_slow_restart"
+      conversion = "int"