[SWDEV-488303] Updated CU occupancy for per-process retrieval (#243)

marifamd · web-flow · commit 24c9bfb69254 · 2025-05-29T20:35:27.000-05:00
Change-Id: I2990597c6dd4b2e8cf3e11ce60f72049ebdd9a8c Signed-off-by: Maisam Arif <Maisam.Arif@amd.com> [ROCm/amdsmi commit: 0fdaebd]
diff --git a/projects/amdsmi/CHANGELOG.md b/projects/amdsmi/CHANGELOG.md
@@ -12,6 +12,11 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr
 
 ### Changed
 
+- **Added Compute Unit Occupancy information per process**  
+  Measuring compute units are the best way currently to determine gfx usage on a per process basis  
+  - Added `CU_OCCUPANCY` to `amd-smi process` output.
+  - Added `CU%` to `amd-smi monitor -q`
+
 - **Expanded Violation Status tracking for GPU metrics 1.8.**
   - The driver will no longer be supporting existing single-value GFX Clk Below Host Limit fields (`acc_gfx_clk_below_host_limit`, `per_gfx_clk_below_host_limit`, `active_gfx_clk_below_host_limit`), they are now changed in favor of new per-XCP/XCC arrays.
   - Added new fields to `amdsmi_violation_status_t` and related interfaces for enhanced violation breakdown:
@@ -54,11 +59,8 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr
 
 ### Resolved issues
 
-- N/A
-
-### Upcoming changes
-
-- N/A
+- **Corrected VRAM memory calculation in `amdsmi_get_gpu_process_list`.**  
+  - Previously, the VRAM memory usage reported by `amdsmi_get_gpu_process_list` was inaccurate and calculated using KB vs KiB.
 
 ### Known issues
 
diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py
@@ -3300,8 +3300,21 @@ def process(self, args, multiple_devices=False, watching_output=False,
 
         filtered_process_values = []
         for process_info in process_list:
-            process_info['mem_usage'] = process_info.pop('mem')
-            process_info['usage'] = process_info.pop('engine_usage')
+            process_info = {
+                "name": process_info["name"],
+                "pid": process_info["pid"],
+                "memory_usage": {
+                    "gtt_mem": process_info["memory_usage"]["gtt_mem"],
+                    "cpu_mem": process_info["memory_usage"]["cpu_mem"],
+                    "vram_mem": process_info["memory_usage"]["vram_mem"],
+                },
+                "mem_usage": process_info["mem"],
+                "usage": {
+                    "gfx": process_info["engine_usage"]["gfx"],
+                    "enc": process_info["engine_usage"]["enc"],
+                },
+                "cu_occupancy": process_info["cu_occupancy"]
+            }
 
             engine_usage_unit = "ns"
             memory_usage_unit = "B"
@@ -5714,35 +5727,43 @@ def monitor(self, args, multiple_devices=False, watching_output=False, gpu=None,
                 logging.debug("Failed to get process list for gpu %s | %s", gpu_id, e.get_error_info())
                 raise e
 
+            try:
+                num_compute_units = amdsmi_interface.amdsmi_get_gpu_asic_info(args.gpu)['num_compute_units']
+            except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
+                num_compute_units = "N/A"
+                logging.debug("Failed to get num compute units for gpu %s | %s", gpu_id, e.get_error_info())
+
             # Clean processes dictionary
             filtered_process_values = []
             for process_info in process_list:
-                process_info['mem_usage'] = process_info.pop('mem')
-                process_info['usage'] = process_info.pop('engine_usage')
+                process_info.pop('mem')  # Remove 'mem' value
+                process_info.pop('engine_usage')  # Remove 'engine_usage' value
 
-                engine_usage_unit = "ns"
                 memory_usage_unit = "B"
-
                 if self.logger.is_human_readable_format():
-                    process_info['mem_usage'] = self.helpers.convert_bytes_to_readable(process_info['mem_usage'])
                     for usage_metric in process_info['memory_usage']:
                         process_info["memory_usage"][usage_metric] = self.helpers.convert_bytes_to_readable(process_info["memory_usage"][usage_metric])
                     memory_usage_unit = ""
 
-                process_info['mem_usage'] = self.helpers.unit_format(self.logger,
-                                                                     process_info['mem_usage'],
-                                                                     memory_usage_unit)
-
-                for usage_metric in process_info['usage']:
-                    process_info['usage'][usage_metric] = self.helpers.unit_format(self.logger,
-                                                                                   process_info['usage'][usage_metric],
-                                                                                   engine_usage_unit)
-
                 for usage_metric in process_info['memory_usage']:
                     process_info['memory_usage'][usage_metric] = self.helpers.unit_format(self.logger,
                                                                                           process_info['memory_usage'][usage_metric],
                                                                                           memory_usage_unit)
 
+                if 'cu_occupancy' in process_info:
+                    try:
+                        cu_occupancy = process_info['cu_occupancy']
+                        if num_compute_units != "N/A" and num_compute_units > 0:
+                            cu_percentage = round((cu_occupancy / num_compute_units) * 100, 1)
+                            process_info['cu_occupancy'] = self.helpers.unit_format(self.logger,
+                                                                                    cu_percentage,
+                                                                                    '%')
+                        else:
+                            process_info['cu_occupancy'] = "N/A"
+                    except Exception as e:
+                        process_info['cu_occupancy'] = "N/A"
+                        logging.debug("Failed to calculate cu_occupancy percentage for GPU %s | %s", gpu_id, str(e))
+
                 filtered_process_values.append({'process_info': process_info})
 
             # If no processes are populated then we populate an N/A placeholder
@@ -5757,8 +5778,7 @@ def monitor(self, args, multiple_devices=False, watching_output=False, gpu=None,
             # Build the process table's title and header
             self.logger.secondary_table_title = "PROCESS INFO"
             self.logger.secondary_table_header = 'GPU'.rjust(3) + "NAME".rjust(22) + "PID".rjust(9) + "GTT_MEM".rjust(10) + \
-                                                "CPU_MEM".rjust(10) + "VRAM_MEM".rjust(10) + "MEM_USAGE".rjust(11) + \
-                                                "GFX".rjust(8) + "ENC".rjust(8)
+                                                "CPU_MEM".rjust(10) + "VRAM_MEM".rjust(10) + "CU%".rjust(9)
 
             if watching_output:
                 self.logger.secondary_table_header = 'TIMESTAMP'.rjust(10) + '  ' + self.logger.secondary_table_header
diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py
@@ -214,9 +214,9 @@ def _convert_json_to_tabular(self, json_object: Dict[str, any], dynamic=False):
                     if process_dict['process_info'] == "No running processes detected":
                         # Add N/A for empty process_info
                         table_values += "N/A".rjust(20) + "N/A".rjust(9) + "N/A".rjust(10) + \
-                                        "N/A".rjust(10) + "N/A".rjust(10) + "N/A".rjust(11) + \
-                                        "N/A".rjust(8) + "N/A".rjust(8) + '\n'
+                                        "N/A".rjust(10) + "N/A".rjust(10) + "N/A".rjust(9) + '\n'
                     else:
+                        #Fix this herre
                         for process_key, process_value in process_dict['process_info'].items():
                             string_process_value = str(process_value)
                             if process_key == "name":
@@ -230,11 +230,8 @@ def _convert_json_to_tabular(self, json_object: Dict[str, any], dynamic=False):
                             elif process_key == "memory_usage":
                                 for memory_key, memory_value in process_value.items():
                                     table_values += str(memory_value).rjust(10)
-                            elif process_key == "mem_usage":
-                                table_values += string_process_value.rjust(11)
-                            elif process_key == "usage":
-                                for usage_key, usage_value in process_value.items():
-                                    table_values += str(usage_value).rjust(8)
+                            elif process_key == "cu_occupancy":
+                                table_values += string_process_value.rjust(9)
                                 # Add the stored gpu and stored timestamp to the next line
                                 table_values += '\n'
                                 if stored_timestamp:
@@ -486,20 +483,6 @@ def _store_output_amdsmi(self, gpu_id, argument, data):
             raise amdsmi_cli_exceptions(self, "Invalid output format given, only json, csv, and human_readable supported")
 
 
-    def _store_output_rocmsmi(self, gpu_id, argument, data):
-        if self.is_json_format():
-            # put output into self.json_output
-            pass
-        elif self.is_csv_format():
-            # put output into self.csv_output
-            pass
-        elif self.is_human_readable_format():
-            # put output into self.human_readable_output
-            pass
-        else:
-            raise amdsmi_cli_exceptions(self, "Invalid output format given, only json, csv, and human_readable supported")
-
-
     def store_multiple_device_output(self):
         """ Store the current output into the multiple_device_output
                 then clear the current output
diff --git a/projects/amdsmi/docs/reference/amdsmi-py-api.md b/projects/amdsmi/docs/reference/amdsmi-py-api.md
@@ -1093,7 +1093,6 @@ except AmdSmiException as e:
     print(e)
 ```
 
-
 ### amdsmi_get_gpu_process_list
 
 Description: Returns the list of processes running on the target GPU; Requires root level access to display root process names; otherwise will return "N/A"
@@ -1111,6 +1110,7 @@ Field | Description
 `mem` | Process memory usage
 `engine_usage` | <table><thead><tr> <th> Subfield </th> <th> Description</th> </tr></thead><tbody><tr><td>`gfx`</td><td>GFX engine usage in ns</td></tr><tr><td>`enc`</td><td>Encode engine usage in ns</td></tr></tbody></table>
 `memory_usage` | <table><thead><tr> <th> Subfield </th> <th> Description</th> </tr></thead><tbody><tr><td>`gtt_mem`</td><td>GTT memory usage</td></tr><tr><td>`cpu_mem`</td><td>CPU memory usage</td></tr><tr><td>`vram_mem`</td><td>VRAM memory usage</td></tr> </tbody></table>
+`cu_occupancy` | Number of Compute Units utilized
 
 Exceptions that can be thrown by `amdsmi_get_gpu_process_list` function:
 
diff --git a/projects/amdsmi/example/amd_smi_drm_example.cc b/projects/amdsmi/example/amd_smi_drm_example.cc
@@ -817,6 +817,7 @@ int main() {
                 amdsmi_proc_info_t process = {};
                 uint64_t mem = 0, gtt_mem = 0, cpu_mem = 0, vram_mem = 0;
                 uint64_t gfx = 0, enc = 0;
+                uint32_t cu_occupancy = 0;
                 char bdf_str[20];
                 sprintf(bdf_str, "%04" PRIx64 ":%02" PRIx32 ":%02" PRIx32 ".%" PRIu32,
                    static_cast<uint64_t>(bdf.domain_number),
@@ -837,7 +838,7 @@ int main() {
                 printf(
                     "| pid   | name             | user       | gpu bdf      | "
                     "fb usage    | gtt memory  | cpu memory  | vram memory  | "
-                    "engine usage (ns)                       |\n");
+                    "engine usage (ns)  | cu occupancy       |\n");
                 printf("|       |                  |            |              "
                        "|             |             |             |            "
                        "  | gfx     enc     |\n");
@@ -855,30 +856,34 @@ int main() {
                     pwd = getpwuid(st.st_uid);
                     if (!pwd)
                         printf("| %5d | %16s | %10d | %s | %7ld KiB | %7ld KiB "
-                               "| %7ld KiB | %7ld KiB  | %lu  %lu |\n",
+                               "| %7ld KiB | %7ld KiB  | %lu  %lu  | %u |\n",
                                process_info_list[it].pid, process_info_list[it].name, st.st_uid,
                                bdf_str, process_info_list[it].mem / 1024,
                                process_info_list[it].memory_usage.gtt_mem / 1024,
                                process_info_list[it].memory_usage.cpu_mem / 1024,
                                process_info_list[it].memory_usage.vram_mem / 1024,
                                process_info_list[it].engine_usage.gfx,
-                               process_info_list[it].engine_usage.enc);
+                               process_info_list[it].engine_usage.enc,
+                               process_info_list[it].cu_occupancy);
                     else
                         printf("| %5d | %16s | %10s | %s | %7ld KiB | %7ld KiB "
-                               "| %7ld KiB | %7ld KiB  | %lu  %lu |\n",
+                               "| %7ld KiB | %7ld KiB  | %lu  %lu  | %u |\n",
                                process_info_list[it].pid, process_info_list[it].name,
                                pwd->pw_name, bdf_str, process_info_list[it].mem / 1024,
                                process_info_list[it].memory_usage.gtt_mem / 1024,
                                process_info_list[it].memory_usage.cpu_mem / 1024,
                                process_info_list[it].memory_usage.vram_mem / 1024,
                                process_info_list[it].engine_usage.gfx,
-                               process_info_list[it].engine_usage.enc);
+                               process_info_list[it].engine_usage.enc,
+                               process_info_list[it].cu_occupancy);
+
                     mem += process_info_list[it].mem / 1024;
                     gtt_mem += process_info_list[it].memory_usage.gtt_mem / 1024;
                     cpu_mem += process_info_list[it].memory_usage.cpu_mem / 1024;
                     vram_mem += process_info_list[it].memory_usage.vram_mem / 1024;
                     gfx = process_info_list[it].engine_usage.gfx;
                     enc = process_info_list[it].engine_usage.enc;
+                    cu_occupancy = process_info_list[it].cu_occupancy;
                     printf(
                         "+-------+------------------+------------+-------------"
                         "-+-------------+-------------+-------------+----------"
@@ -887,10 +892,9 @@ int main() {
                 // TODO: To remove compiler warning, the last 3 values in this printf were
                 //       set to 0L.  Need to find out what these values need to be.
                 printf("|                                 TOTAL:| %s | %7ld "
-                       "KiB | %7ld KiB | %7ld KiB | %7ld KiB | %lu  %lu  "
-                       "%lu  %lu  %lu   |\n",
+                       "KiB | %7ld KiB | %7ld KiB | %7ld KiB | %lu  %lu | %u |\n",
                        bdf_str, mem, gtt_mem, cpu_mem, vram_mem, gfx,
-                       enc, 0L, 0L, 0L);
+                       enc, cu_occupancy, 0L);
                 printf("+=======+==================+============+=============="
                        "+=============+=============+=============+============"
                        "=+==========================================+\n");
diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h
@@ -1093,7 +1093,8 @@ typedef struct {
         uint32_t reserved[10];
     } memory_usage;  //!< in bytes
     char container_name[AMDSMI_MAX_STRING_LENGTH];
-    uint32_t reserved[12];
+    uint32_t cu_occupancy;  //!< Num CUs utilized
+    uint32_t reserved[11];
 } amdsmi_proc_info_t;
 
 /**
diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py
@@ -2691,6 +2691,7 @@ def amdsmi_get_gpu_process_list(
                 "cpu_mem": process_list[index].memory_usage.cpu_mem,
                 "vram_mem": process_list[index].memory_usage.vram_mem,
             },
+            "cu_occupancy": process_list[index].cu_occupancy
         })
 
     return result
diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py
@@ -1289,7 +1289,8 @@ class struct_engine_usage_(Structure):
     ('engine_usage', struct_engine_usage_),
     ('memory_usage', struct_memory_usage_),
     ('container_name', ctypes.c_char * 256),
-    ('reserved', ctypes.c_uint32 * 12),
+    ('cu_occupancy', ctypes.c_uint32),
+    ('PADDING_1', ctypes.c_ubyte * 4),
 ]
 
 amdsmi_proc_info_t = struct_amdsmi_proc_info_t
diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi_kfd.cc b/projects/amdsmi/rocm_smi/src/rocm_smi_kfd.cc
@@ -456,7 +456,6 @@ int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc,
   proc->sdma_usage = 0;
   proc->cu_occupancy = 0;
 
-  uint32_t cu_count = 0;
   static amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();
   static std::map<uint64_t, std::shared_ptr<KFDNode>>& kfd_node_map =
                                                            smi.kfd_node_map();
@@ -510,23 +509,15 @@ int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc,
     }
     else if(sysfs_data_errcode==0){
       // Update CU usage by the process
-      proc->cu_occupancy += std::stoi(tmp);
-      // Collect count of compute units
-      cu_count += kfd_node_map[gpu_id]->cu_count();
+      proc->cu_occupancy = std::stoi(tmp);
     }
     else {
       // Some GFX revisions do not provide cu_occupancy debugfs method
       // which may cause ENOENT
       proc->cu_occupancy = CU_OCCUPANCY_INVALID;
-      cu_count = 0;
     }
   }
 
-  // Adjust CU occupancy to percent.
-  if (cu_count > 0) {
-    proc->cu_occupancy = ((proc->cu_occupancy * 100) / cu_count);
-  }
-
   return 0;
 }
 
diff --git a/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc b/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc
@@ -216,6 +216,9 @@ int32_t AMDSmiGPUDevice::get_compute_process_list_impl(GPUComputeProcessList_t&
 
     /**
      * Complete the process information
+     * This is where we copy rsmi_process_info_t into the larger amdsmi_proc_info_t
+     * Then populate the remaining fields with the gpuvsmi_get_pid_info()
+     * TODO FIX HERE TO GRAB KFD VRAM if /proc is inconsistent
      */
     auto get_process_info = [&](const rsmi_process_info_t& rsmi_proc_info, amdsmi_proc_info_t& asmi_proc_info) {
         auto status_code = gpuvsmi_get_pid_info(get_bdf(), rsmi_proc_info.process_id, asmi_proc_info);
@@ -225,6 +228,9 @@ int32_t AMDSmiGPUDevice::get_compute_process_list_impl(GPUComputeProcessList_t&
             asmi_proc_info.memory_usage.vram_mem = rsmi_proc_info.vram_usage;
         }
 
+        // Copy the cu occupancy from rsmi_process_info_t to amdsmi_proc_info_t
+        asmi_proc_info.cu_occupancy = rsmi_proc_info.cu_occupancy;
+
         return status_code;
     };
 
diff --git a/projects/amdsmi/src/amd_smi/fdinfo.cc b/projects/amdsmi/src/amd_smi/fdinfo.cc
diff --git a/projects/amdsmi/tools/amdsmi_quick_start.py b/projects/amdsmi/tools/amdsmi_quick_start.py

Original file line number	Diff line number	Diff line change
`@@ -1289,7 +1289,8 @@ class struct_engine_usage_(Structure):`
`1289`	`1289`	`('engine_usage', struct_engine_usage_),`
`1290`	`1290`	`('memory_usage', struct_memory_usage_),`
`1291`	`1291`	`('container_name', ctypes.c_char * 256),`
`1292`		`- ('reserved', ctypes.c_uint32 * 12),`
	`1292`	`+ ('cu_occupancy', ctypes.c_uint32),`
	`1293`	`+ ('PADDING_1', ctypes.c_ubyte * 4),`
`1293`	`1294`	`]`
`1294`	`1295`
`1295`	`1296`	`amdsmi_proc_info_t = struct_amdsmi_proc_info_t`
Original file line number	Diff line number	Diff line change
`@@ -456,7 +456,6 @@ int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc,`
`456`	`456`	`proc->sdma_usage = 0;`
`457`	`457`	`proc->cu_occupancy = 0;`
`458`	`458`
`459`		`- uint32_t cu_count = 0;`
`460`	`459`	`static amd::smi::RocmSMI& smi = amd::smi::RocmSMI::getInstance();`
`461`	`460`	`static std::map<uint64_t, std::shared_ptr<KFDNode>>& kfd_node_map =`
`462`	`461`	`smi.kfd_node_map();`
`@@ -510,23 +509,15 @@ int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc,`
`510`	`509`	`}`
`511`	`510`	`else if(sysfs_data_errcode==0){`
`512`	`511`	`// Update CU usage by the process`
`513`		`- proc->cu_occupancy += std::stoi(tmp);`
`514`		`- // Collect count of compute units`
`515`		`- cu_count += kfd_node_map[gpu_id]->cu_count();`
	`512`	`+ proc->cu_occupancy = std::stoi(tmp);`
`516`	`513`	`}`
`517`	`514`	`else {`
`518`	`515`	`// Some GFX revisions do not provide cu_occupancy debugfs method`
`519`	`516`	`// which may cause ENOENT`
`520`	`517`	`proc->cu_occupancy = CU_OCCUPANCY_INVALID;`
`521`		`- cu_count = 0;`
`522`	`518`	`}`
`523`	`519`	`}`
`524`	`520`
`525`		`- // Adjust CU occupancy to percent.`
`526`		`- if (cu_count > 0) {`
`527`		`- proc->cu_occupancy = ((proc->cu_occupancy * 100) / cu_count);`
`528`		`- }`
`529`		`-`
`530`	`521`	`return 0;`
`531`	`522`	`}`
`532`	`523`