[rocpd] Adding summary module to generate summaries from rocpd database + query submodule + rocpd command-line tools (#488)

* adding summary.py to generate tmp <category_region>_summary views

* migrating CSV summary to SDK method of writing CSVs

  - Add domain_view to summary.py
  - omit the C++ code of writing CSV because it gets revered later anyway

* Add summary subparser and write_sql_view_to_csv function

* adding all <>_summary views generation to summary.py

* add summary_per_rank feature

* add --summary-per-rank

* reconstruct generate_summary_view and create_domain_view

-introduce by_rank

* remove sqr and variance in summary views

* use RocpdImportData instead of connection

* two fixes on summary.py

--modify the generate_summary_view function to return a tuple with view name and sql code

add if_not_exits parameter to generete_summary_view

* Refactor summary.py to allow output path and filename args, and apply time_window
- clean up summary table column headers
- only generate by-rank views if that param is specified

* Add ProcessID to Hostname output and csv, so users can identify the system in the by-rank summaries

* Summary.py, just add hostname to by-rank summaries, instead of creating mapping table

* Summary - migrate csv writer to pandas, for more future flexibility

* Adding a few simple tests for summary.py

* Linting fixes

* add region_categories to summary options

  -  Automatically retrieve region categories from the database if argument is None

* add backticks for view_names

* fix tests after rebase

* Made code review changes
- fixed whitespace in CMakelists.txt
- adding query.py module & subparser in __main__.py
- refactor summary function to return query
- used query.py to output csv
- used query.py to also output summary to console
- provided new command line options to select summary output to csv or console

* Made fix to jinja template in query.py, as suggested by copilot

* Consolidated output calls to query in export_view function based on feedback
- refactored: helpers, query functions, create view functions
- extended formats to include what query supports (md, html, pdf, json)
- added json format to query, and changed orient=records
- adding jinja2 and reportlab to requirements.txt

* Add version_info for rocpd and roctx

* Add rocpd commandline tool

* Add executable permissions to source/bin/rocpd.py

* Removed rocpd2query, and cleaned up --help examples

---------

Co-authored-by: acanadas <acanadas@amd.com>
Co-authored-by: Jin Tao <jintao12@amd.com>
Co-authored-by: a-canadasruiz <Araceli.CanadasRuiz@amd.com>
Co-authored-by: Jonathan R. Madsen <Jonathan.Madsen@amd.com>

Author: 4 people

requirements.txt

@@ -5,10 +5,12 @@ cmake>=3.21.0
 cmake-format
 dataclasses
 flake8
+jinja2
 numpy
 otf2
 pandas
 perfetto
 pycobertura
 pytest
 pyyaml
+reportlab

source/bin/CMakeLists.txt

@@ -25,3 +25,39 @@ install(
     PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ
                 WORLD_EXECUTE
     COMPONENT tools)
+
+# for each entry here there must be a ROCPD_BIN_<entry> list
+set(ROCPD_EXECUTABLES "all" "csv" "otf2" "pftrace" "summary")
+
+# format is list: [<exe-name>, <python-module>, <extra-args-to-python-module>]
+set(ROCPD_BIN_all "rocpd" "rocpd" "")
+set(ROCPD_BIN_csv "rocpd2csv" "rocpd.csv" "")
+set(ROCPD_BIN_otf2 "rocpd2otf2" "rocpd.otf2" "")
+set(ROCPD_BIN_pftrace "rocpd2pftrace" "rocpd.pftrace" "")
+set(ROCPD_BIN_summary "rocpd2summary" "rocpd.summary" "")
+
+foreach(_EXE IN LISTS ROCPD_EXECUTABLES)
+    list(GET ROCPD_BIN_${_EXE} 0 ROCPD_EXE_NAME)
+    list(GET ROCPD_BIN_${_EXE} 1 ROCPD_EXE_MODULE)
+    list(GET ROCPD_BIN_${_EXE} 2 ROCPD_EXE_MODULE_ARGS)
+
+    if(NOT ROCPD_EXE_NAME
+       OR NOT ROCPD_EXE_MODULE
+       OR (NOT ROCPD_EXE_MODULE_ARGS AND NOT ROCPD_EXE_MODULE_ARGS STREQUAL ""))
+        message(
+            FATAL_ERROR "ROCPD_BIN_${_EXE} not properly defined: ${ROCPD_BIN_${_EXE}}")
+    endif()
+
+    set(ROCPD_EXE_OUTPUT_FILE
+        ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/${ROCPD_EXE_NAME})
+
+    # Adding main rocpd
+    configure_file(rocpd.py ${ROCPD_EXE_OUTPUT_FILE} @ONLY)
+
+    install(
+        FILES ${ROCPD_EXE_OUTPUT_FILE}
+        DESTINATION ${CMAKE_INSTALL_BINDIR}
+        PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE
+                    WORLD_READ WORLD_EXECUTE
+        COMPONENT rocpd)
+endforeach()

source/bin/rocpd.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import os
+import sys
+
+"""
+Simple Python executable script for invoking `python3 -m @ROCPD_EXE_MODULE@`
+"""
+
+
+def main(argv=sys.argv[1:], environ=dict(os.environ)):
+    """
+    Executes {sys.executable} -m @ROCPD_EXE_MODULE@ @ROCPD_EXE_MODULE_ARGS@
+    """
+
+    ROCPD_SUPPORTED_PYTHON_VERSIONS = [
+        ".".join(itr.split(".")[:2]) for itr in "@ROCPROFILER_PYTHON_VERSIONS@".split(";")
+    ]
+    ROCPD_MODULE_ARGS = [f"{itr}" for itr in "@ROCPD_EXE_MODULE_ARGS@".split(" ") if itr]
+
+    this_dir = os.path.dirname(os.path.realpath(__file__))
+    this_python_ver = f"{sys.version_info.major}.{sys.version_info.minor}"
+    if this_python_ver not in ROCPD_SUPPORTED_PYTHON_VERSIONS:
+        raise ImportError(
+            "@ROCPD_EXE_NAME@ not supported for Python version {} (sys.executable='{}').\n@ROCPD_EXE_NAME@ supported python versions: {}".format(
+                this_python_ver,
+                sys.executable,
+                ", ".join(ROCPD_SUPPORTED_PYTHON_VERSIONS),
+            )
+        )
+
+    module_path = os.path.join(
+        this_dir,
+        "..",
+        "@CMAKE_INSTALL_LIBDIR@",
+        f"python{this_python_ver}",
+        "site-packages",
+    )
+
+    python_path = [module_path] + os.environ.get("PYTHONPATH", "").split(":")
+
+    # update PYTHONPATH environment variable
+    environ["PYTHONPATH"] = ":".join(python_path)
+
+    args = [f"{sys.executable}", "-m", "@ROCPD_EXE_MODULE@"] + ROCPD_MODULE_ARGS + argv
+
+    # does not return
+    os.execvpe(args[0], args, env=environ)
+
+
+if __name__ == "__main__":
+    main()

source/lib/python/rocpd/__init__.py

@@ -46,8 +46,24 @@
     "write_csv",
     "write_otf2",
     "RocpdImportData",
+    "version_info",
 ]
 
+version_info = {
+    "version": "@PROJECT_VERSION@",
+    "major": int("@PROJECT_VERSION_MAJOR@"),
+    "minor": int("@PROJECT_VERSION_MINOR@"),
+    "patch": int("@PROJECT_VERSION_PATCH@"),
+    "git_revision": "@ROCPROFILER_SDK_GIT_REVISION@",
+    "library_arch": "@CMAKE_LIBRARY_ARCHITECTURE@",
+    "system_name": "@CMAKE_SYSTEM_NAME@",
+    "system_processor": "@CMAKE_SYSTEM_PROCESSOR@",
+    "system_version": "@CMAKE_SYSTEM_VERSION@",
+    "compiler_id": "@CMAKE_CXX_COMPILER_ID@",
+    "compiler_version": "@CMAKE_CXX_COMPILER_VERSION@",
+    "rocm_version": "@rocm_version_FULL_VERSION@",
+}
+
 
 def format_path(path, tag=os.path.basename(sys.executable)):
     return libpyrocpd.format_path(path, tag)