Import Python Coverage (#1203)

* Added the ability to turn on the Python coverage model.  
* Added SmartTextFormatter class to allow better formatting of text output to console.

Author: feldergast

.clang-format

@@ -204,5 +204,9 @@ WhitespaceSensitiveMacros:
   - CF_SWIFT_NAME
   - SST_SER
   - SST_SER_AS_PTR
+  - SST_TP_VECTORCALL
+  - SST_TP_PRINT_DEP
+  - SST_TP_WATCHED
+  - SST_TP_VERSIONS_USED
 ...
 

src/sst/core/Makefile.am

@@ -331,6 +331,7 @@ include model/Makefile.inc
 include shared/Makefile.inc
 include impl/Makefile.inc
 include sync/Makefile.inc
+include util/Makefile.inc
 include testingframework/Makefile.inc
 include testElements/Makefile.inc
 

src/sst/core/config.cc

@@ -373,6 +373,14 @@ class ConfigHelper
         return 0;
     }
 
+#if PY_MINOR_VERSION >= 9
+    // enable Python coverage
+    static int enablePythonCoverage(Config* cfg, const std::string& UNUSED(arg))
+    {
+        cfg->enable_python_coverage_ = true;
+        return 0;
+    }
+#endif
 
     // Advanced options - profiling
     static int enableProfiling(Config* cfg, const std::string& arg)
@@ -408,6 +416,35 @@ class ConfigHelper
         return msg;
     }
 
+#if PY_MINOR_VERSION >= 9
+    static std::string getPythonCoverageExtHelp()
+    {
+        std::string msg = "Python Coverage (EXPERIMENTAL):\n\n";
+
+        msg.append("NOTE: This feature is considered experimental until we can complete further testing.\n\n");
+
+        msg.append("If you are using python configuration (model definition) files as part of a larger project and are "
+                   "interested in measuring code coverage of a test/example/application suite, you can instruct sst to "
+                   "enable the python coverage module when launching python configuration files as part of a "
+                   "simulation invocation.  To do so, you need three things:\n\n");
+
+        msg.append("\t1.\t\vInstall python’s coverage module (via an OS package or pip) on your system "
+                   "<https://pypi.org/project/coverage/>\n");
+
+        msg.append("\t2.\t\vEnsure that the \"coverage\" command is in your path and that you can invoke the python "
+                   "that SST uses and import the coverage module without error.\n");
+
+        msg.append(
+            "\t3.\t\vSet the environment variable SST_CONFIG_PYTHON_COVERAGE to a value of 1, yes, on, true or t; or "
+            "invoke coverage on the command line by using the command line option --enable-python-coverage.\n\n");
+
+        msg.append("Then invoke SST as normal using the python model configuration file for which you want to measure "
+                   "coverage.\n");
+
+        return msg;
+    }
+#endif
+
     static std::string getProfilingExtHelp()
     {
         std::string msg = "Profiling Points [EXPERIMENTAL]:\n\n";
@@ -802,6 +839,10 @@ Config::Config(uint32_t num_ranks, bool first_rank) : ConfigShared(!first_rank,
 #endif
     debugFile_ = "/dev/null";
 
+#if PY_MINOR_VERSION >= 9
+    enable_python_coverage_ = false;
+#endif
+
     // Advance Options - Profiling
     enabled_profiling_ = "";
     profiling_output_  = "stdout";
@@ -990,6 +1031,16 @@ Config::insertOptions()
         true);
     addLibraryPathOptions();
 
+#if PY_MINOR_VERSION >= 9
+    DEF_FLAG_EH(
+        "enable-python-coverage", 0,
+        "[EXPERIMENTAL] Causes the base Python interpreter to activate the coverage.Coverage object. This option can "
+        "also be turned "
+        "on by setting the environment variable SST_CONFIG_PYTHON_COVERAGE to true.",
+        std::bind(&ConfigHelper::enablePythonCoverage, this, _1), std::bind(&ConfigHelper::getPythonCoverageExtHelp),
+        false);
+#endif
+
     /* Advanced Features - Profiling */
     DEF_SECTION_HEADING("Advanced Options - Profiling (EXPERIMENTAL)");
     DEF_ARG_EH(

src/sst/core/config.h

@@ -16,6 +16,8 @@
 #include "sst/core/serialization/serializable.h"
 #include "sst/core/sst_types.h"
 
+// Pull in the patchlevel for Python so we can check Python version
+#include <patchlevel.h>
 #include <string>
 
 /* Forward declare for Friendship */
@@ -288,6 +290,14 @@ class Config : public ConfigShared, public SST::Core::Serialization::serializabl
        const std::string& addLibPath() const { return addLibPath_; }
     */
 
+
+#if PY_MINOR_VERSION >= 9
+    /**
+       Controls whether the Python coverage object will be loaded
+     */
+    bool enablePythonCoverage() const { return enable_python_coverage_; }
+#endif
+
     // Advanced options - Profiling
 
     /**
@@ -437,6 +447,9 @@ class Config : public ConfigShared, public SST::Core::Serialization::serializabl
         ser& debugFile_;
         ser& libpath_;
         ser& addlibpath_;
+#if PY_MINOR_VERSION >= 9
+        ser& enable_python_coverage_;
+#endif
         ser& enabled_profiling_;
         ser& profiling_output_;
         ser& runMode_;
@@ -534,6 +547,9 @@ class Config : public ConfigShared, public SST::Core::Serialization::serializabl
     std::string debugFile_; /*!< File to which debug information should be written */
     // std::string libpath_;  ** in ConfigShared
     // std::string addLibPath_; ** in ConfigShared
+#if PY_MINOR_VERSION >= 9
+    bool enable_python_coverage_; /*!< Enable the Python coverage module */
+#endif
 
     // Advanced options - profiling
     std::string enabled_profiling_; /*!< Enabled default profiling points */

src/sst/core/configBase.cc

@@ -14,6 +14,7 @@
 #include "sst/core/configBase.h"
 
 #include "sst/core/env/envquery.h"
+#include "sst/core/util/smartTextFormatter.h"
 #include "sst/core/warnmacros.h"
 
 #include <cstdlib>
@@ -284,9 +285,12 @@ ConfigBase::printExtHelp(const std::string& option)
         fprintf(stderr, "No additional help found for option \"%s\"\n", option.c_str());
     }
     else {
+        Util::SmartTextFormatter formatter({ 2, 5, 8 }, 1);
+
         std::function<std::string(void)>& func = extra_help_map[option];
         std::string                       help = func();
-        fprintf(stderr, "%s\n", help.c_str());
+        formatter.append(help);
+        fprintf(stderr, "%s\n", formatter.str().c_str());
     }
 
     return 1; /* Should not continue */

src/sst/core/configBase.h

@@ -86,6 +86,9 @@ struct AnnotationInfo
     addOption({ longName, optional_argument, 0, shortName }, "[" argName "]", text, func, { __VA_ARGS__ });
 
 // Macros that include extended help
+#define DEF_FLAG_EH(longName, shortName, text, func, eh, ...) \
+    addOption({ longName, no_argument, 0, shortName }, "", text, func, { __VA_ARGS__ }, eh);
+
 #define DEF_ARG_EH(longName, shortName, argName, text, func, eh, ...) \
     addOption({ longName, required_argument, 0, shortName }, argName, text, func, { __VA_ARGS__ }, eh);
 

src/sst/core/from_string.h

@@ -51,7 +51,7 @@ from_string(const std::string& input)
                 return false;
             }
             else {
-                throw new std::invalid_argument("from_string: no valid conversion");
+                throw std::invalid_argument("from_string: no valid conversion");
             }
         }
         else if ( std::is_same<unsigned long, T>::value ) {

src/sst/core/model/python/pymodel.cc

@@ -19,6 +19,7 @@
 #include "sst/core/configGraph.h"
 #include "sst/core/cputimer.h"
 #include "sst/core/factory.h"
+#include "sst/core/from_string.h"
 #include "sst/core/memuse.h"
 #include "sst/core/model/element_python.h"
 #include "sst/core/model/python/pymacros.h"
@@ -88,57 +89,57 @@ DISABLE_WARN_DEPRECATED_DECLARATION
 #endif
 #endif
 static PyTypeObject ModuleLoaderType = {
-    SST_PY_OBJ_HEAD "ModuleLoader",  /* tp_name */
-    sizeof(ModuleLoaderPy_t),        /* tp_basicsize */
-    0,                               /* tp_itemsize */
-    nullptr,                         /* tp_dealloc */
-    0,                               /* tp_vectorcall_offset */
-    nullptr,                         /* tp_getattr */
-    nullptr,                         /* tp_setattr */
-    nullptr,                         /* tp_as_sync */
-    nullptr,                         /* tp_repr */
-    nullptr,                         /* tp_as_number */
-    nullptr,                         /* tp_as_sequence */
-    nullptr,                         /* tp_as_mapping */
-    nullptr,                         /* tp_hash */
-    nullptr,                         /* tp_call */
-    nullptr,                         /* tp_str */
-    nullptr,                         /* tp_getattro */
-    nullptr,                         /* tp_setattro */
-    nullptr,                         /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,              /* tp_flags */
-    "SST Module Loader",             /* tp_doc */
-    nullptr,                         /* tp_traverse */
-    nullptr,                         /* tp_clear */
-    nullptr,                         /* tp_rich_compare */
-    0,                               /* tp_weaklistoffset */
-    nullptr,                         /* tp_iter */
-    nullptr,                         /* tp_iternext */
-    mlMethods,                       /* tp_methods */
-    nullptr,                         /* tp_members */
-    nullptr,                         /* tp_getset */
-    nullptr,                         /* tp_base */
-    nullptr,                         /* tp_dict */
-    nullptr,                         /* tp_descr_get */
-    nullptr,                         /* tp_descr_set */
-    0,                               /* tp_dictoffset */
-    nullptr,                         /* tp_init */
-    nullptr,                         /* tp_alloc */
-    nullptr,                         /* tp_new */
-    nullptr,                         /* tp_free */
-    nullptr,                         /* tp_is_gc */
-    nullptr,                         /* tp_bases */
-    nullptr,                         /* tp_mro */
-    nullptr,                         /* tp_cache */
-    nullptr,                         /* tp_subclasses */
-    nullptr,                         /* tp_weaklist */
-    nullptr,                         /* tp_del */
-    0,                               /* tp_version_tag */
-    nullptr,                         /* tp_finalize */
-    SST_TP_VECTORCALL                /* Python3.8+ */
-        SST_TP_PRINT_DEP             /* Python3.8 only */
-            SST_TP_WATCHED           /* Python3.12+ */
-                SST_TP_VERSIONS_USED /* Python3.13+ */
+    SST_PY_OBJ_HEAD "ModuleLoader", /* tp_name */
+    sizeof(ModuleLoaderPy_t),       /* tp_basicsize */
+    0,                              /* tp_itemsize */
+    nullptr,                        /* tp_dealloc */
+    0,                              /* tp_vectorcall_offset */
+    nullptr,                        /* tp_getattr */
+    nullptr,                        /* tp_setattr */
+    nullptr,                        /* tp_as_sync */
+    nullptr,                        /* tp_repr */
+    nullptr,                        /* tp_as_number */
+    nullptr,                        /* tp_as_sequence */
+    nullptr,                        /* tp_as_mapping */
+    nullptr,                        /* tp_hash */
+    nullptr,                        /* tp_call */
+    nullptr,                        /* tp_str */
+    nullptr,                        /* tp_getattro */
+    nullptr,                        /* tp_setattro */
+    nullptr,                        /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,             /* tp_flags */
+    "SST Module Loader",            /* tp_doc */
+    nullptr,                        /* tp_traverse */
+    nullptr,                        /* tp_clear */
+    nullptr,                        /* tp_rich_compare */
+    0,                              /* tp_weaklistoffset */
+    nullptr,                        /* tp_iter */
+    nullptr,                        /* tp_iternext */
+    mlMethods,                      /* tp_methods */
+    nullptr,                        /* tp_members */
+    nullptr,                        /* tp_getset */
+    nullptr,                        /* tp_base */
+    nullptr,                        /* tp_dict */
+    nullptr,                        /* tp_descr_get */
+    nullptr,                        /* tp_descr_set */
+    0,                              /* tp_dictoffset */
+    nullptr,                        /* tp_init */
+    nullptr,                        /* tp_alloc */
+    nullptr,                        /* tp_new */
+    nullptr,                        /* tp_free */
+    nullptr,                        /* tp_is_gc */
+    nullptr,                        /* tp_bases */
+    nullptr,                        /* tp_mro */
+    nullptr,                        /* tp_cache */
+    nullptr,                        /* tp_subclasses */
+    nullptr,                        /* tp_weaklist */
+    nullptr,                        /* tp_del */
+    0,                              /* tp_version_tag */
+    nullptr,                        /* tp_finalize */
+    SST_TP_VECTORCALL               /* Python3.8+ */
+    SST_TP_PRINT_DEP                /* Python3.8 only */
+    SST_TP_WATCHED                  /* Python3.12+ */
+    SST_TP_VERSIONS_USED            /* Python3.13+ */
 };
 #if PY_MAJOR_VERSION == 3
 #if PY_MINOR_VERSION == 8
@@ -1063,7 +1064,7 @@ PyInit_sst(void)
 
 void
 SSTPythonModelDefinition::initModel(
-    const std::string& script_file, int verbosity, Config* UNUSED(config), int argc, char** argv)
+    const std::string& script_file, int verbosity, Config* config, int argc, char** argv)
 {
     output = new Output("SSTPythonModel: ", verbosity, 0, SST::Output::STDOUT);
 
@@ -1144,6 +1145,28 @@ SSTPythonModelDefinition::initModel(
     // // directory to the path
     PyRun_SimpleString("sys.meta_path.append(sst.ModuleLoader())\n");
 #endif
+
+
+    // Check to see if we need to import the coverage module (only works in Python >=3.9
+#if PY_MINOR_VERSION >= 9
+    if ( config->enablePythonCoverage() ) { enablePythonCoverage = true; }
+    else {
+        const char* envConfigCoverage = getenv("SST_CONFIG_PYTHON_COVERAGE");
+        if ( nullptr != envConfigCoverage ) {
+            std::string value(envConfigCoverage);
+            try {
+                enablePythonCoverage = SST::Core::from_string<bool>(value);
+            }
+            catch ( std::invalid_argument& e ) {
+                output->fatal(
+                    CALL_INFO, 1,
+                    "ERROR: Invalid format for SST_CONFIG_PYTHON_COVERAGE. Valid boolean pairs are true/false, t/f, "
+                    "yes/no, y/n, on/off, or 1/0\n");
+                enablePythonCoverage = false;
+            }
+        }
+    }
+#endif
 }
 
 SSTPythonModelDefinition::SSTPythonModelDefinition(
@@ -1236,8 +1259,29 @@ SSTPythonModelDefinition::createConfigGraph()
 {
     output->verbose(CALL_INFO, 1, 0, "Creating config graph for SST using Python model...\n");
 
+#if PY_MINOR_VERSION >= 9
+    if ( enablePythonCoverage ) {
+        // Create coverage object with a name unlikely to be used in the user script
+        int startcoverageReturn = PyRun_SimpleString("import coverage\n"
+                                                     "zzzSSTPythonCoverageModule = coverage.Coverage()\n"
+                                                     "zzzSSTPythonCoverageModule.start()\n");
+        if ( nullptr != PyErr_Occurred() ) {
+            // Print the Python error and then let SST exit as a fatal-stop.
+            output->fatal(
+                CALL_INFO, 1,
+                "ERROR: Error occurred starting test coverage of the SST model script. Reported error:\n");
+            PyErr_Print();
+        }
+        if ( -1 == startcoverageReturn ) {
+            output->fatal(CALL_INFO, 1, "Execution of starting test coverage failed\n%s", loadErrors.c_str());
+        }
+    }
+#endif
+
+    // Open the input script
     FILE* fp = fopen(scriptName.c_str(), "r");
     if ( !fp ) { output->fatal(CALL_INFO, 1, "Unable to open python script %s\n", scriptName.c_str()); }
+    PyErr_Clear();
     int createReturn = PyRun_AnyFileEx(fp, scriptName.c_str(), 1);
 
     if ( nullptr != PyErr_Occurred() ) {
@@ -1256,6 +1300,24 @@ SSTPythonModelDefinition::createConfigGraph()
         output->fatal(CALL_INFO, 1, "Error occured handling the creation of the component graph in Python.\n");
     }
 
+#if PY_MINOR_VERSION >= 9
+    // If coverage was enabled, stop the module and output the results
+    if ( enablePythonCoverage ) {
+        PyErr_Clear();
+        int stopcoverageReturn = PyRun_SimpleString("zzzSSTPythonCoverageModule.stop()\n"
+                                                    "zzzSSTPythonCoverageModule.save()\n");
+
+        if ( nullptr != PyErr_Occurred() ) {
+            // Print the Python error and then let SST exit as a fatal-stop.
+            output->fatal(
+                CALL_INFO, 1, "ERROR: Error occurred stopping coverage of the SST model script. Reported error:\n");
+            PyErr_Print();
+        }
+        if ( -1 == stopcoverageReturn ) {
+            output->fatal(CALL_INFO, 1, "Execution of stopping coverage failed\n%s", loadErrors.c_str());
+        }
+    }
+#endif
     return graph;
 }