Provide a mechanism for obtaining memory profiles of daemons and application profiles for use in studying our memory footprint. Setting OMPI_MEMPROFILE=N causes mpirun to set a timer for N seconds. When the timer fires, mpirun will query each daemon in the job to report its own memory usage plus the average memory usage of its child processes. The Proportional Set Size (PSS) is used for this purpose.

Author: Ralph Castain

opal/dss/dss_copy.c

@@ -219,6 +219,7 @@ int opal_dss_copy_pstat(opal_pstats_t **dest, opal_pstats_t *src,
     p->time = src->time;
     p->priority = src->priority;
     p->num_threads = src->num_threads;
+    p->pss = src->pss;
     p->vsize = src->vsize;
     p->rss = src->rss;
     p->peak_vsize = src->peak_vsize;

opal/dss/dss_open_close.c

@@ -156,6 +156,7 @@ static void opal_pstat_construct(opal_pstats_t *obj)
     obj->time.tv_usec = 0;
     obj->priority = -1;
     obj->num_threads = -1;
+    obj->pss = 0.0;
     obj->vsize = 0.0;
     obj->rss = 0.0;
     obj->peak_vsize = 0.0;

opal/dss/dss_pack.c

@@ -499,6 +499,9 @@ int opal_dss_pack_pstat(opal_buffer_t *buffer, const void *src,
         if (OPAL_SUCCESS != (ret = opal_dss_pack_buffer(buffer, &ptr[i]->num_threads, 1, OPAL_INT16))) {
             return ret;
         }
+        if (OPAL_SUCCESS != (ret = opal_dss_pack_float(buffer, &ptr[i]->pss, 1, OPAL_FLOAT))) {
+            return ret;
+        }
         if (OPAL_SUCCESS != (ret = opal_dss_pack_float(buffer, &ptr[i]->vsize, 1, OPAL_FLOAT))) {
             return ret;
         }

opal/dss/dss_print.c

@@ -654,10 +654,10 @@ int opal_dss_print_pstat(char **output, char *prefix, opal_pstats_t *src, opal_d
         return OPAL_SUCCESS;
     }
     asprintf(output, "%sOPAL_PSTATS SAMPLED AT: %ld.%06ld\n%snode: %s rank: %d pid: %d cmd: %s state: %c pri: %d #threads: %d Processor: %d\n"
-             "%s\ttime: %ld.%06ld cpu: %5.2f VMsize: %8.2f PeakVMSize: %8.2f RSS: %8.2f\n",
+             "%s\ttime: %ld.%06ld cpu: %5.2f  PSS: %8.2f  VMsize: %8.2f PeakVMSize: %8.2f RSS: %8.2f\n",
              prefx, (long)src->sample_time.tv_sec, (long)src->sample_time.tv_usec,
              prefx, src->node, src->rank, src->pid, src->cmd, src->state[0], src->priority, src->num_threads, src->processor,
-             prefx, (long)src->time.tv_sec, (long)src->time.tv_usec, src->percent_cpu, src->vsize, src->peak_vsize, src->rss);
+             prefx, (long)src->time.tv_sec, (long)src->time.tv_usec, src->percent_cpu, src->pss, src->vsize, src->peak_vsize, src->rss);
     if (prefx != prefix) {
         free(prefx);
     }

opal/dss/dss_types.h

@@ -182,6 +182,7 @@ typedef struct {
     float percent_cpu;
     int32_t priority;
     int16_t num_threads;
+    float pss;   /* in MBytes */
     float vsize;  /* in MBytes */
     float rss;  /* in MBytes */
     float peak_vsize;  /* in MBytes */

opal/dss/dss_unpack.c

@@ -643,6 +643,11 @@ int opal_dss_unpack_pstat(opal_buffer_t *buffer, void *dest,
             return ret;
         }
         m=1;
+        if (OPAL_SUCCESS != (ret = opal_dss_unpack_float(buffer, &ptr[i]->pss, &m, OPAL_FLOAT))) {
+            OPAL_ERROR_LOG(ret);
+            return ret;
+        }
+        m=1;
         if (OPAL_SUCCESS != (ret = opal_dss_unpack_float(buffer, &ptr[i]->vsize, &m, OPAL_FLOAT))) {
             OPAL_ERROR_LOG(ret);
             return ret;

opal/mca/pstat/linux/pstat_linux_module.c

@@ -310,6 +310,31 @@ static int query(pid_t pid,
             }
         }
         fclose(fp);
+
+        /* now create the smaps filename for this proc */
+        memset(data, 0, sizeof(data));
+        numchars = snprintf(data, sizeof(data), "/proc/%d/smaps", pid);
+        if (numchars >= sizeof(data)) {
+            return OPAL_ERR_VALUE_OUT_OF_BOUNDS;
+        }
+
+        if (NULL == (fp = fopen(data, "r"))) {
+            /* ignore this */
+            return OPAL_SUCCESS;
+        }
+
+        /* parse it to find lines that start with "Pss" */
+        while (NULL != (dptr = local_getline(fp))) {
+            if (NULL == (value = local_stripper(dptr))) {
+                /* cannot process */
+                continue;
+            }
+            /* look for Pss */
+            if (0 == strncmp(dptr, "Pss", strlen("Pss"))) {
+                stats->pss += convert_value(value);
+            }
+        }
+        fclose(fp);
     }
 
     if (NULL != nstats) {

orte/mca/odls/odls_types.h

@@ -83,6 +83,9 @@ typedef uint8_t orte_daemon_cmd_flag_t;
 /* for debug purposes, get stack traces from all application procs */
 #define ORTE_DAEMON_GET_STACK_TRACES        (orte_daemon_cmd_flag_t) 31
 
+/* for memory profiling */
+#define ORTE_DAEMON_GET_MEMPROFILE          (orte_daemon_cmd_flag_t) 32
+
 /*
  * Struct written up the pipe from the child to the parent.
  */

orte/mca/rml/rml_types.h

@@ -166,6 +166,9 @@ BEGIN_C_DECLS
 /* stacktrace for debug */
 #define ORTE_RML_TAG_STACK_TRACE            60
 
+/* memory profile */
+#define ORTE_RML_TAG_MEMPROFILE             61
+
 #define ORTE_RML_TAG_MAX                   100
 
 #define ORTE_RML_TAG_NTOH(t) ntohl(t)

orte/orted/orted_comm.c

@@ -45,6 +45,7 @@
 
 #include "opal/mca/event/event.h"
 #include "opal/mca/base/base.h"
+#include "opal/mca/pstat/pstat.h"
 #include "opal/util/output.h"
 #include "opal/util/opal_environ.h"
 #include "opal/util/path.h"
@@ -115,6 +116,8 @@ void orte_daemon_recv(int status, orte_process_name_t* sender,
     FILE *fp;
     char gscmd[256], path[1035], *pathptr;
     char string[256], *string_ptr = string;
+    float pss;
+    opal_pstats_t pstat;
 
     /* unpack the command */
     n = 1;
@@ -1151,6 +1154,44 @@ void orte_daemon_recv(int status, orte_process_name_t* sender,
         }
         break;
 
+    case ORTE_DAEMON_GET_MEMPROFILE:
+        answer = OBJ_NEW(opal_buffer_t);
+        /* pack our hostname so they know where it came from */
+        opal_dss.pack(answer, &orte_process_info.nodename, 1, OPAL_STRING);
+        /* collect my memory usage */
+        OBJ_CONSTRUCT(&pstat, opal_pstats_t);
+        opal_pstat.query(orte_process_info.pid, &pstat, NULL);
+        opal_dss.pack(answer, &pstat.pss, 1, OPAL_FLOAT);
+        OBJ_DESTRUCT(&pstat);
+        /* collect the memory usage of all my children */
+        pss = 0.0;
+        num_replies = 0;
+        for (i=0; i < orte_local_children->size; i++) {
+            if (NULL != (proct = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i)) &&
+                ORTE_FLAG_TEST(proct, ORTE_PROC_FLAG_ALIVE)) {
+                /* collect the stats on this proc */
+                OBJ_CONSTRUCT(&pstat, opal_pstats_t);
+                if (OPAL_SUCCESS == opal_pstat.query(proct->pid, &pstat, NULL)) {
+                    pss += pstat.pss;
+                    ++num_replies;
+                }
+                OBJ_DESTRUCT(&pstat);
+            }
+        }
+        /* compute the average value */
+        if (0 < num_replies) {
+            pss /= (float)num_replies;
+        }
+        opal_dss.pack(answer, &pss, 1, OPAL_FLOAT);
+        /* send it back */
+        if (0 > (ret = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, answer,
+                                               ORTE_RML_TAG_MEMPROFILE,
+                                               orte_rml_send_callback, NULL))) {
+            ORTE_ERROR_LOG(ret);
+            OBJ_RELEASE(answer);
+        }
+        break;
+
     default:
         ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
     }
@@ -1222,6 +1263,9 @@ static char *get_orted_comm_cmd_str(int command)
     case ORTE_DAEMON_GET_STACK_TRACES:
         return strdup("ORTE_DAEMON_GET_STACK_TRACES");
 
+    case ORTE_DAEMON_GET_MEMPROFILE:
+        return strdup("ORTE_DAEMON_GET_MEMPROFILE");
+
     default:
         return strdup("Unknown Command!");
     }