[libomptarget] [ompt] Added envars controlling buffer flush.

LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL and
LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN are added to control when
OMPT buffers with trace records will be flushed by the runtime.
By default, both envars are true.

Author: dhruvachak

offload/include/OpenMP/OMPT/OmptTracingBuffer.h

@@ -31,6 +31,8 @@
 
 #include <omp-tools.h>
 
+#include "Shared/EnvironmentVar.h"
+
 // Maximum number of devices supported in device tracing. No device tracing
 // will be performed for any device-id larger than 1023.
 #define MAX_NUM_DEVICES 1024
@@ -102,6 +104,12 @@ class OmptTracingBufferMgr {
   using BufPtr = std::shared_ptr<Buffer>;
 
 private:
+  /// Envar to control whether a buffer should be flushed when it gets full.
+  BoolEnvar OMPX_FlushOnBufferFull;
+
+  /// Envar to control whether all buffers should be flushed during shutdown.
+  BoolEnvar OMPX_FlushOnShutdown;
+
   // Internal variable for tracking threads to wait for flush
   uint32_t ThreadFlushTracker;
 
@@ -216,8 +224,8 @@ class OmptTracingBufferMgr {
 
   /// Called when a buffer \p Buf may be flushed with \p Cursor as the
   /// last allocated trace record in the buffer.
-  /// setComplete should be called without holding any lock.
-  void setComplete(void *Cursor, BufPtr Buf);
+  /// triggerFlushOnBufferFull should be called without holding any lock.
+  void triggerFlushOnBufferFull(void *Cursor, BufPtr Buf);
 
   // Called to dispatch buffer-completion callbacks for the trace records in
   // this buffer
@@ -350,7 +358,14 @@ class OmptTracingBufferMgr {
   void destroyHelperThreads();
 
 public:
-  OmptTracingBufferMgr();
+  OmptTracingBufferMgr()
+      : OMPX_FlushOnBufferFull("LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL", true),
+        OMPX_FlushOnShutdown("LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN", true) {
+    // no need to hold locks for init() since object is getting constructed
+    // here.
+    init();
+  }
+
   OmptTracingBufferMgr(const OmptTracingBufferMgr &) = delete;
   OmptTracingBufferMgr &operator=(const OmptTracingBufferMgr &) = delete;
 

offload/src/OpenMP/OMPT/OmptTracingBuffer.cpp

@@ -129,8 +129,8 @@ void *OmptTracingBufferMgr::assignCursor(ompt_callbacks_t Type,
   lck.unlock();
 
   // Schedule the full buffer for flushing till the corresponding cursor.
-  if (ToBeFlushedCursor)
-    setComplete(ToBeFlushedCursor, ToBeFlushedBuf);
+  if (OMPX_FlushOnBufferFull && ToBeFlushedCursor)
+    triggerFlushOnBufferFull(ToBeFlushedCursor, ToBeFlushedBuf);
 
   DP("Thread %lu: Assigned %lu bytes at %p in new buffer with id %lu for "
      "device %ld\n",
@@ -148,10 +148,10 @@ void *OmptTracingBufferMgr::assignCursor(ompt_callbacks_t Type,
  * called without holding any lock.
  * Note lock order: buf_lock -> flush_lock
  */
-void OmptTracingBufferMgr::setComplete(void *cursor, BufPtr Buf) {
+void OmptTracingBufferMgr::triggerFlushOnBufferFull(void *cursor, BufPtr Buf) {
   std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
 
-  // Between calling setComplete and this check, a flush-all may have
+  // Between calling this function and this check, a flush-all may have
   // delivered this buffer to the tool and deleted it. So the buffer
   // may not exist.
   if (Id2BufferMap.find(Buf->Id) == Id2BufferMap.end())
@@ -729,7 +729,8 @@ void OmptTracingBufferMgr::shutdownHelperThreads() {
 void OmptTracingBufferMgr::flushAndShutdownHelperThreads() {
   std::unique_lock<std::mutex> Lock(llvm::omp::target::ompt::TraceControlMutex);
   // Flush buffers for all devices.
-  flushAllBuffers(MAX_NUM_DEVICES);
+  if (OMPX_FlushOnShutdown)
+    flushAllBuffers(MAX_NUM_DEVICES);
   shutdownHelperThreads();
 }
 
@@ -747,9 +748,4 @@ void OmptTracingBufferMgr::destroyHelperThreads() {
   CompletionThreads.clear();
   HelperThreadIdMap.clear();
 }
-
-OmptTracingBufferMgr::OmptTracingBufferMgr() {
-  // no need to hold locks for init() since object is getting constructed here
-  init();
-}
 #endif