[{roc,hip}fft] Revisions to accounting of system memory usage and its limit-enforcing logic

Author: regan-amd

projects/hipfft/clients/tests/gtest_main.cpp

@@ -490,7 +490,7 @@ int main(int argc, char* argv[])
         "--version",
         "Print queryable version information from the hipfft library's backend (and return)");
     app.add_option("--R", ramgb, "RAM limit in GiB for tests")
-        ->default_val(host_memory::singleton().get_total_gbytes());
+        ->default_val(system_memory::singleton().get_total_gbytes());
     app.add_option("--V", vramgb, "VRAM limit in GiB for tests")->default_val(0);
     app.add_option("--half_epsilon", half_epsilon)->default_val(9.77e-4);
     app.add_option("--single_epsilon", single_epsilon)->default_val(3.75e-5);
@@ -635,10 +635,9 @@ int main(int argc, char* argv[])
 #endif
 
     // Set host memory limit from command-line options (if more restrictive)
-    const auto usable_bytes = host_memory::singleton().get_usable_bytes();
-    if(ramgb * ONE_GiB < usable_bytes)
-        host_memory::singleton().set_limit_gbytes(ramgb);
-    std::cout << "Usable host memory: " << bytes_to_GiB(host_memory::singleton().get_usable_bytes())
+    if(ramgb * ONE_GiB < system_memory::singleton().get_limit_bytes())
+        system_memory::singleton().set_limit_gbytes(ramgb);
+    std::cout << "Usable system memory: " << system_memory::singleton().get_usable_gbytes()
               << " GiB" << std::endl;
 
     if(use_fftw_wisdom)

projects/hipfft/clients/tests/hipfft_accuracy_test.cpp

@@ -153,7 +153,7 @@ TEST_P(accuracy_test, vs_fftw)
             last_cpu_fft_data = last_cpu_fft_cache();
             GTEST_SKIP() << "host memory allocation failure";
         }
-        catch(const HOSTBUF_MEM_USAGE& e)
+        catch(const SYS_MEM_USAGE& e)
         {
             // explicitly clear cache
             last_cpu_fft_data = last_cpu_fft_cache();

projects/hipfft/clients/tests/hipfftw_test.cpp

@@ -1983,7 +1983,7 @@ namespace
                         plan_execution_output.alloc(output_data_size);
                 }
             }
-            catch(const HOSTBUF_MEM_USAGE& e)
+            catch(const SYS_MEM_USAGE& e)
             {
                 GTEST_SKIP() << e.what();
             }
@@ -2739,7 +2739,7 @@ namespace
                 else
                     GTEST_FAIL() << e.what() << "\nError code: " << e.hip_error << ".";
             }
-            catch(const HOSTBUF_MEM_USAGE& e)
+            catch(const SYS_MEM_USAGE& e)
             {
                 GTEST_SKIP() << e.what();
             }

projects/hipfft/clients/tests/multi_stream_test.cpp

@@ -623,7 +623,7 @@ TEST_P(multiStreamTest, impulseSignalOnOutput)
     {
         GTEST_SKIP() << "host memory allocation failure";
     }
-    catch(const HOSTBUF_MEM_USAGE& e)
+    catch(const SYS_MEM_USAGE& e)
     {
         GTEST_SKIP() << e.what();
     }

projects/hipfft/shared/gpubuf.h

@@ -21,8 +21,11 @@
 #ifndef ROCFFT_GPUBUF_H
 #define ROCFFT_GPUBUF_H
 
+#include "device_properties.h"
 #include "rocfft_hip.h"
+#include "sys_mem.h"
 #include <cstdlib>
+#include <sstream>
 
 // Simple RAII class for GPU buffers.  T is the type of pointer that
 // data() returns
@@ -74,6 +77,19 @@ class gpubuf_t
 
     hipError_t alloc(const size_t size, bool make_it_shared = false)
     {
+        free();
+        const auto dev_prop = get_curr_device_prop();
+        if(dev_prop.integrated && size > system_memory::singleton().get_usable_bytes())
+        {
+            std::stringstream msg;
+            msg << "Unauthorized (integrated) device allocation.\n"
+                << "\tRequested byte size is " << size << "\n"
+                << "\tUsable byte size is " << system_memory::singleton().get_usable_bytes() << "\n"
+                << "\tFree system memory: " << system_memory::singleton().get_free_bytes() << "\n"
+                << "\tUsed system memory: " << system_memory::singleton().get_used_bytes() << "\n"
+                << "\tEnforced usage limit: " << system_memory::singleton().get_limit_bytes();
+            throw SYS_MEM_USAGE{msg.str()};
+        }
         // remember the device that was current as of alloc, so we can
         // free on the correct device
         auto ret = hipGetDevice(&device);
@@ -82,13 +98,16 @@ class gpubuf_t
 
         bsize             = size;
         is_managed_memory = use_alloc_managed() || make_it_shared;
-        free();
         ret = is_managed_memory ? hipMallocManaged(&buf, bsize) : hipMalloc(&buf, bsize);
         if(ret != hipSuccess)
         {
             buf   = nullptr;
             bsize = 0;
         }
+
+        if(dev_prop.integrated)
+            system_memory::singleton().record_used_bytes(bsize);
+
         return ret;
     }
 
@@ -106,6 +125,10 @@ class gpubuf_t
                 // free on the device we allocated on
                 rocfft_scoped_device dev(device);
                 (void)hipFree(buf);
+
+                const auto dev_prop = get_curr_device_prop();
+                if(dev_prop.integrated)
+                    system_memory::singleton().release_used_bytes(bsize);
             }
             buf   = nullptr;
             bsize = 0;

projects/hipfft/shared/hostbuf.h

@@ -23,22 +23,17 @@
 
 #include "arithmetic.h"
 #include "sys_mem.h"
-#include <atomic>
 #include <cstdlib>
 #include <cstring>
-#include <iostream>
+#include <sstream>
+
 #include <new>
 
 #ifndef _WIN32
 #include <stdlib.h>
 #include <sys/mman.h>
 #endif
 
-struct HOSTBUF_MEM_USAGE : public std::runtime_error
-{
-    using std::runtime_error::runtime_error;
-};
-
 // Simple RAII class for host buffers.  T is the type of pointer that
 // data() returns
 template <class T = void>
@@ -87,18 +82,19 @@ class hostbuf_t
     {
         free();
 
-        bsize = size;
-
-        auto usable_mem = host_memory::singleton().get_usable_bytes();
-        if(total_used_mem + size > usable_mem)
+        if(size > system_memory::singleton().get_usable_bytes())
         {
             std::stringstream msg;
-            msg << "Host memory usage limit exceed (used mem: "
-                << bytes_to_GiB(total_used_mem + size)
-                << "GiB, free mem: " << bytes_to_GiB(usable_mem) << " GiB)";
-            throw HOSTBUF_MEM_USAGE{msg.str()};
+            msg << "Unauthorized host allocation.\n"
+                << "\tRequested byte size is " << size << "\n"
+                << "\tUsable byte size is " << system_memory::singleton().get_usable_bytes() << "\n"
+                << "\tFree system memory: " << system_memory::singleton().get_free_bytes() << "\n"
+                << "\tUsed system memory: " << system_memory::singleton().get_used_bytes() << "\n"
+                << "\tEnforced usage limit: " << system_memory::singleton().get_limit_bytes();
+            throw SYS_MEM_USAGE{msg.str()};
         }
 
+        bsize = size;
         if(make_it_pinned)
         {
             if(hipHostMalloc(&buf, size) != hipSuccess)
@@ -142,7 +138,7 @@ class hostbuf_t
 
         is_pinned_memory = make_it_pinned;
         bsize_track      = size;
-        total_used_mem += bsize_track;
+        system_memory::singleton().record_used_bytes(bsize_track);
     }
 
     size_t size() const
@@ -161,7 +157,6 @@ class hostbuf_t
         {
             if(owned)
             {
-                total_used_mem -= bsize_track;
                 if(is_pinned_memory)
                 {
                     (void)hipHostFree(buf);
@@ -174,6 +169,7 @@ class hostbuf_t
                     std::free(buf);
 #endif
                 }
+                system_memory::singleton().release_used_bytes(bsize_track);
             }
             buf   = nullptr;
             bsize = bsize_track = 0;
@@ -238,9 +234,6 @@ class hostbuf_t
     // Buffer size for tracking total memory usage.
     // When buffer is shrunk in place, bsize_track is not changed.
     size_t bsize_track = 0;
-
-    // Keeps track of total used memory for all hostbufs
-    inline static std::atomic<size_t> total_used_mem = 0;
 };
 
 // default hostbuf that gives out void* pointers