Prevent ninja starting new job when physical memory become low.

CMakeLists.txt

@@ -22,7 +22,7 @@ endif()
 # --- compiler flags
 if(MSVC)
 	set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
-	string(REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+	string(REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 	# Note that these settings are separately specified in configure.py, and
 	# these lists should be kept in sync.
 	add_compile_options(/W4 /wd4100 /wd4267 /wd4706 /wd4702 /wd4244 /GR- /Zc:__cplusplus)

src/build.h

@@ -187,6 +187,7 @@ struct BuildConfig {
   int parallelism = 1;
   bool disable_jobserver_client = false;
   int failures_allowed = 1;
+  long desired_free_ram = 0;
   /// The maximum load average we must not exceed. A negative value
   /// means that we do not have any limit.
   double max_load_average = -0.0f;

src/ninja.cc

@@ -246,7 +246,8 @@ void Usage(const BuildConfig& config) {
 "  -d MODE  enable debugging (use '-d list' to list modes)\n"
 "  -t TOOL  run a subtool (use '-t list' to list subtools)\n"
 "    terminates toplevel options; further flags are passed to the tool\n"
-"  -w FLAG  adjust warnings (use '-w list' to list warnings)\n",
+"  -w FLAG  adjust warnings (use '-w list' to list warnings)\n"
+"  --keep-free-memory AMOUNT the amount of ram required to start a job\n",
           kNinjaVersion, config.parallelism);
 }
 
@@ -1684,18 +1685,36 @@ class DeferGuessParallelism {
   ~DeferGuessParallelism() { Refresh(); }
 };
 
+static inline long GetUnitToByteRatio(char unit) {
+  switch (unit) {
+  case 'G':
+  case 'g':
+    return 1e+9;
+  case 'M':
+  case 'm':
+    return 1e+6;
+  case 'K':
+  case 'k':
+    return 1e+3;
+  case 'B':
+    return 1;
+  }
+  return 0;
+}
+
 /// Parse argv for command-line options.
 /// Returns an exit code, or -1 if Ninja should continue.
 int ReadFlags(int* argc, char*** argv,
               Options* options, BuildConfig* config) {
   DeferGuessParallelism deferGuessParallelism(config);
 
-  enum { OPT_VERSION = 1, OPT_QUIET = 2 };
+  enum { OPT_VERSION = 1, OPT_QUIET = 2, OPT_FREE_MEM = 3 };
   const option kLongOptions[] = {
     { "help", no_argument, NULL, 'h' },
     { "version", no_argument, NULL, OPT_VERSION },
     { "verbose", no_argument, NULL, 'v' },
     { "quiet", no_argument, NULL, OPT_QUIET },
+    { "keep-free-memory", required_argument, NULL, OPT_FREE_MEM},
     { NULL, 0, NULL, 0 }
   };
 
@@ -1769,6 +1788,19 @@ int ReadFlags(int* argc, char*** argv,
       case OPT_VERSION:
         printf("%s\n", kNinjaVersion);
         return 0;
+      case OPT_FREE_MEM: {
+        char * end;
+        long value = strtol(optarg, &end, 10);
+        long multiplier = GetUnitToByteRatio(*end);
+        value *= multiplier;
+        //last char was the unit
+        if(multiplier > 0)
+          end++;
+        if (*end != 0 || value < 0 || multiplier == 0)
+          Fatal("invalid --keep-free-memory parameter");
+        config->desired_free_ram = value;
+        break;
+      }
       case 'h':
       default:
         deferGuessParallelism.Refresh();

src/real_command_runner.cc

@@ -74,6 +74,13 @@ size_t RealCommandRunner::CanRunMore() const {
       capacity = load_capacity;
   }
 
+  //this just pause the creation of new processes when we run out of ram
+  if (config_.desired_free_ram > 0) {
+    long memory_capacity = GetFreeMemory() / config_.desired_free_ram;
+    if (memory_capacity < capacity)
+      capacity = memory_capacity;
+  }
+
   if (capacity < 0)
     capacity = 0;
 

src/util.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "util.h"
+#include <cstddef>
 
 #ifdef __CYGWIN__
 #include <windows.h>
@@ -41,9 +42,11 @@
 
 #include <algorithm>
 #include <vector>
+#include <climits>
 
 #if defined(__APPLE__) || defined(__FreeBSD__)
 #include <sys/sysctl.h>
+#include <mach/mach.h>
 #elif defined(__SVR4) && defined(__sun)
 #include <unistd.h>
 #include <sys/loadavg.h>
@@ -1026,3 +1029,63 @@ int platformAwareUnlink(const char* filename) {
 		return unlink(filename);
 	#endif
 }
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+long GetFreeMemory() {
+  static long committed_idle = LONG_MAX;
+  MEMORYSTATUSEX status;
+  status.dwLength = sizeof(status);
+  GlobalMemoryStatusEx(&status);
+  const long committed = (status.ullTotalPageFile - status.ullAvailPageFile);
+  //since system use committed memory normaly, we store the smallest amount we have seen to guess how much
+  // paging is non-ninja related
+  committed_idle = std::min(committed_idle, committed);
+
+  return status.ullAvailPhys - (committed - committed_idle);
+}
+#elif defined(__APPLE__)
+long GetFreeMemory() {
+  static uint64_t swapped_idle = LONG_MAX;
+  vm_size_t page_size;
+  vm_statistics64_data_t vm_stats;
+
+  mach_port_t port = mach_host_self();
+  mach_msg_type_number_t count = sizeof(vm_stats) / sizeof(natural_t);
+
+  size_t swap_stats_size = sizeof(xsw_usage);
+  xsw_usage swap_stats;
+
+  int ctl[] = {CTL_VM, VM_SWAPUSAGE};
+
+  int result = host_page_size(port, &page_size);
+  result += host_statistics64(port, HOST_VM_INFO,
+    reinterpret_cast<host_info64_t>(&vm_stats), &count);
+  result -= sysctl(ctl, 2, &swap_stats, &swap_stats_size, NULL, 0);
+
+  if (KERN_SUCCESS != result)
+    //information not available
+    return std::numeric_limits<long>::max();
+
+  //inactive is memory is marked to be moved to swap or is fs cache and should be considered as free
+  int64_t free_memory = (vm_stats.free_count + vm_stats.inactive_count ) * page_size;
+  //since inactive memory can be moved to the swap this value will be inexact, and i don't belive that there is a clean solution for that.
+  uint64_t used_swap = swap_stats.xsu_used;
+  swapped_idle = std::min(used_swap, swapped_idle);
+  return free_memory - (used_swap - swapped_idle);
+}
+#elif defined(__linux__)
+long GetFreeMemory() {
+  static long swapped_idle = LONG_MAX;
+  struct sysinfo infos;
+  sysinfo(&infos);
+  const long swapped = (infos.totalswap - infos.freeswap);
+  //since system use commited memory normaly, we store the smallest amount we have seen to guess how much
+  // paging is non-ninja related
+  swapped_idle = std::min(swapped_idle, swapped);
+  return infos.freeram - (swapped - swapped_idle);
+}
+#else
+long GetFreeMemory() {
+  return std::numeric_limits<long>::max();
+}
+#endif

src/util.h

@@ -102,6 +102,11 @@ int GetProcessorCount();
 /// on error.
 double GetLoadAverage();
 
+//Swapped memory is very likely caused by the compiler process running out of ram,
+//as such we consider swapped memory as used memory, therefore negative values are to be expected
+/// @return the Available memory on the machine. If no supported it will return LONG_MAX
+long GetFreeMemory();
+
 /// a wrapper for getcwd()
 std::string GetWorkingDirectory();