diff --git a/CMakeLists.txt b/CMakeLists.txt index f96b5489f6..e82102e24b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ endif() # --- compiler flags if(MSVC) set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") - string(REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + string(REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") # Note that these settings are separately specified in configure.py, and # these lists should be kept in sync. add_compile_options(/W4 /wd4100 /wd4267 /wd4706 /wd4702 /wd4244 /GR- /Zc:__cplusplus) diff --git a/src/build.h b/src/build.h index 0531747be1..f97217d1e6 100644 --- a/src/build.h +++ b/src/build.h @@ -187,6 +187,7 @@ struct BuildConfig { int parallelism = 1; bool disable_jobserver_client = false; int failures_allowed = 1; + long desired_free_ram = 0; /// The maximum load average we must not exceed. A negative value /// means that we do not have any limit. double max_load_average = -0.0f; diff --git a/src/ninja.cc b/src/ninja.cc index 85ae6eb0b9..0a1a2e2e23 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -246,7 +246,8 @@ void Usage(const BuildConfig& config) { " -d MODE enable debugging (use '-d list' to list modes)\n" " -t TOOL run a subtool (use '-t list' to list subtools)\n" " terminates toplevel options; further flags are passed to the tool\n" -" -w FLAG adjust warnings (use '-w list' to list warnings)\n", +" -w FLAG adjust warnings (use '-w list' to list warnings)\n" +" --keep-free-memory AMOUNT the amount of ram required to start a job\n", kNinjaVersion, config.parallelism); } @@ -1684,18 +1685,36 @@ class DeferGuessParallelism { ~DeferGuessParallelism() { Refresh(); } }; +static inline long GetUnitToByteRatio(char unit) { + switch (unit) { + case 'G': + case 'g': + return 1e+9; + case 'M': + case 'm': + return 1e+6; + case 'K': + case 'k': + return 1e+3; + case 'B': + return 1; + } + return 0; +} + /// Parse argv for command-line options. /// Returns an exit code, or -1 if Ninja should continue. int ReadFlags(int* argc, char*** argv, Options* options, BuildConfig* config) { DeferGuessParallelism deferGuessParallelism(config); - enum { OPT_VERSION = 1, OPT_QUIET = 2 }; + enum { OPT_VERSION = 1, OPT_QUIET = 2, OPT_FREE_MEM = 3 }; const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, OPT_VERSION }, { "verbose", no_argument, NULL, 'v' }, { "quiet", no_argument, NULL, OPT_QUIET }, + { "keep-free-memory", required_argument, NULL, OPT_FREE_MEM}, { NULL, 0, NULL, 0 } }; @@ -1769,6 +1788,19 @@ int ReadFlags(int* argc, char*** argv, case OPT_VERSION: printf("%s\n", kNinjaVersion); return 0; + case OPT_FREE_MEM: { + char * end; + long value = strtol(optarg, &end, 10); + long multiplier = GetUnitToByteRatio(*end); + value *= multiplier; + //last char was the unit + if(multiplier > 0) + end++; + if (*end != 0 || value < 0 || multiplier == 0) + Fatal("invalid --keep-free-memory parameter"); + config->desired_free_ram = value; + break; + } case 'h': default: deferGuessParallelism.Refresh(); diff --git a/src/real_command_runner.cc b/src/real_command_runner.cc index 4a01276c0f..06e0ac482d 100644 --- a/src/real_command_runner.cc +++ b/src/real_command_runner.cc @@ -74,6 +74,13 @@ size_t RealCommandRunner::CanRunMore() const { capacity = load_capacity; } + //this just pause the creation of new processes when we run out of ram + if (config_.desired_free_ram > 0) { + long memory_capacity = GetFreeMemory() / config_.desired_free_ram; + if (memory_capacity < capacity) + capacity = memory_capacity; + } + if (capacity < 0) capacity = 0; diff --git a/src/util.cc b/src/util.cc index d88a6ba2ec..c23f31c7ab 100644 --- a/src/util.cc +++ b/src/util.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "util.h" +#include #ifdef __CYGWIN__ #include @@ -41,9 +42,11 @@ #include #include +#include #if defined(__APPLE__) || defined(__FreeBSD__) #include +#include #elif defined(__SVR4) && defined(__sun) #include #include @@ -1026,3 +1029,63 @@ int platformAwareUnlink(const char* filename) { return unlink(filename); #endif } + +#if defined(_WIN32) || defined(__CYGWIN__) +long GetFreeMemory() { + static long committed_idle = LONG_MAX; + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + GlobalMemoryStatusEx(&status); + const long committed = (status.ullTotalPageFile - status.ullAvailPageFile); + //since system use committed memory normaly, we store the smallest amount we have seen to guess how much + // paging is non-ninja related + committed_idle = std::min(committed_idle, committed); + + return status.ullAvailPhys - (committed - committed_idle); +} +#elif defined(__APPLE__) +long GetFreeMemory() { + static uint64_t swapped_idle = LONG_MAX; + vm_size_t page_size; + vm_statistics64_data_t vm_stats; + + mach_port_t port = mach_host_self(); + mach_msg_type_number_t count = sizeof(vm_stats) / sizeof(natural_t); + + size_t swap_stats_size = sizeof(xsw_usage); + xsw_usage swap_stats; + + int ctl[] = {CTL_VM, VM_SWAPUSAGE}; + + int result = host_page_size(port, &page_size); + result += host_statistics64(port, HOST_VM_INFO, + reinterpret_cast(&vm_stats), &count); + result -= sysctl(ctl, 2, &swap_stats, &swap_stats_size, NULL, 0); + + if (KERN_SUCCESS != result) + //information not available + return std::numeric_limits::max(); + + //inactive is memory is marked to be moved to swap or is fs cache and should be considered as free + int64_t free_memory = (vm_stats.free_count + vm_stats.inactive_count ) * page_size; + //since inactive memory can be moved to the swap this value will be inexact, and i don't belive that there is a clean solution for that. + uint64_t used_swap = swap_stats.xsu_used; + swapped_idle = std::min(used_swap, swapped_idle); + return free_memory - (used_swap - swapped_idle); +} +#elif defined(__linux__) +long GetFreeMemory() { + static long swapped_idle = LONG_MAX; + struct sysinfo infos; + sysinfo(&infos); + const long swapped = (infos.totalswap - infos.freeswap); + //since system use commited memory normaly, we store the smallest amount we have seen to guess how much + // paging is non-ninja related + swapped_idle = std::min(swapped_idle, swapped); + return infos.freeram - (swapped - swapped_idle); +} +#else +long GetFreeMemory() { + return std::numeric_limits::max(); +} +#endif diff --git a/src/util.h b/src/util.h index 02c2418396..5e2c250ae1 100644 --- a/src/util.h +++ b/src/util.h @@ -102,6 +102,11 @@ int GetProcessorCount(); /// on error. double GetLoadAverage(); +//Swapped memory is very likely caused by the compiler process running out of ram, +//as such we consider swapped memory as used memory, therefore negative values are to be expected +/// @return the Available memory on the machine. If no supported it will return LONG_MAX +long GetFreeMemory(); + /// a wrapper for getcwd() std::string GetWorkingDirectory();