Merge pull request #1259 from rabbitmq/rabbitmq-server-1223

Use system process rss memory when reporting vm memory

Author: michaelklishin

Makefile

@@ -16,6 +16,7 @@ define PROJECT_ENV
 	    {ssl_options, []},
 	    {vm_memory_high_watermark, 0.4},
 	    {vm_memory_high_watermark_paging_ratio, 0.5},
+	    {vm_memory_calculation_strategy, rss},
 	    {memory_monitor_interval, 2500},
 	    {disk_free_limit, 50000000}, %% 50MB
 	    {msg_store_index_module, rabbit_msg_store_ets_index},

src/rabbit_vm.erl

@@ -16,7 +16,7 @@
 
 -module(rabbit_vm).
 
--export([memory/0, binary/0, ets_tables_memory/1]).
+-export([memory/0, total_memory/0, binary/0, ets_tables_memory/1]).
 
 -define(MAGIC_PLUGINS, ["cowboy", "sockjs", "rfc4627_jsonrpc"]).
 
@@ -30,7 +30,6 @@
 
 %%----------------------------------------------------------------------------
 
-%% Like erlang:memory(), but with awareness of rabbit-y things
 memory() ->
     All = interesting_sups(),
     {Sums, _Other} = sum_processes(
@@ -41,7 +40,7 @@ memory() ->
         [aggregate(Names, Sums, memory, fun (X) -> X end)
          || Names <- distinguished_interesting_sups()],
 
-    Mnesia              = mnesia_memory(),
+    MnesiaETS           = mnesia_memory(),
     MsgIndexETS         = ets_memory([msg_store_persistent, msg_store_transient]),
     MetricsETS          = ets_memory([rabbit_metrics]),
     MetricsProc  = try
@@ -52,8 +51,10 @@ memory() ->
 			   0
 		   end,
     MgmtDbETS           = ets_memory([rabbit_mgmt_storage]),
+    OsTotal             = total_memory(),
 
-    [{total,     Total},
+
+    [{total,     ErlangTotal},
      {processes, Processes},
      {ets,       ETS},
      {atom,      Atom},
@@ -66,30 +67,137 @@ memory() ->
         - ConnsReader - ConnsWriter - ConnsChannel - ConnsOther
         - Qs - QsSlave - MsgIndexProc - Plugins - MgmtDbProc - MetricsProc,
 
-    [{total,              Total},
+    [
+     %% Connections
      {connection_readers,  ConnsReader},
      {connection_writers,  ConnsWriter},
      {connection_channels, ConnsChannel},
      {connection_other,    ConnsOther},
+
+     %% Queues
      {queue_procs,         Qs},
      {queue_slave_procs,   QsSlave},
+
+     %% Processes
      {plugins,             Plugins},
      {other_proc,          lists:max([0, OtherProc])}, %% [1]
-     {mnesia,              Mnesia},
+
+     %% Metrics
      {metrics,             MetricsETS + MetricsProc},
      {mgmt_db,             MgmtDbETS + MgmtDbProc},
-     {msg_index,           MsgIndexETS + MsgIndexProc},
-     {other_ets,           ETS - Mnesia - MsgIndexETS - MgmtDbETS},
+
+     %% ETS
+     {mnesia,              MnesiaETS},
+     {other_ets,           ETS - MnesiaETS - MetricsETS - MgmtDbETS - MsgIndexETS},
+
+     %% Messages (mostly, some binaries are not messages)
      {binary,              Bin},
+     {msg_index,           MsgIndexETS + MsgIndexProc},
+
+     %% System
      {code,                Code},
      {atom,                Atom},
-     {other_system,        System - ETS - Atom - Bin - Code}].
+     {other_system,        System - ETS - Bin - Code - Atom + (OsTotal - ErlangTotal)},
 
+     {total,               OsTotal}
+    ].
 %% [1] - erlang:memory(processes) can be less than the sum of its
 %% parts. Rather than display something nonsensical, just silence any
 %% claims about negative memory. See
 %% http://erlang.org/pipermail/erlang-questions/2012-September/069320.html
 
+%% Memory reported by erlang:memory(total) is not supposed to
+%% be equal to the total size of all pages mapped to the emulator,
+%% according to http://erlang.org/doc/man/erlang.html#memory-0
+%% erlang:memory(total) under-reports memory usage by around 20%
+-spec total_memory() -> Bytes :: integer().
+total_memory() ->
+    case get_memory_calculation_strategy() of
+        rss ->
+            case get_system_process_resident_memory() of
+                {ok, MemInBytes} ->
+                    MemInBytes;
+                {error, Reason}  ->
+                    rabbit_log:debug("Unable to get system memory used. Reason: ~p."
+                                     " Falling back to erlang memory reporting",
+                                     [Reason]),
+                    erlang:memory(total)
+            end;
+        erlang ->
+            erlang:memory(total)
+    end.
+
+-spec get_memory_calculation_strategy() -> rss | erlang.
+get_memory_calculation_strategy() ->
+    case application:get_env(rabbit, vm_memory_calculation_strategy, rss) of
+        erlang ->
+            erlang;
+        rss ->
+            rss;
+        UnsupportedValue ->
+            rabbit_log:warning(
+              "Unsupported value '~p' for vm_memory_calculation_strategy. "
+              "Supported values: (rss|erlang). "
+              "Defaulting to 'rss'",
+              [UnsupportedValue]
+            ),
+            rss
+    end.
+
+-spec get_system_process_resident_memory() -> {ok, Bytes :: integer()} | {error, term()}.
+get_system_process_resident_memory() ->
+    try
+        get_system_process_resident_memory(os:type())
+    catch _:Error ->
+            {error, {"Failed to get process resident memory", Error}}
+    end.
+
+get_system_process_resident_memory({unix,darwin}) ->
+    get_ps_memory();
+
+get_system_process_resident_memory({unix, linux}) ->
+    get_ps_memory();
+
+get_system_process_resident_memory({unix,freebsd}) ->
+    get_ps_memory();
+
+get_system_process_resident_memory({unix,openbsd}) ->
+    get_ps_memory();
+
+get_system_process_resident_memory({win32,_OSname}) ->
+    OsPid = os:getpid(),
+    Cmd = " tasklist /fi \"pid eq " ++ OsPid ++ "\" /fo LIST 2>&1 ",
+    CmdOutput = os:cmd(Cmd),
+    %% Memory usage is displayed in kilobytes
+    %% with comma-separated thousands
+    case re:run(CmdOutput, "Mem Usage:\\s+([0-9,]+)\\s+K", [{capture, all_but_first, list}]) of
+        {match, [Match]} ->
+            NoCommas = [ N || N <- Match, N =/= $, ],
+            {ok, list_to_integer(NoCommas) * 1024};
+        _ ->
+            {error, {unexpected_output_from_command, Cmd, CmdOutput}}
+    end;
+
+get_system_process_resident_memory({unix, sunos}) ->
+    get_ps_memory();
+
+get_system_process_resident_memory({unix, aix}) ->
+    get_ps_memory();
+
+get_system_process_resident_memory(_OsType) ->
+    {error, not_implemented_for_os}.
+
+get_ps_memory() ->
+    OsPid = os:getpid(),
+    Cmd = "ps -p " ++ OsPid ++ " -o rss=",
+    CmdOutput = os:cmd(Cmd),
+    case re:run(CmdOutput, "[0-9]+", [{capture, first, list}]) of
+        {match, [Match]} ->
+            {ok, list_to_integer(Match) * 1024};
+        _ ->
+            {error, {unexpected_output_from_command, Cmd, CmdOutput}}
+    end.
+
 binary() ->
     All = interesting_sups(),
     {Sums, Rest} =

src/vm_memory_monitor.erl

@@ -117,14 +117,14 @@ get_memory_limit() ->
 
 get_memory_use(bytes) ->
     MemoryLimit = get_memory_limit(),
-    {erlang:memory(total), case MemoryLimit > 0.0 of
-                               true  -> MemoryLimit;
-                               false -> infinity
-                           end};
+    {rabbit_vm:total_memory(), case MemoryLimit > 0.0 of
+                            true  -> MemoryLimit;
+                            false -> infinity
+                        end};
 get_memory_use(ratio) ->
     MemoryLimit = get_memory_limit(),
     case MemoryLimit > 0.0 of
-        true  -> erlang:memory(total) / MemoryLimit;
+        true  -> rabbit_vm:total_memory() / MemoryLimit;
         false -> infinity
     end.
 
@@ -268,7 +268,7 @@ parse_mem_limit(_) ->
 internal_update(State = #state { memory_limit = MemLimit,
                                  alarmed      = Alarmed,
                                  alarm_funs   = {AlarmSet, AlarmClear} }) ->
-    MemUsed = erlang:memory(total),
+    MemUsed = rabbit_vm:total_memory(),
     NewAlarmed = MemUsed > MemLimit,
     case {Alarmed, NewAlarmed} of
         {false, true} -> emit_update_info(set, MemUsed, MemLimit),
@@ -365,6 +365,7 @@ get_total_memory({unix, aix}) ->
 get_total_memory(_OsType) ->
     unknown.
 
+
 %% A line looks like "Foo bar: 123456."
 parse_line_mach(Line) ->
     [Name, RHS | _Rest] = string:tokens(Line, ":"),