Skip to content

Commit a0151e4

Browse files
Daniil FedotovGerhard Lazu
Daniil Fedotov
authored and
Gerhard Lazu
committed
Move function that reports total used memory
Tests were failing because they required vm_memory_monitor process to be running re #1259 [#145451399] Signed-off-by: Gerhard Lazu <[email protected]>
1 parent bc0309d commit a0151e4

File tree

2 files changed

+97
-100
lines changed

2 files changed

+97
-100
lines changed

‎src/rabbit_vm.erl‎

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
-module(rabbit_vm).
1818

19-
-export([memory/0, binary/0, ets_tables_memory/1]).
19+
-export([memory/0, total_memory/0, binary/0, ets_tables_memory/1]).
2020

2121
-define(MAGIC_PLUGINS, ["cowboy", "sockjs", "rfc4627_jsonrpc"]).
2222

@@ -51,7 +51,7 @@ memory() ->
5151
0
5252
end,
5353
MgmtDbETS = ets_memory([rabbit_mgmt_storage]),
54-
{OsTotal, _} = vm_memory_monitor:get_memory_use(bytes),
54+
OsTotal = total_memory(),
5555

5656

5757
[{total, ErlangTotal},
@@ -101,12 +101,103 @@ memory() ->
101101

102102
{total, OsTotal}
103103
].
104-
105104
%% [1] - erlang:memory(processes) can be less than the sum of its
106105
%% parts. Rather than display something nonsensical, just silence any
107106
%% claims about negative memory. See
108107
%% http://erlang.org/pipermail/erlang-questions/2012-September/069320.html
109108

109+
%% Memory reported by erlang:memory(total) is not supposed to
110+
%% be equal to the total size of all pages mapped to the emulator,
111+
%% according to http://erlang.org/doc/man/erlang.html#memory-0
112+
%% erlang:memory(total) under-reports memory usage by around 20%
113+
-spec total_memory() -> Bytes :: integer().
114+
total_memory() ->
115+
case get_memory_calculation_strategy() of
116+
rss ->
117+
case get_system_process_resident_memory() of
118+
{ok, MemInBytes} ->
119+
MemInBytes;
120+
{error, Reason} ->
121+
rabbit_log:debug("Unable to get system memory used. Reason: ~p."
122+
" Falling back to erlang memory reporting",
123+
[Reason]),
124+
erlang:memory(total)
125+
end;
126+
erlang ->
127+
erlang:memory(total)
128+
end.
129+
130+
-spec get_memory_calculation_strategy() -> rss | erlang.
131+
get_memory_calculation_strategy() ->
132+
case application:get_env(rabbit, vm_memory_calculation_strategy, rss) of
133+
erlang ->
134+
erlang;
135+
rss ->
136+
rss;
137+
UnsupportedValue ->
138+
rabbit_log:warning(
139+
"Unsupported value '~p' for vm_memory_calculation_strategy. "
140+
"Supported values: (rss|erlang). "
141+
"Defaulting to 'rss'",
142+
[UnsupportedValue]
143+
),
144+
rss
145+
end.
146+
147+
-spec get_system_process_resident_memory() -> {ok, Bytes :: integer()} | {error, term()}.
148+
get_system_process_resident_memory() ->
149+
try
150+
get_system_process_resident_memory(os:type())
151+
catch _:Error ->
152+
{error, {"Failed to get process resident memory", Error}}
153+
end.
154+
155+
get_system_process_resident_memory({unix,darwin}) ->
156+
get_ps_memory();
157+
158+
get_system_process_resident_memory({unix, linux}) ->
159+
get_ps_memory();
160+
161+
get_system_process_resident_memory({unix,freebsd}) ->
162+
get_ps_memory();
163+
164+
get_system_process_resident_memory({unix,openbsd}) ->
165+
get_ps_memory();
166+
167+
get_system_process_resident_memory({win32,_OSname}) ->
168+
OsPid = os:getpid(),
169+
Cmd = " tasklist /fi \"pid eq " ++ OsPid ++ "\" /fo LIST 2>&1 ",
170+
CmdOutput = os:cmd(Cmd),
171+
%% Memory usage is displayed in kilobytes
172+
%% with comma-separated thousands
173+
case re:run(CmdOutput, "Mem Usage:\\s+([0-9,]+)\\s+K", [{capture, all_but_first, list}]) of
174+
{match, [Match]} ->
175+
NoCommas = [ N || N <- Match, N =/= $, ],
176+
{ok, list_to_integer(NoCommas) * 1024};
177+
_ ->
178+
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
179+
end;
180+
181+
get_system_process_resident_memory({unix, sunos}) ->
182+
get_ps_memory();
183+
184+
get_system_process_resident_memory({unix, aix}) ->
185+
get_ps_memory();
186+
187+
get_system_process_resident_memory(_OsType) ->
188+
{error, not_implemented_for_os}.
189+
190+
get_ps_memory() ->
191+
OsPid = os:getpid(),
192+
Cmd = "ps -p " ++ OsPid ++ " -o rss=",
193+
CmdOutput = os:cmd(Cmd),
194+
case re:run(CmdOutput, "[0-9]+", [{capture, first, list}]) of
195+
{match, [Match]} ->
196+
{ok, list_to_integer(Match) * 1024};
197+
_ ->
198+
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
199+
end.
200+
110201
binary() ->
111202
All = interesting_sups(),
112203
{Sums, Rest} =

‎src/vm_memory_monitor.erl‎

Lines changed: 3 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -117,14 +117,14 @@ get_memory_limit() ->
117117

118118
get_memory_use(bytes) ->
119119
MemoryLimit = get_memory_limit(),
120-
{get_used_memory(), case MemoryLimit > 0.0 of
120+
{rabbit_vm:total_memory(), case MemoryLimit > 0.0 of
121121
true -> MemoryLimit;
122122
false -> infinity
123123
end};
124124
get_memory_use(ratio) ->
125125
MemoryLimit = get_memory_limit(),
126126
case MemoryLimit > 0.0 of
127-
true -> get_used_memory() / MemoryLimit;
127+
true -> rabbit_vm:total_memory() / MemoryLimit;
128128
false -> infinity
129129
end.
130130

@@ -268,7 +268,7 @@ parse_mem_limit(_) ->
268268
internal_update(State = #state { memory_limit = MemLimit,
269269
alarmed = Alarmed,
270270
alarm_funs = {AlarmSet, AlarmClear} }) ->
271-
MemUsed = get_used_memory(),
271+
MemUsed = rabbit_vm:total_memory(),
272272
NewAlarmed = MemUsed > MemLimit,
273273
case {Alarmed, NewAlarmed} of
274274
{false, true} -> emit_update_info(set, MemUsed, MemLimit),
@@ -366,52 +366,6 @@ get_total_memory(_OsType) ->
366366
unknown.
367367

368368

369-
get_ps_memory() ->
370-
OsPid = os:getpid(),
371-
Cmd = "ps -p " ++ OsPid ++ " -o rss=",
372-
CmdOutput = cmd(Cmd),
373-
case re:run(CmdOutput, "[0-9]+", [{capture, first, list}]) of
374-
{match, [Match]} ->
375-
{ok, list_to_integer(Match) * 1024};
376-
_ ->
377-
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
378-
end.
379-
380-
get_system_process_resident_memory({unix,darwin}) ->
381-
get_ps_memory();
382-
383-
get_system_process_resident_memory({unix, linux}) ->
384-
get_ps_memory();
385-
386-
get_system_process_resident_memory({unix,freebsd}) ->
387-
get_ps_memory();
388-
389-
get_system_process_resident_memory({unix,openbsd}) ->
390-
get_ps_memory();
391-
392-
get_system_process_resident_memory({win32,_OSname}) ->
393-
OsPid = os:getpid(),
394-
Cmd = " tasklist /fi \"pid eq " ++ OsPid ++ "\" /fo LIST 2>&1 ",
395-
CmdOutput = os:cmd(Cmd),
396-
%% Memory usage is displayed in kilobytes
397-
%% with comma-separated thousands
398-
case re:run(CmdOutput, "Mem Usage:\\s+([0-9,]+)\\s+K", [{capture, all_but_first, list}]) of
399-
{match, [Match]} ->
400-
NoCommas = [ N || N <- Match, N =/= $, ],
401-
{ok, list_to_integer(NoCommas) * 1024};
402-
_ ->
403-
{error, {unexpected_output_from_command, Cmd, CmdOutput}}
404-
end;
405-
406-
get_system_process_resident_memory({unix, sunos}) ->
407-
get_ps_memory();
408-
409-
get_system_process_resident_memory({unix, aix}) ->
410-
get_ps_memory();
411-
412-
get_system_process_resident_memory(_OsType) ->
413-
{error, not_implemented_for_os}.
414-
415369
%% A line looks like "Foo bar: 123456."
416370
parse_line_mach(Line) ->
417371
[Name, RHS | _Rest] = string:tokens(Line, ":"),
@@ -494,51 +448,3 @@ read_proc_file(IoDevice, Acc) ->
494448
{ok, Res} -> read_proc_file(IoDevice, [Res | Acc]);
495449
eof -> Acc
496450
end.
497-
498-
-spec get_memory_calculation_strategy() -> rss | erlang.
499-
get_memory_calculation_strategy() ->
500-
case application:get_env(rabbit, vm_memory_calculation_strategy, rss) of
501-
erlang ->
502-
erlang;
503-
rss ->
504-
rss;
505-
UnsupportedValue ->
506-
rabbit_log:warning(
507-
"Unsupported value '~p' for vm_memory_calculation_strategy. "
508-
"Supported values: (rss|erlang). "
509-
"Defaulting to 'rss'",
510-
[UnsupportedValue]
511-
),
512-
rss
513-
end.
514-
515-
516-
%% Memory reported by erlang:memory(total) is not supposed to
517-
%% be equal to the total size of all pages mapped to the emulator,
518-
%% according to http://erlang.org/doc/man/erlang.html#memory-0
519-
%% erlang:memory(total) under-reports memory usage by around 20%
520-
-spec get_used_memory() -> Bytes :: integer().
521-
get_used_memory() ->
522-
case get_memory_calculation_strategy() of
523-
rss ->
524-
case get_system_process_resident_memory() of
525-
{ok, MemInBytes} ->
526-
MemInBytes;
527-
{error, Reason} ->
528-
rabbit_log:debug("Unable to get system memory used. Reason: ~p."
529-
" Falling back to erlang memory reporting",
530-
[Reason]),
531-
erlang:memory(total)
532-
end;
533-
erlang ->
534-
erlang:memory(total)
535-
end.
536-
537-
538-
-spec get_system_process_resident_memory() -> {ok, Bytes :: integer()} | {error, term()}.
539-
get_system_process_resident_memory() ->
540-
try
541-
get_system_process_resident_memory(os:type())
542-
catch _:Error ->
543-
{error, {"Failed to get process resident memory", Error}}
544-
end.

0 commit comments

Comments
 (0)