Skip to content

Commit 9e9e734

Browse files
russelldbrusselldb
authored
Fix brops intermittent test failures (#911)
* Add some extra output in case of monitor timeout On ubuntu 16.04/basho10 there is a reliable and repeatable failure of bprops_eqc. The call to stop_id fails with a timeout. This change adds some extra information to that error message to help fix the test * Kill pid that won't exit * Wait for killed pid to exit Avoid "already_started" errors in setup
1 parent b8a11b4 commit 9e9e734

File tree

3 files changed

+78
-59
lines changed

3 files changed

+78
-59
lines changed

‎rebar.config‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@
1111

1212
{deps, [
1313
{lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "3.2.4"}}},
14-
{poolboy, ".*", {git, "git://github.com/basho/poolboy.git", {tag, "0.8.1p3"}}},
15-
{basho_stats, ".*", {git, "git://github.com/basho/basho_stats.git", {tag, "1.0.3"}}},
14+
{poolboy, ".*", {git, "git://github.com/basho/poolboy.git", {branch, "develop-2.2.5"}}},
15+
{basho_stats, ".*", {git, "git://github.com/basho/basho_stats.git", {branch, "master"}}},
1616
{riak_sysmon, ".*", {git, "https://github.com/basho/riak_sysmon.git", {tag, "2.1.5"}}},
17-
{eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {tag, "2.0.34"}}},
18-
{riak_ensemble, ".*", {git, "https://github.com/basho/riak_ensemble", {tag, "2.1.8"}}},
19-
{pbkdf2, ".*", {git, "git://github.com/basho/erlang-pbkdf2.git", {tag, "2.0.0"}}},
20-
{exometer_core, ".*", {git, "git://github.com/basho/exometer_core.git", {tag, "1.0.0-basho9"}}},
21-
{clique, ".*", {git, "https://github.com/basho/clique.git", {tag, "0.3.9"}}}
17+
{eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {branch, "2.0"}}},
18+
{riak_ensemble, ".*", {git, "https://github.com/basho/riak_ensemble", {branch, "develop-2.2"}}},
19+
{pbkdf2, ".*", {git, "git://github.com/basho/erlang-pbkdf2.git", {branch, "master"}}},
20+
{exometer_core, ".*", {git, "git://github.com/basho/exometer_core.git", {branch, "master"}}},
21+
{clique, ".*", {git, "https://github.com/basho/clique.git", {branch, "develop-2.2"}}}
2222
]}.

‎test/bprops_eqc.erl‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,8 @@ prop_buckets() ->
247247
application:set_env(riak_core, metadata_hashtree_timer, 4294967295),
248248
application:set_env(riak_core, default_bucket_props, ?DEFAULT_BPROPS),
249249
application:set_env(riak_core, cluster_name, "riak_core_bucket_eqc"),
250-
stop_pid(whereis(riak_core_ring_events)),
251-
stop_pid(whereis(riak_core_ring_manager)),
250+
stop_pid(riak_core_ring_events, whereis(riak_core_ring_events)),
251+
stop_pid(riak_core_ring_manager, whereis(riak_core_ring_manager)),
252252
{ok, RingEvents} = riak_core_ring_events:start_link(),
253253
{ok, _RingMgr} = riak_core_ring_manager:start_link(test),
254254
{ok, Claimant} = riak_core_claimant:start_link(),
@@ -261,12 +261,12 @@ prop_buckets() ->
261261
%%
262262
%% shut down
263263
%%
264-
stop_pid(Broadcast),
265-
stop_pid(Hashtree),
266-
stop_pid(MetaMgr),
267-
stop_pid(Claimant),
264+
stop_pid(riak_core_broadcast, Broadcast),
265+
stop_pid(riak_core_metadata_hashtree, Hashtree),
266+
stop_pid(riak_core_metadata_manager, MetaMgr),
267+
stop_pid(riak_core_claimant, Claimant),
268268
riak_core_ring_manager:stop(),
269-
stop_pid(RingEvents),
269+
stop_pid(riak_core_ring_events2, RingEvents),
270270

271271
eqc_statem:pretty_commands(
272272
?MODULE, Cmds,
@@ -300,21 +300,24 @@ setup_cleanup() ->
300300
%% internal helper functions
301301
%%
302302

303-
stop_pid(Other) when not is_pid(Other) ->
303+
stop_pid(_Tag, Other) when not is_pid(Other) ->
304304
ok;
305-
stop_pid(Pid) ->
305+
stop_pid(Tag, Pid) ->
306306
unlink(Pid),
307307
exit(Pid, shutdown),
308-
ok = wait_for_pid(Pid).
308+
ok = wait_for_pid(Tag, Pid).
309309

310-
wait_for_pid(Pid) ->
310+
wait_for_pid(Tag, Pid) ->
311311
Mref = erlang:monitor(process, Pid),
312312
receive
313313
{'DOWN', Mref, process, _, _} ->
314314
ok
315315
after
316316
5000 ->
317-
{error, didnotexit}
317+
demonitor(Mref, [flush]),
318+
exit(Pid, kill),
319+
wait_for_pid(Tag, Pid)
320+
318321
end.
319322

320323
-endif.

‎test/btypes_eqc.erl‎

Lines changed: 56 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -338,55 +338,71 @@ weight(_S, _Cmd) -> 1.
338338

339339
%% @doc the property
340340
prop_btype_invariant() ->
341-
?FORALL(Cmds, commands(?MODULE),
342-
aggregate(command_names(Cmds),
343-
?TRAPEXIT(
344-
begin
345-
meck:new(riak_core_capability, []),
346-
meck:expect(riak_core_capability, get,
347-
fun({riak_core, bucket_types}) -> true;
348-
(X) -> meck:passthrough([X]) end),
349-
os:cmd("rm -r ./btypes_eqc_meta"),
350-
application:set_env(riak_core, claimant_tick, 4294967295),
351-
application:set_env(riak_core, broadcast_lazy_timer, 4294967295),
352-
application:set_env(riak_core, broadcast_exchange_timer, 4294967295),
353-
application:set_env(riak_core, metadata_hashtree_timer, 4294967295),
354-
stop_pid(whereis(riak_core_ring_events)),
355-
stop_pid(whereis(riak_core_ring_manager)),
356-
{ok, RingEvents} = riak_core_ring_events:start_link(),
357-
{ok, _RingMgr} = riak_core_ring_manager:start_link(test),
358-
{ok, Claimant} = riak_core_claimant:start_link(),
359-
{ok, MetaMgr} = riak_core_metadata_manager:start_link([{data_dir, "./btypes_eqc_meta"}]),
360-
{ok, Hashtree} = riak_core_metadata_hashtree:start_link("./btypes_eqc_meta/trees"),
361-
{ok, Broadcast} = riak_core_broadcast:start_link(),
362-
{H, S, Res} = run_commands(?MODULE,Cmds),
363-
stop_pid(Broadcast),
364-
stop_pid(Hashtree),
365-
stop_pid(MetaMgr),
366-
stop_pid(Claimant),
367-
riak_core_ring_manager:stop(),
368-
stop_pid(RingEvents),
369-
os:cmd("rm -r ./btypes_eqc_meta"),
370-
meck:unload(riak_core_capability),
371-
pretty_commands(?MODULE, Cmds, {H, S, Res},
372-
Res == ok)
373-
end))).
374-
375-
stop_pid(Other) when not is_pid(Other) ->
341+
?SETUP(
342+
fun setup_cleanup/0,
343+
?FORALL(Cmds, commands(?MODULE),
344+
aggregate(command_names(Cmds),
345+
?TRAPEXIT(
346+
try
347+
os:cmd("rm -r ./btypes_eqc_meta"),
348+
application:set_env(riak_core, claimant_tick, 4294967295),
349+
application:set_env(riak_core, broadcast_lazy_timer, 4294967295),
350+
application:set_env(riak_core, broadcast_exchange_timer, 4294967295),
351+
application:set_env(riak_core, metadata_hashtree_timer, 4294967295),
352+
stop_pid(riak_core_ring_events, whereis(riak_core_ring_events)),
353+
stop_pid(riak_core_ring_manager, whereis(riak_core_ring_manager)),
354+
{ok, RingEvents} = riak_core_ring_events:start_link(),
355+
{ok, _RingMgr} = riak_core_ring_manager:start_link(test),
356+
{ok, Claimant} = riak_core_claimant:start_link(),
357+
{ok, MetaMgr} = riak_core_metadata_manager:start_link([{data_dir, "./btypes_eqc_meta"}]),
358+
{ok, Hashtree} = riak_core_metadata_hashtree:start_link("./btypes_eqc_meta/trees"),
359+
{ok, Broadcast} = riak_core_broadcast:start_link(),
360+
{H, S, Res} = run_commands(?MODULE,Cmds),
361+
stop_pid(riak_core_broadcast, Broadcast),
362+
stop_pid(riak_core_metadata_hashtree, Hashtree),
363+
stop_pid(riak_core_metadata_manager, MetaMgr),
364+
stop_pid(riak_core_claimant, Claimant),
365+
riak_core_ring_manager:stop(),
366+
stop_pid(riak_core_ring_events2, RingEvents),
367+
pretty_commands(?MODULE, Cmds, {H, S, Res},
368+
Res == ok)
369+
after
370+
os:cmd("rm -r ./btypes_eqc_meta")
371+
end
372+
)
373+
)
374+
)
375+
).
376+
377+
setup_cleanup() ->
378+
meck:new(riak_core_capability, []),
379+
meck:expect(
380+
riak_core_capability, get,
381+
fun({riak_core, bucket_types}) -> true;
382+
(X) -> meck:passthrough([X])
383+
end
384+
),
385+
fun() ->
386+
meck:unload(riak_core_capability)
387+
end.
388+
389+
stop_pid(_Tag, Other) when not is_pid(Other) ->
376390
ok;
377-
stop_pid(Pid) ->
391+
stop_pid(Tag, Pid) ->
378392
unlink(Pid),
379393
exit(Pid, shutdown),
380-
ok = wait_for_pid(Pid).
394+
ok = wait_for_pid(Tag, Pid).
381395

382-
wait_for_pid(Pid) ->
396+
wait_for_pid(Tag, Pid) ->
383397
Mref = erlang:monitor(process, Pid),
384398
receive
385399
{'DOWN', Mref, process, _, _} ->
386400
ok
387401
after
388-
5000 ->
389-
{error, didnotexit}
402+
5000 ->
403+
demonitor(Mref, [flush]),
404+
exit(Pid, kill),
405+
wait_for_pid(Tag, Pid)
390406
end.
391407

392408
-endif.

0 commit comments

Comments
 (0)