Fix specification of backends

- Default backend in rt_planner is now undefined
- All lists of backends are now atoms
- Command-line limits backends when using GiddyUp
- Command-line now properly generates multiple tests when a list of
  backends is specified
- "multi" backend added for verify_2i_aae-multi for GiddyUp

Author: Brett Hazen

src/giddyup.erl

@@ -267,7 +267,8 @@ fetch_all_test_plans(Platform, Product, VersionNumber, DefaultVersion, Host) ->
             Module = binary_to_atom(kvc:path(name, Test), utf8),
             Plan0 = rt_test_plan:new([{id, Id}, {module, Module}, {project, Project}, {platform, Platform}, {version, VersionNumber}]),
             {ok, Plan1} = case kvc:path('tags.backend', Test) of
-                          [] -> {ok, Plan0};
+                          %% Bitcask is the default version
+                          [] -> rt_test_plan:set(backend, bitcask, Plan0);
                           Backend -> rt_test_plan:set(backend, binary_to_atom(Backend, utf8), Plan0)
                       end,
             {ok, Plan2} = case kvc:path('tags.upgrade_version', Test) of

src/riak_test_escript.erl

@@ -25,6 +25,10 @@
 -export([main/1]).
 -export([add_deps/1]).
 
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+-endif.
+
 main(Args) ->
     %% TODO Should we use clique? -jsb
     %% Parse command line arguments ...
@@ -77,7 +81,7 @@ cli_options() ->
  {skip,                   $x, "skip",     string,     "list of tests to skip in a directory"},
  {verbose,                $v, "verbose",  undefined,  "verbose output"},
  {outdir,                 $o, "outdir",   string,     "output directory"},
- {backend,                $b, "backend",  atom,       "backend to test [memory | bitcask | eleveldb]"},
+ {backend,                $b, "backend",  atom,       "backend to test [memory | bitcask | eleveldb | multi]"},
  {keep,            undefined, "keep",     boolean,    "do not teardown cluster"},
  {continue_on_fail,       $n, "continue", boolean,    "continues executing tests on failure"},
  {report,                 $r, "report",   string,     "you're reporting an official test run, provide platform info (e.g. ubuntu-1404-64)\nUse 'config' if you want to pull from ~/.riak_test.config"},
@@ -104,12 +108,25 @@ generate_test_lists(UseGiddyUp, ParsedArgs) ->
     %% test metadata
 
     TestData = compose_test_data(ParsedArgs),
-    Backends = [proplists:get_value(backend, ParsedArgs, bitcask)],
+    CmdLineBackends = rt_util:backend_to_atom_list(proplists:get_value(backend, ParsedArgs)),
+    Backends = determine_backends(CmdLineBackends, UseGiddyUp),
     {Tests, NonTests} = wrap_test_in_test_plan(UseGiddyUp, Backends, TestData),
     Offset = rt_config:get(offset, undefined),
     Workers = rt_config:get(workers, undefined),
     shuffle_tests(Tests, NonTests, Offset, Workers).
 
+%% @doc Which backends should be tested?
+%% Use the command-line specified backend, otherwise default to bitcask
+%% If running under GiddyUp, then default to ALL backends
+%% First argument is a list of command-line backends and second is whether or not in GiddyUp mode
+-spec(determine_backends(atom(), boolean()) -> list()).
+determine_backends(undefined, true) ->
+    [memory, bitcask, eleveldb, multi];
+determine_backends(undefined, _) ->
+    [bitcask];
+determine_backends(Backends, _) ->
+    Backends.
+
 %% @doc Set values in the configuration with values specified on the command line
 maybe_override_setting(Argument, Value, Arguments) ->
     maybe_override_setting(proplists:is_defined(Argument, Arguments), Argument,
@@ -385,16 +402,6 @@ match_group_attributes(Attributes, Groups) ->
                            || Group <- Groups, TestType <- TestTypes ])
     end.
 
-backend_list(Backend) when is_atom(Backend) ->
-    atom_to_list(Backend);
-backend_list(Backends) when is_list(Backends) ->
-    FoldFun = fun(X, []) ->
-                      atom_to_list(X);
-                 (X, Acc) ->
-                      Acc ++ "," ++ atom_to_list(X)
-              end,
-    lists:foldl(FoldFun, [], Backends).
-
 load_tests_in_dir(Dir, Groups, SkipTests) ->
     case filelib:is_dir(Dir) of
         true ->
@@ -470,3 +477,19 @@ stop_giddyup(true) ->
 stop_giddyup(_) ->
     ok.
 
+-ifdef(TEST).
+%% Make sure that bitcask is the default backend
+default_backend_test() ->
+    ?assertEqual([bitcask], determine_backends(undefined, false)).
+
+%% Make sure that GiddyUp supports all backends
+default_giddyup_backend_test() ->
+    ?assertEqual([bitcask, eleveldb, memory], lists:sort(determine_backends(undefined, true))).
+
+%% Command-line backends should always rule
+cmdline_backend_test() ->
+    ?assertEqual([memory], determine_backends([memory], false)),
+    ?assertEqual([memory], determine_backends([memory], true)),
+    ?assertEqual([eleveldb, memory], lists:sort(determine_backends([memory, eleveldb], false))),
+    ?assertEqual([eleveldb, memory], lists:sort(determine_backends([memory, eleveldb], true))).
+-endif.

src/rt_planner.erl

@@ -92,7 +92,7 @@ load_from_giddyup(Backends, CommandLineTests) ->
 %%
 %% @end
 %%--------------------------------------------------------------------
--spec(add_test_plan(string(), string(), [string()], rt_properties2:product_version(), rt_properties2:properties()) -> ok).
+-spec(add_test_plan(string(), string(), [atom()], rt_properties2:product_version(), rt_properties2:properties()) -> ok).
 add_test_plan(Module, Platform, Backends, Version, Properties) ->
     gen_server:call(?MODULE, {add_test_plan, Module, Platform, Backends, Version, Properties}).
 
@@ -183,7 +183,8 @@ init([]) ->
     {noreply, NewState :: #state{}, timeout() | hibernate} |
     {stop, Reason :: term(), Reply :: term(), NewState :: #state{}} |
     {stop, Reason :: term(), NewState :: #state{}}).
-%% Run only those GiddyUp tests which are specified on the command line
+%% Run only those GiddyUp tests which are specified on the command line and are
+%% included in the specified backends.
 %% If none are specified, run everything
 handle_call({load_from_giddyup, Backends, CommandLineTests}, _From, State) ->
     AllGiddyupTests = giddyup:get_test_plans(),
@@ -207,7 +208,7 @@ handle_call({load_from_giddyup, Backends, CommandLineTests}, _From, State) ->
     State1 = lists:foldl(fun sort_and_queue/2, State, Included1),
     State2 = lists:foldl(fun exclude_test_plan/2, State1, Excluded1),
     {reply, ok, State2};
-%% Add a single test plan to the queue
+%% Add a single test plan for each backend to the queue
 handle_call({add_test_plan, Module, Platform, Backends, _Version, _Properties}, _From, State) ->
     State1 = lists:foldl(fun(Backend, AccState) ->
             TestPlan = rt_test_plan:new([{module, Module}, {platform, Platform}, {backend, Backend}]),
@@ -326,6 +327,7 @@ sort_and_queue(TestPlan, State) ->
                 non_runnable_test_plans=QNR2}.
 
 %% Check for api compatibility
+%% TODO: Move into "harness" or "driver" since it might be on a remote node.
 is_runnable_test_plan(TestPlan) ->
     TestModule = rt_test_plan:get_module(TestPlan),
     {Mod, Fun} = riak_test_runner:function_name(confirm, TestModule),

src/rt_properties2.erl

@@ -51,7 +51,7 @@
 -type topology() :: #rt_cluster_topology_v1{}.
 -type feature_flag() :: strong_consistency | yokozuna | jmx | snmp | security.
 -type product_version() :: string() | atom().
--type storage_backend() :: all | riak_kv_bitcask_backend | riak_kv_eleveldb_backend | riak_kv_memory_backend | riak_kv_multi_backend.
+-type storage_backend() :: all | bitcask | eleveldb | memory | multi.
 -type index() :: {binary(), binary(), binary()}.
 
 -export_type([properties/0,

src/rt_test_plan.erl

@@ -42,7 +42,7 @@
     project=rt_config:get_default_version_product() :: atom() | binary(),
     platform :: string(),
     version=rt_config:get_default_version_number() :: string(),
-    backend=bitcask :: atom(),
+    backend=undefined :: atom(),
     upgrade_path=[] :: [rt_properties2:product_version()],
     properties :: rt_properties2:properties()
 }).

src/rt_util.erl

@@ -5,3 +5,38 @@
 
 -export_type([error/0,
               result/0]).
+-export([backend_to_atom_list/1]).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+-endif.
+
+%% @doc Convert string or atom to list of atoms
+-spec(backend_to_atom_list(atom()|string()) -> undefined | list()).
+backend_to_atom_list(undefined) ->
+    undefined;
+backend_to_atom_list(Backends) when is_atom(Backends) ->
+    ListBackends = atom_to_list(Backends),
+    case lists:member($, , ListBackends) of
+        true ->
+            [list_to_atom(X) || X <- string:tokens(ListBackends, ", ")];
+        _ ->
+            [Backends]
+    end;
+backend_to_atom_list(Backends) when is_list(Backends) ->
+    case lists:member($, , Backends) of
+        true ->
+            [list_to_atom(X) || X <- string:tokens(Backends, ", ")];
+        _ ->
+            [list_to_atom(Backends)]
+    end.
+
+-ifdef(TEST).
+%% Properly covert backends to atoms
+backend_to_atom_list_test() ->
+    ?assertEqual(undefined, backend_to_atom_list(undefined)),
+    ?assertEqual([memory], backend_to_atom_list(memory)),
+    ?assertEqual([memory], backend_to_atom_list("memory")),
+    ?assertEqual([bitcask, eleveldb, memory], lists:sort(backend_to_atom_list("memory, bitcask,eleveldb"))),
+    ?assertEqual([bitcask, eleveldb, memory], lists:sort(backend_to_atom_list('memory, bitcask,eleveldb'))).
+-endif.