Initial introduction of rt_test_plan record to wrap tests

Author: Brett Hazen

src/giddyup.erl

@@ -58,7 +58,7 @@ get_schema(Platform) ->
 
 get_schema(Platform, Retries) ->
     Host = rt_config:get(giddyup_host),
-    Project = rt_config:get(rt_project),
+    Project = rt_config:get(giddyup_project),
     Version = rt:get_version(),
     URL = lists:flatten(io_lib:format("http://~s/projects/~s?platform=~s&version=~s", [Host, Project, Platform, Version])),
     lager:info("giddyup url: ~s", [URL]),

src/riak_test_escript.erl

@@ -44,38 +44,37 @@ main(Args) ->
     finalize(Results, ParsedArgs).
 
 prepare(ParsedArgs, Tests, NonTests) ->
-    lager:notice("Tests to run: ~p~n", [Tests]),
+    lager:notice("Test to run: ~p", [[rt_test_plan:get_module(Test) || Test <- Tests]]),
     case NonTests of
         [] ->
             ok;
         _ ->
-            lager:notice("These modules are not runnable tests: ~p~n",
-                      [[NTMod || {NTMod, _} <- NonTests]])
+            lager:notice("Test not to run: ~p", [[rt_test_plan:get_module(Test) || Test <- NonTests]])
     end,
     ok = erlang_setup(ParsedArgs),
     test_setup().
 
-execute(Tests, ParsedArgs, _HarnessArgs) ->
+execute(TestPlans, ParsedArgs, _HarnessArgs) ->
     OutDir = proplists:get_value(outdir, ParsedArgs),
     Report = report(ParsedArgs),
     UpgradeList = upgrade_list(
                     proplists:get_value(upgrade_path, ParsedArgs)),
-    Backend = proplists:get_value(backend, ParsedArgs, bitcask),
 
-    {ok, Executor} = riak_test_executor:start_link(Tests,
-                                                   Backend,
+    {ok, Executor} = riak_test_executor:start_link(TestPlans,
                                                    OutDir,
                                                    Report,
                                                    UpgradeList,
                                                    self()),
-    wait_for_results(Executor, [], length(Tests), 0).
+    wait_for_results(Executor, [], length(TestPlans), 0).
 
 
 report_results(Results, Verbose) ->
     %% TODO: Good place to also do giddyup reporting and provide a
     %% place for extending to any other reporting sources that might
     %% be useful.
-    print_summary(Results, undefined, Verbose),
+
+    ModuleResults = [{rt_test_plan:get_module(TestPlan), PassFail, Duration} || {TestPlan, PassFail, Duration} <- Results],
+    print_summary(ModuleResults, undefined, Verbose),
     ok.
 
     %% TestResults = run_tests(Tests, Outdir, Report, HarnessArgs),
@@ -121,6 +120,7 @@ finalize(TestResults, Args) ->
     Teardown = not proplists:get_value(keep, Args, false),
     maybe_teardown(Teardown, TestResults),
     ok.
+
 %% Option Name, Short Code, Long Code, Argument Spec, Help Message
 cli_options() ->
 [
@@ -184,7 +184,7 @@ report(ParsedArgs) ->
         undefined ->
             undefined;
         "config" ->
-            rt_config:get(platform, undefined);
+            rt_config:get(giddyup_platform, undefined);
         R ->
             R
     end.
@@ -212,7 +212,8 @@ help_or_parse_tests(ParsedArgs, HarnessArgs, false) ->
     maybe_override_setting(continue_on_fail, true, ParsedArgs),
 
     TestData = compose_test_data(ParsedArgs),
-    {Tests, NonTests} = which_tests_to_run(report(ParsedArgs), TestData),
+    Backend = proplists:get_value(backend, ParsedArgs, bitcask),
+    {Tests, NonTests} = wrap_test_in_test_plan(report(ParsedArgs), Backend, TestData),
     Offset = rt_config:get(offset, undefined),
     Workers = rt_config:get(workers, undefined),
     shuffle_tests(ParsedArgs, HarnessArgs, Tests, NonTests, Offset, Workers).
@@ -235,6 +236,7 @@ load_initial_config(ParsedArgs) ->
                    proplists:get_value(file, ParsedArgs)).
 
 shuffle_tests(_, _, [], _, _, _) ->
+    io:format("No tests are scheduled to run~n"),
     lager:error("No tests are scheduled to run~n"),
     halt(1);
 shuffle_tests(ParsedArgs, HarnessArgs, Tests, NonTests, undefined, _) ->
@@ -364,49 +366,41 @@ extract_test_names(Test, {CodePaths, TestNames}) ->
     {[filename:dirname(Test) | CodePaths],
      [list_to_atom(filename:rootname(filename:basename(Test))) | TestNames]}.
 
-which_tests_to_run(undefined, CommandLineTests) ->
-    lists:partition(fun is_runnable_test/1, CommandLineTests);
-which_tests_to_run(Platform, []) ->
-    giddyup:get_suite(Platform);
-which_tests_to_run(Platform, CommandLineTests) ->
-    Suite = filter_zip_suite(Platform, CommandLineTests),
-    lists:partition(fun is_runnable_test/1,
-                    lists:foldr(fun filter_merge_tests/2, [], Suite)).
-
-filter_zip_suite(Platform, CommandLineTests) ->
-    [ {SModule, SMeta, CMeta} || {SModule, SMeta} <- giddyup:get_suite(Platform),
-                                 {CModule, CMeta} <- CommandLineTests,
-                                 SModule =:= CModule].
-
-filter_merge_tests({Module, SMeta, CMeta}, Tests) ->
-    case filter_merge_meta(SMeta, CMeta, [backend, upgrade_version]) of
-        false ->
-            Tests;
-        Meta ->
-            [{Module, Meta}|Tests]
-    end.
-
-filter_merge_meta(SMeta, _CMeta, []) ->
-    SMeta;
-filter_merge_meta(SMeta, CMeta, [Field|Rest]) ->
-    case {kvc:value(Field, SMeta, undefined), kvc:value(Field, CMeta, undefined)} of
-        {X, X} ->
-            filter_merge_meta(SMeta, CMeta, Rest);
-        {_, undefined} ->
-            filter_merge_meta(SMeta, CMeta, Rest);
-        {undefined, X} ->
-            filter_merge_meta(lists:keystore(Field, 1, SMeta, {Field, X}), CMeta, Rest);
+%% @doc Determine which tests to run based on command-line argument
+%%      If the platform is defined, consult GiddyUp, otherwise just shovel
+%%      the whole thing into the Planner
+-spec(load_up_test_planner(string() | undefined, string(), list()) -> list()).
+load_up_test_planner(undefined, Backend, CommandLineTests) ->
+    [rt_planner:add_test_plan(Name, undefined, Backend, undefined, undefined) || Name <- CommandLineTests];
+%% GiddyUp Flavor
+load_up_test_planner(Platform, Backend, CommandLineTests) ->
+    rt_planner:load_from_giddyup(Platform, Backend, CommandLineTests).
+
+%% @doc Push all of the test into the Planner for now and wrap them in an `rt_test_plan'
+%% TODO: Let the Planner do the work, not the riak_test_executor
+-spec(wrap_test_in_test_plan(string(), string(), [atom()]) -> {list(), list()}).
+wrap_test_in_test_plan(Platform, Backend, CommandLineTests) ->
+    %% ibrowse neededfor GiddyUp
+    load_and_start(ibrowse),
+    {ok, _Pid} = rt_planner:start_link(),
+    load_up_test_planner(Platform, Backend, CommandLineTests),
+    TestPlans = [rt_planner:fetch_test_plan() || _ <- lists:seq(1, rt_planner:number_of_plans())],
+    NonRunnableTestPlans = [rt_planner:fetch_test_non_runnable_plan() || _ <- lists:seq(1, rt_planner:number_of_non_runable_plans())],
+    rt_planner:stop(),
+    {TestPlans, NonRunnableTestPlans}.
+
+%% @doc Pull all jobs from the Planner
+%%      Better than using rt_planner:number_of_plans/0
+-spec(fetch_all_test_plans(list()) -> list()).
+fetch_all_test_plans(Acc) ->
+    Plan = rt_planner:fetch_test_plan(),
+    case Plan of
+        empty ->
+            Acc;
         _ ->
-            false
+            fetch_all_test_plans([Plan|Acc])
     end.
 
-%% Check for api compatibility
-is_runnable_test(TestModule) ->
-    {Mod, Fun} = riak_test_runner:function_name(confirm, TestModule),
-    code:ensure_loaded(Mod),
-    erlang:function_exported(Mod, Fun, 0) orelse
-        erlang:function_exported(Mod, Fun, 1).
-
 get_group_tests(Tests, Groups) ->
     lists:filter(fun(Test) ->
                          Mod = list_to_atom(Test),
@@ -546,6 +540,8 @@ print_summary(TestResults, _CoverResult, Verbose) ->
         true ->
             Rows =
                 [format_test_row(Result, Width) || Result <- TestResults],
+            %% TODO: Remove once clique table is fixed
+            [lager:debug("ROW ~p", [Row]) || Row <- Rows],
             Table = clique_table:autosize_create_table(?HEADER, Rows),
             io:format("~ts~n", [Table]);
         false ->

src/riak_test_executor.erl

@@ -3,7 +3,7 @@
 -behavior(gen_fsm).
 
 %% API
--export([start_link/6,
+-export([start_link/5,
          send_event/1,
          stop/0]).
 
@@ -28,7 +28,6 @@
                 running_tests=[] :: [atom()],
                 waiting_tests=[] :: [atom()],
                 notify_pid :: pid(),
-                backend :: atom(),
                 upgrade_list :: [string()],
                 test_properties :: [proplists:proplist()],
                 runner_pids=[] :: [pid()],
@@ -42,9 +41,9 @@
 %%%===================================================================
 
 %% @doc Start the test executor
--spec start_link(atom(), atom(), string(), string(), [string()], pid()) -> {ok, pid()} | ignore | {error, term()}.
-start_link(Tests, Backend, LogDir, ReportInfo, UpgradeList, NotifyPid) ->
-    Args = [Tests, Backend, LogDir, ReportInfo, UpgradeList, NotifyPid],
+-spec start_link(atom(), string(), string(), [string()], pid()) -> {ok, pid()} | ignore | {error, term()}.
+start_link(Tests, LogDir, ReportInfo, UpgradeList, NotifyPid) ->
+    Args = [Tests, LogDir, ReportInfo, UpgradeList, NotifyPid],
     gen_fsm:start_link({local, ?MODULE}, ?MODULE, Args, []).
 
 send_event(Msg) ->
@@ -59,19 +58,14 @@ stop() ->
 %%% gen_fsm callbacks
 %%%===================================================================
 
-init([Tests, Backend, LogDir, ReportInfo, UpgradeList, NotifyPid]) ->
+init([Tests, LogDir, ReportInfo, UpgradeList, NotifyPid]) ->
     %% TODO Change the default when parallel execution support is implemented -jsb
     ExecutionMode = rt_config:get(rt_execution_mode, serial),
 
-    %% TODO: Remove after all tests ported 2.0 -- workaround to support 
-    %% backend command line argument fo v1 cluster provisioning -jsb
-    rt_config:set(rt_backend, Backend),
-
     ContinueOnFail = rt_config:get(continue_on_fail),
 
     lager:notice("Starting the Riak Test executor in ~p execution mode", [ExecutionMode]),
     State = #state{pending_tests=Tests,
-                   backend=Backend,
                    log_dir=LogDir,
                    report_info=ReportInfo,
                    upgrade_list=UpgradeList,
@@ -165,34 +159,34 @@ launch_test({nodes, Nodes, NodeMap}, State) ->
     %% Spawn a test runner for the head of pending. If pending is now
     %% empty transition to `wait_for_completion'; otherwise,
     %% transition to `request_nodes'.
-    #state{pending_tests=[NextTest | RestPending],
+    #state{pending_tests=[NextTestPlan | RestPending],
            execution_mode=ExecutionMode,
-           backend=Backend,
            test_properties=PropertiesList,
            runner_pids=Pids,
            running_tests=Running,
            continue_on_fail=ContinueOnFail} = State,
-    lager:debug("Executing test ~p in mode ~p", [NextTest, ExecutionMode]),
-    {NextTest, TestProps} = lists:keyfind(NextTest, 1, PropertiesList),
+    NextTestModule = rt_test_plan:get_module(NextTestPlan),
+    lager:debug("Executing test ~p in mode ~p", [NextTestModule, ExecutionMode]),
+    {NextTestPlan, TestProps} = lists:keyfind(NextTestPlan, 1, PropertiesList),
     UpdTestProps = rt_properties:set([{node_map, NodeMap}, {node_ids, Nodes}],
                                      TestProps),
-    {RunnerPids, RunningTests} = run_test(ExecutionMode, NextTest, Backend, UpdTestProps, 
+    {RunnerPids, RunningTests} = run_test(ExecutionMode, NextTestPlan, UpdTestProps,
                                           Pids, Running, ContinueOnFail),
     UpdState = State#state{pending_tests=RestPending,
                            execution_mode=ExecutionMode,
                            runner_pids=RunnerPids,
                            running_tests=RunningTests},
 
     launch_test_transition(UpdState);
-launch_test({test_complete, Test, Pid, Results, Duration}, State) ->
+launch_test({test_complete, TestPlan, Pid, Results, Duration}, State) ->
     #state{pending_tests=Pending,
            waiting_tests=Waiting,
            running_tests=Running,
            runner_pids=Pids,
            execution_mode=ExecutionMode} = State,
     %% Report results
-    report_results(Test, Results, Duration, State),
-    UpdState = State#state{running_tests=lists:delete(Test, Running),
+    report_results(TestPlan, Results, Duration, State),
+    UpdState = State#state{running_tests=lists:delete(TestPlan, Running),
                            runner_pids=lists:delete(Pid, Pids),
                            pending_tests=Pending++Waiting,
                            waiting_tests=[],
@@ -202,26 +196,26 @@ launch_test(Event, State) ->
     lager:error("Unknown event ~p with state ~p.", [Event, State]),
     ok.
 
-maybe_reserve_nodes(NextTest, TestProps) ->
+maybe_reserve_nodes(NextTestPlan, TestProps) ->
     VersionsToTest = versions_to_test(TestProps),
-    maybe_reserve_nodes(erlang:function_exported(NextTest, confirm, 1), 
-                        NextTest, VersionsToTest, TestProps).
+    maybe_reserve_nodes(erlang:function_exported(rt_test_plan:get_module(NextTestPlan), confirm, 1),
+                        NextTestPlan, VersionsToTest, TestProps).
 
 maybe_reserve_nodes(true, NextTest, VersionsToTest, TestProps) ->
     NodeCount = rt_properties:get(node_count, TestProps),
 
     %% Send async request to node manager
-    lager:notice("Requesting ~p nodes for the next test, ~p", [NodeCount, NextTest]),
+    lager:notice("Requesting ~p nodes for the next test, ~p", [NodeCount, rt_test_plan:get_module(NextTest)]),
     node_manager:reserve_nodes(NodeCount,
                                VersionsToTest,
                                reservation_notify_fun());
 maybe_reserve_nodes(false, NextTest, VersionsToTest, _TestProps) ->
-    lager:warning("~p is an old style test that requires conversion.", [NextTest]),
+    lager:warning("~p is an old style test that requires conversion.", [rt_test_plan:get_module(NextTest)]),
     node_manager:reserve_nodes(0, VersionsToTest, reservation_notify_fun()),
     ok.
 
 wait_for_completion({test_complete, Test, Pid, Results, Duration}, State) ->
-    lager:debug("Test ~p complete", [Test]),
+    lager:debug("Test ~p complete", [rt_test_plan:get_module(Test)]),
     #state{pending_tests=Pending,
            waiting_tests=Waiting,
            running_tests=Running,
@@ -256,8 +250,8 @@ wait_for_completion(_Event, _From, _State) ->
 %%% Internal functions
 %%%===================================================================
 
-report_results(Test, Results, Duration, #state{notify_pid=NotifyPid}) ->
-    NotifyPid ! {self(), {test_result, {Test, Results, Duration}}},
+report_results(TestPlan, Results, Duration, #state{notify_pid=NotifyPid}) ->
+    NotifyPid ! {self(), {test_result, {TestPlan, Results, Duration}}},
     ok.
 
 report_done(#state{notify_pid=NotifyPid}) ->
@@ -276,7 +270,8 @@ wait_for_completion_transition(_Result, State) ->
 
 launch_test_transition(State=#state{pending_tests=PendingTests,
                                     execution_mode=ExecutionMode}) when PendingTests == [] orelse ExecutionMode == serial ->
-    lager:debug("Waiting for completion: execution mode ~p with pending tests ~p", [ExecutionMode, PendingTests]),
+    PendingModules = [rt_test_plan:get_module(Test) || Test <- PendingTests],
+    lager:debug("Waiting for completion: execution mode ~p with pending tests ~p", [ExecutionMode, PendingModules]),
     {next_state, wait_for_completion, State};
 launch_test_transition(State) ->
     {next_state, request_nodes, State, 0}.
@@ -293,13 +288,13 @@ test_properties(Tests, OverriddenProps) ->
     lists:foldl(test_property_fun(OverriddenProps), [], Tests).
 
 test_property_fun(OverrideProps) ->
-    fun(TestModule, Acc) ->
+    fun(TestPlan, Acc) ->
             {PropsMod, PropsFun} = riak_test_runner:function_name(properties,
-                                                                  TestModule,
+                                                                  rt_test_plan:get_module(TestPlan),
                                                                   0,
                                                                   rt_cluster),
             Properties = rt_properties:set(OverrideProps, PropsMod:PropsFun()),
-            [{TestModule, Properties} | Acc]
+            [{TestPlan, Properties} | Acc]
     end.
 
 versions_to_test(Properties) ->
@@ -330,11 +325,11 @@ override_props(State) ->
             [{upgrade_path, UpgradeList}]
     end.
 
--spec run_test(parallel | serial, atom(), atom(), proplists:proplist(), [pid()], [atom()], boolean()) -> {[pid()], [atom()]}.
-run_test(parallel, Test, Backend, Properties, RunningPids, RunningTests, ContinueOnFail) ->
-    Pid = spawn_link(riak_test_runner, start, [Test, Backend, Properties, ContinueOnFail]),
-    {[Pid | RunningPids], [Test | RunningTests]};
-run_test(serial, Test, Backend, Properties, RunningPids, RunningTests, ContinueOnFail) ->
-    riak_test_runner:start(Test, Backend, Properties, ContinueOnFail),
+-spec run_test(parallel | serial, atom(), proplists:proplist(), [pid()], [atom()], boolean()) -> {[pid()], [atom()]}.
+run_test(parallel, TestPlan, Properties, RunningPids, RunningTests, ContinueOnFail) ->
+    Pid = spawn_link(riak_test_runner, start, [TestPlan, Properties, ContinueOnFail]),
+    {[Pid | RunningPids], [TestPlan | RunningTests]};
+run_test(serial, TestPlan, Properties, RunningPids, RunningTests, ContinueOnFail) ->
+    riak_test_runner:start(TestPlan, Properties, ContinueOnFail),
     {RunningPids, RunningTests}.