Merge pull request #1979 from ferd/ct-failfast

Add a --fail_fast switch to common test

src/cth_fail_fast.erl

@@ -0,0 +1,118 @@
+-module(cth_fail_fast).
+
+%% Callbacks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3, on_tc_skip/4]).
+
+-export([terminate/1]).
+
+%% We work by setting an 'abort' variable on each test case that fails
+%% and then triggering the failure before starting the next test. This
+%% ensures that all other hooks have run for the same event, and
+%% simplifies error reporting.
+-record(state, {abort=false}).
+
+%% @doc Return a unique id for this CTH.
+id(_Opts) ->
+    {?MODULE, make_ref()}.
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state.
+init(_Id, _Opts) ->
+    {ok, #state{}}.
+
+%% @doc Called before init_per_suite is called.
+pre_init_per_suite(_Suite,_Config,#state{abort=true}) ->
+    abort();
+pre_init_per_suite(_Suite,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after init_per_suite.
+post_init_per_suite(_Suite,_Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before end_per_suite.
+pre_end_per_suite(_Suite,_Config,#state{abort=true}) ->
+    abort();
+pre_end_per_suite(_Suite,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after end_per_suite.
+post_end_per_suite(_Suite,_Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before each init_per_group.
+pre_init_per_group(_Group,_Config,#state{abort=true}) ->
+    abort();
+pre_init_per_group(_Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each init_per_group.
+post_init_per_group(_Group,_Config,Return, State) ->
+    {Return, State}.
+
+%% @doc Called after each end_per_group.
+pre_end_per_group(_Group,_Config,#state{abort=true}) ->
+    abort();
+pre_end_per_group(_Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each end_per_group.
+post_end_per_group(_Group,_Config,Return, State) ->
+    {Return, State}.
+
+%% @doc Called before each test case.
+pre_init_per_testcase(_TC,_Config,#state{abort=true}) ->
+    abort();
+pre_init_per_testcase(_TC,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each test case.
+post_end_per_testcase(_TC,_Config,ok,State) ->
+    {ok, State};
+post_end_per_testcase(_TC,_Config,Error,State) ->
+    {Error, State#state{abort=true}}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
+on_tc_fail(_TC, _Reason, State) ->
+    State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. (>= 19.3)
+on_tc_skip(_Suite, _TC, {tc_auto_skip, _}, State) ->
+    State#state{abort=true};
+on_tc_skip(_Suite, _TC, _Reason, State) ->
+    State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. (Pre-19.3)
+on_tc_skip(_TC, {tc_auto_skip, _}, State) ->
+    State#state{abort=true};
+on_tc_skip(_TC, _Reason, State) ->
+    State.
+
+%% @doc Called when the scope of the CTH is done
+terminate(#state{}) ->
+    ok.
+
+%%% Helpers
+abort() ->
+    io:format(user, "Detected test failure. Aborting~n", []),
+    halt(1).

src/rebar_prv_common_test.erl

@@ -171,6 +171,9 @@ transform_opts([{cover, _}|Rest], Acc) ->
 %% drop verbose from opts, ct doesn't care about it
 transform_opts([{verbose, _}|Rest], Acc) ->
     transform_opts(Rest, Acc);
+%% drop fail_fast from opts, ct doesn't care about it
+transform_opts([{fail_fast, _}|Rest], Acc) ->
+    transform_opts(Rest, Acc);
 %% getopt should handle anything else
 transform_opts([Opt|Rest], Acc) ->
     transform_opts(Rest, [Opt|Acc]).
@@ -224,15 +227,21 @@ ensure_opts([V|Rest], Acc) ->
     ensure_opts(Rest, [V|Acc]).
 
 add_hooks(Opts, State) ->
+    FailFast = case fails_fast(State) of
+        true -> [cth_fail_fast];
+        false -> []
+    end,
     case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of
         {false, _} ->
             Opts;
         {Other, false} ->
-            [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), cth_retry]} | Opts];
+            [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other),
+                         cth_retry] ++ FailFast} | Opts];
         {Other, {ct_hooks, Hooks}} ->
             %% Make sure hooks are there once only.
-            ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), cth_retry],
-            AllReadableHooks = [cth_readable_failonly, cth_retry,
+            ReadableHooks = [cth_readable_failonly, readable_shell_type(Other),
+                             cth_retry] ++ FailFast,
+            AllReadableHooks = [cth_readable_failonly, cth_retry, cth_fail_fast,
                                 cth_readable_shell, cth_readable_compact_shell],
             NewHooks =  (Hooks -- AllReadableHooks) ++ ReadableHooks,
             lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks})
@@ -445,6 +454,10 @@ readable(State) ->
         undefined -> rebar_state:get(State, ct_readable, compact)
     end.
 
+fails_fast(State) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    proplists:get_value(fail_fast, RawOpts) == true.
+
 test_dirs(State, Apps, Opts) ->
     case proplists:get_value(spec, Opts) of
         undefined ->
@@ -773,7 +786,8 @@ ct_opts(_State) ->
      {setcookie, undefined, "setcookie", atom, help(setcookie)},
      {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list
      {compile_only, undefined, "compile_only", boolean, help(compile_only)},
-     {retry, undefined, "retry", boolean, help(retry)}
+     {retry, undefined, "retry", boolean, help(retry)},
+     {fail_fast, undefined, "fail_fast", {boolean, false}, help(fail_fast)}
     ].
 
 help(compile_only) ->
@@ -846,5 +860,9 @@ help(setcookie) ->
     "Sets the cookie if the node is distributed";
 help(retry) ->
     "Experimental feature. If any specification for previously failing test is found, runs them.";
+help(fail_fast) ->
+    "Experimental feature. If any test fails, the run is aborted. Since common test does not "
+    "support this natively, we abort the rebar3 run on a failure. This May break CT's disk logging and "
+    "other rebar3 features.";
 help(_) ->
     "".