Add a --fail_fast switch to common test

This switch aborts the rebar3 run after the first test failure is
reported.

Since Common Test does not support this feature and it can be useful for
interactive development (a demand from one of the teams I work with), I
decided to add the feature as an experimental CT hook.

The hook notices any test failure, and forcefully aborts the current
script execution, but only during the initialization phase of the next
test; this ensure that all other hooks for a post-test failure have the
time to do things right.

The CT logs _will_ be interupted, and hooks of all kinds may also
suffer.

Since this might be a bit tricky to support internally, I will be fine
with a review that results in "no, Fred. Please test this in a project
plugin first" or if I get told to use profiles to mature the feature
(i.e. `rebar3 as failfast ct ...`) with a custom hook declaration.

src/cth_fail_fast.erl

@@ -0,0 +1,118 @@
+-module(cth_fail_fast).
+
+%% Callbacks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3, on_tc_skip/4]).
+
+-export([terminate/1]).
+
+%% We work by setting an 'abort' variable on each test case that fails
+%% and then triggering the failure before starting the next test. This
+%% ensures that all other hooks have run for the same event, and
+%% simplifies error reporting.
+-record(state, {abort=false}).
+
+%% @doc Return a unique id for this CTH.
+id(_Opts) ->
+    {?MODULE, make_ref()}.
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state.
+init(_Id, _Opts) ->
+    {ok, #state{}}.
+
+%% @doc Called before init_per_suite is called.
+pre_init_per_suite(_Suite,_Config,#state{abort=true}) ->
+    abort();
+pre_init_per_suite(_Suite,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after init_per_suite.
+post_init_per_suite(_Suite,_Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before end_per_suite.
+pre_end_per_suite(_Suite,_Config,#state{abort=true}) ->
+    abort();
+pre_end_per_suite(_Suite,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after end_per_suite.
+post_end_per_suite(_Suite,_Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before each init_per_group.
+pre_init_per_group(_Group,_Config,#state{abort=true}) ->
+    abort();
+pre_init_per_group(_Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each init_per_group.
+post_init_per_group(_Group,_Config,Return, State) ->
+    {Return, State}.
+
+%% @doc Called after each end_per_group.
+pre_end_per_group(_Group,_Config,#state{abort=true}) ->
+    abort();
+pre_end_per_group(_Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each end_per_group.
+post_end_per_group(_Group,_Config,Return, State) ->
+    {Return, State}.
+
+%% @doc Called before each test case.
+pre_init_per_testcase(_TC,_Config,#state{abort=true}) ->
+    abort();
+pre_init_per_testcase(_TC,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each test case.
+post_end_per_testcase(_TC,_Config,ok,State) ->
+    {ok, State};
+post_end_per_testcase(_TC,_Config,Error,State) ->
+    {Error, State#state{abort=true}}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
+on_tc_fail(_TC, _Reason, State) ->
+    State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. (>= 19.3)
+on_tc_skip(_Suite, _TC, {tc_auto_skip, _}, State) ->
+    State#state{abort=true};
+on_tc_skip(_Suite, _TC, _Reason, State) ->
+    State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. (Pre-19.3)
+on_tc_skip(_TC, {tc_auto_skip, _}, State) ->
+    State#state{abort=true};
+on_tc_skip(_TC, _Reason, State) ->
+    State.
+
+%% @doc Called when the scope of the CTH is done
+terminate(#state{}) ->
+    ok.
+
+%%% Helpers
+abort() ->
+    io:format(user, "Detected test failure. Aborting~n", []),
+    halt(1).

src/rebar_prv_common_test.erl

@@ -171,6 +171,9 @@ transform_opts([{cover, _}|Rest], Acc) ->
 %% drop verbose from opts, ct doesn't care about it
 transform_opts([{verbose, _}|Rest], Acc) ->
     transform_opts(Rest, Acc);
+%% drop fail_fast from opts, ct doesn't care about it
+transform_opts([{fail_fast, _}|Rest], Acc) ->
+    transform_opts(Rest, Acc);
 %% getopt should handle anything else
 transform_opts([Opt|Rest], Acc) ->
     transform_opts(Rest, [Opt|Acc]).
@@ -224,15 +227,21 @@ ensure_opts([V|Rest], Acc) ->
     ensure_opts(Rest, [V|Acc]).
 
 add_hooks(Opts, State) ->
+    FailFast = case fails_fast(State) of
+        true -> [cth_fail_fast];
+        false -> []
+    end,
     case {readable(State), lists:keyfind(ct_hooks, 1, Opts)} of
         {false, _} ->
             Opts;
         {Other, false} ->
-            [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other), cth_retry]} | Opts];
+            [{ct_hooks, [cth_readable_failonly, readable_shell_type(Other),
+                         cth_retry] ++ FailFast} | Opts];
         {Other, {ct_hooks, Hooks}} ->
             %% Make sure hooks are there once only.
-            ReadableHooks = [cth_readable_failonly, readable_shell_type(Other), cth_retry],
-            AllReadableHooks = [cth_readable_failonly, cth_retry,
+            ReadableHooks = [cth_readable_failonly, readable_shell_type(Other),
+                             cth_retry] ++ FailFast,
+            AllReadableHooks = [cth_readable_failonly, cth_retry, cth_fail_fast,
                                 cth_readable_shell, cth_readable_compact_shell],
             NewHooks =  (Hooks -- AllReadableHooks) ++ ReadableHooks,
             lists:keyreplace(ct_hooks, 1, Opts, {ct_hooks, NewHooks})
@@ -445,6 +454,10 @@ readable(State) ->
         undefined -> rebar_state:get(State, ct_readable, compact)
     end.
 
+fails_fast(State) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    proplists:get_value(fail_fast, RawOpts) == true.
+
 test_dirs(State, Apps, Opts) ->
     case proplists:get_value(spec, Opts) of
         undefined ->
@@ -773,7 +786,8 @@ ct_opts(_State) ->
      {setcookie, undefined, "setcookie", atom, help(setcookie)},
      {sys_config, undefined, "sys_config", string, help(sys_config)}, %% comma-separated list
      {compile_only, undefined, "compile_only", boolean, help(compile_only)},
-     {retry, undefined, "retry", boolean, help(retry)}
+     {retry, undefined, "retry", boolean, help(retry)},
+     {fail_fast, undefined, "fail_fast", {boolean, false}, help(fail_fast)}
     ].
 
 help(compile_only) ->
@@ -846,5 +860,9 @@ help(setcookie) ->
     "Sets the cookie if the node is distributed";
 help(retry) ->
     "Experimental feature. If any specification for previously failing test is found, runs them.";
+help(fail_fast) ->
+    "Experimental feature. If any test fails, the run is aborted. Since common test does not "
+    "support this natively, we abort the rebar3 run on a failure. This May break CT's disk logging and "
+    "other rebar3 features.";
 help(_) ->
     "".