add `error_on_warning' option to eunit provider

src/rebar_prv_eunit.erl

@@ -79,6 +79,9 @@ format_error(unknown_error) ->
     io_lib:format("Error running tests", []);
 format_error({error_running_tests, Reason}) ->
     io_lib:format("Error running tests: ~p", [Reason]);
+format_error({eunit_test_errors, Errors}) ->
+    io_lib:format(lists:concat(["Error Running EUnit Tests:"] ++
+                               lists:map(fun(Error) -> "~n  " ++ Error end, Errors)), []);
 format_error({error, Error}) ->
     format_error({error_running_tests, Error}).
 
@@ -141,22 +144,39 @@ select_tests(ProjectApps, [], []) -> default_tests(ProjectApps);
 select_tests(_ProjectApps, A, B)  -> A ++ B.
 
 validate_tests(State, ProjectApps, Tests) ->
-    {ok, lists:filter(fun(Elem) -> validate(State, ProjectApps, Elem) end, Tests)}.
+    gather_tests(fun(Elem) -> validate(State, ProjectApps, Elem) end, Tests, []).
+
+gather_tests(_F, [], Acc) -> {ok, lists:reverse(Acc)};
+gather_tests(F, [Test|Rest], Acc) ->
+    case F(Test) of
+        true  -> gather_tests(F, Rest, [Test|Acc]);
+        false -> gather_tests(F, Rest, Acc);
+        %% failure mode, switch to gathering errors
+        Error -> gather_errors(F, Rest, [Error])
+    end.
+
+gather_errors(_F, [], Acc) -> ?PRV_ERROR({eunit_test_errors, lists:reverse(Acc)});
+gather_errors(F, [Test|Rest], Acc) ->
+    case F(Test) of
+        true  -> gather_errors(F, Rest, Acc);
+        false -> gather_errors(F, Rest, Acc);
+        Error -> gather_errors(F, Rest, [Error|Acc])
+    end.
 
-validate(_State, ProjectApps, {application, App}) ->
-    validate_app(App, ProjectApps);
+validate(State, ProjectApps, {application, App}) ->
+    validate_app(State, ProjectApps, App);
 validate(State, _ProjectApps, {dir, Dir}) ->
     ok = maybe_compile_dir(State, Dir),
-    validate_dir(Dir);
+    validate_dir(State, Dir);
 validate(State, _ProjectApps, {file, File}) ->
     ok = maybe_compile_file(State, File),
-    validate_file(File);
-validate(_State, ProjectApps, {module, Module}) ->
-    validate_module(Module, ProjectApps);
-validate(_State, ProjectApps, {suite, Module}) ->
-    validate_module(Module, ProjectApps);
-validate(_State, ProjectApps, Module) when is_atom(Module) ->
-    validate_module(Module, ProjectApps);
+    validate_file(State, File);
+validate(State, ProjectApps, {module, Module}) ->
+    validate_module(State, ProjectApps, Module);
+validate(State, ProjectApps, {suite, Module}) ->
+    validate_module(State, ProjectApps, Module);
+validate(State, ProjectApps, Module) when is_atom(Module) ->
+    validate_module(State, ProjectApps, Module);
 validate(State, ProjectApps, Path) when is_list(Path) ->
     case ec_file:is_dir(Path) of
         true  -> validate(State, ProjectApps, {dir, Path});
@@ -167,32 +187,39 @@ validate(State, ProjectApps, Path) when is_list(Path) ->
 %% unvalidatable
 validate(_State, _ProjectApps, _Test) -> true.
 
-validate_app(AppName, []) ->
-    ?WARN(lists:concat(["Application `", AppName, "' not found in project."]), []),
-    false;
-validate_app(AppName, [App|Rest]) ->
+validate_app(State, [], AppName) ->
+    warn_or_error(State, lists:concat(["Application `", AppName, "' not found in project."]));
+validate_app(State, [App|Rest], AppName) ->
     case AppName == binary_to_atom(rebar_app_info:name(App), unicode) of
         true  -> true;
-        false -> validate_app(AppName, Rest)
+        false -> validate_app(State, Rest, AppName)
     end.
 
-validate_dir(Dir) ->
+validate_dir(State, Dir) ->
     case ec_file:is_dir(Dir) of
         true  -> true;
-        false -> ?WARN(lists:concat(["Directory `", Dir, "' not found."]), []), false
+        false -> warn_or_error(State, lists:concat(["Directory `", Dir, "' not found."]))
     end.
 
-validate_file(File) ->
+validate_file(State, File) ->
     case ec_file:exists(File) of
         true  -> true;
-        false -> ?WARN(lists:concat(["File `", File, "' not found."]), []), false
+        false -> warn_or_error(State, lists:concat(["File `", File, "' not found."]))
     end.
 
-validate_module(Module, Apps) ->
+validate_module(State, Apps, Module) ->
     AppModules = app_modules([binary_to_atom(rebar_app_info:name(A), unicode) || A <- Apps], []),
     case lists:member(Module, AppModules) of
         true  -> true;
-        false -> ?WARN(lists:concat(["Module `", Module, "' not found in applications."]), []), false
+        false -> warn_or_error(State, lists:concat(["Module `", Module, "' not found in applications."]))
+    end.
+
+warn_or_error(State, Msg) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    Error = proplists:get_value(error_on_warning, RawOpts, false),
+    case Error of
+        true  -> Msg;
+        false -> ?WARN(Msg, []), false
     end.
 
 project_apps(State) ->
@@ -267,6 +294,7 @@ eunit_opts(_State) ->
     [{app, undefined, "app", string, help(app)},
      {cover, $c, "cover", boolean, help(cover)},
      {dir, undefined, "dir", string, help(dir)},
+     {error_on_warning, $e, "error_on_warning", boolean, help(error)},
      {file, undefined, "file", string, help(file)},
      {module, undefined, "module", string, help(module)},
      {suite, undefined, "suite", string, help(suite)},
@@ -275,6 +303,7 @@ eunit_opts(_State) ->
 help(app)     -> "Comma separated list of application test suites to run. Equivalent to `[{application, App}]`.";
 help(cover)   -> "Generate cover data. Defaults to false.";
 help(dir)     -> "Comma separated list of dirs to load tests from. Equivalent to `[{dir, Dir}]`.";
+help(error)   -> "Error on invalid test specifications instead of warning.";
 help(file)    -> "Comma separated list of files to load tests from. Equivalent to `[{file, File}]`.";
 help(module)  -> "Comma separated list of modules to load tests from. Equivalent to `[{module, Module}]`.";
 help(suite)   -> "Comma separated list of test suites to run. Equivalent to `[{module, Suite}]`.";

test/rebar_eunit_SUITE.erl

@@ -15,10 +15,12 @@
          test_single_app_flag/1,
          test_multiple_app_flag/1,
          test_single_suite_flag/1,
-         test_suite_in_app_flag/1,
+         test_nonexistent_suite_flag/1,
          test_single_file_flag/1,
          test_multiple_file_flag/1,
-         test_config_tests/1]).
+         test_nonexistent_file_flag/1,
+         test_config_tests/1,
+         test_nonexistent_tests/1]).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
@@ -41,9 +43,9 @@ all() ->
      test_basic_exports, test_multi_exports,
      test_basic_defines, test_multi_defines,
      test_single_app_flag, test_multiple_app_flag,
-     test_single_suite_flag, test_suite_in_app_flag,
-     test_single_file_flag, test_multiple_file_flag,
-     test_config_tests].
+     test_single_suite_flag, test_nonexistent_suite_flag,
+     test_single_file_flag, test_multiple_file_flag, test_nonexistent_file_flag,
+     test_config_tests, test_nonexistent_tests].
 
 test_basic_app(Config) ->
     AppDir = ?config(apps, Config),
@@ -303,7 +305,7 @@ test_single_suite_flag(Config) ->
     Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
     {module, Suite1} = code:ensure_loaded(Suite1).
 
-test_suite_in_app_flag(Config) ->
+test_nonexistent_suite_flag(Config) ->
     AppDir = ?config(apps, Config),
 
     Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
@@ -320,17 +322,12 @@ test_suite_in_app_flag(Config) ->
                                       [kernel, stdlib]),
 
     RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit",
-                                    "--app=" ++ Name1,
-                                    "--suite=not_a_real_src_" ++ Name1],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
+    {error, {rebar_prv_eunit, Error}} = rebar_test_utils:run_and_check(Config,
+                                                                       RebarConfig,
+                                                                       ["eunit", "-e", "--suite=not_a_real_module"],
+                                                                       return),
 
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    {error, nofile} = code:ensure_loaded(Suite2).
+    Error = {eunit_test_errors, ["Module `not_a_real_module' not found in applications."]}.
 
 test_single_file_flag(Config) ->
     AppDir = ?config(apps, Config),
@@ -339,14 +336,16 @@ test_single_file_flag(Config) ->
     Vsn = rebar_test_utils:create_random_vsn(),
     rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
 
+    File = filename:join([AppDir, "_build", "test", "lib", Name, "ebin", "not_a_real_src_" ++ Name ++ "_tests.beam"]),
+
     RebarConfig = [{erl_opts, [{d, some_define}]}],
     rebar_test_utils:run_and_check(Config,
                                    RebarConfig,
-                                   ["eunit", "--file=" ++ AppDir ++ "/_build/test/lib/" ++ Name ++ "/ebin/not_a_real_src_" ++ Name ++ "_tests.beam"],
+                                   ["eunit", "--file=" ++ File],
                                    {ok, [{app, Name}]}),
 
-    File = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    {module, File} = code:ensure_loaded(File).
+    Mod = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
+    {module, Mod} = code:ensure_loaded(Mod).
 
 test_multiple_file_flag(Config) ->
     AppDir = ?config(apps, Config),
@@ -355,17 +354,39 @@ test_multiple_file_flag(Config) ->
     Vsn = rebar_test_utils:create_random_vsn(),
     rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
 
+    File1 = filename:join([AppDir, "_build", "test", "lib", Name, "ebin", "not_a_real_src_" ++ Name ++ "_tests.beam"]),
+    File2 = filename:join([AppDir, "_build", "test", "lib", Name, "ebin", "not_a_real_src_" ++ Name ++ ".beam"]),
+
+
     RebarConfig = [{erl_opts, [{d, some_define}]}],
     rebar_test_utils:run_and_check(Config,
                                    RebarConfig,
-                                   ["eunit", "--file=" ++ AppDir ++ "/_build/test/lib/" ++ Name ++ "/ebin/not_a_real_src_" ++ Name ++ "_tests.beam," ++ AppDir ++ "/_build/test/lib/" ++ Name ++ "/ebin/not_a_real_src_" ++ Name ++ ".beam"],
+                                   ["eunit", "--file=" ++ File1 ++ "," ++ File2],
                                    {ok, [{app, Name}]}),
 
-    File1 = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    {module, File1} = code:ensure_loaded(File1),
+    Mod1 = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
+    {module, Mod1} = code:ensure_loaded(Mod1),
+
+    Mod2 = list_to_atom("not_a_real_src_" ++ Name),
+    {module, Mod2} = code:ensure_loaded(Mod2).
+
+test_nonexistent_file_flag(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("nonexistent_file_flag_app_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(AppDir,
+                                      Name,
+                                      Vsn,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    {error, {rebar_prv_eunit, Error}} = rebar_test_utils:run_and_check(Config,
+                                                                       RebarConfig,
+                                                                       ["eunit", "-e", "--file=not_a_real_file.beam"],
+                                                                       return),
 
-    File2 = list_to_atom("not_a_real_src_" ++ Name),
-    {module, File2} = code:ensure_loaded(File2).
+    Error = {eunit_test_errors, ["File `not_a_real_file.beam' not found."]}.
 
 test_config_tests(Config) ->
     AppDir = ?config(apps, Config),
@@ -404,3 +425,36 @@ test_config_tests(Config) ->
     {error, nofile} = code:ensure_loaded(Suite2),
     {error, nofile} = code:ensure_loaded(all_tests).
 
+test_nonexistent_tests(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
+                                      Name1,
+                                      Vsn1,
+                                      [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
+                                      Name2,
+                                      Vsn2,
+                                      [kernel, stdlib]),
+
+    RebarConfig = [{erl_opts, [{d, some_define}]}],
+    {error, {rebar_prv_eunit, Error}} = rebar_test_utils:run_and_check(Config,
+                                                                       RebarConfig,
+                                                                       ["eunit",
+                                                                        "-e",
+                                                                        "--app=not_a_real_app",
+                                                                        "--module=not_a_real_module",
+                                                                        "--suite=not_a_real_suite",
+                                                                        "--file=not_a_real_file.beam",
+                                                                        "--dir=not_a_real_dir"],
+                                                                       return),
+
+    Error = {eunit_test_errors, ["Application `not_a_real_app' not found in project.",
+                                 "Directory `not_a_real_dir' not found.",
+                                 "File `not_a_real_file.beam' not found.",
+                                 "Module `not_a_real_module' not found in applications.",
+                                 "Module `not_a_real_suite' not found in applications."]}.