shiny and new test suite for eunit provider

src/rebar_prv_eunit.erl

@@ -8,6 +8,8 @@
 -export([init/1,
          do/1,
          format_error/1]).
+%% exported solely for tests
+-export([compile/1, prepare_tests/1, eunit_opts/1]).
 
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
@@ -37,11 +39,8 @@ init(State) ->
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
-    %% inject the `TEST` macro, `eunit_first_files` and `eunit_compile_opts`
-    %% into the applications to be compiled
-    NewState = inject_eunit_state(State),
-
-    case rebar_prv_compile:do(NewState) of
+    case compile(State) of
+        %% successfully compiled apps
         {ok, S} -> do_tests(S);
         %% this should look like a compiler error, not an eunit error
         Error   -> Error
@@ -49,9 +48,10 @@ do(State) ->
 
 do_tests(State) ->
     ?INFO("Performing EUnit tests...", []),
+
     rebar_utils:update_code(rebar_state:code_paths(State, all_deps)),
 
-    %% Run compile provider prehooks
+    %% Run eunit provider prehooks
     Providers = rebar_state:providers(State),
     Cwd = rebar_dir:get_cwd(),
     rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State),
@@ -101,8 +101,32 @@ format_error({error, Error}) ->
 %% Internal functions
 %% ===================================================================
 
-%% currently only add the `extra_drc_dirs` on provider init
-test_state(_State) -> [{extra_src_dirs, ["test"]}].
+test_state(State) ->
+    ErlOpts = rebar_state:get(State, erl_opts, []),
+    TestOpts = safe_define_test_macro(ErlOpts),
+    [{extra_src_dirs, ["test"]}, {erl_opts, TestOpts}].
+
+safe_define_test_macro(Opts) ->
+    %% defining a compile macro twice results in an exception so
+    %% make sure 'TEST' is only defined once
+    case test_defined(Opts) of
+       true -> Opts;
+       false -> [{d, 'TEST'}] ++ Opts
+    end.
+
+compile(State) ->
+    %% inject `eunit_first_files` and `eunit_compile_opts` into the applications to be compiled
+    NewState = inject_eunit_state(State),
+
+    case rebar_prv_compile:do(NewState) of
+        %% successfully compiled apps
+        {ok, S} ->
+            _ = maybe_cover_compile(S),
+            _ = maybe_compile_bare_testdir(S),
+            {ok, S};
+        %% this should look like a compiler error, not an eunit error
+        Error   -> Error
+    end.
 
 inject_eunit_state(State) ->
     Apps = project_apps(State),
@@ -110,11 +134,10 @@ inject_eunit_state(State) ->
     rebar_state:project_apps(State, ModdedApps).
 
 inject(State, App) ->
-    %% append `eunit_compile_opts` to app defined `erl_opts` and define
-    %% the `TEST` macro if not already compiled
+    %% append `eunit_compile_opts` to app defined `erl_opts`
     ErlOpts = rebar_app_info:get(App, erl_opts, []),
     EUnitOpts = rebar_state:get(State, eunit_compile_opts, []),
-    NewOpts = safe_define_test_macro(EUnitOpts ++ ErlOpts),
+    NewOpts = EUnitOpts ++ ErlOpts,
     %% append `eunit_first_files` to app defined `erl_first_files`
     FirstFiles = rebar_app_info:get(App, erl_first_files, []),
     EUnitFirstFiles = rebar_state:get(State, eunit_first_files, []),
@@ -124,29 +147,27 @@ inject(State, App) ->
                 App,
                 [{erl_opts, NewOpts}, {erl_first_files, NewFirstFiles}]).
 
-safe_define_test_macro(Opts) ->
-    %% defining a compile macro twice results in an exception so
-    %% make sure 'TEST' is only defined once
-    case test_defined(Opts) of
-       true -> Opts;
-       false -> [{d, 'TEST'}] ++ Opts
-    end.
-
 test_defined([{d, 'TEST'}|_]) -> true;
 test_defined([{d, 'TEST', true}|_]) -> true;
 test_defined([_|Rest]) -> test_defined(Rest);
 test_defined([]) -> false.
 
 prepare_tests(State) ->
-    {RawOpts, _} = rebar_state:command_parsed_args(State),
-    ok = maybe_cover_compile(State, RawOpts),
-    CmdTests = resolve_tests(RawOpts),
+    %% parse and translate command line tests
+    CmdTests = resolve_tests(State),
     CfgTests = rebar_state:get(State, eunit_tests, []),
     ProjectApps = project_apps(State),
-    Tests = select_tests(ProjectApps, CmdTests, CfgTests),
+    %% prioritize tests to run first trying any command line specified
+    %% tests falling back to tests specified in the config file finally
+    %% running a default set if no other tests are present
+    Tests = select_tests(State, ProjectApps, CmdTests, CfgTests),
+    %% check applications for existence in project, modules for existence
+    %% in applications, files and dirs for existence on disk and allow
+    %% any unrecognized tests through for eunit to handle
     validate_tests(State, ProjectApps, Tests).
 
-resolve_tests(RawOpts) ->
+resolve_tests(State) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
     Apps         = resolve(app, application, RawOpts),
     Applications = resolve(application, RawOpts),
     Dirs         = resolve(dir, RawOpts),
@@ -166,8 +187,38 @@ resolve(Flag, EUnitKey, RawOpts) ->
 normalize(Key, Value) when Key == dir; Key == file -> {Key, Value};
 normalize(Key, Value) -> {Key, list_to_atom(Value)}.
 
-select_tests(ProjectApps, [], []) -> default_tests(ProjectApps);
-select_tests(_ProjectApps, A, B)  -> A ++ B.
+project_apps(State) ->
+    filter_checkouts(rebar_state:project_apps(State)).
+
+filter_checkouts(Apps) -> filter_checkouts(Apps, []).
+
+filter_checkouts([], Acc) -> lists:reverse(Acc);
+filter_checkouts([App|Rest], Acc) ->
+    case rebar_app_info:is_checkout(App) of
+        true  -> filter_checkouts(Rest, Acc);
+        false -> filter_checkouts(Rest, [App|Acc])
+    end.
+
+select_tests(State, ProjectApps, [], []) -> default_tests(State, ProjectApps);
+select_tests(_State, _ProjectApps, [], Tests)  -> Tests;
+select_tests(_State, _ProjectApps, Tests, _) -> Tests.
+
+default_tests(State, Apps) ->
+    Tests = set_apps(Apps, []),
+    BareTest = filename:join([rebar_state:dir(State), "test"]),
+    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
+    case filelib:is_dir(BareTest) andalso not lists:any(F, Apps) of
+        %% `test` dir at root of project is already scheduled to be
+        %%  included or `test` does not exist
+        false -> lists:reverse(Tests);
+        %% need to add `test` dir at root to dirs to be included
+        true  -> lists:reverse([{dir, filename:join([rebar_dir:base_dir(State), "test"])}|Tests])
+    end.
+
+set_apps([], Acc) -> Acc;
+set_apps([App|Rest], Acc) ->
+    AppName = list_to_atom(binary_to_list(rebar_app_info:name(App))),
+    set_apps(Rest, [{application, AppName}|Acc]).
 
 validate_tests(State, ProjectApps, Tests) ->
     gather_tests(fun(Elem) -> validate(State, ProjectApps, Elem) end, Tests, []).
@@ -175,34 +226,30 @@ validate_tests(State, ProjectApps, Tests) ->
 gather_tests(_F, [], Acc) -> {ok, lists:reverse(Acc)};
 gather_tests(F, [Test|Rest], Acc) ->
     case F(Test) of
-        true  -> gather_tests(F, Rest, [Test|Acc]);
-        false -> gather_tests(F, Rest, Acc);
+        ok             -> gather_tests(F, Rest, [Test|Acc]);
         %% failure mode, switch to gathering errors
-        Error -> gather_errors(F, Rest, [Error])
+        {error, Error} -> gather_errors(F, Rest, [Error])
     end.
 
 gather_errors(_F, [], Acc) -> ?PRV_ERROR({eunit_test_errors, lists:reverse(Acc)});
 gather_errors(F, [Test|Rest], Acc) ->
     case F(Test) of
-        true  -> gather_errors(F, Rest, Acc);
-        false -> gather_errors(F, Rest, Acc);
-        Error -> gather_errors(F, Rest, [Error|Acc])
+        ok             -> gather_errors(F, Rest, Acc);
+        {error, Error} -> gather_errors(F, Rest, [Error|Acc])
     end.
 
 validate(State, ProjectApps, {application, App}) ->
     validate_app(State, ProjectApps, App);
 validate(State, _ProjectApps, {dir, Dir}) ->
-    ok = maybe_compile_dir(State, Dir),
     validate_dir(State, Dir);
 validate(State, _ProjectApps, {file, File}) ->
-    ok = maybe_compile_file(State, File),
     validate_file(State, File);
-validate(State, ProjectApps, {module, Module}) ->
-    validate_module(State, ProjectApps, Module);
-validate(State, ProjectApps, {suite, Module}) ->
-    validate_module(State, ProjectApps, Module);
-validate(State, ProjectApps, Module) when is_atom(Module) ->
-    validate_module(State, ProjectApps, Module);
+validate(State, _ProjectApps, {module, Module}) ->
+    validate_module(State, Module);
+validate(State, _ProjectApps, {suite, Module}) ->
+    validate_module(State, Module);
+validate(State, _ProjectApps, Module) when is_atom(Module) ->
+    validate_module(State, Module);
 validate(State, ProjectApps, Path) when is_list(Path) ->
     case ec_file:is_dir(Path) of
         true  -> validate(State, ProjectApps, {dir, Path});
@@ -211,80 +258,35 @@ validate(State, ProjectApps, Path) when is_list(Path) ->
 %% unrecognized tests should be included. if they're invalid eunit will error
 %% and rebar.config may contain arbitrarily complex tests that are effectively
 %% unvalidatable
-validate(_State, _ProjectApps, _Test) -> true.
+validate(_State, _ProjectApps, _Test) -> ok.
 
-validate_app(State, [], AppName) ->
-    warn_or_error(State, lists:concat(["Application `", AppName, "' not found in project."]));
+validate_app(_State, [], AppName) ->
+    {error, lists:concat(["Application `", AppName, "' not found in project."])};
 validate_app(State, [App|Rest], AppName) ->
     case AppName == binary_to_atom(rebar_app_info:name(App), unicode) of
-        true  -> true;
+        true  -> ok;
         false -> validate_app(State, Rest, AppName)
     end.
 
-validate_dir(State, Dir) ->
+validate_dir(_State, Dir) ->
     case ec_file:is_dir(Dir) of
-        true  -> true;
-        false -> warn_or_error(State, lists:concat(["Directory `", Dir, "' not found."]))
+        true  -> ok;
+        false -> {error, lists:concat(["Directory `", Dir, "' not found."])}
     end.
 
-validate_file(State, File) ->
+validate_file(_State, File) ->
     case ec_file:exists(File) of
-        true  -> true;
-        false -> warn_or_error(State, lists:concat(["File `", File, "' not found."]))
-    end.
-
-validate_module(State, Apps, Module) ->
-    AppModules = app_modules([binary_to_atom(rebar_app_info:name(A), unicode) || A <- Apps], []),
-    case lists:member(Module, AppModules) of
-        true  -> true;
-        false -> warn_or_error(State, lists:concat(["Module `", Module, "' not found in applications."]))
-    end.
-
-warn_or_error(State, Msg) ->
-    {RawOpts, _} = rebar_state:command_parsed_args(State),
-    Error = proplists:get_value(error_on_warning, RawOpts, false),
-    case Error of
-        true  -> Msg;
-        false -> ?WARN(Msg, []), false
-    end.
-
-project_apps(State) ->
-    filter_checkouts(rebar_state:project_apps(State)).
-
-filter_checkouts(Apps) -> filter_checkouts(Apps, []).
-
-filter_checkouts([], Acc) -> lists:reverse(Acc);
-filter_checkouts([App|Rest], Acc) ->
-    case rebar_app_info:is_checkout(App) of
-        true  -> filter_checkouts(Rest, Acc);
-        false -> filter_checkouts(Rest, [App|Acc])
+        true  -> ok;
+        false -> {error, lists:concat(["File `", File, "' not found."])}
     end.
 
-app_modules([], Acc) -> Acc;
-app_modules([App|Rest], Acc) ->
-    Unload = case application:load(App) of
-        ok                           -> true;
-        {error, {already_loaded, _}} -> false
-    end,
-    NewAcc = case application:get_key(App, modules) of
-        {ok, Modules} -> Modules ++ Acc;
-        undefined     -> Acc
-    end,
-    case Unload of
-        true  ->
-            application:unload(App),
-            app_modules(Rest, NewAcc);
-        false ->
-            app_modules(Rest, NewAcc)
+validate_module(_State, Module) ->
+    Path = code:which(Module),
+    case beam_lib:chunks(Path, [exports]) of
+        {ok, _}              -> ok;
+        {error, beam_lib, _} -> {error, lists:concat(["Module `", Module, "' not found in project."])}
     end.
 
-default_tests(Apps) -> set_apps(Apps, []).
-
-set_apps([], Acc) -> lists:reverse(Acc);
-set_apps([App|Rest], Acc) ->
-    AppName = list_to_atom(binary_to_list(rebar_app_info:name(App))),
-    set_apps(Rest, [{application, AppName}|Acc]).
-
 resolve_eunit_opts(State) ->
     {Opts, _} = rebar_state:command_parsed_args(State),
     EUnitOpts = rebar_state:get(State, eunit_opts, []),
@@ -300,16 +302,33 @@ set_verbose(Opts) ->
         false -> [verbose] ++ Opts
     end.
 
-maybe_compile_dir(_, _) -> ok.
-maybe_compile_file(_, _) -> ok.
-
-maybe_cover_compile(State, Opts) ->
-    State1 = case proplists:get_value(cover, Opts, false) of
+maybe_cover_compile(State) ->
+    State1 = case rebar_state:get(State, cover, false) of
         true  -> rebar_state:set(State, cover_enabled, true);
         false -> State
     end,
     rebar_prv_cover:maybe_cover_compile(State1).
 
+maybe_compile_bare_testdir(State) ->
+    case ec_file:is_dir(filename:join([rebar_state:dir(State), "test"])) of
+        true ->
+            ErlOpts = rebar_state:get(State, erl_opts, []),
+            EUnitOpts = rebar_state:get(State, eunit_compile_opts, []),
+            NewErlOpts = safe_define_test_macro(EUnitOpts ++ ErlOpts),
+            %% append `eunit_first_files` to app defined `erl_first_files`
+            FirstFiles = rebar_state:get(State, erl_first_files, []),
+            EUnitFirstFiles = rebar_state:get(State, eunit_first_files, []),
+            NewFirstFiles = EUnitFirstFiles ++ FirstFiles,
+            Opts = rebar_state:opts(State),
+            NewOpts = lists:foldl(fun({K, V}, Dict) -> rebar_opts:set(Dict, K, V) end,
+                                  Opts,
+                                  [{erl_opts, NewErlOpts}, {erl_first_files, NewFirstFiles}, {src_dirs, ["test"]}]),
+            OutDir = filename:join([rebar_dir:base_dir(State), "test"]),
+            filelib:ensure_dir(filename:join([OutDir, "dummy.beam"])),
+            rebar_erlc_compiler:compile(NewOpts, rebar_state:dir(State), OutDir);
+        false -> ok
+    end.
+
 handle_results(ok) -> ok;
 handle_results(error) ->
     {error, unknown_error};
@@ -321,7 +340,6 @@ eunit_opts(_State) ->
      {application, undefined, "application", string, help(app)},
      {cover, $c, "cover", boolean, help(cover)},
      {dir, undefined, "dir", string, help(dir)},
-     {error_on_warning, $e, "error_on_warning", boolean, help(error)},
      {file, undefined, "file", string, help(file)},
      {module, undefined, "module", string, help(module)},
      {suite, undefined, "suite", string, help(module)},
@@ -330,7 +348,6 @@ eunit_opts(_State) ->
 help(app)     -> "Comma separated list of application test suites to run. Equivalent to `[{application, App}]`.";
 help(cover)   -> "Generate cover data. Defaults to false.";
 help(dir)     -> "Comma separated list of dirs to load tests from. Equivalent to `[{dir, Dir}]`.";
-help(error)   -> "Error on invalid test specifications instead of warning.";
 help(file)    -> "Comma separated list of files to load tests from. Equivalent to `[{file, File}]`.";
 help(module)  -> "Comma separated list of modules to load tests from. Equivalent to `[{module, Module}]`.";
 help(verbose) -> "Verbose output. Defaults to false.".

test/rebar_eunit_SUITE.erl

@@ -1,598 +1,472 @@
 -module(rebar_eunit_SUITE).
 
--export([suite/0,
-         init_per_suite/1,
-         end_per_suite/1,
-         init_per_testcase/2,
-         all/0]).
--export([basic_app/1,
-         multi_app/1,
-         profile/1,
-         basic_exports/1,
-         multi_exports/1,
-         basic_defines/1,
-         multi_defines/1,
-         single_app_flag/1,
-         multiple_app_flag/1,
-         single_module_flag/1,
-         nonexistent_module_flag/1,
-         single_file_flag/1,
-         multiple_file_flag/1,
-         nonexistent_file_flag/1,
-         single_dir_flag/1,
-         multiple_dir_flag/1,
-         nonexistent_dir_flag/1,
-         config_tests/1,
-         nonexistent_tests/1,
-         eunit_compile_opts/1,
-         eunit_first_files/1]).
+-export([all/0, groups/0]).
+-export([init_per_suite/1, init_per_group/2, end_per_group/2]).
+-export([basic_app_compiles/1, basic_app_files/1, basic_app_exports/1, basic_app_testset/1]).
+-export([multi_app_compiles/1, multi_app_files/1, multi_app_exports/1, multi_app_testset/1]).
+-export([eunit_tests/1, eunit_opts/1, eunit_first_files/1]).
+-export([single_application_arg/1, multi_application_arg/1, missing_application_arg/1]).
+-export([single_module_arg/1, multi_module_arg/1, missing_module_arg/1]).
+-export([single_suite_arg/1, multi_suite_arg/1, missing_suite_arg/1]).
+-export([single_file_arg/1, multi_file_arg/1, missing_file_arg/1]).
+-export([single_dir_arg/1, multi_dir_arg/1, missing_dir_arg/1]).
+-export([multiple_arg_composition/1, multiple_arg_errors/1]).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("kernel/include/file.hrl").
 
-suite() ->
-    [].
-
+all() ->
+    [{group, basic_app}, {group, multi_app}, {group, cmd_line_args}].
+
+groups() ->
+    [{basic_app, [sequence], [basic_app_compiles, {group, basic_app_results}]},
+     {basic_app_results, [], [basic_app_files, basic_app_exports, basic_app_testset]},
+     {multi_app, [sequence], [multi_app_compiles, {group, multi_app_results}]},
+     {multi_app_results, [], [multi_app_files, multi_app_exports, multi_app_testset]},
+     {cmd_line_args, [], [eunit_tests, eunit_opts, eunit_first_files,
+                          single_application_arg, multi_application_arg, missing_application_arg,
+                          single_module_arg, multi_module_arg, missing_module_arg,
+                          single_suite_arg, multi_suite_arg, missing_suite_arg,
+                          single_file_arg, multi_file_arg, missing_file_arg,
+                          single_dir_arg, multi_dir_arg, missing_dir_arg,
+                          multiple_arg_composition, multiple_arg_errors]}].
+
+%% this just unzips the example apps used by tests to the priv dir for later use
 init_per_suite(Config) ->
+    PrivDir = ?config(priv_dir, Config),
+    DataDir = ?config(data_dir, Config),
+    {ok, Cwd} = file:get_cwd(),
+    file:set_cwd(PrivDir),
+    ok = ec_file:copy(filename:join([DataDir, "basic_app.zip"]), filename:join([PrivDir, "basic_app.zip"])),
+    {ok, _} = zip:extract(filename:join([PrivDir, "basic_app.zip"])),
+    ok = ec_file:copy(filename:join([DataDir, "multi_app.zip"]), filename:join([PrivDir, "multi_app.zip"])),
+    {ok, _} = zip:extract(filename:join([PrivDir, "multi_app.zip"])),
+    file:set_cwd(Cwd),
     Config.
 
-end_per_suite(_Config) ->
-    ok.
+init_per_group(basic_app, Config) ->
+    GroupState = rebar_test_utils:init_rebar_state(Config, "basic_app_"),
 
-init_per_testcase(_, Config) ->
-    rebar_test_utils:init_rebar_state(Config, "eunit_").
+    AppDir = ?config(apps, GroupState),
+    PrivDir = ?config(priv_dir, GroupState),
 
-all() ->
-    [basic_app, multi_app, profile,
-     basic_exports, multi_exports,
-     basic_defines, multi_defines,
-     single_app_flag, multiple_app_flag,
-     single_module_flag, nonexistent_module_flag,
-     single_file_flag, multiple_file_flag, nonexistent_file_flag,
-     single_dir_flag, multiple_dir_flag, nonexistent_dir_flag,
-     config_tests, nonexistent_tests,
-     eunit_compile_opts, eunit_first_files].
-
-basic_app(Config) ->
-    AppDir = ?config(apps, Config),
+    AppDirs = ["src", "include", "test"],
 
-    Name = rebar_test_utils:create_random_name("basic_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    lists:foreach(fun(F) -> ec_file:copy(filename:join([PrivDir, "basic_app", F]),
+                                         filename:join([AppDir, F]),
+                                         [recursive]) end, AppDirs),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config, RebarConfig, ["eunit"], {ok, [{app, Name}]}).
+    RebarConfig = [{erl_opts, [{d, config_define}]}, {eunit_compile_opts, [{d, eunit_compile_define}]}],
 
-multi_app(Config) ->
-    AppDir = ?config(apps, Config),
+    {ok, State} = rebar_test_utils:run_and_check(GroupState, RebarConfig, ["as", "test", "lock"], return),
 
-    Name1 = rebar_test_utils:create_random_name("multi_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name1}, {app, Name2}]}).
-
-profile(Config) ->
-    AppDir = ?config(apps, Config),
+    [{result, State}|GroupState];
+init_per_group(multi_app, Config) ->
+    GroupState = rebar_test_utils:init_rebar_state(Config, "multi_app_"),
 
-    Name = rebar_test_utils:create_random_name("profile_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    AppDir = ?config(apps, GroupState),
+    PrivDir = ?config(priv_dir, GroupState),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]},
-                  {profiles, [{test, [{erl_opts, [debug_info]}]}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["as", "test", "eunit"],
-                                   {ok, [{app, Name}]}).
+    AppDirs = ["apps", "test"],
 
-basic_exports(Config) ->
-    AppDir = ?config(apps, Config),
+    lists:foreach(fun(F) -> ec_file:copy(filename:join([PrivDir, "multi_app", F]),
+                                         filename:join([AppDir, F]),
+                                         [recursive]) end, AppDirs),
 
-    Name = rebar_test_utils:create_random_name("basic_exports_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name}]}),
-
-    App = list_to_atom("not_a_real_src_" ++ Name),
-    Suite = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    AppExports = App:module_info(exports),
-    SuiteExports = Suite:module_info(exports),
-    AppExpect = [{some_test_, 0}],
-    SuiteExpect = [{some_test_, 0}, {define_test_, 0}],
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, AppExports) end, AppExpect),
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, SuiteExports) end, SuiteExpect).
-
-multi_exports(Config) ->
-    AppDir = ?config(apps, Config),
+    RebarConfig = [{erl_opts, [{d, config_define}]}, {eunit_compile_opts, [{d, eunit_compile_define}]}],
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    App1 = list_to_atom("not_a_real_src_" ++ Name1),
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    AppExports1 = App1:module_info(exports),
-    SuiteExports1 = Suite1:module_info(exports),
-    App2 = list_to_atom("not_a_real_src_" ++ Name2),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    AppExports2 = App2:module_info(exports),
-    SuiteExports2 = Suite2:module_info(exports),
-    AppExpect = [{some_test_, 0}],
-    SuiteExpect = [{some_test_, 0}, {define_test_, 0}],
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, AppExports1) end, AppExpect),
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, SuiteExports1) end, SuiteExpect),
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, AppExports2) end, AppExpect),
-    lists:foreach(fun(Expect) -> true = lists:member(Expect, SuiteExports2) end, SuiteExpect).
-
-basic_defines(Config) ->
-    AppDir = ?config(apps, Config),
+    {ok, State} = rebar_test_utils:run_and_check(GroupState, RebarConfig, ["as", "test", "lock"], return),
+
+    [{result, State}|GroupState];
+init_per_group(cmd_line_args, Config) ->
+    GroupState = rebar_test_utils:init_rebar_state(Config, "cmd_line_args_"),
+
+    AppDir = ?config(apps, GroupState),
+    PrivDir = ?config(priv_dir, GroupState),
+
+    AppDirs = ["apps", "test"],
+
+    lists:foreach(fun(F) -> ec_file:copy(filename:join([PrivDir, "multi_app", F]),
+                                         filename:join([AppDir, F]),
+                                         [recursive]) end, AppDirs),
+
+    RebarConfig = [{erl_opts, [{d, config_define}]},
+                   {eunit_compile_opts, [{d, eunit_compile_define}]},
+                   %% test set not supported by cmd line args
+                   {eunit_tests, [{test, multi_app_bar, sanity_test},
+                                  {test, multi_app_baz, sanity_test}]},
+                   {eunit_opts, [verbose]},
+                   {eunit_first_files, [filename:join(["apps", "multi_app_bar", "test", "multi_app_bar_tests_helper.erl"]),
+                                        filename:join(["apps", "multi_app_baz", "test", "multi_app_baz_tests_helper.erl"])]}],
+
+    {ok, State} = rebar_test_utils:run_and_check(GroupState, RebarConfig, ["eunit"], return),
+
+    [{result, State}|GroupState];
+init_per_group(_, Config) -> Config.
 
-    Name = rebar_test_utils:create_random_name("basic_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+end_per_group(_, Config) -> Config.
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config, RebarConfig, ["eunit"], {ok, [{app, Name}]}),
 
-    App = list_to_atom("not_a_real_src_" ++ Name),
-    Suite = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    AppOpts = proplists:get_value(options, App:module_info(compile), []),
-    SuiteOpts = proplists:get_value(options, Suite:module_info(compile), []),
-    Expect = [{d, some_define}],
-    lists:foreach(fun(E) -> true = lists:member(E, AppOpts) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, SuiteOpts) end, Expect).
 
-multi_defines(Config) ->
+%% === tests for a single application at the root of a project ===
+
+%% check that project compiles properly
+basic_app_compiles(Config) ->
     AppDir = ?config(apps, Config),
+    State = ?config(result, Config),
+
+    {ok, _} = rebar_prv_eunit:do(State),
 
-    Name1 = rebar_test_utils:create_random_name("multi_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    App1 = list_to_atom("not_a_real_src_" ++ Name1),
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    AppOpts1 = proplists:get_value(options, App1:module_info(compile), []),
-    SuiteOpts1 = proplists:get_value(options, Suite1:module_info(compile), []),
-    App2 = list_to_atom("not_a_real_src_" ++ Name2),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    AppOpts2 = proplists:get_value(options, App2:module_info(compile), []),
-    SuiteOpts2 = proplists:get_value(options, Suite2:module_info(compile), []),
-    Expect = [{d, some_define}],
-    lists:foreach(fun(E) -> true = lists:member(E, AppOpts1) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, SuiteOpts1) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, AppOpts2) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, SuiteOpts2) end, Expect).
-
-single_app_flag(Config) ->
+    rebar_test_utils:check_results(AppDir, [{app, "basic_app"}], "*").
+
+%% check that all files expected to be present are present
+basic_app_files(Config) ->
     AppDir = ?config(apps, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    BareSuite = io_lib:format("-module(all_tests).\n";
-                              "-compile(export_all).\n"
-                              "-include_lib(\"eunit/include/eunit.hrl\").\n"
-                              "some_test_() -> ?_assert(true).\n"
-                              "define_test_() -> ?_assertEqual(true, ?some_define).\n", []),
-    FileName = filename:join([AppDir, "test", "all_tests.erl"]),
-    ok = filelibn>:ensure_dir(FileName),
-    ok = ec_file:write(FileName, BareSuite),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--application=" ++ Name1],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    {error, nofile} = code:ensure_loaded(Suite2),
-    {error, nofile} = code:ensure_loaded(all_tests).
-
-multiple_app_flag(Config) ->
+    lists:foreach(fun(F) -> true = ec_file:exists(filename:join([AppDir, "_build", "test", "lib", "basic_app", "ebin", F])) end,
+                  ["basic_app.app", "basic_app.beam", "basic_app_tests.beam", "basic_app_tests_helper.beam"]).
+
+%% check that the correct tests are exported from modules for project
+%% note that this implies `TEST` is set correctly
+basic_app_exports(_Config) ->
+    Tests = fun(Mod) ->
+        begin
+            Path = code:which(Mod),
+            {ok, {Mod, [{exports, Ex}]}} = beam_lib:chunks(Path, [exports]),
+            true = lists:member({sanity_test, 0}, Ex)
+        end
+    end,
+    Helpers = fun(Mod) ->
+        begin
+            Path = code:which(Mod),
+            {ok, {Mod, [{exports, Ex}]}} = beam_lib:chunks(Path, [exports]),
+            true = lists:member({help, 0}, Ex)
+        end
+    end,
+    lists:foreach(Tests, [basic_app, basic_app_tests]),
+    lists:foreach(Helpers, [basic_app_tests_helper]).
+
+%% check that the correct tests are schedule to run for project
+basic_app_testset(Config) ->
+    Result = ?config(result, Config),
+
+    {ok, [{application, basic_app}]} = rebar_prv_eunit:prepare_tests(Result).
+
+
+
+%% === tests for multiple applications in the `apps' directory of a project ===
+
+%% check that project compiles properly
+multi_app_compiles(Config) ->
     AppDir = ?config(apps, Config),
+    State = ?config(result, Config),
+
+    {ok, _} = rebar_prv_eunit:do(State),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    BareSuite = io_lib:format("-module(all_tests).\n"
-                              "-compile(export_all).\n"
-                              "-include_lib(\"eunit/include/eunit.hrl\").\n"
-                              "some_test_() -> ?_assert(true).\n"
-                              "define_test_() -> ?_assertEqual(true, ?some_define).\n", []),
-    FileName = filename:join([AppDir, "test", "all_tests.erl"]),
-    ok = filelib:ensure_dir(FileName),
-    ok = ec_file:write(FileName, BareSuite),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--application=" ++ Name1 ++ "," ++ Name2],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    {module, Suite2} = code:ensure_loaded(Suite2),
-    {error, nofile} = code:ensure_loaded(all_tests).
-
-single_module_flag(Config) ->
+    rebar_test_utils:check_results(AppDir, [{app, "multi_app_bar"}, {app, "multi_app_baz"}], "*").
+
+%% check that all files expected to be present are present
+multi_app_files(Config) ->
     AppDir = ?config(apps, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--module=not_a_real_src_" ++ Name1],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1).
-
-nonexistent_module_flag(Config) ->
+    lists:foreach(fun(F) -> true = ec_file:exists(filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin", F])) end,
+                  ["multi_app_bar.app", "multi_app_bar.beam", "multi_app_bar_tests.beam", "multi_app_bar_tests_helper.beam"]),
+    lists:foreach(fun(F) -> true = ec_file:exists(filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin", F])) end,
+                  ["multi_app_baz.app", "multi_app_baz.beam", "multi_app_baz_tests.beam", "multi_app_baz_tests_helper.beam"]),
+    lists:foreach(fun(F) -> true = ec_file:exists(filename:join([AppDir, "_build", "test", "test", F])) end,
+                  ["multi_app_tests.beam", "multi_app_tests_helper.beam"]).
+
+%% check that the correct tests are exported from modules for project
+%% note that this implies `TEST` is set correctly
+multi_app_exports(_Config) ->
+    Tests = fun(Mod) ->
+        begin
+            Ex = Mod:module_info(exports),
+            true = lists:member({sanity_test, 0}, Ex)
+        end
+    end,
+    Helpers = fun(Mod) ->
+        begin
+            Ex = Mod:module_info(exports),
+            true = lists:member({help, 0}, Ex)
+        end
+    end,
+    lists:foreach(Tests, [multi_app_bar, multi_app_bar_tests,
+                          multi_app_baz, multi_app_baz_tests,
+                          multi_app_tests]),
+    lists:foreach(Helpers, [multi_app_bar_tests_helper, multi_app_baz_tests_helper, multi_app_tests_helper]).
+
+%% check that the correct tests are schedule to run for project
+multi_app_testset(Config) ->
     AppDir = ?config(apps, Config),
+    Result = ?config(result, Config),
+
+    Set = {ok, [{application, multi_app_bar}, {application, multi_app_baz}, {dir, filename:join([AppDir, "_build", "test", "test"])}]},
+    Set = rebar_prv_eunit:prepare_tests(Result).
+
+
+
+%% === tests for command line arguments ===
+
+%% no explicit test for cmd line args taking precedence over the rebar.config since
+%% almost every single test implies it
+
+%% check tests in the rebar.config are run if no cmd line opts are specified
+eunit_tests(Config) ->
+    State = ?config(result, Config),
+
+    Expect = {ok, [{test, multi_app_bar, sanity_test}, {test, multi_app_baz, sanity_test}]},
+    Expect = rebar_prv_eunit:prepare_tests(State).
+
+%% check eunit_opts from the rebar.config are respected
+eunit_opts(Config) ->
+    State = ?config(result, Config),
+
+    Apps = rebar_state:project_apps(State),
+    lists:foreach(fun(App) -> [verbose] = rebar_app_info:get(App, eunit_opts) end,
+                  Apps).
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    {error, {rebar_prv_eunit, Error}} = rebar_test_utils:run_and_check(Config,
-                                                                       RebarConfig,
-                                                                       ["eunit", "-e", "--module=not_a_real_module"],
-                                                                       return),
-
-    Error = {eunit_test_errors, ["Module `not_a_real_module' not found in applications."]}.
-
-single_file_flag(Config) ->
+%% check eunit_first_files from the rebar.config are respected
+eunit_first_files(Config) ->
+    State = ?config(result, Config),
+
+    FirstFiles = [filename:join(["apps", "multi_app_bar", "test", "multi_app_bar_tests_helper.erl"]),
+                  filename:join(["apps", "multi_app_baz", "test", "multi_app_baz_tests_helper.erl"])],
+
+    Apps = rebar_state:project_apps(State),
+    lists:foreach(fun(App) -> FirstFiles = rebar_app_info:get(App, eunit_first_files) end,
+                  Apps).
+
+%% check that the --application cmd line opt generates the correct test set
+single_application_arg(Config) ->
+    S = ?config(result, Config),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=multi_app_bar"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{application, multi_app_bar}]} = rebar_prv_eunit:prepare_tests(State).
+
+multi_application_arg(Config) ->
+    S = ?config(result, Config),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=multi_app_bar,multi_app_baz"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{application, multi_app_bar}, {application, multi_app_baz}]} = rebar_prv_eunit:prepare_tests(State).
+
+%% check that an invalid --application cmd line opt generates an error
+missing_application_arg(Config) ->
+    S = ?config(result, Config),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=missing_app"]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["Application `missing_app' not found in project."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
+
+%% check that the --module cmd line opt generates the correct test set
+single_module_arg(Config) ->
     AppDir = ?config(apps, Config),
+    S = ?config(result, Config),
 
-    Name = rebar_test_utils:create_random_name("single_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
 
-    File = filename:join([AppDir, "_build", "test", "lib", Name, "ebin", "not_a_real_src_" ++ Name ++ "_tests.beam"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--module=multi_app_bar"]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--file=" ++ File],
-                                   {ok, [{app, Name}]}),
+    {ok, [{module, multi_app_bar}]} = rebar_prv_eunit:prepare_tests(State),
 
-    Mod = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    {module, Mod} = code:ensure_loaded(Mod).
+    %% restore path
+    code:set_path(Path).
 
-multiple_file_flag(Config) ->
+multi_module_arg(Config) ->
     AppDir = ?config(apps, Config),
+    S = ?config(result, Config),
+
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin"])]),
 
-    Name = rebar_test_utils:create_random_name("multiple_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--module=multi_app_bar,multi_app_baz"]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    File1 = filename:join([AppDir, "_build", "test", "lib", Name, "ebin", "not_a_real_src_" ++ Name ++ "_tests.beam"]),
-    File2 = filename:join([AppDir, "_build", "test", "lib", Name, "ebin", "not_a_real_src_" ++ Name ++ ".beam"]),
+    {ok, [{module, multi_app_bar}, {module, multi_app_baz}]} = rebar_prv_eunit:prepare_tests(State),
 
+    %% restore path
+    code:set_path(Path).
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--file=" ++ File1 ++ "," ++ File2],
-                                   {ok, [{app, Name}]}),
+%% check that an invalid --module cmd line opt generates an error
+missing_module_arg(Config) ->
+    S = ?config(result, Config),
 
-    Mod1 = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    {module, Mod1} = code:ensure_loaded(Mod1),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--module=missing_app"]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    Mod2 = list_to_atom("not_a_real_src_" ++ Name),
-    {module, Mod2} = code:ensure_loaded(Mod2).
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["Module `missing_app' not found in project."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
 
-nonexistent_file_flag(Config) ->
+%% check that the --suite cmd line opt generates the correct test set
+single_suite_arg(Config) ->
     AppDir = ?config(apps, Config),
+    S = ?config(result, Config),
 
-    Name = rebar_test_utils:create_random_name("nonexistent_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir,
-                                      Name,
-                                      Vsn,
-                                      [kernel, stdlib]),
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    {error, {rebar_prv_eunit, Error}} = rebar_test_utils:run_and_check(Config,
-                                                                       RebarConfig,
-                                                                       ["eunit", "-e", "--file=not_a_real_file.beam"],
-                                                                       return),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--suite=multi_app_bar"]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    Error = {eunit_test_errors, ["File `not_a_real_file.beam' not found."]}.
+    {ok, [{module, multi_app_bar}]} = rebar_prv_eunit:prepare_tests(State),
 
-single_dir_flag(Config) ->
+    %% restore path
+    code:set_path(Path).
+
+multi_suite_arg(Config) ->
     AppDir = ?config(apps, Config),
+    S = ?config(result, Config),
+
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin"])]),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--suite=multi_app_bar,multi_app_baz"]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    Name = rebar_test_utils:create_random_name("single_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    {ok, [{module, multi_app_bar}, {module, multi_app_baz}]} = rebar_prv_eunit:prepare_tests(State),
 
-    Dir = filename:join([AppDir, "_build", "test", "lib", Name, "ebin"]),
+    %% restore path
+    code:set_path(Path).
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--dir=" ++ Dir],
-                                   {ok, [{app, Name}]}),
+%% check that an invalid --suite cmd line opt generates an error
+missing_suite_arg(Config) ->
+    S = ?config(result, Config),
 
-    Mod = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    {module, Mod} = code:ensure_loaded(Mod).
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--suite=missing_app"]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-multiple_dir_flag(Config) ->
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["Module `missing_app' not found in project."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
+
+%% check that the --file cmd line opt generates the correct test set
+single_file_arg(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name = rebar_test_utils:create_random_name("multiple_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    Path = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin", "multi_app_bar.beam"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--file=" ++ Path]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{file, Path}]} = rebar_prv_eunit:prepare_tests(State).
+
+multi_file_arg(Config) ->
+    S = ?config(result, Config),
+    AppDir = ?config(apps, Config),
 
-    Dir1 = filename:join([AppDir, "_build", "test", "lib", Name, "ebin"]),
-    Dir2 = filename:join([AppDir, "_build", "test", "lib", Name, "ebin"]),
+    BarPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin", "multi_app_bar.beam"]),
+    BazPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin", "multi_app_baz.beam"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--file=" ++ BarPath ++ "," ++ BazPath]),
+    State = rebar_state:command_parsed_args(S, Args),
 
+    {ok, [{file, BarPath}, {file, BazPath}]} = rebar_prv_eunit:prepare_tests(State).
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit", "--dir=" ++ Dir1 ++ "," ++ Dir2],
-                                   {ok, [{app, Name}]}),
+%% check that an invalid --file cmd line opt generates an error
+missing_file_arg(Config) ->
+    S = ?config(result, Config),
+    AppDir = ?config(apps, Config),
 
-    Mod1 = list_to_atom("not_a_real_src_" ++ Name ++ "_tests"),
-    {module, Mod1} = code:ensure_loaded(Mod1),
+    Path = filename:join([AppDir, "_build", "test", "lib", "missing_app", "ebin", "missing_app.beam"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--file=" ++ Path]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    Mod2 = list_to_atom("not_a_real_src_" ++ Name),
-    {module, Mod2} = code:ensure_loaded(Mod2).
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["File `" ++ Path ++"' not found."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
 
-nonexistent_dir_flag(Config) ->
+%% check that the --dir cmd line opt generates the correct test set
+single_dir_arg(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name = rebar_test_utils:create_random_name("nonexistent_file_flag_app_"),
-    Vsn = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(AppDir,
-                                      Name,
-                                      Vsn,
-                                      [kernel, stdlib]),
+    Path = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--dir=" ++ Path]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    {ok, [{dir, Path}]} = rebar_prv_eunit:prepare_tests(State).
+
+multi_dir_arg(Config) ->
+    S = ?config(result, Config),
+    AppDir = ?config(apps, Config),
 
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    {error, {rebar_prv_eunit, Error}} = rebar_test_utils:run_and_check(Config,
-                                                                       RebarConfig,
-                                                                       ["eunit", "-e", "--dir=not_a_real_dir"],
-                                                                       return),
+    BarPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"]),
+    BazPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_baz", "ebin"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--dir=" ++ BarPath ++ "," ++ BazPath]),
+    State = rebar_state:command_parsed_args(S, Args),
 
-    Error = {eunit_test_errors, ["Directory `not_a_real_dir' not found."]}.
+    {ok, [{dir, BarPath}, {dir, BazPath}]} = rebar_prv_eunit:prepare_tests(State).
 
-config_tests(Config) ->
+%% check that an invalid --dir cmd line opt generates an error
+missing_dir_arg(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name1 = rebar_test_utils:create_random_name("config_tests_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("config_tests_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    BareSuite = io_lib:format("-module(all_tests).\n"
-                              "-compile(export_all).\n"
-                              "-include_lib(\"eunit/include/eunit.hrl\").\n"
-                              "some_test_() -> ?_assert(true).\n"
-                              "define_test_() -> ?_assertEqual(true, ?some_define).\n", []),
-    FileName = filename:join([AppDir, "test", "all_tests.erl"]),
-    ok = filelib:ensure_dir(FileName),
-    ok = ec_file:write(FileName, BareSuite),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}, {eunit_tests, [{application, list_to_atom(Name1)}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    {module, Suite1} = code:ensure_loaded(Suite1),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    {error, nofile} = code:ensure_loaded(Suite2),
-    {error, nofile} = code:ensure_loaded(all_tests).
-
-nonexistent_tests(Config) ->
+    Path = filename:join([AppDir, "_build", "test", "lib", "missing_app", "ebin"]),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--dir=" ++ Path]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Error = {error, {rebar_prv_eunit, {eunit_test_errors, ["Directory `" ++ Path ++"' not found."]}}},
+    Error = rebar_prv_eunit:prepare_tests(State).
+
+%% check that multiple args are composed
+multiple_arg_composition(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_exports_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_exports_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}],
-    {error, {rebar_prv_eunit, Error}} = rebar_test_utils:run_and_check(Config,
-                                                                       RebarConfig,
-                                                                       ["eunit",
-                                                                        "-e",
-                                                                        "--app=not_a_real_app",
-                                                                        "--application=not_a_real_application",
-                                                                        "--module=not_a_real_module",
-                                                                        "--suite=not_a_real_suite",
-                                                                        "--file=not_a_real_file.beam",
-                                                                        "--dir=not_a_real_dir"],
-                                                                       return),
-
-    Error = {eunit_test_errors, ["Application `not_a_real_app' not found in project.",
-                                 "Application `not_a_real_application' not found in project.",
-                                 "Directory `not_a_real_dir' not found.",
-                                 "File `not_a_real_file.beam' not found.",
-                                 "Module `not_a_real_module' not found in applications.",
-                                 "Module `not_a_real_suite' not found in applications."]}.
-
-eunit_compile_opts(Config) ->
+    %% necessary to fix paths
+    Path = code:get_path(),
+    code:add_paths([filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"])]),
+    FilePath = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin", "multi_app_bar.beam"]),
+    DirPath = filename:join([AppDir, "_build", "test", "lib", "multi_app_bar", "ebin"]),
+
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=multi_app_bar",
+                                                              "--module=multi_app_bar",
+                                                              "--suite=multi_app_bar",
+                                                              "--file=" ++ FilePath,
+                                                              "--dir=" ++ DirPath]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Expect = [{application, multi_app_bar},
+              {dir, DirPath},
+              {file, FilePath},
+              {module, multi_app_bar},
+              {module, multi_app_bar}],
+
+    {ok, Expect} = rebar_prv_eunit:prepare_tests(State),
+
+    %% restore path
+    code:set_path(Path).
+
+%% check that multiple errors are reported
+multiple_arg_errors(Config) ->
+    S = ?config(result, Config),
     AppDir = ?config(apps, Config),
 
-    Name1 = rebar_test_utils:create_random_name("multi_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    RebarConfig = [{erl_opts, [{d, some_define}]}, {eunit_compile_opts, [{d, some_other_define}]}],
-    rebar_test_utils:run_and_check(Config,
-                                   RebarConfig,
-                                   ["eunit"],
-                                   {ok, [{app, Name1}, {app, Name2}]}),
-
-    App1 = list_to_atom("not_a_real_src_" ++ Name1),
-    Suite1 = list_to_atom("not_a_real_src_" ++ Name1 ++ "_tests"),
-    AppOpts1 = proplists:get_value(options, App1:module_info(compile), []),
-    SuiteOpts1 = proplists:get_value(options, Suite1:module_info(compile), []),
-
-    App2 = list_to_atom("not_a_real_src_" ++ Name2),
-    Suite2 = list_to_atom("not_a_real_src_" ++ Name2 ++ "_tests"),
-    AppOpts2 = proplists:get_value(options, App2:module_info(compile), []),
-    SuiteOpts2 = proplists:get_value(options, Suite2:module_info(compile), []),
-
-    Expect = [{d, some_other_define}, {d, some_define}],
-    lists:foreach(fun(E) -> true = lists:member(E, AppOpts1) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, SuiteOpts1) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, AppOpts2) end, Expect),
-    lists:foreach(fun(E) -> true = lists:member(E, SuiteOpts2) end, Expect).
+    FilePath = filename:join([AppDir, "_build", "test", "lib", "missing_app", "ebin", "missing_app.beam"]),
+    DirPath = filename:join([AppDir, "_build", "test", "lib", "missing_app", "ebin"]),
 
-eunit_first_files(Config) ->
-    AppDir = ?config(apps, Config),
+    {ok, Args} = getopt:parse(rebar_prv_eunit:eunit_opts(S), ["--application=missing_app",
+                                                              "--module=missing_app",
+                                                              "--suite=missing_app",
+                                                              "--file=" ++ FilePath,
+                                                              "--dir=" ++ DirPath]),
+    State = rebar_state:command_parsed_args(S, Args),
+
+    Expect = ["Application `missing_app' not found in project.",
+              "Directory `" ++ DirPath ++ "' not found.",
+              "File `" ++ FilePath ++ "' not found.",
+              "Module `missing_app' not found in project.",
+              "Module `missing_app' not found in project."],
+
+    {error, {rebar_prv_eunit, {eunit_test_errors, Expect}}} = rebar_prv_eunit:prepare_tests(State).
 
-    Name1 = rebar_test_utils:create_random_name("multi_app1_"),
-    Vsn1 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name1]),
-                                      Name1,
-                                      Vsn1,
-                                      [kernel, stdlib]),
-    Name2 = rebar_test_utils:create_random_name("multi_app2_"),
-    Vsn2 = rebar_test_utils:create_random_vsn(),
-    rebar_test_utils:create_eunit_app(filename:join([AppDir,"apps",Name2]),
-                                      Name2,
-                                      Vsn2,
-                                      [kernel, stdlib]),
-
-    ErlFirstFiles = ["not_a_real_src_" ++ Name1, "not_a_real_src_" ++ Name2],
-    EUnitFirstFiles = ["not_a_real_src_" ++ Name1 ++ "_tests", "not_a_real_src_" ++ Name2 ++ "_tests"],
-
-    RebarConfig = [{erl_opts, [{d, some_define}]},
-                   {erl_first_files, ErlFirstFiles},
-                   {eunit_first_files, EUnitFirstFiles}],
-    {ok, State} = rebar_test_utils:run_and_check(Config,
-                                                 RebarConfig,
-                                                 ["eunit"],
-                                                 {ok, [{app, Name1}, {app, Name2}]}),
-
-    AllFirstFiles = EUnitFirstFiles ++ ErlFirstFiles,
-    Apps = rebar_state:project_apps(State),
-    lists:foreach(fun(App) -> AllFirstFiles = rebar_app_info:get(App, erl_first_files) end,
-                  Apps).

test/rebar_eunit_SUITE_data/deflate

@@ -0,0 +1,15 @@
+#!/usr/bin/env escript
+
+main(["basic_app"]) ->
+  case filelib:is_dir("basic_app") of
+    true  -> zip:create("basic_app.zip", ["basic_app"]), halt(0);
+    false -> io:format("unable to locate basic_app directory~n", []), halt(1)
+  end;
+main(["multi_app"]) ->
+  case filelib:is_dir("multi_app") of
+    true  -> zip:create("multi_app.zip", ["multi_app"]), halt(0);
+    false -> io:format("unable to locate multi_app directory~n", []), halt(1)
+  end,
+  halt(0);
+main(_) ->
+  io:format("usage: deflate basic_app | multi_app~n", []), halt(1).

test/rebar_eunit_SUITE_data/inflate

@@ -0,0 +1,15 @@
+#!/usr/bin/env escript
+
+main(["basic_app"]) ->
+  case filelib:is_file("basic_app.zip") of
+    true  -> zip:unzip("basic_app.zip"), halt(0);
+    false -> io:format("unable to locate basic_app.zip~n", []), halt(1)
+  end;
+main(["multi_app"]) ->
+  case filelib:is_file("multi_app.zip") of
+    true  -> zip:unzip("multi_app.zip"), halt(0);
+    false -> io:format("unable to locate multi_app.zip~n", []), halt(1)
+  end,
+  halt(0);
+main(_) ->
+  io:format("usage: inflate basic_app | multi_app~n", []), halt(1).

test/rebar_test_utils.erl

@@ -1,7 +1,7 @@
 -module(rebar_test_utils).
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
--export([init_rebar_state/1, init_rebar_state/2, run_and_check/4]).
+-export([init_rebar_state/1, init_rebar_state/2, run_and_check/4, check_results/3]).
 -export([expand_deps/2, flat_deps/1, top_level_deps/1]).
 -export([create_app/4, create_eunit_app/4, create_empty_app/4, create_config/2,
          package_app/3]).