Merge pull request #305 from talentdeficit/better_ct

modify `ct` provider to copy selected directories and compile them

src/rebar_prv_common_test.erl

@@ -2,12 +2,13 @@
 %% ex: ts=4 sw=4 et
 
 -module(rebar_prv_common_test).
-
 -behaviour(provider).
 
 -export([init/1,
          do/1,
          format_error/1]).
+%% exported for test purposes, consider private
+-export([setup_ct/1]).
 
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
@@ -37,54 +38,352 @@ init(State) ->
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
     ?INFO("Running Common Test suites...", []),
-    {RawOpts, _} = rebar_state:command_parsed_args(State),
-    Opts = transform_opts(RawOpts),
-    TestApps = filter_checkouts(rebar_state:project_apps(State)),
-    ok = create_dirs(Opts),
-    InDirs = in_dirs(State, RawOpts),
-    ok = compile_tests(State, TestApps, InDirs),
-    case resolve_ct_opts(State, TestApps, Opts) of
-        {ok, CTOpts} ->
-            run_test(State, RawOpts, CTOpts);
-        {error, Reason} ->
-            ?PRV_ERROR(Reason)
-    end.
-
-run_test(State, RawOpts, CTOpts) ->
-    ok = maybe_cover_compile(State, RawOpts),
-    Verbose = proplists:get_value(verbose, RawOpts, false),
-    Result = run_test(CTOpts, Verbose),
-    ok = rebar_prv_cover:maybe_write_coverdata(State, ?PROVIDER),
-    case Result of
-        {error, Reason} ->
-            {error, {?MODULE, Reason}};
-        ok ->
-            {ok, State}
+    try
+        case setup_ct(State) of
+            {error, {no_tests_specified, Opts}} ->
+                ?WARN("No tests specified in opts: ~p", [Opts]),
+                {ok, State};
+            Opts ->
+                Opts1 = setup_logdir(State, Opts),
+                ?DEBUG("common test opts: ~p", [Opts1]),
+                run_test(State, Opts1)
+        end
+    catch error:Reason -> ?PRV_ERROR(Reason)
     end.
 
 -spec format_error(any()) -> iolist().
+format_error({multiple_dirs_and_suites, Opts}) ->
+    io_lib:format("Multiple dirs declared alongside suite in opts: ~p", [Opts]);
+format_error({bad_dir_or_suite, Opts}) ->
+    io_lib:format("Bad value for dir or suite in opts: ~p", [Opts]);
 format_error({failures_running_tests, {Failed, AutoSkipped}}) ->
     io_lib:format("Failures occured running tests: ~b", [Failed+AutoSkipped]);
 format_error({error_running_tests, Reason}) ->
     io_lib:format("Error running tests: ~p", [Reason]);
-format_error({error_processing_options, Reason}) ->
-    io_lib:format("Error processing options: ~p", [Reason]).
+format_error({error, Reason}) ->
+    io_lib:format("Unknown error: ~p", [Reason]).
+
+%% ===================================================================
+%% Internal functions
+%% ===================================================================
+
+run_test(State, Opts) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    Result = case proplists:get_value(verbose, RawOpts, false) of
+        true  -> run_test(Opts);
+        false -> run_test_quiet(Opts)
+    end,
+    ok = rebar_prv_cover:maybe_write_coverdata(State, ?PROVIDER),
+    case Result of
+        ok    -> {ok, State};
+        Error -> Error
+    end.
+
+run_test(Opts) -> handle_results(ct:run_test(Opts)).
 
-run_test(CTOpts, true) ->
-    handle_results(ct:run_test(CTOpts));
-run_test(CTOpts, false) ->
+run_test_quiet(Opts) ->
     Pid = self(),
-    LogDir = proplists:get_value(logdir, CTOpts),
+    LogDir = proplists:get_value(logdir, Opts),
     erlang:spawn_monitor(fun() ->
         {ok, F} = file:open(filename:join([LogDir, "ct.latest.log"]),
                             [write]),
         true = group_leader(F, self()),
-        Pid ! ct:run_test(CTOpts)
+        Pid ! ct:run_test(Opts)
     end),
-    receive Result -> handle_quiet_results(CTOpts, Result) end.
+    receive Result -> handle_quiet_results(Opts, Result) end.
+
+handle_results(Results) when is_list(Results) ->
+    Result = lists:foldl(fun sum_results/2, {0, 0, {0,0}}, Results),
+    handle_results(Result);
+handle_results({_, Failed, {_, AutoSkipped}})
+  when Failed > 0 orelse AutoSkipped > 0 ->
+    ?PRV_ERROR({failures_running_tests, {Failed, AutoSkipped}});
+handle_results({error, Reason}) ->
+    ?PRV_ERROR({error_running_tests, Reason});
+handle_results(_) ->
+    ok.
+
+sum_results({Passed, Failed, {UserSkipped, AutoSkipped}},
+            {Passed2, Failed2, {UserSkipped2, AutoSkipped2}}) ->
+    {Passed+Passed2, Failed+Failed2,
+     {UserSkipped+UserSkipped2, AutoSkipped+AutoSkipped2}}.
+
+handle_quiet_results(_, {error, _} = Result) ->
+    handle_results(Result);
+handle_quiet_results(_, {'DOWN', _, _, _, Reason}) ->
+    handle_results(?PRV_ERROR(Reason));
+handle_quiet_results(CTOpts, Results) when is_list(Results) ->
+    _ = [format_result(Result) || Result <- Results],
+    case handle_results(Results) of
+        ?PRV_ERROR({failures_running_tests, _}) = Error ->
+            LogDir = proplists:get_value(logdir, CTOpts),
+            Index = filename:join([LogDir, "index.html"]),
+            ?CONSOLE("Results written to ~p.", [Index]),
+            Error;
+        Other ->
+            Other
+    end;
+handle_quiet_results(CTOpts, Result) ->
+    handle_quiet_results(CTOpts, [Result]).
+
+format_result({Passed, 0, {0, 0}}) ->
+    ?CONSOLE("All ~p tests passed.", [Passed]);
+format_result({Passed, Failed, Skipped}) ->
+    Format = [format_failed(Failed), format_skipped(Skipped),
+              format_passed(Passed)],
+    ?CONSOLE("~s", [Format]).
+
+format_failed(0) ->
+    [];
+format_failed(Failed) ->
+    io_lib:format("Failed ~p tests. ", [Failed]).
+
+format_passed(Passed) ->
+    io_lib:format("Passed ~p tests. ", [Passed]).
+
+format_skipped({0, 0}) ->
+    [];
+format_skipped({User, Auto}) ->
+    io_lib:format("Skipped ~p (~p, ~p) tests. ", [User+Auto, User, Auto]).
+
+test_state(State) ->
+    TestOpts = case rebar_state:get(State, ct_compile_opts, []) of
+        []    -> [];
+        Opts  -> [{erl_opts, Opts}]
+    end,
+    [first_files(State)|TestOpts].
+
+first_files(State) ->
+    CTFirst = rebar_state:get(State, ct_first_files, []),
+    {erl_first_files, CTFirst}.
+
+setup_ct(State) ->
+    Opts = resolve_ct_opts(State),
+    Opts1 = discover_tests(State, Opts),
+    copy_and_compile_tests(State, Opts1).
+
+resolve_ct_opts(State) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    CmdOpts = transform_opts(RawOpts),
+    CfgOpts = rebar_state:get(State, ct_opts, []),
+    Merged = lists:ukeymerge(1,
+                             lists:ukeysort(1, CmdOpts),
+                             lists:ukeysort(1, CfgOpts)),
+    %% make sure `dir` and/or `suite` from command line go in as
+    %%  a pair overriding both `dir` and `suite` from config if
+    %%  they exist
+    case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of
+        {undefined, undefined} -> Merged;
+        {_Suite, undefined}    -> lists:keydelete(dir, 1, Merged);
+        {undefined, _Dir}      -> lists:keydelete(suite, 1, Merged);
+        {_Suite, _Dir}         -> Merged
+    end.
+
+discover_tests(State, Opts) ->
+    case proplists:get_value(spec, Opts) of
+        undefined -> discover_dirs_and_suites(State, Opts);
+        TestSpec  -> discover_testspec(TestSpec, Opts)
+    end.
+
+discover_dirs_and_suites(State, Opts) ->
+    case {proplists:get_value(dir, Opts), proplists:get_value(suite, Opts)} of
+        %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test`
+        %%  as suites
+        {undefined, undefined} -> test_dirs(State, Opts);
+        %% no dirs defined
+        {undefined, _} -> Opts;
+        %% no suites defined
+        {_, undefined} -> Opts;
+        %% a single dir defined, this is ok
+        {Dirs, Suites} when is_integer(hd(Dirs)), is_list(Suites) -> Opts;
+        %% still a single dir defined, adjust to make acceptable to ct
+        {[Dir], Suites} when is_integer(hd(Dir)), is_list(Suites) ->
+            [{dir, Dir}|lists:keydelete(dir, 1, Opts)];
+        %% multiple dirs and suites, error now to simplify later steps
+        {_, _} -> erlang:error({multiple_dirs_and_suites, Opts})
+    end.
+
+discover_testspec(_TestSpec, Opts) ->
+    lists:keydelete(auto_compile, 1, Opts).
+
+copy_and_compile_tests(State, Opts) ->
+    %% possibly enable cover
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    State1 = case proplists:get_value(cover, RawOpts, false) of
+        true  -> rebar_state:set(State, cover_enabled, true);
+        false -> State
+    end,
+    copy_and_compile_test_suites(State1, Opts).
+
+copy_and_compile_test_suites(State, Opts) ->
+    case proplists:get_value(suite, Opts) of
+        %% no suites, try dirs
+        undefined -> copy_and_compile_test_dirs(State, Opts);
+        Suites    ->
+            Dir = proplists:get_value(dir, Opts, undefined),
+            AllSuites = join(Dir, Suites),
+            Dirs = find_suite_dirs(AllSuites),
+            lists:foreach(fun(S) ->
+                NewPath = copy(State, S),
+                compile_dir(State, NewPath)
+            end, Dirs),
+            NewSuites = lists:map(fun(S) -> retarget_path(State, S) end, AllSuites),
+            [{suite, NewSuites}|lists:keydelete(suite, 1, Opts)]
+    end.
+
+copy_and_compile_test_dirs(State, Opts) ->
+    case proplists:get_value(dir, Opts) of
+        undefined -> {error, {no_tests_specified, Opts}};
+        %% dir is a single directory
+        Dir when is_list(Dir), is_integer(hd(Dir)) ->
+            NewPath = copy(State, Dir),
+            [{dir, compile_dir(State, NewPath)}|lists:keydelete(dir, 1, Opts)];
+        %% dir is a list of directories
+        Dirs when is_list(Dirs) ->
+            NewDirs = lists:map(fun(Dir) ->
+                NewPath = copy(State, Dir),
+                compile_dir(State, NewPath)
+            end, Dirs),
+            [{dir, NewDirs}|lists:keydelete(dir, 1, Opts)]
+    end.
+
+join(undefined, Suites) -> Suites;
+join(Dir, Suites) when is_list(Dir), is_integer(hd(Dir)) ->
+    lists:map(fun(S) -> filename:join([Dir, S]) end, Suites);
+%% multiple dirs or a bad dir argument, try to continue anyways
+join(_, Suites) -> Suites. 
+
+find_suite_dirs(Suites) ->
+    AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites),
+    %% eliminate duplicates
+    lists:usort(AllDirs).
+
+copy(State, Target) ->
+    case retarget_path(State, Target) of
+        %% directory lies outside of our project's file structure so
+        %%  don't copy it
+        Target    -> Target;
+        NewTarget ->
+            %% unlink the directory if it's a symlink
+            case ec_file:is_symlink(NewTarget) of
+                true  -> ok = ec_file:remove(NewTarget);
+                false -> ok
+            end,
+            ok = ec_file:copy(Target, NewTarget, [recursive]),
+            NewTarget
+    end.
+
+compile_dir(State, Dir) ->
+    NewState = replace_src_dirs(State, [Dir]),
+    ok = rebar_erlc_compiler:compile(NewState, rebar_dir:base_dir(State), Dir),
+    ok = maybe_cover_compile(State, Dir),
+    Dir.
+
+retarget_path(State, Path) ->
+    ProjectApps = rebar_state:project_apps(State),
+    retarget_path(State, Path, ProjectApps).
+
+%% not relative to any apps in project, check to see it's relative to
+%%  project root
+retarget_path(State, Path, []) ->
+    case relative_path(reduce_path(Path), rebar_state:dir(State)) of
+        {ok, NewPath}         -> filename:join([rebar_dir:base_dir(State), NewPath]);
+        %% not relative to project root, don't modify
+        {error, not_relative} -> Path
+    end;
+%% relative to current app, retarget to the same dir relative to
+%%  the app's out_dir
+retarget_path(State, Path, [App|Rest]) ->
+    case relative_path(reduce_path(Path), rebar_app_info:dir(App)) of
+        {ok, NewPath}         -> filename:join([rebar_app_info:out_dir(App), NewPath]);
+        {error, not_relative} -> retarget_path(State, Path, Rest)
+    end.
+
+relative_path(Target, To) ->
+    relative_path1(filename:split(filename:absname(Target)),
+                   filename:split(filename:absname(To))).
+
+relative_path1([Part|Target], [Part|To]) -> relative_path1(Target, To);
+relative_path1([], [])                   -> {ok, ""};
+relative_path1(Target, [])               -> {ok, filename:join(Target)};
+relative_path1(_, _)                     -> {error, not_relative}.
+
+reduce_path(Dir) -> reduce_path([], filename:split(filename:absname(Dir))).
+
+reduce_path([], [])                -> filename:nativename("/");
+reduce_path(Acc, [])               -> filename:join(lists:reverse(Acc));
+reduce_path(Acc, ["."|Rest])       -> reduce_path(Acc, Rest);
+reduce_path([_|Acc], [".."|Rest])  -> reduce_path(Acc, Rest);
+reduce_path([], [".."|Rest])       -> reduce_path([], Rest);
+reduce_path(Acc, [Component|Rest]) -> reduce_path([Component|Acc], Rest).
+
+replace_src_dirs(State, Dirs) ->
+    %% replace any `src_dirs` with the test dirs
+    ErlOpts = rebar_state:get(State, erl_opts, []),
+    StrippedOpts = lists:keydelete(src_dirs, 1, ErlOpts),
+    rebar_state:set(State, erl_opts, [{src_dirs, Dirs}|StrippedOpts]).
+
+test_dirs(State, Opts) ->
+    BareTest = filename:join([rebar_state:dir(State), "test"]),
+    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
+    TestApps = project_apps(State),
+    case filelib:is_dir(BareTest) andalso not lists:any(F, TestApps) of
+        %% `test` dir at root of project is already scheduled to be
+        %%  included or `test` does not exist
+        false -> application_dirs(TestApps, Opts, []);
+        %% need to add `test` dir at root to dirs to be included
+        true  -> application_dirs(TestApps, Opts, [BareTest])
+    end.
+
+project_apps(State) ->
+    filter_checkouts(rebar_state:project_apps(State)).
+
+filter_checkouts(Apps) -> filter_checkouts(Apps, []).
+
+filter_checkouts([], Acc) -> lists:reverse(Acc);
+filter_checkouts([App|Rest], Acc) ->
+    case rebar_app_info:is_checkout(App) of
+        true  -> filter_checkouts(Rest, Acc);
+        false -> filter_checkouts(Rest, [App|Acc])
+    end.
+
+application_dirs([], Opts, []) -> Opts;
+application_dirs([], Opts, [Acc]) -> [{dir, Acc}|Opts];
+application_dirs([], Opts, Acc) -> [{dir, lists:reverse(Acc)}|Opts];
+application_dirs([App|Rest], Opts, Acc) ->
+    TestDir = filename:join([rebar_app_info:dir(App), "test"]),
+    case filelib:is_dir(TestDir) of
+        true  -> application_dirs(Rest, Opts, [TestDir|Acc]);
+        false -> application_dirs(Rest, Opts, Acc)
+    end.
+
+setup_logdir(State, Opts) ->
+    Logdir = case proplists:get_value(logdir, Opts) of
+        undefined -> filename:join([rebar_dir:base_dir(State), "logs"]);
+        Dir       -> Dir
+    end,
+    ensure_dir([Logdir]),
+    [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)].
+
+ensure_dir([]) -> ok;
+ensure_dir([Dir|Rest]) ->
+    case ec_file:is_dir(Dir) of
+        true ->
+            ok;
+        false ->
+            ec_file:mkdir_path(Dir)
+    end,
+    ensure_dir(Rest).
+
+maybe_cover_compile(State, Dir) ->
+    {Opts, _} = rebar_state:command_parsed_args(State),
+    State1 = case proplists:get_value(cover, Opts, false) of
+        true  -> rebar_state:set(State, cover_enabled, true);
+        false -> State
+    end,
+    rebar_prv_cover:maybe_cover_compile(State1, [Dir]).
 
 ct_opts(_State) ->
-    DefaultLogsDir = filename:join([rebar_dir:get_cwd(), "_build", "logs"]),
     [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list
      {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list
      {group, undefined, "group", string, help(group)}, %% comma-seperated list
@@ -95,13 +394,13 @@ ct_opts(_State) ->
      {config, undefined, "config", string, help(config)}, %% comma-seperated list
      {userconfig, undefined, "userconfig", string, help(userconfig)}, %% [{CallbackMod, CfgStrings}] | {CallbackMod, CfgStrings}
      {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool
-     {logdir, undefined, "logdir", {string, DefaultLogsDir}, help(logdir)}, %% dir
+     {logdir, undefined, "logdir", string, help(logdir)}, %% dir
      {logopts, undefined, "logopts", string, help(logopts)}, %% enum, no_nl | no_src
      {verbosity, undefined, "verbosity", string, help(verbosity)}, %% Integer OR [{Category, VLevel}]
      {silent_connections, undefined, "silent_connections", string,
       help(silent_connections)}, % all OR %% comma-seperated list
      {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% file
-     {cover, $c, "cover", boolean, help(cover)},
+     {cover, $c, "cover", {boolean, false}, help(cover)},
      {cover_spec, undefined, "cover_spec", string, help(cover_spec)}, %% file
      {cover_stop, undefined, "cover_stop", boolean, help(cover_stop)}, %% Boolean
      {event_handler, undefined, "event_handler", string, help(event_handler)}, %% EH | [EH] WHERE EH atom() | {atom(), InitArgs} | {[atom()], InitArgs}
@@ -118,6 +417,7 @@ ct_opts(_State) ->
      {force_stop, undefined, "force_stop", string, help(force_stop)}, % enum: skip_rest, bool
      {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Booloean
      {ct_hooks, undefined, "ct_hooks", string, help(ct_hooks)}, %% List: [CTHModule | {CTHModule, CTHInitArgs}] where CTHModule is atom CthInitArgs is term
+     {auto_compile, undefined, "auto_compile", {boolean, false}, help(auto_compile)},
      {verbose, $v, "verbose", boolean, help(verbose)}
     ].
 
@@ -131,42 +431,26 @@ help(testcase) ->
     "List of test cases to run";
 help(spec) ->
     "List of test specs to run";
-help(join_specs) ->
-    ""; %% ??
 help(label) ->
     "Test label";
 help(config) ->
     "List of config files";
-help(allow_user_terms) ->
-    ""; %% ??
 help(logdir) ->
     "Log folder";
-help(logopts) ->
-    ""; %% ??
 help(verbosity) ->
     "Verbosity";
-help(silent_connections) ->
-    ""; %% ??
 help(stylesheet) ->
     "Stylesheet to use for test results";
 help(cover) ->
     "Generate cover data";
 help(cover_spec) ->
     "Cover file to use";
-help(cover_stop) ->
-    ""; %% ??
 help(event_handler) ->
     "Event handlers to attach to the runner";
 help(include) ->
     "Include folder";
 help(abort_if_missing_suites) ->
     "Abort if suites are missing";
-help(multiply_timetraps) ->
-    ""; %% ??
-help(scale_timetraps) ->
-    ""; %% ??
-help(create_priv_dir) ->
-    ""; %% ??
 help(repeat) ->
     "How often to repeat tests";
 help(duration) ->
@@ -177,12 +461,10 @@ help(force_stop) ->
     "Force stop after time";
 help(basic_html) ->
     "Show basic HTML";
-help(ct_hooks) ->
-    "";
-help(userconfig) ->
-    "";
 help(verbose) ->
-    "Verbose output".
+    "Verbose output";
+help(_) ->
+    "".
 
 transform_opts(Opts) ->
     transform_opts(Opts, []).
@@ -255,210 +537,4 @@ parse_term(String) ->
             Terms;
         Term ->
             Term
-    end.
-
-filter_checkouts(Apps) -> filter_checkouts(Apps, []).
-
-filter_checkouts([], Acc) -> lists:reverse(Acc);
-filter_checkouts([App|Rest], Acc) ->
-    AppDir = filename:absname(rebar_app_info:dir(App)),
-    CheckoutsDir = filename:absname("_checkouts"),
-    case lists:prefix(CheckoutsDir, AppDir) of
-        true -> filter_checkouts(Rest, Acc);
-        false -> filter_checkouts(Rest, [App|Acc])
-    end.
-
-create_dirs(Opts) ->
-    LogDir = proplists:get_value(logdir, Opts),
-    ensure_dir([LogDir]),
-    ok.
-
-ensure_dir([]) -> ok;
-ensure_dir([Dir|Rest]) ->
-    case ec_file:is_dir(Dir) of
-        true ->
-            ok;
-        false ->
-            ec_file:mkdir_path(Dir)
-    end,
-    ensure_dir(Rest).
-
-in_dirs(State, Opts) ->
-    %% preserve the override nature of command line opts by only checking
-    %% `rebar.config` defined additional test dirs if none are defined via
-    %% command line flag
-    case proplists:get_value(dir, Opts) of
-        undefined ->
-            CTOpts = rebar_state:get(State, ct_opts, []),
-            proplists:get_value(dir, CTOpts, []);
-        Dirs -> split_string(Dirs)
-    end.
-
-test_dirs(State, TestApps) ->
-    %% we need to add "./ebin" if it exists but only if it's not already
-    %%  due to be added
-    F = fun(App) -> rebar_app_info:dir(App) =/= rebar_dir:get_cwd() end,
-    BareEbin = filename:join([rebar_dir:base_dir(State), "ebin"]),
-    case lists:any(F, TestApps) andalso filelib:is_dir(BareEbin) of
-        false -> application_dirs(TestApps, []);
-        true  -> [BareEbin|application_dirs(TestApps, [])]
-    end.
-
-application_dirs([], Acc) -> lists:reverse(Acc);
-application_dirs([App|Rest], Acc) ->
-    application_dirs(Rest, [rebar_app_info:ebin_dir(App)|Acc]).
-
-test_state(State) ->
-    TestOpts = case rebar_state:get(State, ct_compile_opts, []) of
-        []    -> [];
-        Opts  -> [{erl_opts, Opts}]
-    end,
-    [first_files(State)|TestOpts].
-
-first_files(State) ->
-    CTFirst = rebar_state:get(State, ct_first_files, []),
-    {erl_first_files, CTFirst}.
-
-resolve_ct_opts(State, TestApps, CmdLineOpts) ->
-    CTOpts = rebar_state:get(State, ct_opts, []),
-    Opts = lists:ukeymerge(1,
-                    lists:ukeysort(1, CmdLineOpts),
-                    lists:ukeysort(1, CTOpts)),
-    TestDirs = test_dirs(State, TestApps),
-    try resolve_ct_opts(TestDirs, Opts) of
-        Opts2 ->
-            %% disable `auto_compile`
-            {ok, [{auto_compile, false} | Opts2]}
-    catch
-        throw:{error, Reason}->
-            {error, {error_processing_options, Reason}}
-    end.
-
-resolve_ct_opts(Dirs, Opts) ->
-    Opts2 = lists:keydelete(dir, 1, Opts),
-    case lists:keytake(suite, 1, Opts2) of
-        {value, {suite, Suites}, Opts3} ->
-            %% Find full path to suites so that test names are consistent with
-            %% names when testing all dirs.
-            Suites2 = [resolve_suite(Dirs, Suite) || Suite <- Suites],
-            [{suite, Suites2} | Opts3];
-        false ->
-            %% No suites, test all dirs.
-            [{dir, Dirs} | Opts2]
-    end.
-
-resolve_suite(Dirs, Suite) ->
-    File = Suite ++ code:objfile_extension(),
-    case [Path || Dir <- Dirs,
-                  Path <- [filename:join(Dir, File)],
-                  filelib:is_file(Path)] of
-        [Suite2] ->
-            Suite2;
-        [] ->
-            throw({error, {unknown_suite, File}});
-        Suites ->
-            throw({error, {duplicate_suites, Suites}})
-    end.
-
-compile_tests(State, TestApps, InDirs) ->
-    F = fun(AppInfo) ->
-        AppDir = rebar_app_info:dir(AppInfo),
-        S = case rebar_app_info:state(AppInfo) of
-            undefined ->
-                C = rebar_config:consult(AppDir),
-                rebar_state:new(State, C, AppDir);
-            AppState ->
-                AppState
-        end,
-        ok = rebar_erlc_compiler:compile(replace_src_dirs(S, ["test"]),
-                                         ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)))
-    end,
-    lists:foreach(F, TestApps),
-    compile_extra_tests(State, TestApps, InDirs).
-
-%% extra directories containing tests can be passed to ct via the `dir` option
-compile_extra_tests(State, TestApps, InDirs) ->
-    F = fun(App) -> rebar_app_info:dir(App) == rebar_dir:get_cwd() end,
-    TestDirs = case lists:filter(F, TestApps) of
-        %% add `test` to indirs if it exists at the root of the project and
-        %%  it hasn't already been compiled
-        [] -> ["test"|InDirs];
-        %% already compiled `./test` so do nothing
-        _  -> InDirs
-    end,
-    %% symlink each of the extra dirs
-    lists:foreach(fun(Dir) ->
-        Source = filename:join([rebar_dir:get_cwd(), Dir]),
-        Target = filename:join([rebar_dir:base_dir(State), Dir]),
-        ok = rebar_file_utils:symlink_or_copy(Source, Target)
-    end, TestDirs),
-    rebar_erlc_compiler:compile(replace_src_dirs(State, TestDirs),
-                                rebar_dir:base_dir(State),
-                                filename:join([rebar_dir:base_dir(State), "ebin"])).
-
-replace_src_dirs(State, Dirs) ->
-    %% replace any `src_dirs` with the test dirs
-    ErlOpts = rebar_state:get(State, erl_opts, []),
-    StrippedOpts = lists:keydelete(src_dirs, 1, ErlOpts),
-    rebar_state:set(State, erl_opts, [{src_dirs, Dirs}|StrippedOpts]).
-
-maybe_cover_compile(State, Opts) ->
-    State1 = case proplists:get_value(cover, Opts, false) of
-        true  -> rebar_state:set(State, cover_enabled, true);
-        false -> State
-    end,
-    rebar_prv_cover:maybe_cover_compile(State1).
-
-handle_results(Results) when is_list(Results) ->
-    Result = lists:foldl(fun sum_results/2, {0, 0, {0,0}}, Results),
-    handle_results(Result);
-handle_results({_, Failed, {_, AutoSkipped}})
-  when Failed > 0 orelse AutoSkipped > 0 ->
-    {error, {failures_running_tests, {Failed, AutoSkipped}}};
-handle_results({error, Reason}) ->
-    {error, {error_running_tests, Reason}};
-handle_results(_) ->
-    ok.
-
-sum_results({Passed, Failed, {UserSkipped, AutoSkipped}},
-            {Passed2, Failed2, {UserSkipped2, AutoSkipped2}}) ->
-    {Passed+Passed2, Failed+Failed2,
-     {UserSkipped+UserSkipped2, AutoSkipped+AutoSkipped2}}.
-
-handle_quiet_results(_, {error, _} = Result) ->
-    handle_results(Result);
-handle_quiet_results(_, {'DOWN', _, _, _, Reason}) ->
-    handle_results({error, Reason});
-handle_quiet_results(CTOpts, Results) when is_list(Results) ->
-    _ = [format_result(Result) || Result <- Results],
-    case handle_results(Results) of
-        {error, {failures_running_tests, _}} = Error ->
-            LogDir = proplists:get_value(logdir, CTOpts),
-            Index = filename:join([LogDir, "index.html"]),
-            ?CONSOLE("Results written to ~p.", [Index]),
-            Error;
-        Other ->
-            Other
-    end;
-handle_quiet_results(CTOpts, Result) ->
-    handle_quiet_results(CTOpts, [Result]).
-
-format_result({Passed, 0, {0, 0}}) ->
-    ?CONSOLE("All ~p tests passed.", [Passed]);
-format_result({Passed, Failed, Skipped}) ->
-    Format = [format_failed(Failed), format_skipped(Skipped),
-              format_passed(Passed)],
-    ?CONSOLE("~s", [Format]).
-
-format_failed(0) ->
-    [];
-format_failed(Failed) ->
-    io_lib:format("Failed ~p tests. ", [Failed]).
-
-format_passed(Passed) ->
-    io_lib:format("Passed ~p tests. ", [Passed]).
-
-format_skipped({0, 0}) ->
-    [];
-format_skipped({User, Auto}) ->
-    io_lib:format("Skipped ~p (~p, ~p) tests. ", [User+Auto, User, Auto]).
+    end.

src/rebar_prv_cover.erl

@@ -8,6 +8,7 @@
 -export([init/1,
          do/1,
          maybe_cover_compile/1,
+         maybe_cover_compile/2,
          maybe_write_coverdata/2,
          format_error/1]).
 
@@ -43,8 +44,12 @@ do(State) ->
 
 -spec maybe_cover_compile(rebar_state:t()) -> ok.
 maybe_cover_compile(State) ->
+    maybe_cover_compile(State, []).
+
+-spec maybe_cover_compile(rebar_state:t(), [file:name()]) -> ok.
+maybe_cover_compile(State, ExtraDirs) ->
     case rebar_state:get(State, cover_enabled, false) of
-        true  -> cover_compile(State);
+        true  -> cover_compile(State, ExtraDirs);
         false -> ok
     end.
 
@@ -151,9 +156,10 @@ analysis(State, Task) ->
 restore_cover_paths(State) ->
     lists:foreach(fun(App) ->
         AppDir = rebar_app_info:out_dir(App),
-        _ = code:add_path(filename:join([AppDir, "ebin"]))
+        _ = code:add_path(filename:join([AppDir, "ebin"])),
+        _ = code:add_path(filename:join([AppDir, "test"]))
     end, rebar_state:project_apps(State)),
-    _ = code:add_path(filename:join([rebar_dir:base_dir(State), "ebin"])),
+    _ = code:add_path(filename:join([rebar_dir:base_dir(State), "test"])),
     ok.
 
 analyze_to_file(Mod, State, Task) ->
@@ -269,7 +275,7 @@ strip_coverdir(File) ->
     filename:join(lists:reverse(lists:sublist(lists:reverse(filename:split(File)),
                                               2))).
 
-cover_compile(State) ->
+cover_compile(State, ExtraDirs) ->
     %% start the cover server if necessary
     {ok, CoverPid} = start_cover(),
     %% redirect cover output
@@ -277,7 +283,7 @@ cover_compile(State) ->
     %% cover compile the modules we just compiled
     Apps = filter_checkouts(rebar_state:project_apps(State)),
     CompileResult = compile_beam_directories(Apps, []) ++
-                    compile_bare_test_directory(State),
+                    compile_extras(ExtraDirs, []),
     %% print any warnings about modules that failed to cover compile
     lists:foreach(fun print_cover_warnings/1, CompileResult).
 
@@ -285,10 +291,8 @@ filter_checkouts(Apps) -> filter_checkouts(Apps, []).
 
 filter_checkouts([], Acc) -> lists:reverse(Acc);
 filter_checkouts([App|Rest], Acc) ->
-    AppDir = filename:absname(rebar_app_info:dir(App)),
-    CheckoutsDir = filename:absname("_checkouts"),
-    case lists:prefix(CheckoutsDir, AppDir) of
-        true -> filter_checkouts(Rest, Acc);
+    case rebar_app_info:is_checkout(App) of
+        true  -> filter_checkouts(Rest, Acc);
         false -> filter_checkouts(Rest, [App|Acc])
     end.
 
@@ -298,13 +302,10 @@ compile_beam_directories([App|Rest], Acc) ->
                                                         "ebin"])),
     compile_beam_directories(Rest, Acc ++ Result).
 
-compile_bare_test_directory(State) ->
-    case cover:compile_beam_directory(filename:join([rebar_dir:base_dir(State),
-                                                     "ebin"])) of
-        %% if directory doesn't exist just return empty result set
-        {error, enoent} -> [];
-        Result     -> Result
-    end.
+compile_extras([], Acc) -> Acc;
+compile_extras([Dir|Rest], Acc) ->
+    Result = cover:compile_beam_directory(Dir),
+    compile_extras(Rest, Acc ++ Result).
 
 start_cover() ->
     case cover:start() of

src/rebar_prv_shell.erl

@@ -57,7 +57,7 @@ init(State) ->
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(Config) ->
-    shell(),
+    shell(Config),
     {ok, Config}.
 
 -spec format_error(any()) -> iolist().
@@ -71,7 +71,7 @@ format_error(Reason) ->
 %% it also lacks the ctrl-c interrupt handler that `erl` features. ctrl-c will
 %% immediately kill the script. ctrl-g, however, works fine
 
-shell() ->
+shell(State) ->
     %% scan all processes for any with references to the old user and save them to
     %% update later
     NeedsUpdate = [Pid || Pid <- erlang:processes(),
@@ -92,6 +92,8 @@ shell() ->
     %% times). removes at most the error_logger added by init and the
     %% error_logger added by the tty handler
     ok = remove_error_handler(3),
+    %% add test paths
+    ok = add_test_paths(State),
     %% this call never returns (until user quits shell)
     timer:sleep(infinity).
 
@@ -116,3 +118,13 @@ wait_until_user_started(Timeout) ->
         undefined -> timer:sleep(100), wait_until_user_started(Timeout - 100);
         _ -> ok
     end.
+
+add_test_paths(State) ->
+    lists:foreach(fun(App) ->
+        AppDir = rebar_app_info:out_dir(App),
+        %% ignore errors resulting from non-existent directories
+        _ = code:add_path(filename:join([AppDir, "ebin"])),
+        _ = code:add_path(filename:join([AppDir, "test"]))
+    end, rebar_state:project_apps(State)),
+    _ = code:add_path(filename:join([rebar_dir:base_dir(State), "test"])),
+    ok.

test/rebar_ct_SUITE.erl

@@ -0,0 +1,520 @@
+-module(rebar_ct_SUITE).
+
+-export([all/0,
+         groups/0,
+         init_per_group/2,
+         end_per_group/2]).
+-export([basic_app_default_dirs/1,
+         basic_app_default_beams/1,
+         multi_app_default_dirs/1,
+         multi_app_default_beams/1,
+         single_app_dir/1,
+         single_extra_dir/1,
+         single_unmanaged_dir/1,
+         single_suite/1,
+         single_extra_suite/1,
+         single_unmanaged_suite/1,
+         multi_suite/1,
+         all_suite/1,
+         single_dir_and_single_suite/1]).
+
+-include_lib("common_test/include/ct.hrl").
+
+all() -> [{group, basic_app},
+          {group, multi_app},
+          {group, dirs_and_suites}].
+
+groups() -> [{basic_app, [], [basic_app_default_dirs,
+                              basic_app_default_beams]},
+             {multi_app, [], [multi_app_default_dirs,
+                              multi_app_default_beams]},
+             {dirs_and_suites, [], [single_app_dir,
+                                    single_extra_dir,
+                                    single_unmanaged_dir,
+                                    single_suite,
+                                    single_extra_suite,
+                                    single_unmanaged_suite,
+                                    multi_suite,
+                                    all_suite,
+                                    single_dir_and_single_suite]}].
+
+init_per_group(basic_app, Config) ->
+    C = rebar_test_utils:init_rebar_state(Config, "ct_"),
+
+    AppDir = ?config(apps, C),
+
+    Name = rebar_test_utils:create_random_name(atom_to_list(basic_app) ++ "_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    Suite = filename:join([AppDir, "test", Name ++ "_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite),
+    ok = file:write_file(Suite, test_suite(Name)),
+
+    {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, []),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    [{result, Result}, {appnames, [Name]}|C];
+init_per_group(multi_app, Config) ->
+    C = rebar_test_utils:init_rebar_state(Config, "ct_"),
+
+    AppDir = ?config(apps, C),
+
+    Name1 = rebar_test_utils:create_random_name(atom_to_list(multi_app) ++ "_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    AppDir1 = filename:join([AppDir, "apps", Name1]),
+    rebar_test_utils:create_app(AppDir1, Name1, Vsn1, [kernel, stdlib]),
+
+    Suite1 = filename:join([AppDir1, "test", Name1 ++ "_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite1),
+    ok = file:write_file(Suite1, test_suite(Name1)),
+
+    Name2 = rebar_test_utils:create_random_name(atom_to_list(multi_app) ++ "_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    AppDir2 = filename:join([AppDir, "apps", Name2]),
+    rebar_test_utils:create_app(AppDir2, Name2, Vsn2, [kernel, stdlib]),
+
+    Suite2 = filename:join([AppDir2, "test", Name2 ++ "_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite2),
+    ok = file:write_file(Suite2, test_suite(Name2)),
+
+    Suite3 = filename:join([AppDir, "test", "extras_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite3),
+    ok = file:write_file(Suite3, test_suite("extras")),
+
+    {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, []),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    [{result, Result}, {appnames, [Name1, Name2]}|C];
+init_per_group(dirs_and_suites, Config) ->
+    C = rebar_test_utils:init_rebar_state(Config, "ct_"),
+
+    AppDir = ?config(apps, C),
+
+    Name1 = rebar_test_utils:create_random_name(atom_to_list(dirs_and_suites) ++ "_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    AppDir1 = filename:join([AppDir, "apps", Name1]),
+    rebar_test_utils:create_app(AppDir1, Name1, Vsn1, [kernel, stdlib]),
+
+    Suite1 = filename:join([AppDir1, "test", Name1 ++ "_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite1),
+    ok = file:write_file(Suite1, test_suite(Name1)),
+
+    Name2 = rebar_test_utils:create_random_name(atom_to_list(dir_and_suites) ++ "_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    AppDir2 = filename:join([AppDir, "apps", Name2]),
+    rebar_test_utils:create_app(AppDir2, Name2, Vsn2, [kernel, stdlib]),
+
+    Suite2 = filename:join([AppDir2, "test", Name2 ++ "_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite2),
+    ok = file:write_file(Suite2, test_suite(Name2)),
+
+    Suite3 = filename:join([AppDir, "test", "extras_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite3),
+    ok = file:write_file(Suite3, test_suite("extras")),
+
+    [{appnames, [Name1, Name2]}|C].
+
+end_per_group(_Group, _Config) -> ok.
+
+basic_app_default_dirs(Config) ->
+    AppDir = ?config(apps, Config),
+    [Name] = ?config(appnames, Config),
+    Result = ?config(result, Config),
+
+    Expect = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name, "test"])),
+    Dir = proplists:get_value(dir, Result),
+
+    Expect = Dir.
+
+basic_app_default_beams(Config) ->
+    AppDir = ?config(apps, Config),
+    [Name] = ?config(appnames, Config),
+
+    File = filename:join([AppDir,
+                          "_build",
+                          "test",
+                          "lib",
+                          Name,
+                          "test",
+                          Name ++ "_SUITE.beam"]),
+
+    true = filelib:is_file(File).
+
+multi_app_default_dirs(Config) ->
+    AppDir = ?config(apps, Config),
+    [Name1, Name2] = ?config(appnames, Config),
+    Result = ?config(result, Config),
+
+    Expect1 = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name1, "test"])),
+    Expect2 = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name2, "test"])),
+    Expect3 = filename:absname(filename:join([AppDir, "_build", "test", "test"])),
+    Dirs = proplists:get_value(dir, Result),
+
+    true = (lists:sort([Expect1, Expect2, Expect3]) == lists:sort(Dirs)).
+
+multi_app_default_beams(Config) ->
+    AppDir = ?config(apps, Config),
+    [Name1, Name2] = ?config(appnames, Config),
+
+    File1 = filename:join([AppDir,
+                           "_build",
+                           "test",
+                           "lib",
+                           Name1,
+                           "test",
+                           Name1 ++ "_SUITE.beam"]),
+    File2 = filename:join([AppDir,
+                           "_build",
+                           "test",
+                           "lib",
+                           Name2,
+                           "test",
+                           Name2 ++ "_SUITE.beam"]),
+    File3 = filename:join([AppDir,
+                           "_build",
+                           "test",
+                           "test",
+                           "extras_SUITE.beam"]),
+
+    true = filelib:is_file(File1),
+    true = filelib:is_file(File2),
+    true = filelib:is_file(File3).
+
+single_app_dir(Config) ->
+    AppDir = ?config(apps, Config),
+    [Name1, _Name2] = ?config(appnames, Config),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec,
+                                      ["--dir=" ++ filename:join([AppDir,
+                                                                  "apps",
+                                                                  Name1,
+                                                                  "test"])]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name1, "test"])),
+    Dir = proplists:get_value(dir, Result),
+
+    Expect = Dir.
+
+single_extra_dir(Config) ->
+    AppDir = ?config(apps, Config),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--dir=" ++ filename:join([AppDir,
+                                                                              "test"])]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect = filename:absname(filename:join([AppDir, "_build", "test", "test"])),
+    Dir = proplists:get_value(dir, Result),
+
+    Expect = Dir.
+
+single_unmanaged_dir(Config) ->
+    PrivDir = ?config(priv_dir, Config),
+
+    Suite = filename:join([PrivDir, "unmanaged_dir", "unmanaged_dir_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite),
+    ok = file:write_file(Suite, test_suite("unmanaged_dir")),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--dir=" ++ filename:absname(filename:join([PrivDir,
+                                                                                               "unmanaged_dir"]))]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect = filename:absname(filename:join([PrivDir, "unmanaged_dir"])),
+    Dir = proplists:get_value(dir, Result),
+
+    Expect = Dir.
+
+single_suite(Config) ->
+    AppDir = ?config(apps, Config),
+    [Name1, _Name2] = ?config(appnames, Config),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec,
+                                      ["--suite=" ++ filename:join([AppDir,
+                                                                    "apps",
+                                                                    Name1,
+                                                                    "test",
+                                                                    Name1 ++ "_SUITE"])]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect = [filename:absname(filename:join([AppDir,
+                                              "_build",
+                                              "test",
+                                              "lib",
+                                              Name1,
+                                              "test",
+                                              Name1 ++ "_SUITE"]))],
+    Suite = proplists:get_value(suite, Result),
+
+    Expect = Suite.
+
+single_extra_suite(Config) ->
+    AppDir = ?config(apps, Config),
+    [_Name1, _Name2] = ?config(appnames, Config),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec,
+                                      ["--suite=" ++ filename:join([AppDir,
+                                                                    "test",
+                                                                    "extra_SUITE"])]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect = [filename:absname(filename:join([AppDir,
+                                              "_build",
+                                              "test",
+                                              "test",
+                                              "extra_SUITE"]))],
+    Suite = proplists:get_value(suite, Result),
+
+    Expect = Suite.
+
+single_unmanaged_suite(Config) ->
+    PrivDir = ?config(priv_dir, Config),
+    [_Name1, _Name2] = ?config(appnames, Config),
+
+    Suite = filename:join([PrivDir, "unmanaged", "unmanaged_SUITE.erl"]),
+    ok = filelib:ensure_dir(Suite),
+    ok = file:write_file(Suite, test_suite("unmanaged")),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec,
+                                      ["--suite=" ++ filename:absname(filename:join([PrivDir,
+                                                                                     "unmanaged",
+                                                                                     "unmanaged_SUITE"]))]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect = [filename:absname(filename:join([PrivDir,
+                                              "unmanaged",
+                                              "unmanaged_SUITE"]))],
+    SuitePath = proplists:get_value(suite, Result),
+
+    Expect = SuitePath.
+
+multi_suite(Config) ->
+    AppDir = ?config(apps, Config),
+    [Name1, Name2] = ?config(appnames, Config),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec,
+                                      ["--suite=" ++ filename:join([AppDir,
+                                                                    "apps",
+                                                                    Name1,
+                                                                    "test",
+                                                                    Name1 ++ "_SUITE," ++ AppDir,
+                                                                    "apps",
+                                                                    Name2,
+                                                                    "test",
+                                                                    Name2 ++ "_SUITE"])]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect1 = filename:absname(filename:join([AppDir,
+                                              "_build",
+                                              "test",
+                                              "lib",
+                                              Name1,
+                                              "test",
+                                              Name1 ++ "_SUITE"])),
+    Expect2 = filename:absname(filename:join([AppDir,
+                                              "_build",
+                                              "test",
+                                              "lib",
+                                              Name2,
+                                              "test",
+                                              Name2 ++ "_SUITE"])),
+    Suites = proplists:get_value(suite, Result),
+
+    true = (lists:sort([Expect1, Expect2]) == lists:sort(Suites)).
+
+all_suite(Config) ->
+    AppDir = ?config(apps, Config),
+    [Name1, Name2] = ?config(appnames, Config),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec,
+                                      ["--suite=" ++ filename:join([AppDir,
+                                                                    "apps",
+                                                                    Name1,
+                                                                    "test",
+                                                                    Name1 ++ "_SUITE," ++ AppDir,
+                                                                    "apps",
+                                                                    Name2,
+                                                                    "test",
+                                                                    Name2 ++ "_SUITE," ++ AppDir,
+                                                                    "test",
+                                                                    "extra_SUITE"])]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect1 = filename:absname(filename:join([AppDir,
+                                              "_build",
+                                              "test",
+                                              "lib",
+                                              Name1,
+                                              "test",
+                                              Name1 ++ "_SUITE"])),
+    Expect2 = filename:absname(filename:join([AppDir,
+                                              "_build",
+                                              "test",
+                                              "lib",
+                                              Name2,
+                                              "test",
+                                              Name2 ++ "_SUITE"])),
+    Expect3 = filename:absname(filename:join([AppDir,
+                                              "_build",
+                                              "test",
+                                              "test",
+                                              "extra_SUITE"])),
+    Suites = proplists:get_value(suite, Result),
+
+    true = (lists:sort([Expect1, Expect2, Expect3]) == lists:sort(Suites)).
+
+single_dir_and_single_suite(Config) ->
+    AppDir = ?config(apps, Config),
+    [_Name1, _Name2] = ?config(appnames, Config),
+
+    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+
+    LibDirs = rebar_dir:lib_dirs(State),
+    State1 = rebar_app_discover:do(State, LibDirs),
+
+    Providers = rebar_state:providers(State1),
+    Namespace = rebar_state:namespace(State1),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec,
+                                      ["--dir=" ++ filename:join([AppDir, "test"]),
+                                       "--suite=extra_SUITE"]),
+
+    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+
+    Result = rebar_prv_common_test:setup_ct(State2),
+
+    Expect = [filename:absname(filename:join([AppDir,
+                                              "_build",
+                                              "test",
+                                              "test",
+                                              "extra_SUITE"]))],
+    Suite = proplists:get_value(suite, Result),
+
+    Expect = Suite.
+
+test_suite(Name) ->
+    io_lib:format("-module(~ts_SUITE).\n"
+                  "-compile(export_all).\n"
+                  "all() -> [some_test].\n"
+                  "some_test(_) -> ok.\n", [Name]).