Merge pull request #894 from talentdeficit/ct_test_specs

rebar_prv_common_test refactor

priv/shell-completion/bash/rebar3

@@ -54,11 +54,7 @@ _rebar3()
             --suite \
             --group \
             --case \
-            --spec \
-            --join_specs \
-            --label \
             --config \
-            --userconfig \
             --allow_user_terms \
             --logdir \
             --logopts \
@@ -66,21 +62,20 @@ _rebar3()
             --silent_connections \
             --stylesheet \
             --cover \
-            --cover_spec \
-            --cover_stop \
-            --event_handler \
-            --include \
-            --abort_if_missing_suites \
-            --multiply_timetraps \
-            --scale_timetraps \
-            --create_priv_dir \
             --repeat \
             --duration \
             --until \
             --force_stop \
             --basic_html \
-            --ct_hooks \
-            --verbose"
+            --stylesheet \
+            --decrypt_key \
+            --decrypt_file \
+            --abort_if_missing_suites \
+            --multiply_timetraps \
+            --scale_timetraps \
+            --create_priv_dir \
+            --verbose" \
+            --auto_compile
     elif [[ ${prev} == deps ]] ; then
         :
     elif [[ ${prev} == dialyzer ]] ; then

priv/shell-completion/fish/rebar3.fish

@@ -90,10 +90,28 @@ complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a ct -d "Run Common Te
 complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l dir -d "Compile and run all test suites in the specified directories."
 complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l suites -d "Compile and run all test suites specified. Must be specified by full path, either absolute or relative to the current directory."
 complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l group -d "Test groups to run."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l label -d "Test label."
 complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l config -d "Config files to use when running tests."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l allow_user_terms -d "Allow user defined terms in config files."
 complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l logdir -d "The directory in which test logs will be written. Default: _build/test/logs"
-complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -s v -l verbose -d "Enable verbose output. Default: false"
-complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -s c -l cover -d "Generate cover data"
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l logopts -d "Options for common test logging."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l verbosity -d "Verbosity."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -s c -l cover -d "Generate cover data."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l include -d "Include folders."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l repeat -d "How often to repeat tests."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l duration -d "Max runtime (format: HHMMSS)."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l until -d "Run until (format: HHMMSS)."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l force_stop -d "Force stop on test timeout."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l basic_html -d "Show basic HTML."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l stylesheet -d "CSS stylesheet to apply to html output."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l decrypt_key -d "Path to key for decrypting config."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l decrypt_file -d "Path to file containing key for decrypting config."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l abort_if_missing_suites -d "Abort if suites are missing."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l multiply_timetraps -d "Multiply timetraps."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l scale_timetraps -d "Scale timetraps."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l create_priv_dir -d "Create priv dir (auto_per_run | auto_per_tc | manual_per_tc)."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -s v -l verbose -d "Enable verbose output. Default: false."
+complete -f -c 'rebar3' -n '__fish_rebar3_using_command ct' -l auto_compile -d "Let common test compile test suites instead of rebar3."
 
 complete -f -c 'rebar3' -n '__fish_rebar3_needs_command' -a deps -d "List dependencies"
 

priv/shell-completion/zsh/_rebar3

@@ -43,33 +43,27 @@ _rebar3 () {
                         '(--suite)--suite[List of test suites to run]:suites' \
                         '(--group)--group[List of test groups to run]:groups' \
                         '(--case)--case[List of test cases to run]:cases' \
-                        '(--spec)--spec[List of test specs to run]:specs' \
-                        '(--join_specs)--join_specs' \
                         '(--label)--label[Test label]:label' \
                         '(--config)--config[List of config files]:config files:_files' \
-                        '(--userconfig)--userconfig' \
                         '(--allow_user_terms)--allow_user_terms' \
                         '(--logdir)--logdir[Log folder]:log folder:_files -/' \
                         '(--logopts)--logopts' \
                         '(--verbosity)--verbosity[Verbosity]:verbosity' \
-                        '(--silent_connections)--silent_connections' \
-                        '(--stylesheet)--stylesheet[Stylesheet to use for test results]:stylesheet:_files' \
                         '(-c --cover)'{-c,--cover}'[Generate cover data]' \
-                        '(--cover_spec)--cover_spec[Cover file to use]:cover file:_files' \
-                        '(--cover_stop)--cover_stop' \
-                        '(--event_handler)--event_handler[Event handlers to attach to the runner]:event handlers' \
-                        '(--include)--include[Include folder]:include directory:_files -/' \
-                        '(--abort_if_missing_suites)--abort_if_missing_suites[Abort if suites are missing]:abort missing suites:(true false)' \
-                        '(--multiply_timetraps)--multiply_timetraps' \
-                        '(--scale_timetraps)--scale_timetraps' \
-                        '(--create_priv_dir)--create_priv_dir' \
                         '(--repeat)--repeat[How often to repeat tests]:repeat test count' \
                         '(--duration)--duration[Max runtime (format: HHMMSS)]:max run time' \
                         '(--until)--until[Run until (format: HHMMSS)]:run until time' \
-                        '(--force_stop)--force_stop[Force stop after time]' \
+                        '(--force_stop)--force_stop[Force stop on test timeout]:skip_rest' \
                         '(--basic_html)--basic_html[Show basic HTML]' \
-                        '(--ct_hooks)--ct_hooks:ct hooks' \
+                        '(--stylesheet)--stylesheet[Stylesheet to use for test results]:stylesheet:_files' \
+                        '(--decrypt_key)--decrypt_key[Path to key for decrypting config]:decrypt key:_files' \
+                        '(--decrypt_file)--decrypt_file[Path to file containing key for decrypting config]:decrypt file:_files' \
+                        '(--abort_if_missing_suites)--abort_if_missing_suites[Abort if suites are missing]:abort missing suites:(true false)' \
+                        '(--multiply_timetraps)--multiply_timetraps' \
+                        '(--scale_timetraps)--scale_timetraps' \
+                        '(--create_priv_dir)--create_priv_dir' \
                         '(-v --verbose)'{-v,--verbose}'[Print coverage analysis]' \
+                        '(--auto_compile)--auto_compile' \
                     && ret=0
                 ;;
                 (deps)

src/rebar_prv_common_test.erl

@@ -2,19 +2,21 @@
 %% ex: ts=4 sw=4 et
 
 -module(rebar_prv_common_test).
+
 -behaviour(provider).
 
 -export([init/1,
          do/1,
          format_error/1]).
 %% exported for test purposes, consider private
--export([setup_ct/1]).
+-export([compile/2, prepare_tests/1, translate_paths/2]).
 
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
 
 -define(PROVIDER, ct).
--define(DEPS, [compile]).
+%% we need to modify app_info state before compile
+-define(DEPS, [lock]).
 
 %% ===================================================================
 %% Public API
@@ -31,12 +33,19 @@ init(State) ->
                                  {desc, "Run Common Tests."},
                                  {opts, ct_opts(State)},
                                  {profiles, [test]}]),
-    State1 = rebar_state:add_provider(State, Provider),
-    State2 = rebar_state:add_to_profile(State1, test, test_state(State1)),
-    {ok, State2}.
+    {ok, rebar_state:add_provider(State, Provider)}.
 
 -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
 do(State) ->
+    Tests = prepare_tests(State),
+    case compile(State, Tests) of
+        %% successfully compiled apps
+        {ok, S} -> do(S, Tests);
+        %% this should look like a compiler error, not a ct error
+        Error   -> Error
+    end.
+
+do(State, Tests) ->
     ?INFO("Running Common Test suites...", []),
     rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]),
 
@@ -45,63 +54,332 @@ do(State) ->
     Cwd = rebar_dir:get_cwd(),
     rebar_hooks:run_all_hooks(Cwd, pre, ?PROVIDER, Providers, State),
 
-    try run_test(State) of
-        {ok, State1} = Result ->
-            %% Run ct provider posthooks
-            rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State1),
-            rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
-            Result;
-        ?PRV_ERROR(_) = Error ->
+    case Tests of
+        {ok, T} ->
+            case run_tests(State, T) of
+                ok    ->
+                    %% Run ct provider posthooks
+                    rebar_hooks:run_all_hooks(Cwd, post, ?PROVIDER, Providers, State),
+                    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+                    {ok, State};
+                Error ->
+                    rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
+                    Error
+            end;
+        Error ->
             rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
             Error
-    catch
-        throw:{error, Reason} ->
-            rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)),
-            ?PRV_ERROR(Reason)
+    end.
+
+run_tests(State, Opts) ->
+    T = translate_paths(State, Opts),
+    Opts1 = setup_logdir(State, T),
+    Opts2 = turn_off_auto_compile(Opts1),
+    ?DEBUG("ct_opts ~p", [Opts2]),
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    ok = maybe_write_coverdata(State),
+    case proplists:get_value(verbose, RawOpts, false) of
+        true  -> run_test_verbose(Opts2);
+        false -> run_test_quiet(Opts2)
     end.
 
 -spec format_error(any()) -> iolist().
-format_error({multiple_dirs_and_suites, Opts}) ->
-    io_lib:format("Multiple dirs declared alongside suite in opts: ~p", [Opts]);
-format_error({bad_dir_or_suite, Opts}) ->
-    io_lib:format("Bad value for dir or suite in opts: ~p", [Opts]);
+format_error({error, Reason}) ->
+    io_lib:format("Error running tests:~n  ~p", [Reason]);
+format_error({error_running_tests, Reason}) ->
+    format_error({error, Reason});
 format_error({failures_running_tests, {Failed, AutoSkipped}}) ->
     io_lib:format("Failures occured running tests: ~b", [Failed+AutoSkipped]);
-format_error({error_running_tests, Reason}) ->
-    io_lib:format("Error running tests: ~p", [Reason]);
-format_error(suite_at_project_root) ->
-    io_lib:format("Test suites can't be located in project root", []);
-format_error({error, Reason}) ->
-    io_lib:format("Unknown error: ~p", [Reason]).
+format_error({multiple_errors, Errors}) ->
+    io_lib:format(lists:concat(["Error running tests:"] ++
+                               lists:map(fun(Error) -> "~n  " ++ Error end, Errors)), []).
 
 %% ===================================================================
 %% Internal functions
 %% ===================================================================
 
-run_test(State) ->
-    case setup_ct(State) of
-        {error, {no_tests_specified, Opts}} ->
-            ?WARN("No tests specified in opts: ~p", [Opts]),
-            {ok, State};
-        Opts ->
-            Opts1 = setup_logdir(State, Opts),
-            ?DEBUG("common test opts: ~p", [Opts1]),
-            run_test(State, Opts1)
-    end.
+prepare_tests(State) ->
+    %% command line test options
+    CmdOpts = cmdopts(State),
+    %% rebar.config test options
+    CfgOpts = cfgopts(State),
+    ProjectApps = rebar_state:project_apps(State),
+    %% prioritize tests to run first trying any command line specified
+    %% tests falling back to tests specified in the config file finally
+    %% running a default set if no other tests are present
+    select_tests(State, ProjectApps, CmdOpts, CfgOpts).
 
-run_test(State, Opts) ->
+cmdopts(State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
-    ok = rebar_prv_cover:maybe_cover_compile(State, apps),
-    Result = case proplists:get_value(verbose, RawOpts, false) of
-        true  -> run_test_verbose(Opts);
-        false -> run_test_quiet(Opts)
+    %% filter out opts common_test doesn't know about and convert
+    %% to ct acceptable forms
+    transform_opts(RawOpts, []).
+
+transform_opts([], Acc) -> lists:reverse(Acc);
+transform_opts([{dir, Dirs}|Rest], Acc) ->
+    transform_opts(Rest, [{dir, split_string(Dirs)}|Acc]);
+transform_opts([{suite, Suites}|Rest], Acc) ->
+    transform_opts(Rest, [{suite, split_string(Suites)}|Acc]);
+transform_opts([{group, Groups}|Rest], Acc) ->
+    transform_opts(Rest, [{group, split_string(Groups)}|Acc]);
+transform_opts([{testcase, Cases}|Rest], Acc) ->
+    transform_opts(Rest, [{testcase, split_string(Cases)}|Acc]);
+transform_opts([{config, Configs}|Rest], Acc) ->
+    transform_opts(Rest, [{config, split_string(Configs)}|Acc]);
+transform_opts([{logopts, LogOpts}|Rest], Acc) ->
+    transform_opts(Rest, [{logopts, lists:map(fun(P) -> list_to_atom(P) end, split_string(LogOpts))}|Acc]);
+transform_opts([{force_stop, "true"}|Rest], Acc) ->
+    transform_opts(Rest, [{force_stop, true}|Acc]);
+transform_opts([{force_stop, "false"}|Rest], Acc) ->
+    transform_opts(Rest, [{force_stop, false}|Acc]);
+transform_opts([{force_stop, "skip_rest"}|Rest], Acc) ->
+    transform_opts(Rest, [{force_stop, skip_rest}|Acc]);
+transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) ->
+    transform_opts(Rest, [{create_priv_dir, list_to_atom(CreatePrivDir)}|Acc]);
+%% drop cover from opts, ct doesn't care about it
+transform_opts([{cover, _}|Rest], Acc) ->
+    transform_opts(Rest, Acc);
+%% drop verbose from opts, ct doesn't care about it
+transform_opts([{verbose, _}|Rest], Acc) ->
+    transform_opts(Rest, Acc);
+%% getopt should handle anything else
+transform_opts([Opt|Rest], Acc) ->
+    transform_opts(Rest, [Opt|Acc]).
+
+split_string(String) ->
+    string:tokens(String, [$,]).
+
+cfgopts(State) ->
+    Opts = rebar_state:get(State, ct_opts, []),
+    rebar_utils:filtermap(fun filter_opts/1, Opts).
+
+filter_opts({test_spec, _}) ->
+    ?WARN("Test specs not supported", []),
+    false;
+filter_opts({auto_compile, _}) ->
+    ?WARN("Auto compile not supported", []),
+    false;
+filter_opts({suite, Suite}) when is_integer(hd(Suite)) -> true;
+filter_opts({suite, Suite}) when is_atom(Suite) ->
+    {true, {suite, atom_to_list(Suite)}};
+filter_opts({suite, Suites}) ->
+    {true, {suite, lists:map(fun(S) when is_atom(S) -> atom_to_list(S);
+                                (S) when is_list(S) -> S
+                             end,
+                             Suites)}};
+filter_opts(_) -> true.
+
+select_tests(State, ProjectApps, CmdOpts, CfgOpts) ->
+    FixedOpts = lists:filter(fun({_, _}) -> true; (V) -> ?WARN("`~p` is not a valid option for `ct_opts`", [V]) end, CfgOpts),
+    Merged = lists:ukeymerge(1,
+                             lists:ukeysort(1, CmdOpts),
+                             lists:ukeysort(1, FixedOpts)),
+    %% make sure `dir` and/or `suite` from command line go in as
+    %% a pair overriding both `dir` and `suite` from config if
+    %% they exist
+    Opts = case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of
+        {undefined, undefined} -> Merged;
+        {_Suite, undefined}    -> lists:keydelete(dir, 1, Merged);
+        {undefined, _Dir}      -> lists:keydelete(suite, 1, Merged);
+        {_Suite, _Dir}         -> Merged
     end,
-    ok = rebar_prv_cover:maybe_write_coverdata(State, ?PROVIDER),
-    case Result of
-        ok    -> {ok, State};
-        Error -> Error
+    discover_tests(State, ProjectApps, Opts).
+
+discover_tests(State, ProjectApps, Opts) ->
+    case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of
+        %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test`
+        %%  as suites
+        {undefined, undefined} -> {ok, [default_tests(State, ProjectApps)|Opts]};
+        {_, _}                 -> {ok, Opts}
+    end.
+
+default_tests(State, ProjectApps) ->
+    BareTest = filename:join([rebar_state:dir(State), "test"]),
+    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
+    AppTests = application_dirs(ProjectApps, []),
+    case filelib:is_dir(BareTest) andalso not lists:any(F, ProjectApps) of
+        %% `test` dir at root of project is already scheduled to be
+        %%  included or `test` does not exist
+        false -> {dir, AppTests};
+        %% need to add `test` dir at root to dirs to be included
+        true  -> {dir, AppTests ++ [BareTest]}
+    end.
+
+application_dirs([], []) -> [];
+application_dirs([], Acc) -> lists:reverse(Acc);
+application_dirs([App|Rest], Acc) ->
+    TestDir = filename:join([rebar_app_info:dir(App), "test"]),
+    case filelib:is_dir(TestDir) of
+        true  -> application_dirs(Rest, [TestDir|Acc]);
+        false -> application_dirs(Rest, Acc)
+    end.
+
+compile(State, {ok, Tests}) ->
+    %% inject `ct_first_files` and `ct_compile_opts` into the applications
+    %% to be compiled
+    case inject_ct_state(State, Tests) of
+        {ok, NewState} -> do_compile(NewState);
+        Error          -> Error
+    end;
+%% maybe compile even in the face of errors?
+compile(_State, Error) -> Error.
+
+do_compile(State) ->
+    case rebar_prv_compile:do(State) of
+        %% successfully compiled apps
+        {ok, S} ->
+            ok = maybe_cover_compile(S),
+            {ok, S};
+        %% this should look like a compiler error, not an eunit error
+        Error   -> Error
+    end.
+
+inject_ct_state(State, Tests) ->
+    Apps = rebar_state:project_apps(State),
+    ModdedApps = lists:map(fun(App) ->
+        NewOpts = inject(rebar_app_info:opts(App), State),
+        rebar_app_info:opts(App, NewOpts)
+    end, Apps),
+    NewOpts = inject(rebar_state:opts(State), State),
+    NewState = rebar_state:opts(State, NewOpts),
+    test_dirs(NewState, ModdedApps, Tests).
+
+inject(Opts, State) ->
+    %% append `ct_compile_opts` to app defined `erl_opts`
+    ErlOpts = rebar_opts:get(Opts, erl_opts, []),
+    CTOpts = rebar_state:get(State, ct_compile_opts, []),
+    NewErlOpts = CTOpts ++ ErlOpts,
+    %% append `ct_first_files` to app defined `erl_first_files`
+    FirstFiles = rebar_opts:get(Opts, erl_first_files, []),
+    CTFirstFiles = rebar_state:get(State, ct_first_files, []),
+    NewFirstFiles = CTFirstFiles ++ FirstFiles,
+    %% insert the new keys into the opts
+    lists:foldl(fun({K, V}, NewOpts) -> rebar_opts:set(NewOpts, K, V) end,
+                Opts,
+                [{erl_opts, NewErlOpts}, {erl_first_files, NewFirstFiles}]).
+
+test_dirs(State, Apps, Opts) ->
+    case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of
+        {Suites, undefined} -> set_compile_dirs(State, Apps, {suite, Suites});
+        {undefined, Dirs}   -> set_compile_dirs(State, Apps, {dir, Dirs});
+        {Suites, Dir} when is_integer(hd(Dir)) ->
+            set_compile_dirs(State, Apps, join(Suites, Dir));
+        {Suites, [Dir]} when is_integer(hd(Dir)) ->
+            set_compile_dirs(State, Apps, join(Suites, Dir));          
+        {_Suites, _Dirs}    -> {error, "Only a single directory may be specified when specifying suites"}
+    end.
+
+join(Suite, Dir) when is_integer(hd(Suite)) ->
+    {suite, [filename:join([Dir, Suite])]};
+join(Suites, Dir) ->
+    {suite, lists:map(fun(S) -> filename:join([Dir, S]) end, Suites)}.
+
+set_compile_dirs(State, Apps, {dir, Dir}) when is_integer(hd(Dir)) ->
+    %% single directory
+    %% insert `Dir` into an app if relative, or the base state if not
+    %% app relative but relative to the root or not at all if outside
+    %% project scope
+    {NewState, NewApps} = maybe_inject_test_dir(State, [], Apps, Dir),
+    {ok, rebar_state:project_apps(NewState, NewApps)};
+set_compile_dirs(State, Apps, {dir, Dirs}) ->
+    %% multiple directories
+    F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end,
+    {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs),
+    {ok, rebar_state:project_apps(NewState, NewApps)};
+set_compile_dirs(State, Apps, {suite, Suites}) ->
+    %% suites with dir component
+    Dirs = find_suite_dirs(Suites),
+    F = fun(Dir, {S, A}) -> maybe_inject_test_dir(S, [], A, Dir) end,
+    {NewState, NewApps} = lists:foldl(F, {State, Apps}, Dirs),
+    {ok, rebar_state:project_apps(NewState, NewApps)}.
+
+find_suite_dirs(Suites) ->
+    AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites),
+    %% eliminate duplicates
+    lists:usort(AllDirs).
+
+maybe_inject_test_dir(State, AppAcc, [App|Rest], Dir) ->
+    case rebar_file_utils:path_from_ancestor(Dir, rebar_app_info:dir(App)) of
+        {ok, Path} ->
+            Opts = inject_test_dir(rebar_app_info:opts(App), Path),
+            {State, AppAcc ++ [rebar_app_info:opts(App, Opts)] ++ Rest};
+        {error, badparent} ->
+            maybe_inject_test_dir(State, AppAcc ++ [App], Rest, Dir)
+    end;
+maybe_inject_test_dir(State, AppAcc, [], Dir) ->
+    case rebar_file_utils:path_from_ancestor(Dir, rebar_state:dir(State)) of
+        {ok, []}   ->
+            ?WARN("Can't have suites in root of project dir, dropping from tests", []),
+            {State, AppAcc};
+        {ok, Path} ->
+            Opts = inject_test_dir(rebar_state:opts(State), Path),
+            {rebar_state:opts(State, Opts), AppAcc};
+        {error, badparent} ->
+            {State, AppAcc}
     end.
 
+inject_test_dir(Opts, Dir) ->
+    %% append specified test targets to app defined `extra_src_dirs`
+    ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []),
+    rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]).
+
+translate_paths(State, Opts) ->
+    case {proplists:get_value(suite, Opts), proplists:get_value(dir, Opts)} of
+        {_Suites, undefined} -> translate_suites(State, Opts, []);
+        {undefined, _Dirs}   -> translate_dirs(State, Opts, []);
+        %% both dirs and suites are defined, only translate dir paths
+        _                    -> translate_dirs(State, Opts, [])
+    end.
+
+translate_dirs(_State, [], Acc) -> lists:reverse(Acc);
+translate_dirs(State, [{dir, Dir}|Rest], Acc) when is_integer(hd(Dir)) ->
+    %% single dir
+    Apps = rebar_state:project_apps(State),
+    translate_dirs(State, Rest, [{dir, translate(State, Apps, Dir)}|Acc]);
+translate_dirs(State, [{dir, Dirs}|Rest], Acc) ->
+    %% multiple dirs
+    Apps = rebar_state:project_apps(State),
+    NewDirs = {dir, lists:map(fun(Dir) -> translate(State, Apps, Dir) end, Dirs)},
+    translate_dirs(State, Rest, [NewDirs|Acc]);
+translate_dirs(State, [Test|Rest], Acc) ->
+    translate_dirs(State, Rest, [Test|Acc]).
+
+translate_suites(_State, [], Acc) -> lists:reverse(Acc);
+translate_suites(State, [{suite, Suite}|Rest], Acc) when is_integer(hd(Suite)) ->
+    %% single suite
+    Apps = rebar_state:project_apps(State),
+    translate_suites(State, Rest, [{suite, translate(State, Apps, Suite)}|Acc]);
+translate_suites(State, [{suite, Suites}|Rest], Acc) ->
+    %% multiple suites
+    Apps = rebar_state:project_apps(State),
+    NewSuites = {suite, lists:map(fun(Suite) -> translate(State, Apps, Suite) end, Suites)},
+    translate_suites(State, Rest, [NewSuites|Acc]);
+translate_suites(State, [Test|Rest], Acc) ->
+    translate_suites(State, Rest, [Test|Acc]).
+
+translate(State, [App|Rest], Path) ->
+    case rebar_file_utils:path_from_ancestor(Path, rebar_app_info:dir(App)) of
+        {ok, P}            -> filename:join([rebar_app_info:out_dir(App), P]);
+        {error, badparent} -> translate(State, Rest, Path)
+    end;
+translate(State, [], Path) ->
+    case rebar_file_utils:path_from_ancestor(Path, rebar_state:dir(State)) of
+        {ok, P}            -> filename:join([rebar_dir:base_dir(State), "extras", P]);
+        %% not relative, leave as is
+        {error, badparent} -> Path
+    end.
+
+setup_logdir(State, Opts) ->
+    Logdir = case proplists:get_value(logdir, Opts) of
+        undefined -> filename:join([rebar_dir:base_dir(State), "logs"]);
+        Dir       -> Dir
+    end,
+    filelib:ensure_dir(filename:join([Logdir, "dummy.beam"])),
+    [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)].
+
+turn_off_auto_compile(Opts) ->
+    [{auto_compile, false}|lists:keydelete(auto_compile, 1, Opts)].
+
 run_test_verbose(Opts) -> handle_results(ct:run_test(Opts)).
 
 run_test_quiet(Opts) ->
@@ -171,272 +449,46 @@ format_skipped({0, 0}) ->
 format_skipped({User, Auto}) ->
     io_lib:format("Skipped ~p (~p, ~p) tests. ", [User+Auto, User, Auto]).
 
-test_state(State) ->
-    TestOpts = case rebar_state:get(State, ct_compile_opts, []) of
-        []    -> [];
-        Opts  -> [{erl_opts, Opts}]
-    end,
-    [first_files(State)|TestOpts].
-
-first_files(State) ->
-    CTFirst = rebar_state:get(State, ct_first_files, []),
-    {erl_first_files, CTFirst}.
-
-setup_ct(State) ->
-    Opts = resolve_ct_opts(State),
-    Opts1 = discover_tests(State, Opts),
-    copy_and_compile_tests(State, Opts1).
-
-resolve_ct_opts(State) ->
-    {RawOpts, _} = rebar_state:command_parsed_args(State),
-    CmdOpts = transform_opts(RawOpts),
-    CfgOpts = rebar_state:get(State, ct_opts, []),
-    Merged = lists:ukeymerge(1,
-                             lists:ukeysort(1, CmdOpts),
-                             lists:ukeysort(1, CfgOpts)),
-    %% make sure `dir` and/or `suite` from command line go in as
-    %%  a pair overriding both `dir` and `suite` from config if
-    %%  they exist
-    case {proplists:get_value(suite, CmdOpts), proplists:get_value(dir, CmdOpts)} of
-        {undefined, undefined} -> Merged;
-        {_Suite, undefined}    -> lists:keydelete(dir, 1, Merged);
-        {undefined, _Dir}      -> lists:keydelete(suite, 1, Merged);
-        {_Suite, _Dir}         -> Merged
-    end.
-
-discover_tests(State, Opts) ->
-    case proplists:get_value(spec, Opts) of
-        undefined -> discover_dirs_and_suites(State, Opts);
-        TestSpec  -> discover_testspec(TestSpec, Opts)
-    end.
-
-discover_dirs_and_suites(State, Opts) ->
-    case {proplists:get_value(dir, Opts), proplists:get_value(suite, Opts)} of
-        %% no dirs or suites defined, try using `$APP/test` and `$ROOT/test`
-        %%  as suites
-        {undefined, undefined} -> test_dirs(State, Opts);
-        %% no dirs defined
-        {undefined, _} -> Opts;
-        %% no suites defined
-        {_, undefined} -> Opts;
-        %% a single dir defined, this is ok
-        {Dirs, Suites} when is_integer(hd(Dirs)), is_list(Suites) -> Opts;
-        %% still a single dir defined, adjust to make acceptable to ct
-        {[Dir], Suites} when is_integer(hd(Dir)), is_list(Suites) ->
-            [{dir, Dir}|lists:keydelete(dir, 1, Opts)];
-        %% multiple dirs and suites, error now to simplify later steps
-        {_, _} -> throw({error, {multiple_dirs_and_suites, Opts}})
-    end.
-
-discover_testspec(_TestSpec, Opts) ->
-    lists:keydelete(auto_compile, 1, Opts).
-
-copy_and_compile_tests(State, Opts) ->
-    %% possibly enable cover
+maybe_cover_compile(State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
     State1 = case proplists:get_value(cover, RawOpts, false) of
         true  -> rebar_state:set(State, cover_enabled, true);
         false -> State
     end,
-    copy_and_compile_test_suites(State1, Opts).
-
-copy_and_compile_test_suites(State, Opts) ->
-    case proplists:get_value(suite, Opts) of
-        %% no suites, try dirs
-        undefined -> copy_and_compile_test_dirs(State, Opts);
-        Suites    ->
-            Dir = proplists:get_value(dir, Opts, undefined),
-            AllSuites = join(Dir, Suites),
-            Dirs = find_suite_dirs(AllSuites),
-            lists:foreach(fun(S) ->
-                NewPath = copy(State, S),
-                compile_dir(State, NewPath)
-            end, Dirs),
-            NewSuites = lists:map(fun(S) -> retarget_path(State, S) end, AllSuites),
-            [{suite, NewSuites}|lists:keydelete(suite, 1, Opts)]
-    end.
-
-copy_and_compile_test_dirs(State, Opts) ->
-    case proplists:get_value(dir, Opts) of
-        undefined -> {error, {no_tests_specified, Opts}};
-        %% dir is a single directory
-        Dir when is_list(Dir), is_integer(hd(Dir)) ->
-            NewPath = copy(State, Dir),
-            [{dir, compile_dir(State, NewPath)}|lists:keydelete(dir, 1, Opts)];
-        %% dir is a list of directories
-        Dirs when is_list(Dirs) ->
-            NewDirs = lists:map(fun(Dir) ->
-                NewPath = copy(State, Dir),
-                compile_dir(State, NewPath)
-            end, Dirs),
-            [{dir, NewDirs}|lists:keydelete(dir, 1, Opts)]
-    end.
-
-join(undefined, Suites) -> Suites;
-join(Dir, Suites) when is_list(Dir), is_integer(hd(Dir)) ->
-    lists:map(fun(S) -> filename:join([Dir, S]) end, Suites);
-%% multiple dirs or a bad dir argument, try to continue anyways
-join(_, Suites) -> Suites.
-
-find_suite_dirs(Suites) ->
-    AllDirs = lists:map(fun(S) -> filename:dirname(filename:absname(S)) end, Suites),
-    %% eliminate duplicates
-    lists:usort(AllDirs).
-
-copy(State, Dir) ->
-    From = reduce_path(Dir),
-    retarget_path(State, From).
-
-compile_dir(State, Dir) ->
-    NewState = replace_src_dirs(State, [filename:absname(Dir)]),
-    ok = rebar_erlc_compiler:compile(rebar_state:opts(NewState), rebar_dir:base_dir(State), Dir),
-    ok = maybe_cover_compile(State, Dir),
-    Dir.
-
-retarget_path(State, Path) ->
-    ProjectApps = rebar_state:project_apps(State),
-    retarget_path(State, Path, ProjectApps).
-
-%% not relative to any apps in project, check to see it's relative to
-%%  project root
-retarget_path(State, Path, []) ->
-    case relative_path(reduce_path(Path), rebar_state:dir(State)) of
-        {ok, NewPath}         -> filename:join([rebar_dir:base_dir(State), NewPath]);
-        %% not relative to project root, don't modify
-        {error, not_relative} -> Path
-    end;
-%% relative to current app, retarget to the same dir relative to
-%%  the app's out_dir
-retarget_path(State, Path, [App|Rest]) ->
-    case relative_path(reduce_path(Path), rebar_app_info:dir(App)) of
-        {ok, NewPath}         -> filename:join([rebar_app_info:out_dir(App), NewPath]);
-        {error, not_relative} -> retarget_path(State, Path, Rest)
-    end.
-
-relative_path(Target, To) ->
-    relative_path1(filename:split(filename:absname(Target)),
-                   filename:split(filename:absname(To))).
-
-relative_path1([Part|Target], [Part|To]) -> relative_path1(Target, To);
-relative_path1([], [])                   -> {ok, ""};
-relative_path1(Target, [])               -> {ok, filename:join(Target)};
-relative_path1(_, _)                     -> {error, not_relative}.
-
-reduce_path(Dir) -> reduce_path([], filename:split(filename:absname(Dir))).
-
-reduce_path([], [])                -> filename:nativename("/");
-reduce_path(Acc, [])               -> filename:join(lists:reverse(Acc));
-reduce_path(Acc, ["."|Rest])       -> reduce_path(Acc, Rest);
-reduce_path([_|Acc], [".."|Rest])  -> reduce_path(Acc, Rest);
-reduce_path([], [".."|Rest])       -> reduce_path([], Rest);
-reduce_path(Acc, [Component|Rest]) -> reduce_path([Component|Acc], Rest).
-
-replace_src_dirs(State, Dirs) ->
-    %% replace any `src_dirs` with the test dirs
-    ErlOpts = rebar_state:get(State, erl_opts, []),
-    StrippedErlOpts = filter_src_dirs(ErlOpts),
-    State1 = rebar_state:set(State, erl_opts, StrippedErlOpts),
-    State2 = rebar_state:set(State1, src_dirs, []),
-    rebar_state:set(State2, extra_src_dirs, Dirs).
-
-filter_src_dirs(ErlOpts) ->
-    lists:filter(fun({src_dirs, _}) -> false; ({extra_src_dirs, _}) -> false; (_) -> true end, ErlOpts).
-
-test_dirs(State, Opts) ->
-    BareTest = filename:join([rebar_state:dir(State), "test"]),
-    F = fun(App) -> rebar_app_info:dir(App) == rebar_state:dir(State) end,
-    TestApps = project_apps(State),
-    case filelib:is_dir(BareTest) andalso not lists:any(F, TestApps) of
-        %% `test` dir at root of project is already scheduled to be
-        %%  included or `test` does not exist
-        false -> application_dirs(TestApps, Opts, []);
-        %% need to add `test` dir at root to dirs to be included
-        true  -> application_dirs(TestApps, Opts, [BareTest])
-    end.
-
-project_apps(State) ->
-    filter_checkouts(rebar_state:project_apps(State)).
-
-filter_checkouts(Apps) -> filter_checkouts(Apps, []).
-
-filter_checkouts([], Acc) -> lists:reverse(Acc);
-filter_checkouts([App|Rest], Acc) ->
-    case rebar_app_info:is_checkout(App) of
-        true  -> filter_checkouts(Rest, Acc);
-        false -> filter_checkouts(Rest, [App|Acc])
-    end.
-
-application_dirs([], Opts, []) -> Opts;
-application_dirs([], Opts, [Acc]) -> [{dir, Acc}|Opts];
-application_dirs([], Opts, Acc) -> [{dir, lists:reverse(Acc)}|Opts];
-application_dirs([App|Rest], Opts, Acc) ->
-    TestDir = filename:join([rebar_app_info:dir(App), "test"]),
-    case filelib:is_dir(TestDir) of
-        true  -> application_dirs(Rest, Opts, [TestDir|Acc]);
-        false -> application_dirs(Rest, Opts, Acc)
-    end.
-
-setup_logdir(State, Opts) ->
-    Logdir = case proplists:get_value(logdir, Opts) of
-        undefined -> filename:join([rebar_dir:base_dir(State), "logs"]);
-        Dir       -> Dir
-    end,
-    ensure_dir([Logdir]),
-    [{logdir, Logdir}|lists:keydelete(logdir, 1, Opts)].
+    rebar_prv_cover:maybe_cover_compile(State1).
 
-ensure_dir([]) -> ok;
-ensure_dir([Dir|Rest]) ->
-    case ec_file:is_dir(Dir) of
-        true ->
-            ok;
-        false ->
-            ec_file:mkdir_path(Dir)
-    end,
-    ensure_dir(Rest).
-
-maybe_cover_compile(State, Dir) ->
-    {Opts, _} = rebar_state:command_parsed_args(State),
-    State1 = case proplists:get_value(cover, Opts, false) of
+maybe_write_coverdata(State) ->
+    {RawOpts, _} = rebar_state:command_parsed_args(State),
+    State1 = case proplists:get_value(cover, RawOpts, false) of
         true  -> rebar_state:set(State, cover_enabled, true);
         false -> State
     end,
-    rebar_prv_cover:maybe_cover_compile(State1, [Dir]).
+    rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER).
 
 ct_opts(_State) ->
     [{dir, undefined, "dir", string, help(dir)}, %% comma-seperated list
      {suite, undefined, "suite", string, help(suite)}, %% comma-seperated list
      {group, undefined, "group", string, help(group)}, %% comma-seperated list
      {testcase, undefined, "case", string, help(testcase)}, %% comma-seperated list
-     {spec, undefined, "spec", string, help(spec)}, %% comma-seperated list
-     {join_specs, undefined, "join_specs", boolean, help(join_specs)}, %% Boolean
      {label, undefined, "label", string, help(label)}, %% String
      {config, undefined, "config", string, help(config)}, %% comma-seperated list
-     {userconfig, undefined, "userconfig", string, help(userconfig)}, %% [{CallbackMod, CfgStrings}] | {CallbackMod, CfgStrings}
      {allow_user_terms, undefined, "allow_user_terms", boolean, help(allow_user_terms)}, %% Bool
      {logdir, undefined, "logdir", string, help(logdir)}, %% dir
-     {logopts, undefined, "logopts", string, help(logopts)}, %% enum, no_nl | no_src
-     {verbosity, undefined, "verbosity", string, help(verbosity)}, %% Integer OR [{Category, VLevel}]
-     {silent_connections, undefined, "silent_connections", string,
-      help(silent_connections)}, % all OR %% comma-seperated list
-     {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% file
+     {logopts, undefined, "logopts", string, help(logopts)}, %% comma seperated list
+     {verbosity, undefined, "verbosity", integer, help(verbosity)}, %% Integer
      {cover, $c, "cover", {boolean, false}, help(cover)},
-     {cover_spec, undefined, "cover_spec", string, help(cover_spec)}, %% file
-     {cover_stop, undefined, "cover_stop", boolean, help(cover_stop)}, %% Boolean
-     {event_handler, undefined, "event_handler", string, help(event_handler)}, %% EH | [EH] WHERE EH atom() | {atom(), InitArgs} | {[atom()], InitArgs}
-     {include, undefined, "include", string, help(include)}, % comma-seperated list
-     {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true},
-      help(abort_if_missing_suites)}, %% boolean
-     {multiply_timetraps, undefined, "multiply_timetraps", integer,
-      help(multiply_timetraps)}, %% integer
-     {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)}, %% Boolean
-     {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)}, %% enum: auto_per_run | auto_per_tc | manual_per_tc
      {repeat, undefined, "repeat", integer, help(repeat)}, %% integer
      {duration, undefined, "duration", string, help(duration)}, % format: HHMMSS
      {until, undefined, "until", string, help(until)}, %% format: YYMoMoDD[HHMMSS]
-     {force_stop, undefined, "force_stop", string, help(force_stop)}, % enum: skip_rest, bool
-     {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Booloean
-     {ct_hooks, undefined, "ct_hooks", string, help(ct_hooks)}, %% List: [CTHModule | {CTHModule, CTHInitArgs}] where CTHModule is atom CthInitArgs is term
-     {auto_compile, undefined, "auto_compile", {boolean, false}, help(auto_compile)},
+     {force_stop, undefined, "force_stop", string, help(force_stop)}, %% String
+     {basic_html, undefined, "basic_html", boolean, help(basic_html)}, %% Boolean
+     {stylesheet, undefined, "stylesheet", string, help(stylesheet)}, %% String
+     {decrypt_key, undefined, "decrypt_key", string, help(decrypt_key)}, %% String
+     {decrypt_file, undefined, "decrypt_file", string, help(decrypt_file)}, %% String
+     {abort_if_missing_suites, undefined, "abort_if_missing_suites", {boolean, true}, help(abort_if_missing_suites)}, %% Boolean
+     {multiply_timetraps, undefined, "multiply_timetraps", integer, help(multiple_timetraps)}, %% Integer
+     {scale_timetraps, undefined, "scale_timetraps", boolean, help(scale_timetraps)},
+     {create_priv_dir, undefined, "create_priv_dir", string, help(create_priv_dir)},
      {verbose, $v, "verbose", boolean, help(verbose)}
     ].
 
@@ -448,28 +500,20 @@ help(group) ->
     "List of test groups to run";
 help(testcase) ->
     "List of test cases to run";
-help(spec) ->
-    "List of test specs to run";
 help(label) ->
     "Test label";
 help(config) ->
     "List of config files";
+help(allow_user_terms) ->
+    "Allow user defined config values in config files";
 help(logdir) ->
     "Log folder";
+help(logopts) ->
+    "Options for common test logging";
 help(verbosity) ->
     "Verbosity";
-help(stylesheet) ->
-    "Stylesheet to use for test results";
 help(cover) ->
     "Generate cover data";
-help(cover_spec) ->
-    "Cover file to use";
-help(event_handler) ->
-    "Event handlers to attach to the runner";
-help(include) ->
-    "Include folder";
-help(abort_if_missing_suites) ->
-    "Abort if suites are missing";
 help(repeat) ->
     "How often to repeat tests";
 help(duration) ->
@@ -477,85 +521,25 @@ help(duration) ->
 help(until) ->
     "Run until (format: HHMMSS)";
 help(force_stop) ->
-    "Force stop after time";
+    "Force stop on test timeout (true | false | skip_rest)";
 help(basic_html) ->
     "Show basic HTML";
+help(stylesheet) ->
+    "CSS stylesheet to apply to html output";
+help(decrypt_key) ->
+    "Path to key for decrypting config";
+help(decrypt_file) ->
+    "Path to file containing key for decrypting config";
+help(abort_if_missing_suites) ->
+    "Abort if suites are missing";
+help(multiply_timetraps) ->
+    "Multiply timetraps";
+help(scale_timetraps) ->
+    "Scale timetraps";
+help(create_priv_dir) ->
+    "Create priv dir (auto_per_run | auto_per_tc | manual_per_tc)";
 help(verbose) ->
     "Verbose output";
 help(_) ->
     "".
 
-transform_opts(Opts) ->
-    transform_opts(Opts, []).
-
-transform_opts([], Acc) -> Acc;
-%% drop `cover` and `verbose` so they're not passed as an option to common_test
-transform_opts([{cover, _}|Rest], Acc) ->
-    transform_opts(Rest, Acc);
-transform_opts([{cover_spec, CoverSpec}|Rest], Acc) ->
-    transform_opts(Rest, [{cover, CoverSpec}|Acc]);
-transform_opts([{verbose, _}|Rest], Acc) ->
-    transform_opts(Rest, Acc);
-transform_opts([{ct_hooks, CtHooks}|Rest], Acc) ->
-    transform_opts(Rest, [{ct_hooks, parse_term(CtHooks)}|Acc]);
-transform_opts([{force_stop, "skip_rest"}|Rest], Acc) ->
-    transform_opts(Rest, [{force_stop, skip_rest}|Acc]);
-transform_opts([{force_stop, _}|Rest], Acc) ->
-    transform_opts(Rest, [{force_stop, true}|Acc]);
-transform_opts([{repeat, Repeat}|Rest], Acc) ->
-    transform_opts(Rest, [{repeat,
-                           ec_cnv:to_integer(Repeat)}|Acc]);
-transform_opts([{create_priv_dir, CreatePrivDir}|Rest], Acc) ->
-    transform_opts(Rest, [{create_priv_dir,
-                           to_atoms(CreatePrivDir)}|Acc]);
-transform_opts([{multiply_timetraps, MultiplyTimetraps}|Rest], Acc) ->
-    transform_opts(Rest, [{multiply_timetraps,
-                           ec_cnv:to_integer(MultiplyTimetraps)}|Acc]);
-transform_opts([{event_handler, EventHandler}|Rest], Acc) ->
-    transform_opts(Rest, [{event_handler, parse_term(EventHandler)}|Acc]);
-transform_opts([{silent_connections, "all"}|Rest], Acc) ->
-    transform_opts(Rest, [{silent_connections, all}|Acc]);
-transform_opts([{silent_connections, SilentConnections}|Rest], Acc) ->
-    transform_opts(Rest, [{silent_connections,
-                           to_atoms(split_string(SilentConnections))}|Acc]);
-transform_opts([{verbosity, Verbosity}|Rest], Acc) ->
-    transform_opts(Rest, [{verbosity, parse_term(Verbosity)}|Acc]);
-transform_opts([{logopts, LogOpts}|Rest], Acc) ->
-    transform_opts(Rest, [{logopts, to_atoms(split_string(LogOpts))}|Acc]);
-transform_opts([{userconfig, UserConfig}|Rest], Acc) ->
-    transform_opts(Rest, [{userconfig, parse_term(UserConfig)}|Acc]);
-transform_opts([{testcase, Testcase}|Rest], Acc) ->
-    transform_opts(Rest, [{testcase, to_atoms(split_string(Testcase))}|Acc]);
-transform_opts([{group, Group}|Rest], Acc) -> % @TODO handle ""
-    % Input is a list or an atom. It can also be a nested list.
-    transform_opts(Rest, [{group, parse_term(Group)}|Acc]);
-transform_opts([{suite, Suite}|Rest], Acc) ->
-    transform_opts(Rest, [{suite, split_string(Suite)}|Acc]);
-transform_opts([{Key, Val}|Rest], Acc) when is_list(Val) ->
-    % Default to splitting a string on comma, that works fine for both flat
-    % lists of which there are many and single-items.
-    Val1 = case split_string(Val) of
-               [Val2] ->
-                   Val2;
-               Val2 ->
-                   Val2
-           end,
-    transform_opts(Rest, [{Key, Val1}|Acc]);
-transform_opts([{Key, Val}|Rest], Acc) ->
-    transform_opts(Rest, [{Key, Val}|Acc]).
-
-to_atoms(List) ->
-    lists:map(fun(X) -> list_to_atom(X) end, List).
-
-split_string(String) ->
-    string:tokens(String, ",").
-
-parse_term(String) ->
-    String1 = "[" ++ String ++ "].",
-    {ok, Tokens, _} = erl_scan:string(String1),
-    case erl_parse:parse_term(Tokens) of
-        {ok, [Terms]} ->
-            Terms;
-        Term ->
-            Term
-    end.

test/rebar_ct_SUITE.erl

@@ -17,15 +17,40 @@
          multi_suite/1,
          all_suite/1,
          single_dir_and_single_suite/1,
-         symlinked_dir_overwritten_fix/1,
-         data_dir_correct/1]).
+         data_dir_correct/1,
+         cmd_label/1,
+         cmd_config/1,
+         cmd_allow_user_terms/1,
+         cmd_logdir/1,
+         cmd_logopts/1,
+         cmd_verbosity/1,
+         cmd_repeat/1,
+         cmd_duration/1,
+         cmd_until/1,
+         cmd_force_stop/1,
+         cmd_basic_html/1,
+         cmd_stylesheet/1,
+         cmd_decrypt_key/1,
+         cmd_decrypt_file/1,
+         cmd_abort_if_missing_suites/1,
+         cmd_multiply_timetraps/1,
+         cmd_scale_timetraps/1,
+         cmd_create_priv_dir/1,
+         cfg_opts/1,
+         cfg_arbitrary_opts/1,
+         cfg_test_spec_filtered/1,
+         cfg_atom_suites/1]).
 
 -include_lib("common_test/include/ct.hrl").
 
 all() -> [{group, basic_app},
           {group, multi_app},
           {group, dirs_and_suites},
-          {group, data_dirs}].
+          {group, data_dirs},
+          {group, ct_opts},
+          cfg_opts, cfg_arbitrary_opts,
+          cfg_test_spec_filtered,
+          cfg_atom_suites].
 
 groups() -> [{basic_app, [], [basic_app_default_dirs,
                               basic_app_default_beams]},
@@ -39,9 +64,26 @@ groups() -> [{basic_app, [], [basic_app_default_dirs,
                                     single_unmanaged_suite,
                                     multi_suite,
                                     all_suite,
-                                    single_dir_and_single_suite,
-                                    symlinked_dir_overwritten_fix]},
-             {data_dirs, [], [data_dir_correct]}].
+                                    single_dir_and_single_suite]},
+             {data_dirs, [], [data_dir_correct]},
+             {ct_opts, [], [cmd_label,
+                            cmd_config,
+                            cmd_allow_user_terms,
+                            cmd_logdir,
+                            cmd_logopts,
+                            cmd_verbosity,
+                            cmd_repeat,
+                            cmd_duration,
+                            cmd_until,
+                            cmd_force_stop,
+                            cmd_basic_html,
+                            cmd_stylesheet,
+                            cmd_decrypt_key,
+                            cmd_decrypt_file,
+                            cmd_abort_if_missing_suites,
+                            cmd_multiply_timetraps,
+                            cmd_scale_timetraps,
+                            cmd_create_priv_dir]}].
 
 init_per_group(basic_app, Config) ->
     C = rebar_test_utils:init_rebar_state(Config, "ct_"),
@@ -56,22 +98,14 @@ init_per_group(basic_app, Config) ->
     ok = filelib:ensure_dir(Suite),
     ok = file:write_file(Suite, test_suite(Name)),
 
-    {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "compile"], return),
+    {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return),
 
-    LibDirs = rebar_dir:lib_dirs(State),
-    State1 = rebar_app_discover:do(State, LibDirs),
-
-    Providers = rebar_state:providers(State1),
-    Namespace = rebar_state:namespace(State1),
-    CommandProvider = providers:get_provider(ct, Providers, Namespace),
-    GetOptSpec = providers:opts(CommandProvider),
-    {ok, GetOptResult} = getopt:parse(GetOptSpec, []),
+    Tests = rebar_prv_common_test:prepare_tests(State),
+    {ok, NewState} = rebar_prv_common_test:compile(State, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
-
-    Result = rebar_prv_common_test:setup_ct(State2),
-
-    [{result, Result}, {appnames, [Name]}|C];
+    [{result, Opts}, {appnames, [Name]}|C];
 init_per_group(multi_app, Config) ->
     C = rebar_test_utils:init_rebar_state(Config, "ct_"),
 
@@ -99,22 +133,14 @@ init_per_group(multi_app, Config) ->
     ok = filelib:ensure_dir(Suite3),
     ok = file:write_file(Suite3, test_suite("extras")),
 
-    {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "compile"], return),
+    {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return),
 
-    LibDirs = rebar_dir:lib_dirs(State),
-    State1 = rebar_app_discover:do(State, LibDirs),
+    Tests = rebar_prv_common_test:prepare_tests(State),
+    {ok, NewState} = rebar_prv_common_test:compile(State, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    Providers = rebar_state:providers(State1),
-    Namespace = rebar_state:namespace(State1),
-    CommandProvider = providers:get_provider(ct, Providers, Namespace),
-    GetOptSpec = providers:opts(CommandProvider),
-    {ok, GetOptResult} = getopt:parse(GetOptSpec, []),
-
-    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
-
-    Result = rebar_prv_common_test:setup_ct(State2),
-
-    [{result, Result}, {appnames, [Name1, Name2]}|C];
+    [{result, Opts}, {appnames, [Name1, Name2]}|C];
 init_per_group(dirs_and_suites, Config) ->
     C = rebar_test_utils:init_rebar_state(Config, "ct_"),
 
@@ -142,7 +168,21 @@ init_per_group(dirs_and_suites, Config) ->
     ok = filelib:ensure_dir(Suite3),
     ok = file:write_file(Suite3, test_suite("extras")),
 
-    [{appnames, [Name1, Name2]}|C];
+    {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return),
+
+    [{s, State}, {appnames, [Name1, Name2]}|C];
+init_per_group(ct_opts, Config) ->
+    C = rebar_test_utils:init_rebar_state(Config, "ct_opts"),
+
+    AppDir = ?config(apps, C),
+
+    Name = rebar_test_utils:create_random_name("ct_opts_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    {ok, State} = rebar_test_utils:run_and_check(C, [], ["as", "test", "lock"], return),
+
+    [{result, State}|C];
 init_per_group(_, Config) -> Config.
 
 end_per_group(_Group, _Config) -> ok.
@@ -152,10 +192,10 @@ basic_app_default_dirs(Config) ->
     [Name] = ?config(appnames, Config),
     Result = ?config(result, Config),
 
-    Expect = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name, "test"])),
+    Expect = filename:join([AppDir, "_build", "test", "lib", Name, "test"]),
     Dir = proplists:get_value(dir, Result),
 
-    Expect = Dir.
+    [Expect] = Dir.
 
 basic_app_default_beams(Config) ->
     AppDir = ?config(apps, Config),
@@ -178,7 +218,7 @@ multi_app_default_dirs(Config) ->
 
     Expect1 = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name1, "test"])),
     Expect2 = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name2, "test"])),
-    Expect3 = filename:absname(filename:join([AppDir, "_build", "test", "test"])),
+    Expect3 = filename:absname(filename:join([AppDir, "_build", "test", "extras", "test"])),
     Dirs = proplists:get_value(dir, Result),
 
     true = (lists:sort([Expect1, Expect2, Expect3]) == lists:sort(Dirs)).
@@ -215,8 +255,7 @@ multi_app_default_beams(Config) ->
 single_app_dir(Config) ->
     AppDir = ?config(apps, Config),
     [Name1, _Name2] = ?config(appnames, Config),
-
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+    State = ?config(s, Config),
 
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
@@ -233,17 +272,19 @@ single_app_dir(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    Expect = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name1, "test"])),
-    Dir = proplists:get_value(dir, Result),
+    Expect = filename:join([AppDir, "_build", "test", "lib", Name1, "test"]),
+    Dir = proplists:get_value(dir, Opts),
 
-    Expect = Dir.
+    [Expect] = Dir.
 
 single_extra_dir(Config) ->
     AppDir = ?config(apps, Config),
-
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+    State = ?config(s, Config),
 
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
@@ -257,22 +298,24 @@ single_extra_dir(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    Expect = filename:absname(filename:join([AppDir, "_build", "test", "test"])),
-    Dir = proplists:get_value(dir, Result),
+    Expect = filename:join([AppDir, "_build", "test", "extras", "test"]),
+    Dir = proplists:get_value(dir, Opts),
 
-    Expect = Dir.
+    [Expect] = Dir.
 
 single_unmanaged_dir(Config) ->
     PrivDir = ?config(priv_dir, Config),
+    State = ?config(s, Config),
 
     Suite = filename:join([PrivDir, "unmanaged_dir", "unmanaged_dir_SUITE.erl"]),
     ok = filelib:ensure_dir(Suite),
     ok = file:write_file(Suite, test_suite("unmanaged_dir")),
 
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
-
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
 
@@ -285,18 +328,20 @@ single_unmanaged_dir(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    Expect = filename:absname(filename:join([PrivDir, "unmanaged_dir"])),
-    Dir = proplists:get_value(dir, Result),
+    Expect = filename:join([PrivDir, "unmanaged_dir"]),
+    Dir = proplists:get_value(dir, Opts),
 
-    Expect = Dir.
+    [Expect] = Dir.
 
 single_suite(Config) ->
     AppDir = ?config(apps, Config),
     [Name1, _Name2] = ?config(appnames, Config),
-
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+    State = ?config(s, Config),
 
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
@@ -314,24 +359,26 @@ single_suite(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    Expect = [filename:absname(filename:join([AppDir,
-                                              "_build",
-                                              "test",
-                                              "lib",
-                                              Name1,
-                                              "test",
-                                              Name1 ++ "_SUITE"]))],
-    Suite = proplists:get_value(suite, Result),
+    Expect = filename:join([AppDir,
+                            "_build",
+                            "test",
+                            "lib",
+                            Name1,
+                            "test",
+                            Name1 ++ "_SUITE"]),
+    Suite = proplists:get_value(suite, Opts),
 
-    Expect = Suite.
+    [Expect] = Suite.
 
 single_extra_suite(Config) ->
     AppDir = ?config(apps, Config),
     [_Name1, _Name2] = ?config(appnames, Config),
-
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+    State = ?config(s, Config),
 
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
@@ -347,27 +394,30 @@ single_extra_suite(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    Expect = [filename:absname(filename:join([AppDir,
-                                              "_build",
-                                              "test",
-                                              "test",
-                                              "extra_SUITE"]))],
-    Suite = proplists:get_value(suite, Result),
+    Expect = filename:join([AppDir,
+                            "_build",
+                            "test",
+                            "extras",
+                            "test",
+                            "extra_SUITE"]),
+    Suite = proplists:get_value(suite, Opts),
 
-    Expect = Suite.
+    [Expect] = Suite.
 
 single_unmanaged_suite(Config) ->
     PrivDir = ?config(priv_dir, Config),
     [_Name1, _Name2] = ?config(appnames, Config),
+    State = ?config(s, Config),
 
     Suite = filename:join([PrivDir, "unmanaged", "unmanaged_SUITE.erl"]),
     ok = filelib:ensure_dir(Suite),
     ok = file:write_file(Suite, test_suite("unmanaged")),
 
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
-
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
 
@@ -382,20 +432,22 @@ single_unmanaged_suite(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    Expect = [filename:absname(filename:join([PrivDir,
-                                              "unmanaged",
-                                              "unmanaged_SUITE"]))],
-    SuitePath = proplists:get_value(suite, Result),
+    Expect = filename:join([PrivDir,
+                            "unmanaged",
+                            "unmanaged_SUITE"]),
+    SuitePath = proplists:get_value(suite, Opts),
 
-    Expect = SuitePath.
+    [Expect] = SuitePath.
 
 multi_suite(Config) ->
     AppDir = ?config(apps, Config),
     [Name1, Name2] = ?config(appnames, Config),
-
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+    State = ?config(s, Config),
 
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
@@ -417,31 +469,33 @@ multi_suite(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
-
-    Expect1 = filename:absname(filename:join([AppDir,
-                                              "_build",
-                                              "test",
-                                              "lib",
-                                              Name1,
-                                              "test",
-                                              Name1 ++ "_SUITE"])),
-    Expect2 = filename:absname(filename:join([AppDir,
-                                              "_build",
-                                              "test",
-                                              "lib",
-                                              Name2,
-                                              "test",
-                                              Name2 ++ "_SUITE"])),
-    Suites = proplists:get_value(suite, Result),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
+
+    Expect1 = filename:join([AppDir,
+                             "_build",
+                             "test",
+                             "lib",
+                             Name1,
+                             "test",
+                             Name1 ++ "_SUITE"]),
+    Expect2 = filename:join([AppDir,
+                             "_build",
+                             "test",
+                             "lib",
+                             Name2,
+                             "test",
+                             Name2 ++ "_SUITE"]),
+    Suites = proplists:get_value(suite, Opts),
 
     true = (lists:sort([Expect1, Expect2]) == lists:sort(Suites)).
 
 all_suite(Config) ->
     AppDir = ?config(apps, Config),
     [Name1, Name2] = ?config(appnames, Config),
-
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+    State = ?config(s, Config),
 
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
@@ -465,36 +519,39 @@ all_suite(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
-
-    Expect1 = filename:absname(filename:join([AppDir,
-                                              "_build",
-                                              "test",
-                                              "lib",
-                                              Name1,
-                                              "test",
-                                              Name1 ++ "_SUITE"])),
-    Expect2 = filename:absname(filename:join([AppDir,
-                                              "_build",
-                                              "test",
-                                              "lib",
-                                              Name2,
-                                              "test",
-                                              Name2 ++ "_SUITE"])),
-    Expect3 = filename:absname(filename:join([AppDir,
-                                              "_build",
-                                              "test",
-                                              "test",
-                                              "extra_SUITE"])),
-    Suites = proplists:get_value(suite, Result),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
+
+    Expect1 = filename:join([AppDir,
+                             "_build",
+                             "test",
+                             "lib",
+                             Name1,
+                             "test",
+                             Name1 ++ "_SUITE"]),
+    Expect2 = filename:join([AppDir,
+                             "_build",
+                             "test",
+                             "lib",
+                             Name2,
+                             "test",
+                             Name2 ++ "_SUITE"]),
+    Expect3 = filename:join([AppDir,
+                             "_build",
+                             "test",
+                             "extras",
+                             "test",
+                             "extra_SUITE"]),
+    Suites = proplists:get_value(suite, Opts),
 
     true = (lists:sort([Expect1, Expect2, Expect3]) == lists:sort(Suites)).
 
 single_dir_and_single_suite(Config) ->
     AppDir = ?config(apps, Config),
     [_Name1, _Name2] = ?config(appnames, Config),
-
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+    State = ?config(s, Config),
 
     LibDirs = rebar_dir:lib_dirs(State),
     State1 = rebar_app_discover:do(State, LibDirs),
@@ -509,52 +566,369 @@ single_dir_and_single_suite(Config) ->
 
     State2 = rebar_state:command_parsed_args(State1, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
+    Tests = rebar_prv_common_test:prepare_tests(State2),
+    {ok, NewState} = rebar_prv_common_test:compile(State2, Tests),
+    {ok, T} = Tests,
+    Opts = rebar_prv_common_test:translate_paths(NewState, T),
 
-    Expect = [filename:absname(filename:join([AppDir,
-                                              "_build",
-                                              "test",
-                                              "test",
-                                              "extra_SUITE"]))],
-    Suite = proplists:get_value(suite, Result),
+    Expect = filename:join([AppDir,
+                            "_build",
+                            "test",
+                            "extras",
+                            "test"]),
+    Dir = proplists:get_value(dir, Opts),
+    [Expect] = Dir,
 
-    Expect = Suite.
+    Suite = proplists:get_value(suite, Opts),
+    ["extra_SUITE"] = Suite.
 
-symlinked_dir_overwritten_fix(Config) ->
-    AppDir = ?config(apps, Config),
-    [Name1, _Name2] = ?config(appnames, Config),
+%% this test probably only fails when this suite is run via rebar3 with the --cover flag
+data_dir_correct(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Parts = filename:split(DataDir),
+    ["rebar_ct_SUITE_data","test","rebar","lib","test","_build"|_] = lists:reverse(Parts).
 
-    {ok, State} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return),
+cmd_label(Config) ->
+    State = ?config(result, Config),
 
-    LibDirs = rebar_dir:lib_dirs(State),
-    State1 = rebar_app_discover:do(State, LibDirs),
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--label=this_is_a_label"]),
 
-    Providers = rebar_state:providers(State1),
-    Namespace = rebar_state:namespace(State1),
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({label, "this_is_a_label"}, TestOpts).
+
+cmd_config(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
     CommandProvider = providers:get_provider(ct, Providers, Namespace),
     GetOptSpec = providers:opts(CommandProvider),
-    {ok, GetOptResult} = getopt:parse(GetOptSpec,
-                                      ["--dir=" ++ filename:join([AppDir,
-                                                                  "apps",
-                                                                  Name1])]),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--config=config/foo,config/bar,config/baz"]),
 
-    State2 = rebar_state:command_parsed_args(State1, GetOptResult),
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
 
-    Result = rebar_prv_common_test:setup_ct(State2),
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
 
-    Expect = filename:absname(filename:join([AppDir, "_build", "test", "lib", Name1])),
-    Dir = proplists:get_value(dir, Result),
+    true = lists:member({config, ["config/foo", "config/bar", "config/baz"]}, TestOpts).
 
-    Expect = Dir,
+cmd_allow_user_terms(Config) ->
+    State = ?config(result, Config),
 
-    {ok, _} = rebar_test_utils:run_and_check(Config, [], ["as", "test", "compile"], return).
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--allow_user_terms=true"]),
 
-%% this test probably only fails when this suite is run via rebar3 with the --cover flag
-data_dir_correct(Config) ->
-    DataDir = ?config(data_dir, Config),
-    Parts = filename:split(DataDir),
-    ["rebar_ct_SUITE_data","test","rebar","lib","test","_build"|_] = lists:reverse(Parts).
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({allow_user_terms, true}, TestOpts).
+
+cmd_logdir(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--logdir=/tmp/ct_logs"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({logdir, "/tmp/ct_logs"}, TestOpts).
+
+cmd_logopts(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--logopts=no_src,no_nl"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({logopts, [no_src, no_nl]}, TestOpts).
+
+cmd_verbosity(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--verbosity=43"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({verbosity, 43}, TestOpts).
+
+cmd_repeat(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--repeat=3"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({repeat, 3}, TestOpts).
+
+cmd_duration(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--duration=001500"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({duration, "001500"}, TestOpts).
+
+cmd_until(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--until=001500"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({until, "001500"}, TestOpts).
+
+cmd_force_stop(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--force_stop=skip_rest"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({force_stop, skip_rest}, TestOpts).
+
+cmd_basic_html(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--basic_html"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({basic_html, true}, TestOpts).
+
+cmd_stylesheet(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--stylesheet=resources/tests.css"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({stylesheet, "resources/tests.css"}, TestOpts).
+
+cmd_decrypt_key(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--decrypt_key==ac467e30"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({decrypt_key, "=ac467e30"}, TestOpts).
+
+cmd_decrypt_file(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--decrypt_file=../keyfile.pem"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({decrypt_file, "../keyfile.pem"}, TestOpts).
+
+cmd_abort_if_missing_suites(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--abort_if_missing_suites"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({abort_if_missing_suites, true}, TestOpts).
+
+cmd_multiply_timetraps(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--multiply_timetraps=3"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({multiply_timetraps, 3}, TestOpts).
+
+cmd_scale_timetraps(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--scale_timetraps"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({scale_timetraps, true}, TestOpts).
+
+cmd_create_priv_dir(Config) ->
+    State = ?config(result, Config),
+
+    Providers = rebar_state:providers(State),
+    Namespace = rebar_state:namespace(State),
+    CommandProvider = providers:get_provider(ct, Providers, Namespace),
+    GetOptSpec = providers:opts(CommandProvider),
+    {ok, GetOptResult} = getopt:parse(GetOptSpec, ["--create_priv_dir=manual_per_tc"]),
+
+    NewState = rebar_state:command_parsed_args(State, GetOptResult),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(NewState),
+
+    true = lists:member({create_priv_dir, manual_per_tc}, TestOpts).
+
+cfg_opts(Config) ->
+    C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_opts_"),
+
+    AppDir = ?config(apps, C),
+
+    Name = rebar_test_utils:create_random_name("ct_cfg_opts_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    RebarConfig = [{ct_opts, [{label, "this_is_a_label"}, {decrypt_file, "../keyfile.pem"}]}],
+
+    {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State),
+
+    true = lists:member({label, "this_is_a_label"}, TestOpts),
+    true = lists:member({decrypt_file, "../keyfile.pem"}, TestOpts).
+
+%% allow even nonsensical opts to be passed to ct_run for futureproofing
+cfg_arbitrary_opts(Config) ->
+    C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_arbitrary_opts_"),
+
+    AppDir = ?config(apps, C),
+
+    Name = rebar_test_utils:create_random_name("ct_cfg_arbitrary_opts_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    RebarConfig = [{ct_opts, [{foo, 1}, {bar, 2}, {baz, 3}]}],
+
+    {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State),
+
+    true = lists:member({foo, 1}, TestOpts),
+    true = lists:member({bar, 2}, TestOpts),
+    true = lists:member({baz, 3}, TestOpts).
+
+cfg_test_spec_filtered(Config) ->
+    C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_test_spec_filtered_opts_"),
+
+    AppDir = ?config(apps, C),
+
+    Name = rebar_test_utils:create_random_name("ct_cfg_test_spec_filtered_opts_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    RebarConfig = [{ct_opts, [{test_spec, "spec/foo.spec"}]}],
+
+    {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State),
+
+    false = lists:keysearch(test_spec, 1, TestOpts).
+
+cfg_atom_suites(Config) ->
+    C = rebar_test_utils:init_rebar_state(Config, "ct_cfg_atom_suites_"),
+
+    AppDir = ?config(apps, C),
+
+    Name = rebar_test_utils:create_random_name("ct_cfg_atom_suites_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    RebarConfig = [{ct_opts, [{suite, [foo, bar, baz]}]}],
+
+    {ok, State} = rebar_test_utils:run_and_check(C, RebarConfig, ["as", "test", "lock"], return),
+
+    {ok, TestOpts} = rebar_prv_common_test:prepare_tests(State),
 
+    true = lists:member({suite, ["foo", "bar", "baz"]}, TestOpts).
 
 %% helper for generating test data
 test_suite(Name) ->