generalize the "test" special dir into an {extra_src_dirs, [...]} option

like `src_dirs`, `extra_src_dirs` are directories to be copied to
the `_build` dir and compiled. unlike `src_dirs` they are not added
to the .app specification

src/rebar_erlc_compiler.erl

@@ -143,7 +143,8 @@ doterl_compile(Config, Dir, OutDir, MoreSources, ErlOpts) ->
     %% Support the src_dirs option allowing multiple directories to
     %% contain erlang source. This might be used, for example, should
     %% eunit tests be separated from the core application source.
-    SrcDirs = [filename:join(Dir, X) || X <- proplists:get_value(src_dirs, ErlOpts, ["src"])],
+    SrcDirs = [filename:join(Dir, X) || X <- proplists:get_value(src_dirs, ErlOpts, ["src"]) ++
+                                             proplists:get_value(extra_src_dirs, ErlOpts, [])],
     AllErlFiles = gather_src(SrcDirs, []) ++ MoreSources,
 
     %% Make sure that ebin/ exists and is on the path

src/rebar_otp_app.erl

@@ -100,7 +100,7 @@ preprocess(State, AppInfo, AppSrcFile) ->
             %% substitute. Note that we include the list of modules available in
             %% ebin/ and update the app data accordingly.
             OutDir = rebar_app_info:out_dir(AppInfo),
-            AppVars = load_app_vars(State) ++ [{modules, ebin_modules(AppInfo, OutDir)}],
+            AppVars = load_app_vars(State) ++ [{modules, ebin_modules(State, AppInfo, OutDir)}],
             A1 = apply_app_vars(AppVars, AppData),
 
             %% AppSrcFile may contain instructions for generating a vsn number
@@ -152,14 +152,25 @@ validate_name(AppName, File) ->
             ?PRV_ERROR({invalid_name, File, AppName})
     end.
 
-ebin_modules(App, Dir) ->
+ebin_modules(State, App, Dir) ->
     Beams = lists:sort(rebar_utils:beams(filename:join(Dir, "ebin"))),
-    F = fun(Beam) -> not lists:prefix(filename:join([rebar_app_info:out_dir(App), "test"]),
-                                      beam_src(Beam))
-    end,
+    ExtraDirs = extra_dirs(State),
+    F = fun(Beam) -> not in_extra_dir(App, Beam, ExtraDirs) end,
     Filtered = lists:filter(F, Beams),
     [rebar_utils:beam_to_mod(N) || N <- Filtered].
 
+extra_dirs(State) ->
+    ErlOpts = rebar_utils:erl_opts(State),
+    Extras = proplists:get_value(extra_src_dirs, ErlOpts, []),
+    SrcDirs = proplists:get_value(src_dirs, ErlOpts, ["src"]),
+    %% remove any dirs that are defined in `src_dirs` from `extra_src_dirs`
+    Extras -- SrcDirs.
+
+in_extra_dir(App, Beam, Dirs) ->
+    lists:any(fun(Dir) -> lists:prefix(filename:join([rebar_app_info:out_dir(App), Dir]),
+                                       beam_src(Beam)) end,
+              Dirs).
+
 beam_src(Beam) ->
     case beam_lib:chunks(Beam, [compile_info]) of
         {ok, {_mod, Chunks}} ->

src/rebar_prv_compile.erl

@@ -120,8 +120,9 @@ copy_app_dirs(State, OldAppDir, AppDir) ->
             filelib:ensure_dir(filename:join(AppDir, "dummy")),
             %% link to src_dirs to be adjacent to ebin is needed for R15 use of cover/xref
             ErlOpts = rebar_utils:erl_opts(State),
-            SrcDirs = proplists:get_value(src_dirs, ErlOpts, ["src"]),
-            [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include", "test"] ++ SrcDirs];
+            SrcDirs = proplists:get_value(src_dirs, ErlOpts, ["src"]) ++
+                      proplists:get_value(extra_src_dirs, ErlOpts, []),
+            [symlink_or_copy(OldAppDir, AppDir, Dir) || Dir <- ["priv", "include"] ++ SrcDirs];
         false ->
             ok
     end.

src/rebar_prv_eunit.erl

@@ -84,7 +84,8 @@ format_error({error_running_tests, Reason}) ->
 test_state(State) ->
     ErlOpts = rebar_state:get(State, eunit_compile_opts, []),
     TestOpts = safe_define_test_macro(ErlOpts),
-    first_files(State) ++ [{erl_opts, TestOpts}].
+    TestDir = [{extra_src_dirs, ["test"]}],
+    first_files(State) ++ [{erl_opts, TestOpts ++ TestDir}].
 
 safe_define_test_macro(Opts) ->
     %% defining a compile macro twice results in an exception so
@@ -105,54 +106,38 @@ first_files(State) ->
 
 prepare_tests(State) ->
     {RawOpts, _} = rebar_state:command_parsed_args(State),
-    resolve_apps(State, RawOpts).
+    ok = maybe_cover_compile(State, RawOpts),
+    ProjectApps = project_apps(State),
+    resolve_apps(ProjectApps, RawOpts).
+
+maybe_cover_compile(State, Opts) ->
+    State1 = case proplists:get_value(cover, Opts, false) of
+        true  -> rebar_state:set(State, cover_enabled, true);
+        false -> State
+    end,
+    rebar_prv_cover:maybe_cover_compile(State1).
 
-resolve_apps(State, RawOpts) ->
+resolve_apps(ProjectApps, RawOpts) ->
     case proplists:get_value(app, RawOpts) of
-        undefined -> resolve_suites(State, project_apps(State), RawOpts);
+        undefined -> resolve_suites(ProjectApps, RawOpts);
         %% convert app name strings to `rebar_app_info` objects
         Apps      -> AppNames = string:tokens(Apps, [$,]),
-                     ProjectApps = project_apps(State),
                      case filter_apps_by_name(AppNames, ProjectApps) of
-                         {ok, TestApps} -> resolve_suites(State, TestApps, RawOpts);
+                         {ok, TestApps} -> resolve_suites(TestApps, RawOpts);
                          Error          -> Error
                      end
     end.
 
-resolve_suites(State, Apps, RawOpts) ->
+resolve_suites(Apps, RawOpts) ->
     case proplists:get_value(suite, RawOpts) of
-        undefined  -> compile_tests(State, Apps, all, RawOpts);
+        undefined  -> test_set(Apps, all);
         Suites     -> SuiteNames = string:tokens(Suites, [$,]),
                       case filter_suites_by_apps(SuiteNames, Apps) of
-                          {ok, S} -> compile_tests(State, Apps, S, RawOpts);
+                          {ok, S} -> test_set(Apps, S);
                           Error   -> Error
                       end
     end.
 
-compile_tests(State, TestApps, Suites, RawOpts) ->
-    F = fun(AppInfo) ->
-        AppDir = rebar_app_info:dir(AppInfo),
-        S = case rebar_app_info:state(AppInfo) of
-            undefined ->
-                C = rebar_config:consult(AppDir),
-                rebar_state:new(State, C, AppDir);
-            AppState ->
-                AppState
-        end,
-        ok = rebar_erlc_compiler:compile(replace_src_dirs(S),
-                                         ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)))
-    end,
-    lists:foreach(F, TestApps),
-    ok = maybe_cover_compile(State, RawOpts),
-    {ok, test_set(TestApps, Suites)}.
-
-maybe_cover_compile(State, Opts) ->
-    State1 = case proplists:get_value(cover, Opts, false) of
-        true  -> rebar_state:set(State, cover_enabled, true);
-        false -> State
-    end,
-    rebar_prv_cover:maybe_cover_compile(State1).
-
 project_apps(State) ->
     filter_checkouts(rebar_state:project_apps(State)).
 
@@ -219,14 +204,8 @@ app_modules([App|Rest], Acc) ->
             app_modules(Rest, NewAcc)
     end.
 
-replace_src_dirs(State) ->
-    %% replace any `src_dirs` with just the `test` dir
-    ErlOpts = rebar_state:get(State, erl_opts, []),
-    StrippedOpts = lists:keydelete(src_dirs, 1, ErlOpts),
-    rebar_state:set(State, erl_opts, [{src_dirs, ["test"]}|StrippedOpts]).
-
-test_set(Apps, all) -> set_apps(Apps, []);
-test_set(_Apps, Suites) -> set_suites(Suites, []).
+test_set(Apps, all) -> {ok, set_apps(Apps, [])};
+test_set(_Apps, Suites) -> {ok, set_suites(Suites, [])}.
 
 set_apps([], Acc) -> lists:reverse(Acc);
 set_apps([App|Rest], Acc) ->

test/rebar_extra_src_dirs_SUITE.erl

@@ -0,0 +1,152 @@
+-module(rebar_extra_src_dirs_SUITE).
+
+-export([suite/0,
+         init_per_suite/1,
+         end_per_suite/1,
+         init_per_testcase/2,
+         end_per_testcase/2,
+         all/0,
+         build_basic_app/1,
+         build_multi_apps/1,
+         src_dir_takes_precedence/1]).
+
+-include_lib("common_test/include/ct.hrl").
+
+suite() ->
+    [].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(_, Config) ->
+    rebar_test_utils:init_rebar_state(Config).
+
+end_per_testcase(_, _Config) -> ok.
+
+all() ->
+    [build_basic_app, build_multi_apps, src_dir_takes_precedence].
+
+build_basic_app(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    Extra = filename:join([AppDir, "extra", "extra.erl"]),
+    ok = filelib:ensure_dir(Extra),
+    Src = io_lib:format("-module(extra).~n-export([x/0]).~nx() -> ok.", []),
+    ok = ec_file:write(Extra, Src),
+
+    RebarConfig = [{erl_opts, [{extra_src_dirs, ["extra"]}]}],
+
+    rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}),
+
+    %% check that `extra.erl` was compiled to the `ebin` dir
+    Ebin = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]),
+    true = filelib:is_file(filename:join([Ebin, "extra.beam"])),
+
+    %% check that `extra.erl` is not in the `modules` key of the app
+    {ok, App} = file:consult(filename:join([AppDir,
+                                            "_build",
+                                            "default",
+                                            "lib",
+                                            Name,
+                                            "ebin",
+                                            Name ++ ".app"])),
+    [{application, _, KVs}] = App,
+    Mods = proplists:get_value(modules, KVs),
+    false = lists:member(extra, Mods).
+
+build_multi_apps(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name1 = rebar_test_utils:create_random_name("app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(filename:join([AppDir,Name1]), Name1, Vsn1, [kernel, stdlib]),
+    Name2 = rebar_test_utils:create_random_name("app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(filename:join([AppDir,Name2]), Name2, Vsn2, [kernel, stdlib]),
+
+    Extra1 = filename:join([AppDir, Name1, "extra", "extra1.erl"]),
+    ok = filelib:ensure_dir(Extra1),
+    Src1 = io_lib:format("-module(extra1).~n-export([x/0]).~nx() -> ok.", []),
+    ok = ec_file:write(Extra1, Src1),
+
+    Extra2 = filename:join([AppDir, Name2, "extra", "extra2.erl"]),
+    ok = filelib:ensure_dir(Extra2),
+    Src2 = io_lib:format("-module(extra2).~n-export([x/0]).~nx() -> ok.", []),
+    ok = ec_file:write(Extra2, Src2),
+
+    RebarConfig = [{erl_opts, [{extra_src_dirs, ["extra"]}]}],
+
+    rebar_test_utils:run_and_check(
+        Config, RebarConfig, ["compile"],
+        {ok, [{app, Name1}, {app, Name2}]}
+    ),
+
+    %% check that `extraX.erl` was compiled to the `ebin` dir
+    Ebin1 = filename:join([AppDir, "_build", "default", "lib", Name1, "ebin"]),
+    true = filelib:is_file(filename:join([Ebin1, "extra1.beam"])),
+    
+    Ebin2 = filename:join([AppDir, "_build", "default", "lib", Name2, "ebin"]),
+    true = filelib:is_file(filename:join([Ebin2, "extra2.beam"])),
+
+    %% check that `extraX.erl` is not in the `modules` key of the app
+    {ok, App1} = file:consult(filename:join([AppDir,
+                                             "_build",
+                                             "default",
+                                             "lib",
+                                             Name1,
+                                             "ebin",
+                                             Name1 ++ ".app"])),
+    [{application, _, KVs1}] = App1,
+    Mods1 = proplists:get_value(modules, KVs1),
+    false = lists:member(extra1, Mods1),
+
+    {ok, App2} = file:consult(filename:join([AppDir,
+                                             "_build",
+                                             "default",
+                                             "lib",
+                                             Name2,
+                                             "ebin",
+                                             Name2 ++ ".app"])),
+    [{application, _, KVs2}] = App2,
+    Mods2 = proplists:get_value(modules, KVs2),
+    false = lists:member(extra2, Mods2).
+
+src_dir_takes_precedence(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    Extra = filename:join([AppDir, "extra", "extra.erl"]),
+    ok = filelib:ensure_dir(Extra),
+    Src = io_lib:format("-module(extra).~n-export([x/0]).~nx() -> ok.", []),
+    ok = ec_file:write(Extra, Src),
+
+    RebarConfig = [{erl_opts, [{src_dirs, ["src", "extra"]}, {extra_src_dirs, ["extra"]}]}],
+
+    rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], {ok, [{app, Name}]}),
+
+    %% check that `extra.erl` was compiled to the `ebin` dir
+    %% check that `extraX.erl` was compiled to the `ebin` dir
+    Ebin = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]),
+    true = filelib:is_file(filename:join([Ebin, "extra.beam"])),
+
+    %% check that `extra.erl` is in the `modules` key of the app
+    {ok, App} = file:consult(filename:join([AppDir,
+                                            "_build",
+                                            "default",
+                                            "lib",
+                                            Name,
+                                            "ebin",
+                                            Name ++ ".app"])),
+    [{application, _, KVs}] = App,
+    Mods = proplists:get_value(modules, KVs),
+    true = lists:member(extra, Mods).