Merge pull request #1382 from sirihansen/siri/recursive-opt

Add 'recursive' option

src/rebar_dir.erl

@@ -22,6 +22,7 @@
          processing_base_dir/2,
          make_relative_path/2,
          src_dirs/1, src_dirs/2,
+         src_dir_opts/2, recursive/2,
          extra_src_dirs/1, extra_src_dirs/2,
          all_src_dirs/1, all_src_dirs/3,
          retarget_path/2]).
@@ -160,28 +161,43 @@ do_make_relative_path(Source, Target) ->
     Base = lists:duplicate(max(length(Target) - 1, 0), ".."),
     filename:join(Base ++ Source).
 
+%%%-----------------------------------------------------------------
+%%% 'src_dirs' and 'extra_src_dirs' can be configured with options
+%%% like this:
+%%%
+%%% {src_dirs,[{"foo",[{recursive,false}]}]}
+%%% {extra_src_dirs,[{"bar",[recursive]}]} (equivalent to {recursive,true})
+%%%
+%%% src_dirs/1,2 and extra_src_dirs/1,2 return only the list of
+%%% directories for the 'src_dirs' and 'extra_src_dirs' options
+%%% respectively, while src_dirs_opts/2 return the options list for
+%%% the given directory, no matter if it is configured as 'src_dirs' or
+%%% 'extra_src_dirs'.
+%%%
 -spec src_dirs(rebar_dict()) -> list(file:filename_all()).
 src_dirs(Opts) -> src_dirs(Opts, []).
 
 -spec src_dirs(rebar_dict(), list(file:filename_all())) -> list(file:filename_all()).
 src_dirs(Opts, Default) ->
-    ErlOpts = rebar_opts:erl_opts(Opts),
-    Vs = proplists:get_all_values(src_dirs, ErlOpts),
-    case lists:append([rebar_opts:get(Opts, src_dirs, []) | Vs]) of
-        []   -> Default;
-        Dirs -> lists:usort(Dirs)
-    end.
+    src_dirs(src_dirs, Opts, Default).
 
 -spec extra_src_dirs(rebar_dict()) -> list(file:filename_all()).
 extra_src_dirs(Opts) -> extra_src_dirs(Opts, []).
 
 -spec extra_src_dirs(rebar_dict(), list(file:filename_all())) -> list(file:filename_all()).
 extra_src_dirs(Opts, Default) ->
+    src_dirs(extra_src_dirs, Opts, Default).
+
+src_dirs(Type, Opts, Default) ->
+    lists:usort([case D0 of {D,_} -> D; _ -> D0 end ||
+                    D0 <- raw_src_dirs(Type,Opts,Default)]).
+
+raw_src_dirs(Type, Opts, Default) ->
     ErlOpts = rebar_opts:erl_opts(Opts),
-    Vs = proplists:get_all_values(extra_src_dirs, ErlOpts),
-    case lists:append([rebar_opts:get(Opts, extra_src_dirs, []) | Vs]) of
+    Vs = proplists:get_all_values(
Type, ErlOpts),
+    case lists:append([rebar_opts:get(Opts, 
Type, []) | Vs]) of
         []   -> Default;
-        Dirs -> lists:usort(Dirs)
+        Dirs -> Dirs
     end.
 
 -spec all_src_dirs(rebar_dict()) -> list(file:filename_all()).
@@ -192,6 +208,32 @@ all_src_dirs(Opts) -> all_src_dirs(Opts, [], []).
 all_src_dirs(Opts, SrcDefault, ExtraDefault) ->
     lists:usort(src_dirs(Opts, SrcDefault) ++ extra_src_dirs(Opts, ExtraDefault)).
 
+%%%-----------------------------------------------------------------
+%%% Return the list of options for the given src directory
+%%% If the same option is given multiple times for a directory in the
+%%% config, the priority order is: first occurence of 'src_dirs'
+%%% followed by first occurence of 'extra_src_dirs'.
+-spec src_dir_opts(rebar_dict(), file:filename_all()) -> [{atom(),term()}].
+src_dir_opts(Opts, Dir) ->
+    RawSrcDirs = raw_src_dirs(src_dirs, Opts, []),
+    RawExtraSrcDirs = raw_src_dirs(extra_src_dirs, Opts, []),
+    AllOpts = [Opt || {D,Opt} <- RawSrcDirs++RawExtraSrcDirs,
+                      D==Dir],
+    lists:ukeysort(1,proplists:unfold(lists:append(AllOpts))).
+
+%%%-----------------------------------------------------------------
+%%% Return the value of the 'recursive' option for the given directory.
+%%% If not given, the value of 'recursive' in the 'erlc_compiler'
+%%% options is used, and finally the default is 'true'.
+-spec recursive(rebar_dict(), file:filename_all()) -> boolean().
+recursive(Opts, Dir) ->
+    DirOpts = src_dir_opts(Opts, Dir),
+    Default = proplists:get_value(recursive,
+                                  rebar_opts:get(Opts, erlc_compiler, []),
+                                  true),
+    R = proplists:get_value(recursive, DirOpts, Default),
+    R.
+
 %% given a path if that path is an ancestor of an app dir return the path relative to that
 %% apps outdir. if the path is not an ancestor to any app dirs but is an ancestor of the
 %% project root return the path relative to the project base_dir. if it is not an ancestor

src/rebar_erlc_compiler.erl

@@ -47,10 +47,6 @@
 -type compile_opts() :: [compile_opt()].
 -type compile_opt() :: {recursive, boolean()}.
 
--record(compile_opts, {
-    recursive = true
-}).
-
 -define(DEFAULT_OUTDIR, "ebin").
 -define(RE_PREFIX, "^[^._]").
 
@@ -99,21 +95,25 @@ compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t ->
     Dir = ec_cnv:to_list(rebar_app_info:out_dir(AppInfo)),
     RebarOpts = rebar_app_info:opts(AppInfo),
 
+    SrcOpts = [check_last_mod,
+               {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}],
+    MibsOpts = [check_last_mod,
+                {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}],
     rebar_base_compiler:run(RebarOpts,
                             check_files([filename:join(Dir, File)
                                          || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]),
                             filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl",
-                            fun compile_xrl/3),
+                            fun compile_xrl/3, SrcOpts),
     rebar_base_compiler:run(RebarOpts,
                             check_files([filename:join(Dir, File)
                                          || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]),
                             filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl",
-                            fun compile_yrl/3),
+                            fun compile_yrl/3, SrcOpts),
     rebar_base_compiler:run(RebarOpts,
                             check_files([filename:join(Dir, File)
                                          || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]),
                             filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin",
-                            compile_mib(AppInfo)),
+                            compile_mib(AppInfo), MibsOpts),
 
     SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end,
                         rebar_dir:src_dirs(RebarOpts, ["src"])),
@@ -182,13 +182,10 @@ compile_dir(RebarOpts, BaseDir, Dir, Opts) ->
                    compile_opts()) -> ok.
 compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t ->
     compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts);
-compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, Opts) ->
-    CompileOpts = parse_opts(Opts),
-  
+compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) ->
     ErlOpts = rebar_opts:erl_opts(RebarOpts),
     ?DEBUG("erlopts ~p", [ErlOpts]),
-    Recursive = CompileOpts#compile_opts.recursive,
-    AllErlFiles = gather_src(SrcDirs, Recursive),
+    AllErlFiles = gather_src(RebarOpts, BaseDir, SrcDirs, CompileOpts),
     ?DEBUG("files to compile ~p", [AllErlFiles]),
 
     %% Make sure that outdir is on the path
@@ -266,12 +263,22 @@ clean_dirs(AppDir, [Dir|Rest]) ->
 %% Internal functions
 %% ===================================================================
 
-gather_src(Dirs, Recursive) ->
-    gather_src(Dirs, [], Recursive).
-
-gather_src([], Srcs, _Recursive) -> Srcs;
-gather_src([Dir|Rest], Srcs, Recursive) ->
-    gather_src(Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", Recursive), Recursive).
+gather_src(Opts, BaseDir, Dirs, CompileOpts) ->
+    gather_src(Opts, filename:split(BaseDir), Dirs, [], CompileOpts).
+
+gather_src(_Opts, _BaseDirParts, [], Srcs, _CompileOpts) -> Srcs;
+gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) ->
+    DirParts = filename:split(Dir),
+    RelDir = case lists:prefix(BaseDirParts,DirParts) of
+                 true ->
+                     case lists:nthtail(length(BaseDirParts),DirParts) of
+                         [] -> ".";
+                         RestParts -> filename:join(RestParts)
+                     end;
+                 false -> Dir
+             end,
+    DirRecursive = dir_recursive(Opts, RelDir, CompileOpts),
+    gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts).
     
 %% Get files which need to be compiled first, i.e. those specified in erl_first_files
 %% and parse_transform options.  Also produce specific erl_opts for these first
@@ -758,8 +765,8 @@ include_abs_dirs(ErlOpts, BaseDir) ->
     InclDirs = ["include"|proplists:get_all_values(i, ErlOpts)],
     lists:map(fun(Incl) -> filename:join([BaseDir, Incl]) end, InclDirs).
 
-parse_opts(Opts) -> parse_opts(Opts, #compile_opts{}).
-
-parse_opts([], CompileOpts) -> CompileOpts;
- class="nf">parse_opts class="p">( class="p">[ class="p">{ class="n">recursive class="p">, Recursive}|Rest], CompileOpts) when Recursive == true; Recursive == false ->
-    parse_opts(Rest, CompileOpts#compile_opts{recursive = Recursive}).
+dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) ->
+    case proplists:get_value(recursive,CompileOpts) of
+        undefined -> rebar_dir:recursive(Opts, Dir);
+        Recursive -> Recursive
+    end.

src/rebar_prv_eunit.erl

@@ -310,7 +310,7 @@ maybe_inject_test_dir(State, AppAcc, [], Dir) ->
 
 inject_test_dir(Opts, Dir) ->
     %% append specified test targets to app defined `extra_src_dirs`
-    ExtraSrcDirs = rebar_dir:extra_src_dirs(Opts),
+    ExtraSrcDirs = rebar_opts:get(Opts, extra_src_dirs, []),
     rebar_opts:set(Opts, extra_src_dirs, ExtraSrcDirs ++ [Dir]).
 
 compile({error, _} = Error) -> Error;

test/rebar_compile_SUITE.erl

@@ -46,7 +46,8 @@
          include_file_in_src_test/1,
          always_recompile_when_erl_compiler_options_set/1,
          recompile_when_parse_transform_inline_changes/1,
-         recompile_when_parse_transform_as_opt_changes/1]).
+         recompile_when_parse_transform_as_opt_changes/1,
+         recursive/1,no_recursive/1]).
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
@@ -72,7 +73,8 @@ all() ->
      include_file_relative_to_working_directory, include_file_in_src,
      include_file_relative_to_working_directory_test, include_file_in_src_test,
      recompile_when_parse_transform_as_opt_changes,
-     recompile_when_parse_transform_inline_changes] ++
+     recompile_when_parse_transform_inline_changes,
+     recursive, no_recursive] ++
      case erlang:function_exported(os, unsetenv, 1) of
          true  -> [always_recompile_when_erl_compiler_options_set];
          false -> []
@@ -1520,3 +1522,39 @@ recompile_when_parse_transform_as_opt_changes(Config) ->
 
     ?assert(ModTime =/= NewModTime).
 
+recursive(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    rebar_test_utils:write_src_file(filename:join(AppDir,src),"rec.erl"),
+
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),
+
+    EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]),
+    {ok, Files} = rebar_utils:list_dir(EbinDir),
+    ?assert(lists:member("rec.beam",Files)).
+
+no_recursive(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+    rebar_test_utils:write_src_file(filename:join(AppDir,src),"rec.erl"),
+
+    RebarConfig1 = [{erlc_compiler,[{recursive,false}]}],
+    rebar_test_utils:run_and_check(Config, RebarConfig1, ["compile"],
+                                   {ok, [{app, Name}]}),
+    EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]),
+    {ok, Files1} = rebar_utils:list_dir(EbinDir),
+    ?assert(false==lists:member("rec.beam",Files1)),
+
+    RebarConfig2 = [{src_dirs,[{"src",[{recursive,false}]}]}],
+    rebar_test_utils:run_and_check(Config, RebarConfig2, ["compile"],
+                                   {ok, [{app, Name}]}),
+    {ok, Files2} = rebar_utils:list_dir(EbinDir),
+    ?assert(false==lists:member("rec.beam",Files2)),
+
+    ok.

test/rebar_dir_SUITE.erl

@@ -3,8 +3,10 @@
 -export([all/0, init_per_testcase/2, end_per_testcase/2]).
 
 -export([default_src_dirs/1, default_extra_src_dirs/1, default_all_src_dirs/1]).
--export([src_dirs/1, extra_src_dirs/1, all_src_dirs/1]).
+-export([src_dirs/1, src_dirs_with_opts/1, extra_src_dirs/1, all_src_dirs/1]).
+-export([src_dir_opts/1, recursive/1]).
 -export([profile_src_dirs/1, profile_extra_src_dirs/1, profile_all_src_dirs/1]).
+-export([profile_src_dir_opts/1]).
 -export([retarget_path/1, alt_base_dir_abs/1, alt_base_dir_rel/1]).
 -export([global_cache_dir/1, default_global_cache_dir/1, overwrite_default_global_cache_dir/1]).
 
@@ -14,8 +16,9 @@
 
 
 all() -> [default_src_dirs, default_extra_src_dirs, default_all_src_dirs,
-          src_dirs, extra_src_dirs, all_src_dirs,
+          src_dirs, extra_src_dirs, all_src_dirs, src_dir_opts, recursive,
           profile_src_dirs, profile_extra_src_dirs, profile_all_src_dirs,
+          profile_src_dir_opts,
           retarget_path, alt_base_dir_abs, alt_base_dir_rel, global_cache_dir,
           default_global_cache_dir, overwrite_default_global_cache_dir].
 
@@ -70,6 +73,13 @@ src_dirs(Config) ->
 
     ["bar", "baz", "foo"] = rebar_dir:src_dirs(rebar_state:opts(State)).
 
+src_dirs_with_opts(Config) ->
+    RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar", "baz"]},
+                               {src_dirs, [{"foo",[{recursive,false}]}, "qux"]}]}],
+    {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return),
+
+    ["bar", "baz", "foo", "qux"] = rebar_dir:src_dirs(rebar_state:opts(State)).
+
 extra_src_dirs(Config) ->
     RebarConfig = [{erl_opts, [{extra_src_dirs, ["foo", "bar", "baz"]}]}],
     {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return),
@@ -77,11 +87,41 @@ extra_src_dirs(Config) ->
     ["bar", "baz", "foo"] = rebar_dir:extra_src_dirs(rebar_state:opts(State)).
 
 all_src_dirs(Config) ->
-    RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar"]}, {extra_src_dirs, ["baz", "qux"]}]}],
+    RebarConfig = [{erl_opts, [{src_dirs, ["foo", "bar"]}, {extra_src_dirs, ["baz", "qux"]}, {src_dirs, [{"foo", [{recursive,false}]}]}]}],
     {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], return),
 
     ["bar", "baz", "foo", "qux"] = rebar_dir:all_src_dirs(rebar_state:opts(State)).
 
+src_dir_opts(Config) ->
+    RebarConfig =
+        [{erl_opts, [{src_dirs, [{"foo",[{recursive,true}]}, "bar"]},
+                     {extra_src_dirs, ["baz", {"foo", [{recursive,false}]}]},
+                     {src_dirs, [{"foo", [{recursive,false}]}]}]}],
+    {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig,
+                                                 ["compile"], return),
+    [{recursive,true}] = rebar_dir:src_dir_opts(rebar_state:opts(State), "foo"),
+    [] = rebar_dir:src_dir_opts(rebar_state:opts(State), "bar"),
+    [] = rebar_dir:src_dir_opts(rebar_state:opts(State), "nonexisting").
+
+recursive(Config) ->
+    RebarConfig1 =
+        [{erl_opts, [{src_dirs, ["foo", "bar"]},
+                     {extra_src_dirs, ["baz", {"foo", [{recursive,true}]}]},
+                     {src_dirs, [{"foo", [{recursive,false}]}]}]}],
+    {ok, State1} = rebar_test_utils:run_and_check(Config, RebarConfig1,
+                                                 ["compile"], return),
+    false = rebar_dir:recursive(rebar_state:opts(State1), "foo"),
+    true = rebar_dir:recursive(rebar_state:opts(State1), "bar"),
+
+    RebarConfig2 = [{erlc_compiler,[{recursive,false}]},
+                    {erl_opts,[{src_dirs,["foo",{"bar",[{recursive,true}]}]}]}],
+    {ok, State2} = rebar_test_utils:run_and_check(Config, RebarConfig2,
+                                                 ["compile"], return),
+    false = rebar_dir:recursive(rebar_state:opts(State2), "foo"),
+    true = rebar_dir:recursive(rebar_state:opts(State2), "bar"),
+
+    ok.
+
 profile_src_dirs(Config) ->
     RebarConfig = [
         {erl_opts, [{src_dirs, ["foo", "bar"]}]},
@@ -118,6 +158,26 @@ profile_all_src_dirs(Config) ->
     R = lists:sort(["foo", "bar", "baz", "qux"]),
     R = rebar_dir:all_src_dirs(rebar_state:opts(State)).
 
+profile_src_dir_opts(Config) ->
+    RebarConfig = [
+        {erl_opts, [{src_dirs, ["foo"]},
+                    {extra_src_dirs, [{"bar",[recursive]}]}]},
+        {profiles, [
+            {more, [{erl_opts, [{src_dirs, [{"bar",[{recursive,false}]}]},
+                                {extra_src_dirs, ["qux"]}]}]}
+        ]}
+    ],
+    {ok, State} = rebar_test_utils:run_and_check(Config, RebarConfig,
+                                                 ["as", "more", "compile"],
+                                                 return),
+
+    [{recursive,false}] = rebar_dir:src_dir_opts(rebar_state:opts(State),"bar"),
+
+    {ok, State1} = rebar_test_utils:run_and_check(Config, RebarConfig,
+                                                 ["compile"], return),
+
+    [{recursive,true}] = rebar_dir:src_dir_opts(rebar_state:opts(State1),"bar").
+
 retarget_path(Config) ->
     {ok, State} = rebar_test_utils:run_and_check(Config, [], ["compile"], return),