allow plugins to add project builders and compilers

rebar_compiler is a new behaviour that a plugin can implement to
be called on any ues of the compile provider to compile source
files and keep track of their dependencies.

rebar.config

@@ -33,6 +33,7 @@
 {erl_opts, [{platform_define, "^[0-9]+", namespaced_types},
             {platform_define, "^(19|2)", rand_only},
             {platform_define, "^2", unicode_str},
+            {platform_define, "^2", filelib_find_source},
             {platform_define, "^(R|1|20)", fun_stacktrace},
             warnings_as_errors
            ]}.

src/rebar.app.src

@@ -40,6 +40,9 @@
                      {pkg, rebar_pkg_resource},
                      {hg, rebar_hg_resource}]},
 
+        {app_compilers, [{erl, rebar_erlc_compiler}]},
+        {compilers, [rebar_compiler_xrl, rebar_compiler_yrl, rebar_compiler_mib, rebar_compiler_erl]},
+
         {providers, [rebar_prv_app_discovery,
                      rebar_prv_as,
                      rebar_prv_bare_compile,

src/rebar3.erl

@@ -139,11 +139,14 @@ run_aux(State, RawArgs) ->
                      rebar_state:set(State1, rebar_packages_cdn, CDN)
              end,
 
+    Compilers = application:get_env(rebar, compilers, []),
+    State0 = rebar_state:compilers(State2, Compilers),
+
     %% TODO: this means use of REBAR_PROFILE=profile will replace the repos with
     %% the repos defined in the profile. But it will not work with `as profile`.
     %% Maybe it shouldn't work with either to be consistent?
     Resources = application:get_env(rebar, resources, []),
-    State2_ = rebar_state:create_resources(Resources, State2),
+    State2_ = rebar_state:create_resources(Resources, State0),
 
     %% bootstrap test profile
     State3 = rebar_state:add_to_profile(State2_, test, test_state(State1)),

src/rebar_app_discover.erl

@@ -427,7 +427,7 @@ try_handle_app_src_file(AppInfo, _, _AppDir, [], _Validate) ->
     %% a .app after this step
     case filelib:is_file(filename:join(rebar_app_info:dir(AppInfo), "mix.exs")) of
         true ->
-            {true, AppInfo};
+            {true, rebar_app_info:project_type(AppInfo, mix)};
         false ->
             false
     end;

src/rebar_app_info.erl

@@ -24,7 +24,6 @@
          parent/2,
          original_vsn/1,
          original_vsn/2,
-         ebin_dir/1,
          priv_dir/1,
          applications/1,
          applications/2,
@@ -38,6 +37,8 @@
          dir/2,
          out_dir/1,
          out_dir/2,
+         ebin_dir/1,
+         ebin_dir/2,
          default/1,
          default/2,
          opts/1,
@@ -47,6 +48,8 @@
          set/3,
          source/1,
          source/2,
+         project_type/1,
+         project_type/2,
          is_lock/1,
          is_lock/2,
          is_checkout/1,
@@ -68,7 +71,10 @@
 -include("rebar.hrl").
 -include_lib("providers/include/providers.hrl").
 
--export_type([t/0]).
+-export_type([t/0,
+              project_type/0]).
+
+-type project_type() :: rebar3 | mix | undefined.
 
 -record(app_info_t, {name               :: binary() | undefined,
                      app_file_src       :: file:filename_all() | undefined,
@@ -85,10 +91,12 @@
                      dep_level=0        :: integer(),
                      dir                :: file:name(),
                      out_dir            :: file:name(),
+                     ebin_dir           :: file:name(),
                      source             :: string() | tuple() | checkout | undefined,
                      is_lock=false      :: boolean(),
                      is_checkout=false  :: boolean(),
                      valid              :: boolean() | undefined,
+                     project_type       :: project_type(),
                      is_available=false :: boolean()}).
 
 %%============================================================================
@@ -125,7 +133,8 @@ new(AppName, Vsn, Dir) ->
     {ok, #app_info_t{name=rebar_utils:to_binary(AppName),
                      original_vsn=rebar_utils:to_binary(Vsn),
                      dir=rebar_utils:to_list(Dir),
-                     out_dir=rebar_utils:to_list(Dir)}}.
+                     out_dir=rebar_utils:to_list(Dir),
+                     ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin")}}.
 
 %% @doc build a complete version of the app info with all fields set.
 -spec new(atom() | binary() | string(), binary() | string(), file:name(), list()) ->
@@ -135,6 +144,7 @@ new(AppName, Vsn, Dir, Deps) ->
                      original_vsn=rebar_utils:to_binary(Vsn),
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
+                     ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"),
                      deps=Deps}}.
 
 %% @doc build a complete version of the app info with all fields set.
@@ -146,6 +156,7 @@ new(Parent, AppName, Vsn, Dir, Deps) ->
                      original_vsn=rebar_utils:to_binary(Vsn),
                      dir=rebar_utils:to_list(Dir),
                      out_dir=rebar_utils:to_list(Dir),
+                     ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"),
                      deps=Deps}}.
 
 %% @doc update the opts based on the contents of a config
@@ -447,12 +458,21 @@ out_dir(#app_info_t{out_dir=OutDir}) ->
 %% should go
 -spec out_dir(t(), file:name()) -> t().
 out_dir(AppInfo=#app_info_t{}, OutDir) ->
-    AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir)}.
+    AppInfo#app_info_t{out_dir=rebar_utils:to_list(OutDir),
+                       ebin_dir=filename:join(rebar_utils:to_list(OutDir), "ebin")}.
 
 %% @doc gets the directory where ebin files for the app should go
 -spec ebin_dir(t()) -> file:name().
-ebin_dir(#app_info_t{out_dir=OutDir}) ->
-    rebar_utils:to_list(filename:join(OutDir, "ebin")).
+ebin_dir(#app_info_t{ebin_dir=undefined,
+                     out_dir=OutDir}) ->
+    filename:join(rebar_utils:to_list(OutDir), "ebin");
+ebin_dir(#app_info_t{ebin_dir=EbinDir}) ->
+    EbinDir.
+
+%% @doc sets the directory where beam files should go
+-spec ebin_dir(t(), file:name()) -> t().
+ebin_dir(AppInfo, EbinDir) ->
+    AppInfo#app_info_t{ebin_dir=EbinDir}.
 
 %% @doc gets the directory where private files for the app should go
 -spec priv_dir(t()) -> file:name().
@@ -500,6 +520,17 @@ is_available(#app_info_t{is_available=IsAvailable}) ->
 is_available(AppInfo=#app_info_t{}, IsAvailable) ->
     AppInfo#app_info_t{is_available=IsAvailable}.
 
+%% @doc
+-spec project_type(t()) -> atom().
+project_type(#app_info_t{project_type=ProjectType}) ->
+    ProjectType.
+
+%% @doc
+-spec project_type(t(), atom()) -> t().
+project_type(AppInfo=#app_info_t{}, ProjectType) ->
+    AppInfo#app_info_t{project_type=ProjectType}.
+
+
 %% @doc returns whether the app is valid (built) or not
 -spec valid(t()) -> boolean().
 valid(AppInfo=#app_info_t{valid=undefined}) ->

src/rebar_app_utils.erl

@@ -273,9 +273,8 @@ update_source(AppInfo, Source, _State) ->
 -spec format_error(any()) -> iolist().
 format_error({missing_package, Name, undefined}) ->
     io_lib:format("Package not found in any repo: ~ts", [rebar_utils:to_binary(Name)]);
-format_error({missing_package, Name, Vsn}) ->
-    io_lib:format("Package not found in any repo: ~ts ~ts", [rebar_utils:to_binary(Name),
-                                                             rebar_utils:to_binary(Vsn)]);
+format_error({missing_package, Name, Constraint}) ->
+    io_lib:format("Package not found in any repo: ~ts ~ts", [Name, Constraint]);
 format_error({parse_dep, Dep}) ->
     io_lib:format("Failed parsing dep ~p", [Dep]);
 format_error({invalid_vsn, Dep, InvalidVsn}) ->

src/rebar_base_compiler.erl

@@ -33,6 +33,8 @@
          run/8,
          ok_tuple/2,
          error_tuple/4,
+         report/1,
+         maybe_report/1,
          format_error_source/2]).
 
 -type desc() :: term().
@@ -94,14 +96,14 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt,
       TargetDir :: file:filename(),
       SourceExt :: string(),
       TargetExt :: string().
-run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt,
+run(Config, FirstFiles, SourceDirs, SourceExt, TargetDir, TargetExt,
     Compile3Fn, Opts) ->
     %% Convert simple extension to proper regex
     SourceExtRe = "^(?!\\._).*\\" ++ SourceExt ++ [$$],
 
     Recursive = proplists:get_value(recursive, Opts, true),
     %% Find all possible source files
-    FoundFiles = rebar_utils:find_files(SourceDir, SourceExtRe, Recursive),
+    FoundFiles = rebar_utils:find_files_in_dirs(SourceDirs, SourceExtRe, Recursive),
     %% Remove first files from found files
     RestFiles = [Source || Source <- FoundFiles,
                            not lists:member(Source, FirstFiles)],
@@ -111,7 +113,7 @@ run(Config, FirstFiles, SourceDir, SourceExt, TargetDir, TargetExt,
 
     run(Config, FirstFiles, RestFiles,
         fun(S, C) ->
-                Target = target_file(S, SourceDir, SourceExt,
+                Target = target_file(S, SourceExt,
                                      TargetDir, TargetExt),
                 simple_compile_wrapper(S, Target, Compile3Fn, C, CheckLastMod)
         end).
@@ -160,32 +162,15 @@ simple_compile_wrapper(Source, Target, Compile3Fn, Config, true) ->
 
 %% @private take a basic source set of file fragments and a target location,
 %% create a file path and name for a compile artifact.
--spec target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) -> File when
+-spec target_file(SourceFile, SourceExt, TargetDir, TargetExt) -> File when
       SourceFile :: file:filename(),
-      SourceDir :: file:filename(),
       TargetDir :: file:filename(),
       SourceExt :: string(),
       TargetExt :: string(),
       File :: file:filename().
-target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) ->
-    BaseFile = remove_common_path(SourceFile, SourceDir),
-    filename:join([TargetDir, filename:basename(BaseFile, SourceExt) ++ TargetExt]).
-
-%% @private removes the common prefix between two file paths.
-%% The remainder of the first file path passed will have its ending returned
-%% when either path starts diverging.
--spec remove_common_path(file:filename(), file:filename()) -> file:filename().
-remove_common_path(Fname, Path) ->
-    remove_common_path1(filename:split(Fname), filename:split(Path)).
-
-%% @private given two lists of file fragments, discard the identical
-%% prefixed sections, and return the final bit of the first operand
-%% as a filename.
--spec remove_common_path1([string()], [string()]) -> file:filename().
-remove_common_path1([Part | RestFilename], [Part | RestPath]) ->
-    remove_common_path1(RestFilename, RestPath);
-remove_common_path1(FilenameParts, _) ->
-    filename:join(FilenameParts).
+target_file(SourceFile, SourceExt, TargetDir, TargetExt) ->
+    %% BaseFile = remove_common_path(SourceFile, SourceDir),
+    filename:join([TargetDir, filename:basename(SourceFile, SourceExt) ++ TargetExt]).
 
 %% @private runs the compile function `CompileFn' on every file
 %% passed internally, along with the related project configuration.

src/rebar_compiler.erl

@@ -0,0 +1,304 @@
+-module(rebar_compiler).
+
+-export([compile_all/2,
+         clean/2,
+
+         ok_tuple/2,
+         error_tuple/4,
+         maybe_report/1,
+         format_error_source/2,
+         report/1]).
+
+-include("rebar.hrl").
+
+-type extension() :: string().
+-type out_mappings() :: [{extension(), file:filename()}].
+
+-callback context(rebar_app_info:t()) -> #{src_dirs     => [file:dirname()],
+                                           include_dirs => [file:dirname()],
+                                           src_ext      => extension(),
+                                           out_mappings => out_mappings()}.
+-callback needed_files(digraph:graph(), [file:filename()], rebar_app_info:t()) -> [file:filename()].
+-callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()].
+-callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
+    ok | {ok, [string()]} | {ok, [string()], [string()]}.
+
+-define(DAG_VSN, 2).
+-define(DAG_FILE, "source.dag").
+-type dag_v() :: {digraph:vertex(), term()} | 'false'.
+-type dag_e() :: {digraph:vertex(), digraph:vertex()}.
+-type dag() :: {list(dag_v()), list(dag_e()), list(string())}.
+-record(dag, {vsn = ?DAG_VSN :: pos_integer(),
+              info = {[], [], []} :: dag()}).
+
+-define(RE_PREFIX, "^(?!\\._)").
+
+compile_all(Compilers, AppInfo) ->
+    OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)),
+
+    %% Make sure that outdir is on the path
+    ok = rebar_file_utils:ensure_dir(OutDir),
+    true = code:add_patha(filename:absname(OutDir)),
+
+    %% necessary for erlang:function_exported/3 to work as expected
+    %% called here for clarity as it's required by both opts_changed/2
+    %% and erl_compiler_opts_set/0 in needed_files
+    _ = code:ensure_loaded(compile),
+
+    lists:foreach(fun(CompilerMod) ->
+                          run(CompilerMod, AppInfo),
+                          run_on_extra_src_dirs(CompilerMod, AppInfo, fun run/2)
+                  end, Compilers),
+    ok.
+
+run(CompilerMod, AppInfo) ->
+    #{src_dirs := SrcDirs,
+      include_dirs := InclDirs,
+      src_ext := SrcExt,
+      out_mappings := Mappings} = CompilerMod:context(AppInfo),
+
+    BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
+    EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
+
+    BaseOpts = rebar_app_info:opts(AppInfo),
+    AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs],
+    FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
+
+    OutDir = rebar_app_info:out_dir(AppInfo),
+    AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
+    G = init_dag(CompilerMod, AbsInclDirs, AbsSrcDirs, FoundFiles, OutDir, EbinDir),
+    {{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, AppInfo),
+    true = digraph:delete(G),
+
+    compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
+    compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod).
+
+compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
+    ok;
+compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
+    case CompilerMod:compile(Source, Outs, Config, Opts) of
+        ok ->
+            ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
+        {ok, Warnings} ->
+            report(Warnings),
+            ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
+        skipped ->
+            ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
+        Error ->
+            NewSource = format_error_source(Source, Config),
+            ?ERROR("Compiling ~ts failed", [NewSource]),
+            maybe_report(Error),
+            ?DEBUG("Compilation failed: ~p", [Error]),
+            ?FAIL
+    end,
+    compile_each(Rest, Opts, Config, Outs, CompilerMod).
+
+%% @doc remove compiled artifacts from an AppDir.
+-spec clean([module()], rebar_app_info:t()) -> 'ok'.
+clean(Compilers, AppInfo) ->
+    lists:foreach(fun(CompilerMod) ->
+                          clean_(CompilerMod, AppInfo),
+                          run_on_extra_src_dirs(CompilerMod, AppInfo, fun clean_/2)
+                  end, Compilers).
+
+clean_(CompilerMod, AppInfo) ->
+    #{src_dirs := SrcDirs,
+      src_ext := SrcExt} = CompilerMod:context(AppInfo),
+    BaseDir = rebar_app_info:dir(AppInfo),
+    Opts = rebar_app_info:opts(AppInfo),
+    EbinDir = rebar_app_info:ebin_dir(AppInfo),
+
+    FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts),
+    CompilerMod:clean(FoundFiles, AppInfo),
+    rebar_file_utils:rm_rf(dag_file(CompilerMod, EbinDir)).
+
+
+run_on_extra_src_dirs(CompilerMod, AppInfo, Fun) ->
+    ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo), []),
+    run_on_extra_src_dirs(ExtraDirs, CompilerMod, AppInfo, Fun).
+
+run_on_extra_src_dirs([], _CompilerMod, _AppInfo, _Fun) ->
+    ok;
+run_on_extra_src_dirs([Dir | Rest], CompilerMod, AppInfo, Fun) ->
+    case filelib:is_dir(filename:join(rebar_app_info:dir(AppInfo), Dir)) of
+        true ->
+            EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir),
+            AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir),
+            AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]),
+            AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, ["src"]),
+            Fun(CompilerMod, AppInfo3);
+        _ ->
+            ok
+    end,
+    run_on_extra_src_dirs(Rest, CompilerMod, AppInfo, Fun).
+
+%% These functions are here for the ultimate goal of getting rid of
+%% rebar_base_compiler. This can't be done because of existing plugins.
+
+ok_tuple(Source, Ws) ->
+    rebar_base_compiler:ok_tuple(Source, Ws).
+
+error_tuple(Source, Es, Ws, Opts) ->
+    rebar_base_compiler:error_tuple(Source, Es, Ws, Opts).
+
+maybe_report(Reportable) ->
+    rebar_base_compiler:maybe_report(Reportable).
+
+format_error_source(Path, Opts) ->
+    rebar_base_compiler:format_error_source(Path, Opts).
+
+report(Messages) ->
+    rebar_base_compiler:report(Messages).
+
+%% private functions
+
+find_source_files(BaseDir, SrcExt, SrcDirs, Opts) ->
+    SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$],
+    lists:flatmap(fun(SrcDir) ->
+                      Recursive = rebar_dir:recursive(Opts, SrcDir),
+                      rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive)
+                  end, SrcDirs).
+
+dag_file(CompilerMod, Dir) ->
+    filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, ?DAG_FILE]).
+
+%% private graph functions
+
+%% Get dependency graph of given Erls files and their dependencies (header files,
+%% parse transforms, behaviours etc.) located in their directories or given
+%% InclDirs. Note that last modification times stored in vertices already respect
+%% dependencies induced by given graph G.
+init_dag(Compiler, InclDirs, SrcDirs, Erls, Dir, EbinDir) ->
+    G = digraph:new([acyclic]),
+    try restore_dag(Compiler, G, InclDirs, Dir)
+    catch
+        _:_ ->
+            ?WARN("Failed to restore ~ts file. Discarding it.~n", [dag_file(Compiler, Dir)]),
+            file:delete(dag_file(Compiler, Dir))
+    end,
+    Dirs = lists:usort(InclDirs ++ SrcDirs),
+    %% A source file may have been renamed or deleted. Remove it from the graph
+    %% and remove any beam file for that source if it exists.
+    Modified = maybe_rm_beams_and_edges(G, EbinDir, Erls),
+    Modified1 = lists:foldl(update_dag_fun(G, Compiler, Dirs), Modified, Erls),
+    if Modified1 -> store_dag(Compiler, G, InclDirs, Dir); not Modified1 -> ok end,
+    G.
+
+maybe_rm_beams_and_edges(G, Dir, Files) ->
+    Vertices = digraph:vertices(G),
+    case lists:filter(fun(File) ->
+                              case filename:extension(File) =:= ".erl" of
+                                  true ->
+                                      maybe_rm_beam_and_edge(G, Dir, File);
+                                  false ->
+                                      false
+                              end
+                      end, lists:sort(Vertices) -- lists:sort(Files)) of
+        [] ->
+            false;
+        _ ->
+            true
+    end.
+
+maybe_rm_beam_and_edge(G, OutDir, Source) ->
+    %% This is NOT a double check it is the only check that the source file is actually gone
+    case filelib:is_regular(Source) of
+        true ->
+            %% Actually exists, don't delete
+            false;
+        false ->
+            Target = target_base(OutDir, Source) ++ ".beam",
+            ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]),
+            file:delete(Target),
+            digraph:del_vertex(G, Source),
+            true
+    end.
+
+
+target_base(OutDir, Source) ->
+    filename:join(OutDir, filename:basename(Source, ".erl")).
+
+restore_dag(Compiler, G, InclDirs, Dir) ->
+    case file:read_file(dag_file(Compiler, Dir)) of
+        {ok, Data} ->
+            % Since externally passed InclDirs can influence dependency graph (see
+            % modify_dag), we have to check here that they didn't change.
+            #dag{vsn=?DAG_VSN, info={Vs, Es, InclDirs}} =
+                binary_to_term(Data),
+            lists:foreach(
+              fun({V, LastUpdated}) ->
+                      digraph:add_vertex(G, V, LastUpdated)
+              end, Vs),
+            lists:foreach(
+              fun({_, V1, V2, _}) ->
+                      digraph:add_edge(G, V1, V2)
+              end, Es);
+        {error, _} ->
+            ok
+    end.
+
+store_dag(Compiler, G, InclDirs, Dir) ->
+    Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
+    Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
+    File = dag_file(Compiler, Dir),
+    ok = filelib:ensure_dir(File),
+    Data = term_to_binary(#dag{info={Vs, Es, InclDirs}}, [{compressed, 2}]),
+    file:write_file(File, Data).
+
+update_dag(G, Compiler, Dirs, Source) ->
+    case digraph:vertex(G, Source) of
+        {_, LastUpdated} ->
+            case filelib:last_modified(Source) of
+                0 ->
+                    %% The file doesn't exist anymore,
+                    %% erase it from the graph.
+                    %% All the edges will be erased automatically.
+                    digraph:del_vertex(G, Source),
+                    modified;
+                LastModified when LastUpdated < LastModified ->
+                    modify_dag(G, Compiler, Source, LastModified, filename:dirname(Source), Dirs);
+                _ ->
+                    Modified = lists:foldl(
+                        update_dag_fun(G, Compiler, Dirs),
+                        false, digraph:out_neighbours(G, Source)),
+                    MaxModified = update_max_modified_deps(G, Source),
+                    case Modified orelse MaxModified > LastUpdated of
+                        true -> modified;
+                        false -> unmodified
+                    end
+            end;
+        false ->
+            modify_dag(G, Compiler, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs)
+    end.
+
+modify_dag(G, Compiler, Source, LastModified, SourceDir, Dirs) ->
+    AbsIncls = Compiler:dependencies(Source, SourceDir, Dirs),
+    digraph:add_vertex(G, Source, LastModified),
+    digraph:del_edges(G, digraph:out_edges(G, Source)),
+    lists:foreach(
+      fun(Incl) ->
+              update_dag(G, Compiler, Dirs, Incl),
+              digraph:add_edge(G, Source, Incl)
+      end, AbsIncls),
+    modified.
+
+update_dag_fun(G, Compiler, Dirs) ->
+    fun(Erl, Modified) ->
+        case update_dag(G, Compiler, Dirs, Erl) of
+            modified -> true;
+            unmodified -> Modified
+        end
+    end.
+
+update_max_modified_deps(G, Source) ->
+    MaxModified =
+        lists:foldl(fun(File, Acc) ->
+                            case digraph:vertex(G, File) of
+                                {_, MaxModified} when MaxModified > Acc ->
+                                    MaxModified;
+                                _ ->
+                                    Acc
+                            end
+                    end, 0, [Source | digraph:out_neighbours(G, Source)]),
+    digraph:add_vertex(G, Source, MaxModified),
+    MaxModified.

src/rebar_compiler_erl.erl

@@ -0,0 +1,368 @@
+-module(rebar_compiler_erl).
+
+-behaviour(rebar_compiler).
+
+-export([context/1,
+         needed_files/3,
+         dependencies/3,
+         compile/4,
+         clean/2]).
+
+-include("rebar.hrl").
+
+context(AppInfo) ->
+    EbinDir = rebar_app_info:ebin_dir(AppInfo),
+    Mappings = [{".beam", EbinDir}],
+
+    OutDir = rebar_app_info:dir(AppInfo),
+    SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]),
+    ExistingSrcDirs = lists:filter(fun(D) ->
+                                           ec_file:is_dir(filename:join(OutDir, D))
+                                   end, SrcDirs),
+
+    RebarOpts = rebar_app_info:opts(AppInfo),
+    ErlOpts = rebar_opts:erl_opts(RebarOpts),
+    ErlOptIncludes = proplists:get_all_values(i, ErlOpts),
+    InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes),
+
+    #{src_dirs => ExistingSrcDirs,
+      include_dirs => [filename:join([OutDir, "include"]) | InclDirs],
+      src_ext => ".erl",
+      out_mappings => Mappings}.
+
+
+needed_files(Graph, FoundFiles, AppInfo) ->
+    OutDir = rebar_app_info:out_dir(AppInfo),
+    Dir = rebar_app_info:dir(AppInfo),
+    EbinDir = rebar_app_info:ebin_dir(AppInfo),
+    RebarOpts = rebar_app_info:opts(AppInfo),
+    ErlOpts = rebar_opts:erl_opts(RebarOpts),
+    ?DEBUG("erlopts ~p", [ErlOpts]),
+    ?DEBUG("files to compile ~p", [FoundFiles]),
+
+    %% Make sure that the ebin dir is on the path
+    ok = rebar_file_utils:ensure_dir(EbinDir),
+    true = code:add_patha(filename:absname(EbinDir)),
+
+    {ParseTransforms, Rest} = split_source_files(FoundFiles, ErlOpts),
+    NeededErlFiles = case needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, ParseTransforms) of
+                         [] ->
+                             needed_files(Graph, ErlOpts, RebarOpts, OutDir, EbinDir, Rest);
+                         _  ->
+                             %% at least one parse transform in the opts needs updating, so recompile all
+                             FoundFiles
+                     end,
+    {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles),
+    SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles),
+    DepErlsOrdered = digraph_utils:topsort(SubGraph),
+    OtherErls = lists:reverse(DepErlsOrdered),
+
+    PrivIncludes = [{i, filename:join(OutDir, Src)}
+                    || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])],
+    AdditionalOpts = PrivIncludes ++ [{i, filename:join(OutDir, "include")}, {i, OutDir}, return],
+
+    true = digraph:delete(SubGraph),
+
+    {{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts},
+     {[Erl || Erl <- OtherErls,
+              not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}.
+
+dependencies(Source, SourceDir, Dirs) ->
+    {ok, Fd} = file:open(Source, [read]),
+    Incls = parse_attrs(Fd, [], SourceDir),
+    AbsIncls = expand_file_names(Incls, Dirs),
+    ok = file:close(Fd),
+    AbsIncls.
+
+compile(Source, [{_, OutDir}], Config, ErlOpts) ->
+    case compile:file(Source, [{outdir, OutDir} | ErlOpts]) of
+        {ok, _Mod} ->
+            ok;
+        {ok, _Mod, []} ->
+            ok;
+        {ok, _Mod, Ws} ->
+            FormattedWs = format_error_sources(Ws, Config),
+            rebar_compiler:ok_tuple(Source, FormattedWs);
+        {error, Es, Ws} ->
+            error_tuple(Source, Es, Ws, Config, ErlOpts);
+        error ->
+            error
+    end.
+
+clean(Files, AppInfo) ->
+    EbinDir = rebar_app_info:ebin_dir(AppInfo),
+    [begin
+         Source = filename:basename(File, ".erl"),
+         Target = target_base(EbinDir, Source) ++ ".beam",
+         file:delete(Target)
+     end || File <- Files].
+
+%%
+
+error_tuple(Module, Es, Ws, AllOpts, Opts) ->
+    FormattedEs = format_error_sources(Es, AllOpts),
+    FormattedWs = format_error_sources(Ws, AllOpts),
+    rebar_compiler:error_tuple(Module, FormattedEs, FormattedWs, Opts).
+
+format_error_sources(Es, Opts) ->
+    [{rebar_compiler:format_error_source(Src, Opts), Desc}
+     || {Src, Desc} <- Es].
+
+%% Get files which need to be compiled first, i.e. those specified in erl_first_files
+%% and parse_transform options.  Also produce specific erl_opts for these first
+%% files, so that yet to be compiled parse transformations are excluded from it.
+erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) ->
+    ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []),
+    valid_erl_first_conf(ErlFirstFilesConf),
+    NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)),
+    %% NOTE: order of files here is important!
+    ErlFirstFiles =
+        [filename:join(Dir, File) || File <- ErlFirstFilesConf,
+                                     lists:member(filename:join(Dir, File), NeededErlFiles)],
+    {ParseTransforms, ParseTransformsErls} =
+        lists:unzip(lists:flatmap(
+                      fun(PT) ->
+                              PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs],
+                              [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)]
+                      end, proplists:get_all_values(parse_transform, ErlOpts))),
+    ErlOptsFirst = lists:filter(fun({parse_transform, PT}) ->
+                                        not lists:member(PT, ParseTransforms);
+                                   (_) ->
+                                        true
+                                end, ErlOpts),
+    {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}.
+
+split_source_files(SourceFiles, ErlOpts) ->
+    ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts),
+    lists:partition(fun(Source) ->
+                            lists:member(filename_to_atom(Source), ParseTransforms)
+                    end, SourceFiles).
+
+filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))).
+
+%% Get subset of SourceFiles which need to be recompiled, respecting
+%% dependencies induced by given graph G.
+needed_files(Graph, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) ->
+    lists:filter(fun(Source) ->
+                         TargetBase = target_base(OutDir, Source),
+                         Target = TargetBase ++ ".beam",
+                         PrivIncludes = [{i, filename:join(Dir, Src)}
+                                         || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])],
+                         AllOpts = [{outdir, filename:dirname(Target)}
+                                   ,{i, filename:join(Dir, "include")}
+                                   ,{i, Dir}] ++ PrivIncludes ++ ErlOpts,
+                         digraph:vertex(Graph, Source) > {Source, filelib:last_modified(Target)}
+                              orelse opts_changed(AllOpts, TargetBase)
+                              orelse erl_compiler_opts_set()
+                 end, SourceFiles).
+
+target_base(OutDir, Source) ->
+    filename:join(OutDir, filename:basename(Source, ".erl")).
+
+opts_changed(NewOpts, Target) ->
+    TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of
+        true  -> NewOpts ++ compile:env_compiler_options();
+        false -> NewOpts
+    end,
+    case compile_info(Target) of
+        {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts));
+        _          -> true
+    end.
+
+effects_code_generation(Option) ->
+    case Option of
+        beam -> false;
+        report_warnings -> false;
+        report_errors -> false;
+        return_errors-> false;
+        return_warnings-> false;
+        report -> false;
+        warnings_as_errors -> false;
+        binary -> false;
+        verbose -> false;
+        {cwd,_} -> false;
+        {outdir, _} -> false;
+        _ -> true
+    end.
+
+compile_info(Target) ->
+    case beam_lib:chunks(Target, [compile_info]) of
+        {ok, {_mod, Chunks}} ->
+            CompileInfo = proplists:get_value(compile_info, Chunks, []),
+            {ok, proplists:get_value(options, CompileInfo, [])};
+        {error, beam_lib, Reason} ->
+            ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]),
+            {error, Reason}
+    end.
+
+erl_compiler_opts_set() ->
+    EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of
+        false -> false;
+        _     -> true
+    end,
+    %% return false if changed env opts would have been caught in opts_changed/2
+    EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0).
+
+valid_erl_first_conf(FileList) ->
+    Strs = filter_file_list(FileList),
+    case rebar_utils:is_list_of_strings(Strs) of
+        true -> true;
+        false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive",
+                        [FileList])
+    end.
+
+filter_file_list(FileList) ->
+    Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList),
+    case Atoms of
+        [] ->
+            FileList;
+        _ ->
+          atoms_in_erl_first_files_warning(Atoms),
+          lists:filter( fun(X) -> not(is_atom(X)) end, FileList)
+     end.
+
+atoms_in_erl_first_files_warning(Atoms) ->
+  W = "You have provided atoms as file entries in erl_first_files; "
+      "erl_first_files only expects lists of filenames as strings. "
+      "The following modules (~p) may not work as expected and it is advised "
+      "that you change these entires to string format "
+      "(e.g., \"src/module.erl\") ",
+  ?WARN(W, [Atoms]).
+
+module_to_erl(Mod) ->
+    atom_to_list(Mod) ++ ".erl".
+
+parse_attrs(Fd, Includes, Dir) ->
+    case io:parse_erl_form(Fd, "") of
+        {ok, Form, _Line} ->
+            case erl_syntax:type(Form) of
+                attribute ->
+                    NewIncludes = process_attr(Form, Includes, Dir),
+                    parse_attrs(Fd, NewIncludes, Dir);
+                _ ->
+                    parse_attrs(Fd, Includes, Dir)
+            end;
+        {eof, _} ->
+            Includes;
+        _Err ->
+            parse_attrs(Fd, Includes, Dir)
+    end.
+
+process_attr(Form, Includes, Dir) ->
+    AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)),
+    process_attr(AttrName, Form, Includes, Dir).
+
+process_attr(import, Form, Includes, _Dir) ->
+    case erl_syntax_lib:analyze_import_attribute(Form) of
+        {Mod, _Funs} ->
+            [module_to_erl(Mod)|Includes];
+        Mod ->
+            [module_to_erl(Mod)|Includes]
+    end;
+process_attr(file, Form, Includes, _Dir) ->
+    {File, _} = erl_syntax_lib:analyze_file_attribute(Form),
+    [File|Includes];
+process_attr(include, Form, Includes, _Dir) ->
+    [FileNode] = erl_syntax:attribute_arguments(Form),
+    File = erl_syntax:string_value(FileNode),
+    [File|Includes];
+process_attr(include_lib, Form, Includes, Dir) ->
+    [FileNode] = erl_syntax:attribute_arguments(Form),
+    RawFile = erl_syntax:string_value(FileNode),
+    maybe_expand_include_lib_path(RawFile, Dir) ++ Includes;
+process_attr(behavior, Form, Includes, _Dir) ->
+    process_attr(behaviour, Form, Includes, _Dir);
+process_attr(behaviour, Form, Includes, _Dir) ->
+    [FileNode] = erl_syntax:attribute_arguments(Form),
+    File = module_to_erl(erl_syntax:atom_value(FileNode)),
+    [File|Includes];
+process_attr(compile, Form, Includes, _Dir) ->
+    [Arg] = erl_syntax:attribute_arguments(Form),
+    case erl_syntax:concrete(Arg) of
+        {parse_transform, Mod} ->
+            [module_to_erl(Mod)|Includes];
+        {core_transform, Mod} ->
+            [module_to_erl(Mod)|Includes];
+        L when is_list(L) ->
+            lists:foldl(
+              fun({parse_transform, Mod}, Acc) ->
+                      [module_to_erl(Mod)|Acc];
+                 ({core_transform, Mod}, Acc) ->
+                      [module_to_erl(Mod)|Acc];
+                 (_, Acc) ->
+                      Acc
+              end, Includes, L);
+        _ ->
+            Includes
+    end;
+process_attr(_, _Form, Includes, _Dir) ->
+    Includes.
+
+%% NOTE: If, for example, one of the entries in Files, refers to
+%% gen_server.erl, that entry will be dropped. It is dropped because
+%% such an entry usually refers to the beam file, and we don't pass a
+%% list of OTP src dirs for finding gen_server.erl's full path. Also,
+%% if gen_server.erl was modified, it's not rebar's task to compile a
+%% new version of the beam file. Therefore, it's reasonable to drop
+%% such entries. Also see process_attr(behaviour, Form, Includes).
+-spec expand_file_names([file:filename()],
+                        [file:filename()]) -> [file:filename()].
+expand_file_names(Files, Dirs) ->
+    %% We check if Files exist by itself or within the directories
+    %% listed in Dirs.
+    %% Return the list of files matched.
+    lists:flatmap(
+      fun(Incl) ->
+              case filelib:is_regular(Incl) of
+                  true ->
+                      [Incl];
+                  false ->
+                      lists:flatmap(
+                        fun(Dir) ->
+                                FullPath = filename:join(Dir, Incl),
+                                case filelib:is_regular(FullPath) of
+                                    true ->
+                                        [FullPath];
+                                    false ->
+                                        []
+                                end
+                        end, Dirs)
+              end
+      end, Files).
+
+%% Given a path like "stdlib/include/erl_compile.hrl", return
+%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl".
+%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should
+%% work, but to not crash when an unusual include_lib path is used,
+%% utilize more elaborate logic.
+maybe_expand_include_lib_path(File, Dir) ->
+    File1 = filename:basename(File),
+    case filename:split(filename:dirname(File)) of
+        [_] ->
+            warn_and_find_path(File, Dir);
+        [Lib | SubDir] ->
+            case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of
+                {error, bad_name} ->
+                    warn_and_find_path(File, Dir);
+                AppDir ->
+                    [filename:join(AppDir, File1)]
+            end
+    end.
+
+%% The use of -include_lib was probably incorrect by the user but lets try to make it work.
+%% We search in the outdir and outdir/../include to see if the header exists.
+warn_and_find_path(File, Dir) ->
+    SrcHeader = filename:join(Dir, File),
+    case filelib:is_regular(SrcHeader) of
+        true ->
+            [SrcHeader];
+        false ->
+            IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]),
+            IncludeHeader = filename:join(IncludeDir, File),
+            case filelib:is_regular(IncludeHeader) of
+                true ->
+                    [filename:join(IncludeDir, File)];
+                false ->
+                    []
+            end
+    end.

src/rebar_compiler_mib.erl

@@ -0,0 +1,70 @@
+-module(rebar_compiler_mib).
+
+-behaviour(rebar_compiler).
+
+-export([context/1,
+         needed_files/3,
+         dependencies/3,
+         compile/4,
+         clean/2]).
+
+-include("rebar.hrl").
+-include_lib("stdlib/include/erl_compile.hrl").
+
+context(AppInfo) ->
+    Dir = rebar_app_info:dir(AppInfo),
+    Mappings = [{".bin", filename:join([Dir, "priv", "mibs"])},
+               {".hrl", filename:join(Dir, "include")}],
+
+    #{src_dirs => ["mibs"],
+      include_dirs => [],
+      src_ext => ".mib",
+      out_mappings => Mappings}.
+
+needed_files(_, FoundFiles, AppInfo) ->
+    FirstFiles = [],
+
+    %% Remove first files from found files
+    RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)],
+
+    Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), mib_opts, []),
+    {{FirstFiles, Opts}, {RestFiles, Opts}}.
+
+dependencies(_, _, _) ->
+    [].
+
+compile(Source, OutDirs, _, Opts) ->
+    {_, BinOut} = lists:keyfind(".bin", 1, OutDirs),
+    {_, HrlOut} = lists:keyfind(".hrl", 1, OutDirs),
+
+    ok = rebar_file_utils:ensure_dir(BinOut),
+    ok = rebar_file_utils:ensure_dir(HrlOut),
+    Mib = filename:join(BinOut, filename:basename(Source, ".mib")),
+    HrlFilename = Mib ++ ".hrl",
+
+    AllOpts = [{outdir, BinOut}, {i, [BinOut]}] ++ Opts,
+
+    case snmpc:compile(Source, AllOpts) of
+        {ok, _} ->
+            MibToHrlOpts =
+                case proplists:get_value(verbosity, AllOpts, undefined) of
+                    undefined ->
+                        #options{specific = [],
+                                 cwd = rebar_dir:get_cwd()};
+                    Verbosity ->
+                        #options{specific = [{verbosity, Verbosity}],
+                                 cwd = rebar_dir:get_cwd()}
+                end,
+            ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts),
+            rebar_file_utils:mv(HrlFilename, HrlOut),
+            ok;
+        {error, compilation_failed} ->
+            ?FAIL
+    end.
+
+clean(MibFiles, AppInfo) ->
+    AppDir = rebar_app_info:dir(AppInfo),
+    MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles],
+    rebar_file_utils:delete_each(
+      [filename:join([AppDir, "include", MIB++".hrl"]) || MIB <- MIBs]),
+    ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])).

src/rebar_compiler_xrl.erl

@@ -0,0 +1,50 @@
+-module(rebar_compiler_xrl).
+
+-behaviour(rebar_compiler).
+
+-export([context/1,
+         needed_files/3,
+         dependencies/3,
+         compile/4,
+         clean/2]).
+
+context(AppInfo) ->
+    Dir = rebar_app_info:dir(AppInfo),
+    Mappings = [{".erl", filename:join([Dir, "src"])}],
+    #{src_dirs => ["src"],
+      include_dirs => [],
+      src_ext => ".xrl",
+      out_mappings => Mappings}.
+
+needed_files(_, FoundFiles, AppInfo) ->
+    FirstFiles = [],
+
+    %% Remove first files from found files
+    RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)],
+
+    Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), xrl_opts, []),
+
+    {{FirstFiles, Opts}, {RestFiles, Opts}}.
+
+dependencies(_, _, _) ->
+    [].
+
+compile(Source, [{_, OutDir}], _, Opts) ->
+    BaseName = filename:basename(Source),
+    Target = filename:join([OutDir, BaseName]),
+    AllOpts = [{parserfile, Target} | Opts],
+    AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts,
+                                                           filename:pathtype(I) =:= relative],
+    case leex:file(Source, AllOpts1 ++ [{return, true}]) of
+        {ok, _} ->
+            ok;
+        {ok, _Mod, Ws} ->
+            rebar_compiler:ok_tuple(Source, Ws);
+        {error, Es, Ws} ->
+            rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1)
+    end.
+
+clean(XrlFiles, _AppInfo) ->
+    rebar_file_utils:delete_each(
+      [rebar_utils:to_list(re:replace(F, "\\.xrl$", ".erl", [unicode]))
+       || F <- XrlFiles]).

src/rebar_compiler_yrl.erl

@@ -0,0 +1,49 @@
+-module(rebar_compiler_yrl).
+
+-behaviour(rebar_compiler).
+
+-export([context/1,
+         needed_files/3,
+         dependencies/3,
+         compile/4,
+         clean/2]).
+
+context(AppInfo) ->
+    Dir = rebar_app_info:dir(AppInfo),
+    Mappings = [{".erl", filename:join([Dir, "src"])}],
+    #{src_dirs => ["src"],
+      include_dirs => [],
+      src_ext => ".yrl",
+      out_mappings => Mappings}.
+
+needed_files(_, FoundFiles, AppInfo) ->
+    FirstFiles = [],
+
+    %% Remove first files from found files
+    RestFiles = [Source || Source <- FoundFiles, not lists:member(Source, FirstFiles)],
+
+    Opts = rebar_opts:get(rebar_app_info:opts(AppInfo), yrl_opts, []),
+    {{FirstFiles, Opts}, {RestFiles, Opts}}.
+
+dependencies(_, _, _) ->
+    [].
+
+compile(Source, [{_, OutDir}], _, Opts) ->
+    BaseName = filename:basename(Source),
+    Target = filename:join([OutDir, BaseName]),
+    AllOpts = [{parserfile, Target} | Opts],
+    AllOpts1 = [{includefile, filename:join(OutDir, I)} || {includefile, I} <- AllOpts,
+                                                           filename:pathtype(I) =:= relative],
+    case yeec:file(Source, AllOpts1 ++ [{return, true}]) of
+        {ok, _} ->
+            ok;
+        {ok, _Mod, Ws} ->
+            rebar_compiler:ok_tuple(Source, Ws);
+        {error, Es, Ws} ->
+            rebar_compiler:error_tuple(Source, Es, Ws, AllOpts1)
+    end.
+
+clean(YrlFiles, _AppInfo) ->
+    rebar_file_utils:delete_each(
+      [rebar_utils:to_list(re:replace(F, "\\.yrl$", ".erl", [unicode]))
+       || F <- YrlFiles]).

src/rebar_dir.erl

@@ -301,9 +301,8 @@ all_src_dirs(Opts, SrcDefault, ExtraDefault) ->
 src_dir_opts(Opts, Dir) ->
     RawSrcDirs = raw_src_dirs(src_dirs, Opts, []),
     RawExtraSrcDirs = raw_src_dirs(extra_src_dirs, Opts, []),
-    AllOpts = [Opt || {D,Opt} <- RawSrcDirs++RawExtraSrcDirs,
-                      D==Dir],
-    lists:ukeysort(1,proplists:unfold(lists:append(AllOpts))).
+    AllOpts = [Opt || {D, Opt} <- RawSrcDirs++RawExtraSrcDirs, D==Dir],
+    lists:ukeysort(1, proplists:unfold(lists:append(AllOpts))).
 
 %%% @doc
 %%% Return the value of the 'recursive' option for the given directory.

src/rebar_erlc_compiler.erl

@@ -1,827 +0,0 @@
-%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
-%% ex: ts=4 sw=4 et
-%% -------------------------------------------------------------------
-%%
-%% rebar: Erlang Build Tools
-%%
-%% Copyright (c) 2009, 2010 Dave Smith (dizzyd@dizzyd.com)
-%%
-%% Permission is hereby granted, free of charge, to any person obtaining a copy
-%% of this software and associated documentation files (the "Software"), to deal
-%% in the Software without restriction, including without limitation the rights
-%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-%% copies of the Software, and to permit persons to whom the Software is
-%% furnished to do so, subject to the following conditions:
-%%
-%% The above copyright notice and this permission notice shall be included in
-%% all copies or substantial portions of the Software.
-%%
-%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-%% THE SOFTWARE.
-%% -------------------------------------------------------------------
--module(rebar_erlc_compiler).
-
--export([compile/1, compile/2, compile/3,
-         compile_dir/3, compile_dir/4,
-         compile_dirs/5,
-         clean/1]).
-
--include("rebar.hrl").
--include_lib("stdlib/include/erl_compile.hrl").
-
--define(ERLCINFO_VSN, 2).
--define(ERLCINFO_FILE, "erlcinfo").
--type erlc_info_v() :: {digraph:vertex(), term()} | 'false'.
--type erlc_info_e() :: {digraph:vertex(), digraph:vertex()}.
--type erlc_info() :: {list(erlc_info_v()), list(erlc_info_e()), list(string())}.
--record(erlcinfo, {
-    vsn = ?ERLCINFO_VSN :: pos_integer(),
-    info = {[], [], []} :: erlc_info()
-}).
-
--type compile_opts() :: [compile_opt()].
--type compile_opt() :: {recursive, boolean()}.
-
--define(DEFAULT_OUTDIR, "ebin").
--define(RE_PREFIX, "^(?!\\._)").
-
-%% ===================================================================
-%% Public API
-%% ===================================================================
-
-%% Supported configuration variables:
-%%
-%% * erl_opts - Erlang list of options passed to compile:file/2
-%%              It is also possible to specify platform specific
-%%              options by specifying a pair or a triplet where the
-%%              first string is a regex that is checked against the
-%%              string
-%%
-%%                OtpRelease ++ "-" ++ SysArch ++ "-" ++ Words.
-%%
-%%              where
-%%
-%%                OtpRelease = erlang:system_info(otp_release).
-%%                SysArch = erlang:system_info(system_architecture).
-%%                Words = integer_to_list(8 *
-%%                            erlang:system_info({wordsize, external})).
-%%
-%%              E.g. to define HAVE_SENDFILE only on systems with
-%%              sendfile(), to define BACKLOG on Linux/FreeBSD as 128,
-%%              and to define 'old_inets' for R13 OTP release do:
-%%
-%%              {erl_opts, [{platform_define,
-%%                           "(linux|solaris|freebsd|darwin)",
-%%                           'HAVE_SENDFILE'},
-%%                          {platform_define, "(linux|freebsd)",
-%%                           'BACKLOG', 128},
-%%                          {platform_define, "R13",
-%%                           'old_inets'}]}.
-%%
-
-%% @equiv compile(AppInfo, [])
--spec compile(rebar_app_info:t()) -> ok.
-compile(AppInfo) when element(1, AppInfo) == app_info_t ->
-    compile(AppInfo, []).
-
-%% @doc compile an individual application.
--spec compile(rebar_app_info:t(), compile_opts()) -> ok.
-compile(AppInfo, CompileOpts) when element(1, AppInfo) == app_info_t ->
-    Dir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)),
-    RebarOpts = rebar_app_info:opts(AppInfo),
-
-    SrcOpts = [check_last_mod,
-               {recursive, dir_recursive(RebarOpts, "src", CompileOpts)}],
-    MibsOpts = [check_last_mod,
-                {recursive, dir_recursive(RebarOpts, "mibs", CompileOpts)}],
-
-    rebar_base_compiler:run(RebarOpts,
-                            check_files([filename:join(Dir, File)
-                                         || File <- rebar_opts:get(RebarOpts, xrl_first_files, [])]),
-                            filename:join(Dir, "src"), ".xrl", filename:join(Dir, "src"), ".erl",
-                            fun compile_xrl/3, SrcOpts),
-    rebar_base_compiler:run(RebarOpts,
-                            check_files([filename:join(Dir, File)
-                                         || File <- rebar_opts:get(RebarOpts, yrl_first_files, [])]),
-                            filename:join(Dir, "src"), ".yrl", filename:join(Dir, "src"), ".erl",
-                            fun compile_yrl/3, SrcOpts),
-    rebar_base_compiler:run(RebarOpts,
-                            check_files([filename:join(Dir, File)
-                                         || File <- rebar_opts:get(RebarOpts, mib_first_files, [])]),
-                            filename:join(Dir, "mibs"), ".mib", filename:join([Dir, "priv", "mibs"]), ".bin",
-                            compile_mib(AppInfo), MibsOpts),
-
-    SrcDirs = lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end,
-                        rebar_dir:src_dirs(RebarOpts, ["src"])),
-    OutDir = filename:join(Dir, outdir(RebarOpts)),
-    compile_dirs(RebarOpts, Dir, SrcDirs, OutDir, CompileOpts),
-
-    ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts),
-    F = fun(D) ->
-        case ec_file:is_dir(filename:join([Dir, D])) of
-            true  -> compile_dirs(RebarOpts, Dir, [D], D, CompileOpts);
-            false -> ok
-        end
-    end,
-    lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(Dir, SrcDir) end, ExtraDirs)).
-
-%% @hidden
-%% these are kept for backwards compatibility but they're bad functions with
-%% bad interfaces you probably shouldn't use
-%% State/RebarOpts have to have src_dirs set and BaseDir must be the parent
-%% directory of those src_dirs
-
--spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok.
-compile(State, BaseDir, OutDir) when element(1, State) == state_t ->
-    compile(rebar_state:opts(State), BaseDir, OutDir, [{recursive, false}]);
-compile(RebarOpts, BaseDir, OutDir) ->
-    compile(RebarOpts, BaseDir, OutDir, [{recursive, false}]).
-
-%% @hidden
-
--spec compile(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok.
-compile(State, BaseDir, OutDir, CompileOpts) when element(1, State) == state_t ->
-    compile(rebar_state:opts(State), BaseDir, OutDir, CompileOpts);
-compile(RebarOpts, BaseDir, OutDir, CompileOpts) ->
-    SrcDirs = lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end,
-                        rebar_dir:src_dirs(RebarOpts, ["src"])),
-    compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts),
-
-    ExtraDirs = rebar_dir:extra_src_dirs(RebarOpts),
-    F = fun(D) ->
-        case ec_file:is_dir(filename:join([BaseDir, D])) of
-            true  -> compile_dirs(RebarOpts, BaseDir, [D], D, CompileOpts);
-            false -> ok
-        end
-    end,
-    lists:foreach(F, lists:map(fun(SrcDir) -> filename:join(BaseDir, SrcDir) end, ExtraDirs)).
-
-%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, [{recursive, false}])
--spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name()) -> ok.
-compile_dir(State, BaseDir, Dir) when element(1, State) == state_t ->
-    compile_dir(rebar_state:opts(State), BaseDir, Dir, [{recursive, false}]);
-compile_dir(RebarOpts, BaseDir, Dir) ->
-    compile_dir(RebarOpts, BaseDir, Dir, [{recursive, false}]).
-
-%% @equiv compile_dirs(Context, BaseDir, [Dir], Dir, Opts)
--spec compile_dir(rebar_dict() | rebar_state:t(), file:name(), file:name(), compile_opts()) -> ok.
-compile_dir(State, BaseDir, Dir, Opts) when element(1, State) == state_t ->
-    compile_dirs(rebar_state:opts(State), BaseDir, [Dir], Dir, Opts);
-compile_dir(RebarOpts, BaseDir, Dir, Opts) ->
-    compile_dirs(RebarOpts, BaseDir, [Dir], Dir, Opts).
-
-%% @doc compile a list of directories with the given opts.
--spec compile_dirs(rebar_dict() | rebar_state:t(),
-                   file:filename(),
-                   [file:filename()],
-                   file:filename(),
-                   compile_opts()) -> ok.
-compile_dirs(State, BaseDir, Dirs, OutDir, CompileOpts) when element(1, State) == state_t ->
-    compile_dirs(rebar_state:opts(State), BaseDir, Dirs, OutDir, CompileOpts);
-compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) ->
-    ErlOpts = rebar_opts:erl_opts(RebarOpts),
-    ?DEBUG("erlopts ~p", [ErlOpts]),
-    AllErlFiles = gather_src(RebarOpts, BaseDir, SrcDirs, CompileOpts),
-    ?DEBUG("files to compile ~p", [AllErlFiles]),
-
-    %% Make sure that outdir is on the path
-    ok = filelib:ensure_dir(filename:join(OutDir, "dummy.beam")),
-    true = code:add_patha(filename:absname(OutDir)),
-
-    G = init_erlcinfo(include_abs_dirs(ErlOpts, BaseDir), AllErlFiles, BaseDir, OutDir),
-
-    {ParseTransforms, Rest} = split_source_files(AllErlFiles, ErlOpts),
-    NeededErlFiles = case needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, ParseTransforms) of
-        [] -> needed_files(G, ErlOpts, RebarOpts, BaseDir, OutDir, Rest);
-        %% at least one parse transform in the opts needs updating, so recompile all
-        _  -> AllErlFiles
-    end,
-
-    {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles),
-    {DepErls, OtherErls} = lists:partition(
-                             fun(Source) -> digraph:in_degree(G, Source) > 0 end,
-                             [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]),
-    SubGraph = digraph_utils:subgraph(G, DepErls),
-    DepErlsOrdered = digraph_utils:topsort(SubGraph),
-    FirstErls = ErlFirstFiles ++ lists:reverse(DepErlsOrdered),
-    try
-        rebar_base_compiler:run(
-            RebarOpts, FirstErls, OtherErls,
-            fun(S, C) ->
-                    ErlOpts1 = case lists:member(S, ErlFirstFiles) of
-                                   true -> ErlOptsFirst;
-                                   false -> ErlOpts
-                               end,
-                    internal_erl_compile(C, BaseDir, S, OutDir, ErlOpts1, RebarOpts)
-            end)
-    after
-        true = digraph:delete(SubGraph),
-        true = digraph:delete(G)
-    end,
-    ok.
-
-%% @doc remove compiled artifacts from an AppDir.
--spec clean(rebar_app_info:t()) -> 'ok'.
-clean(AppInfo) ->
-    AppDir = rebar_app_info:out_dir(AppInfo),
-
-    MibFiles = rebar_utils:find_files(filename:join([AppDir, "mibs"]), ?RE_PREFIX".*\\.mib\$"),
-    MIBs = [filename:rootname(filename:basename(MIB)) || MIB <- MibFiles],
-    rebar_file_utils:delete_each(
-      [filename:join([AppDir, "include",MIB++".hrl"]) || MIB <- MIBs]),
-    ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])),
-
-    YrlFiles = rebar_utils:find_files(filename:join([AppDir, "src"]), ?RE_PREFIX".*\\.[x|y]rl\$"),
-    rebar_file_utils:delete_each(
-      [rebar_utils:to_list(re:replace(F, "\\.[x|y]rl$", ".erl", [unicode]))
-       || F <- YrlFiles]),
-
-    BinDirs = ["ebin"|rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo))],
-    ok = clean_dirs(AppDir, BinDirs),
-
-    %% Delete the build graph, if any
-    rebar_file_utils:rm_rf(erlcinfo_file(AppDir)).
-
-clean_dirs(_AppDir, []) -> ok;
-clean_dirs(AppDir, [Dir|Rest]) ->
-    ok = rebar_file_utils:rm_rf(filename:join([AppDir, Dir, "*.beam"])),
-    %% Erlang compilation is recursive, so it's possible that we have a nested
-    %% directory structure in ebin with .beam files within. As such, we want
-    %% to scan whatever is left in the app's out_dir directory for sub-dirs which
-    %% satisfy our criteria.
-    BeamFiles = rebar_utils:find_files(filename:join([AppDir, Dir]), ?RE_PREFIX".*\\.beam\$"),
-    rebar_file_utils:delete_each(BeamFiles),
-    lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, dirs(filename:join([AppDir, Dir]))),
-    clean_dirs(AppDir, Rest).
-
-
-%% ===================================================================
-%% Internal functions
-%% ===================================================================
-
-gather_src(Opts, BaseDir, Dirs, CompileOpts) ->
-    gather_src(Opts, filename:split(BaseDir), Dirs, [], CompileOpts).
-
-gather_src(_Opts, _BaseDirParts, [], Srcs, _CompileOpts) -> Srcs;
-gather_src(Opts, BaseDirParts, [Dir|Rest], Srcs, CompileOpts) ->
-    DirParts = filename:split(Dir),
-    RelDir = case lists:prefix(BaseDirParts,DirParts) of
-                 true ->
-                     case lists:nthtail(length(BaseDirParts),DirParts) of
-                         [] -> ".";
-                         RestParts -> filename:join(RestParts)
-                     end;
-                 false -> Dir
-             end,
-    DirRecursive = dir_recursive(Opts, RelDir, CompileOpts),
-    gather_src(Opts, BaseDirParts, Rest, Srcs ++ rebar_utils:find_files(Dir, ?RE_PREFIX".*\\.erl\$", DirRecursive), CompileOpts).
-
-%% Get files which need to be compiled first, i.e. those specified in erl_first_files
-%% and parse_transform options.  Also produce specific erl_opts for these first
-%% files, so that yet to be compiled parse transformations are excluded from it.
-erl_first_files(Opts, ErlOpts, Dir, NeededErlFiles) ->
-    ErlFirstFilesConf = rebar_opts:get(Opts, erl_first_files, []),
-    valid_erl_first_conf(ErlFirstFilesConf),
-    NeededSrcDirs = lists:usort(lists:map(fun filename:dirname/1, NeededErlFiles)),
-    %% NOTE: order of files here is important!
-    ErlFirstFiles =
-        [filename:join(Dir, File) || File <- ErlFirstFilesConf,
-                                     lists:member(filename:join(Dir, File), NeededErlFiles)],
-    {ParseTransforms, ParseTransformsErls} =
-        lists:unzip(lists:flatmap(
-                      fun(PT) ->
-                              PTerls = [filename:join(D, module_to_erl(PT)) || D <- NeededSrcDirs],
-                              [{PT, PTerl} || PTerl <- PTerls, lists:member(PTerl, NeededErlFiles)]
-                      end, proplists:get_all_values(parse_transform, ErlOpts))),
-    ErlOptsFirst = lists:filter(fun({parse_transform, PT}) ->
-                                        not lists:member(PT, ParseTransforms);
-                                   (_) ->
-                                        true
-                               end, ErlOpts),
-    {ErlFirstFiles ++ ParseTransformsErls, ErlOptsFirst}.
-
-split_source_files(SourceFiles, ErlOpts) ->
-    ParseTransforms = proplists:get_all_values(parse_transform, ErlOpts),
-    lists:partition(fun(Source) ->
-                            lists:member(filename_to_atom(Source), ParseTransforms)
-                    end, SourceFiles).
-
-filename_to_atom(F) -> list_to_atom(filename:rootname(filename:basename(F))).
-
-%% Get subset of SourceFiles which need to be recompiled, respecting
-%% dependencies induced by given graph G.
-needed_files(G, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) ->
-    lists:filter(fun(Source) ->
-                         TargetBase = target_base(OutDir, Source),
-                         Target = TargetBase ++ ".beam",
-                         PrivIncludes = [{i, filename:join(Dir, Src)}
-                                         || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])],
-                         AllOpts = [{outdir, filename:dirname(Target)}
-                                   ,{i, filename:join(Dir, "include")}
-                                   ,{i, Dir}] ++ PrivIncludes ++ ErlOpts,
-                         %% necessary for erlang:function_exported/3 to work as expected
-                         %% called here for clarity as it's required by both opts_changed/2
-                         %% and erl_compiler_opts_set/0
-                         _ = code:ensure_loaded(compile),
-                         digraph:vertex(G, Source) > {Source, filelib:last_modified(Target)}
-                              orelse opts_changed(AllOpts, TargetBase)
-                              orelse erl_compiler_opts_set()
-                 end, SourceFiles).
-
-maybe_rm_beam_and_edge(G, OutDir, Source) ->
-    %% This is NOT a double check it is the only check that the source file is actually gone
-    case filelib:is_regular(Source) of
-        true ->
-            %% Actually exists, don't delete
-            false;
-        false ->
-            Target = target_base(OutDir, Source) ++ ".beam",
-            ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]),
-            file:delete(Target),
-            digraph:del_vertex(G, Source),
-            true
-    end.
-
-opts_changed(NewOpts, Target) ->
-    TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of
-        true  -> NewOpts ++ compile:env_compiler_options();
-        false -> NewOpts
-    end,
-    case compile_info(Target) of
-        {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts));
-        _          -> true
-    end.
-
-effects_code_generation(Option) ->
-    case Option of
-        beam -> false;
-        report_warnings -> false;
-        report_errors -> false;
-        return_errors-> false;
-        return_warnings-> false;
-        report -> false;
-        warnings_as_errors -> false;
-        binary -> false;
-        verbose -> false;
-        {cwd,_} -> false;
-        {outdir, _} -> false;
-        _ -> true
-    end.
-
-compile_info(Target) ->
-    case beam_lib:chunks(Target, [compile_info]) of
-        {ok, {_mod, Chunks}} ->
-            CompileInfo = proplists:get_value(compile_info, Chunks, []),
-            {ok, proplists:get_value(options, CompileInfo, [])};
-        {error, beam_lib, Reason} ->
-            ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]),
-            {error, Reason}
-    end.
-
-erl_compiler_opts_set() ->
-    EnvSet = case os:getenv("ERL_COMPILER_OPTIONS") of
-        false -> false;
-        _     -> true
-    end,
-    %% return false if changed env opts would have been caught in opts_changed/2
-    EnvSet andalso not erlang:function_exported(compile, env_compiler_options, 0).
-
-erlcinfo_file(Dir) ->
-    filename:join(rebar_dir:local_cache_dir(Dir), ?ERLCINFO_FILE).
-
-%% Get dependency graph of given Erls files and their dependencies (header files,
-%% parse transforms, behaviours etc.) located in their directories or given
-%% InclDirs. Note that last modification times stored in vertices already respect
-%% dependencies induced by given graph G.
-init_erlcinfo(InclDirs, Erls, Dir, OutDir) ->
-    G = digraph:new([acyclic]),
-    try restore_erlcinfo(G, InclDirs, Dir)
-    catch
-        _:_ ->
-            ?WARN("Failed to restore ~ts file. Discarding it.~n", [erlcinfo_file(Dir)]),
-            file:delete(erlcinfo_file(Dir))
-    end,
-    Dirs = source_and_include_dirs(InclDirs, Erls),
-    %% A source file may have been renamed or deleted. Remove it from the graph
-    %% and remove any beam file for that source if it exists.
-    Modified = maybe_rm_beams_and_edges(G, OutDir, Erls),
-    Modified1 = lists:foldl(update_erlcinfo_fun(G, Dirs), Modified, Erls),
-    if Modified1 -> store_erlcinfo(G, InclDirs, Dir); not Modified1 -> ok end,
-    G.
-
-maybe_rm_beams_and_edges(G, Dir, Files) ->
-    Vertices = digraph:vertices(G),
-    case lists:filter(fun(File) ->
-                              case filename:extension(File) =:= ".erl" of
-                                  true ->
-                                      maybe_rm_beam_and_edge(G, Dir, File);
-                                  false ->
-                                      false
-                              end
-                      end, lists:sort(Vertices) -- lists:sort(Files)) of
-        [] ->
-            false;
-        _ ->
-            true
-    end.
-
-source_and_include_dirs(InclDirs, Erls) ->
-    SourceDirs = lists:map(fun filename:dirname/1, Erls),
-    lists:usort(InclDirs ++ SourceDirs).
-
-update_erlcinfo(G, Dirs, Source) ->
-    case digraph:vertex(G, Source) of
-        {_, LastUpdated} ->
-            case filelib:last_modified(Source) of
-                0 ->
-                    %% The file doesn't exist anymore,
-                    %% erase it from the graph.
-                    %% All the edges will be erased automatically.
-                    digraph:del_vertex(G, Source),
-                    modified;
-                LastModified when LastUpdated < LastModified ->
-                    modify_erlcinfo(G, Source, LastModified, filename:dirname(Source), Dirs);
-                _ ->
-                    Modified = lists:foldl(
-                        update_erlcinfo_fun(G, Dirs),
-                        false, digraph:out_neighbours(G, Source)),
-                    MaxModified = update_max_modified_deps(G, Source),
-                    case Modified orelse MaxModified > LastUpdated of
-                        true -> modified;
-                        false -> unmodified
-                    end
-            end;
-        false ->
-            modify_erlcinfo(G, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs)
-    end.
-
-update_erlcinfo_fun(G, Dirs) ->
-    fun(Erl, Modified) ->
-        case update_erlcinfo(G, Dirs, Erl) of
-            modified -> true;
-            unmodified -> Modified
-        end
-    end.
-
-update_max_modified_deps(G, Source) ->
-    MaxModified = lists:max(lists:map(
-        fun(File) -> {_, MaxModified} = digraph:vertex(G, File), MaxModified end,
-        [Source|digraph:out_neighbours(G, Source)])),
-    digraph:add_vertex(G, Source, MaxModified),
-    MaxModified.
-
-modify_erlcinfo(G, Source, LastModified, Dir, Dirs) ->
-    {ok, Fd} = file:open(Source, [read]),
-    Incls = parse_attrs(Fd, [], Dir),
-    AbsIncls = expand_file_names(Incls, Dirs),
-    ok = file:close(Fd),
-    digraph:add_vertex(G, Source, LastModified),
-    digraph:del_edges(G, digraph:out_edges(G, Source)),
-    lists:foreach(
-      fun(Incl) ->
-              update_erlcinfo(G, Dirs, Incl),
-              digraph:add_edge(G, Source, Incl)
-      end, AbsIncls),
-    modified.
-
-restore_erlcinfo(G, InclDirs, Dir) ->
-    case file:read_file(erlcinfo_file(Dir)) of
-        {ok, Data} ->
-            % Since externally passed InclDirs can influence erlcinfo graph (see
-            % modify_erlcinfo), we have to check here that they didn't change.
-            #erlcinfo{vsn=?ERLCINFO_VSN, info={Vs, Es, InclDirs}} =
-                binary_to_term(Data),
-            lists:foreach(
-              fun({V, LastUpdated}) ->
-                      digraph:add_vertex(G, V, LastUpdated)
-              end, Vs),
-            lists:foreach(
-              fun({_, V1, V2, _}) ->
-                      digraph:add_edge(G, V1, V2)
-              end, Es);
-        {error, _} ->
-            ok
-    end.
-
-store_erlcinfo(G, InclDirs, Dir) ->
-    Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
-    Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
-    File = erlcinfo_file(Dir),
-    ok = filelib:ensure_dir(File),
-    Data = term_to_binary(#erlcinfo{info={Vs, Es, InclDirs}}, [{compressed, 2}]),
-    file:write_file(File, Data).
-
-%% NOTE: If, for example, one of the entries in Files, refers to
-%% gen_server.erl, that entry will be dropped. It is dropped because
-%% such an entry usually refers to the beam file, and we don't pass a
-%% list of OTP src dirs for finding gen_server.erl's full path. Also,
-%% if gen_server.erl was modified, it's not rebar's task to compile a
-%% new version of the beam file. Therefore, it's reasonable to drop
-%% such entries. Also see process_attr(behaviour, Form, Includes).
--spec expand_file_names([file:filename()],
-                        [file:filename()]) -> [file:filename()].
-expand_file_names(Files, Dirs) ->
-    %% We check if Files exist by itself or within the directories
-    %% listed in Dirs.
-    %% Return the list of files matched.
-    lists:flatmap(
-      fun(Incl) ->
-              case filelib:is_regular(Incl) of
-                  true ->
-                      [Incl];
-                  false ->
-                      lists:flatmap(
-                        fun(Dir) ->
-                                FullPath = filename:join(Dir, Incl),
-                                case filelib:is_regular(FullPath) of
-                                    true ->
-                                        [FullPath];
-                                    false ->
-                                        []
-                                end
-                        end, Dirs)
-              end
-      end, Files).
-
--spec internal_erl_compile(rebar_dict(), file:filename(), file:filename(),
-                           file:filename(), list(), rebar_dict()) ->
-      ok | {ok, any()} | {error, any(), any()}.
-internal_erl_compile(Opts, Dir, Module, OutDir, ErlOpts, RebarOpts) ->
-    Target = target_base(OutDir, Module) ++ ".beam",
-    ok = filelib:ensure_dir(Target),
-    PrivIncludes = [{i, filename:join(Dir, Src)}
-                    || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])],
-    AllOpts = [{outdir, filename:dirname(Target)}] ++ ErlOpts ++ PrivIncludes ++
-              [{i, filename:join(Dir, "include")}, {i, Dir}, return],
-    case compile:file(Module, AllOpts) of
-        {ok, _Mod} ->
-            ok;
-        {ok, _Mod, Ws} ->
-            FormattedWs = format_error_sources(Ws, Opts),
-            rebar_base_compiler:ok_tuple(Module, FormattedWs);
-        {error, Es, Ws} ->
-            error_tuple(Module, Es, Ws, AllOpts, Opts)
-    end.
-
-error_tuple(Module, Es, Ws, AllOpts, Opts) ->
-    FormattedEs = format_error_sources(Es, Opts),
-    FormattedWs = format_error_sources(Ws, Opts),
-    rebar_base_compiler:error_tuple(Module, FormattedEs, FormattedWs, AllOpts).
-
-format_error_sources(Es, Opts) ->
-    [{rebar_base_compiler:format_error_source(Src, Opts), Desc}
-     || {Src, Desc} <- Es].
-
-target_base(OutDir, Source) ->
-    filename:join(OutDir, filename:basename(Source, ".erl")).
-
--spec compile_mib(rebar_app_info:t()) ->
-    fun((file:filename(), file:filename(), rebar_dict()) -> 'ok').
-compile_mib(AppInfo) ->
-    fun(Source, Target, Opts) ->
-        Dir = filename:dirname(Target),
-        Mib = filename:rootname(Target),
-        HrlFilename = Mib ++ ".hrl",
-
-        AppInclude = filename:join([rebar_app_info:dir(AppInfo), "include"]),
-
-        ok = filelib:ensure_dir(Target),
-        ok = filelib:ensure_dir(filename:join([AppInclude, "dummy.hrl"])),
-
-        AllOpts = [{outdir, Dir}
-                  ,{i, [Dir]}] ++
-            rebar_opts:get(Opts, mib_opts, []),
-
-        case snmpc:compile(Source, AllOpts) of
-            {ok, _} ->
-                MibToHrlOpts =
-                    case proplists:get_value(verbosity, AllOpts, undefined) of
-                        undefined ->
-                            #options{specific = [],
-                                     cwd = rebar_dir:get_cwd()};
-                        Verbosity ->
-                            #options{specific = [{verbosity, Verbosity}],
-                                     cwd = rebar_dir:get_cwd()}
-                    end,
-                ok = snmpc:mib_to_hrl(Mib, Mib, MibToHrlOpts),
-                rebar_file_utils:mv(HrlFilename, AppInclude),
-                ok;
-            {error, compilation_failed} ->
-                ?FAIL
-        end
-    end.
-
--spec compile_xrl(file:filename(), file:filename(),
-                  rebar_dict()) -> 'ok'.
-compile_xrl(Source, Target, Opts) ->
-    AllOpts = [{scannerfile, Target} | rebar_opts:get(Opts, xrl_opts, [])],
-    compile_xrl_yrl(Opts, Source, Target, AllOpts, leex).
-
--spec compile_yrl(file:filename(), file:filename(),
-                  rebar_dict()) -> 'ok'.
-compile_yrl(Source, Target, Opts) ->
-    AllOpts = [{parserfile, Target} | rebar_opts:get(Opts, yrl_opts, [])],
-    compile_xrl_yrl(Opts, Source, Target, AllOpts, yecc).
-
--spec compile_xrl_yrl(rebar_dict(), file:filename(),
-                      file:filename(), list(), module()) -> 'ok'.
-compile_xrl_yrl(_Opts, Source, Target, AllOpts, Mod) ->
-    %% FIX ME: should be the outdir or something
-    Dir = filename:dirname(filename:dirname(Target)),
-    AllOpts1 = [{includefile, filename:join(Dir, I)} || {includefile, I} <- AllOpts,
-                                                        filename:pathtype(I) =:= relative],
-    case needs_compile(Source, Target) of
-        true ->
-            case Mod:file(Source, AllOpts1 ++ [{return, true}]) of
-                {ok, _} ->
-                    ok;
-                {ok, _Mod, Ws} ->
-                    rebar_base_compiler:ok_tuple(Source, Ws);
-                {error, Es, Ws} ->
-                    rebar_base_compiler:error_tuple(Source,
-                                                    Es, Ws, AllOpts1)
-            end;
-        false ->
-            skipped
-    end.
-
-needs_compile(Source, Target) ->
-    filelib:last_modified(Source) > filelib:last_modified(Target).
-
-
-
--spec dirs(file:filename()) -> [file:filename()].
-dirs(Dir) ->
-    [F || F <- filelib:wildcard(filename:join([Dir, "*"])), filelib:is_dir(F)].
-
--spec delete_dir(file:filename(), [string()]) -> 'ok' | {'error', atom()}.
-delete_dir(Dir, []) ->
-    file:del_dir(Dir);
-delete_dir(Dir, Subdirs) ->
-    lists:foreach(fun(D) -> delete_dir(D, dirs(D)) end, Subdirs),
-    file:del_dir(Dir).
-
-parse_attrs(Fd, Includes, Dir) ->
-    case io:parse_erl_form(Fd, "") of
-        {ok, Form, _Line} ->
-            case erl_syntax:type(Form) of
-                attribute ->
-                    NewIncludes = process_attr(Form, Includes, Dir),
-                    parse_attrs(Fd, NewIncludes, Dir);
-                _ ->
-                    parse_attrs(Fd, Includes, Dir)
-            end;
-        {eof, _} ->
-            Includes;
-        _Err ->
-            parse_attrs(Fd, Includes, Dir)
-    end.
-
-process_attr(Form, Includes, Dir) ->
-    AttrName = erl_syntax:atom_value(erl_syntax:attribute_name(Form)),
-    process_attr(AttrName, Form, Includes, Dir).
-
-process_attr(import, Form, Includes, _Dir) ->
-    case erl_syntax_lib:analyze_import_attribute(Form) of
-        {Mod, _Funs} ->
-            [module_to_erl(Mod)|Includes];
-        Mod ->
-            [module_to_erl(Mod)|Includes]
-    end;
-process_attr(file, Form, Includes, _Dir) ->
-    {File, _} = erl_syntax_lib:analyze_file_attribute(Form),
-    [File|Includes];
-process_attr(include, Form, Includes, _Dir) ->
-    [FileNode] = erl_syntax:attribute_arguments(Form),
-    File = erl_syntax:string_value(FileNode),
-    [File|Includes];
-process_attr(include_lib, Form, Includes, Dir) ->
-    [FileNode] = erl_syntax:attribute_arguments(Form),
-    RawFile = erl_syntax:string_value(FileNode),
-    maybe_expand_include_lib_path(RawFile, Dir) ++ Includes;
-process_attr(behavior, Form, Includes, _Dir) ->
-    process_attr(behaviour, Form, Includes, _Dir);
-process_attr(behaviour, Form, Includes, _Dir) ->
-    [FileNode] = erl_syntax:attribute_arguments(Form),
-    File = module_to_erl(erl_syntax:atom_value(FileNode)),
-    [File|Includes];
-process_attr(compile, Form, Includes, _Dir) ->
-    [Arg] = erl_syntax:attribute_arguments(Form),
-    case erl_syntax:concrete(Arg) of
-        {parse_transform, Mod} ->
-            [module_to_erl(Mod)|Includes];
-        {core_transform, Mod} ->
-            [module_to_erl(Mod)|Includes];
-        L when is_list(L) ->
-            lists:foldl(
-              fun({parse_transform, Mod}, Acc) ->
-                      [module_to_erl(Mod)|Acc];
-                 ({core_transform, Mod}, Acc) ->
-                      [module_to_erl(Mod)|Acc];
-                 (_, Acc) ->
-                      Acc
-              end, Includes, L);
-        _ ->
-            Includes
-    end;
-process_attr(_, _Form, Includes, _Dir) ->
-    Includes.
-
-module_to_erl(Mod) ->
-    atom_to_list(Mod) ++ ".erl".
-
-%% Given a path like "stdlib/include/erl_compile.hrl", return
-%% "OTP_INSTALL_DIR/lib/erlang/lib/stdlib-x.y.z/include/erl_compile.hrl".
-%% Usually a simple [Lib, SubDir, File1] = filename:split(File) should
-%% work, but to not crash when an unusual include_lib path is used,
-%% utilize more elaborate logic.
-maybe_expand_include_lib_path(File, Dir) ->
-    File1 = filename:basename(File),
-    case filename:split(filename:dirname(File)) of
-        [_] ->
-            warn_and_find_path(File, Dir);
-        [Lib | SubDir] ->
-            case code:lib_dir(list_to_atom(Lib), list_to_atom(filename:join(SubDir))) of
-                {error, bad_name} ->
-                    warn_and_find_path(File, Dir);
-                AppDir ->
-                    [filename:join(AppDir, File1)]
-            end
-    end.
-
-%% The use of -include_lib was probably incorrect by the user but lets try to make it work.
-%% We search in the outdir and outdir/../include to see if the header exists.
-warn_and_find_path(File, Dir) ->
-    SrcHeader = filename:join(Dir, File),
-    case filelib:is_regular(SrcHeader) of
-        true ->
-            [SrcHeader];
-        false ->
-            IncludeDir = filename:join(rebar_utils:droplast(filename:split(Dir))++["include"]),
-            IncludeHeader = filename:join(IncludeDir, File),
-            case filelib:is_regular(IncludeHeader) of
-                true ->
-                    [filename:join(IncludeDir, File)];
-                false ->
-                    []
-            end
-    end.
-
-%%
-%% Ensure all files in a list are present and abort if one is missing
-%%
--spec check_files([file:filename()]) -> [file:filename()].
-check_files(FileList) ->
-    [check_file(F) || F <- FileList].
-
-check_file(File) ->
-    case filelib:is_regular(File) of
-        false -> ?ABORT("File ~p is missing, aborting\n", [File]);
-        true -> File
-    end.
-
-outdir(RebarOpts) ->
-    ErlOpts = rebar_opts:erl_opts(RebarOpts),
-    proplists:get_value(outdir, ErlOpts, ?DEFAULT_OUTDIR).
-
-include_abs_dirs(ErlOpts, BaseDir) ->
-    ErlOptIncludes = proplists:get_all_values(i, ErlOpts),
-    InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes),
-    [filename:join([BaseDir, "include"])|InclDirs].
-
-dir_recursive(Opts, Dir, CompileOpts) when is_list(CompileOpts) ->
-    case proplists:get_value(recursive,CompileOpts) of
-        undefined -> rebar_dir:recursive(Opts, Dir);
-        Recursive -> Recursive
-    end.
-
-valid_erl_first_conf(FileList) ->
-    Strs = filter_file_list(FileList),
-    case rebar_utils:is_list_of_strings(Strs) of
-        true -> true;
-        false -> ?ABORT("An invalid file list (~p) was provided as part of your erl_first_files directive",
-                        [FileList])
-    end.
-
-filter_file_list(FileList) ->
-    Atoms = lists:filter( fun(X) -> is_atom(X) end, FileList),
-    case Atoms of
-        [] ->
-            FileList;
-        _ ->
-          atoms_in_erl_first_files_warning(Atoms),
-          lists:filter( fun(X) -> not(is_atom(X)) end, FileList)
-     end.
-
-atoms_in_erl_first_files_warning(Atoms) ->
-  W = "You have provided atoms as file entries in erl_first_files; "
-      "erl_first_files only expects lists of filenames as strings. "
-      "The following modules (~p) may not work as expected and it is advised "
-      "that you change these entires to string format "
-      "(e.g., \"src/module.erl\") ",
-  ?WARN(W, [Atoms]).

src/rebar_file_utils.erl

@@ -43,7 +43,8 @@
          path_from_ancestor/2,
          canonical_path/1,
          resolve_link/1,
-         split_dirname/1]).
+         split_dirname/1,
+         ensure_dir/1]).
 
 -include("rebar.hrl").
 
@@ -386,7 +387,7 @@ reset_dir(Path) ->
     %% delete the directory if it exists
     _ = ec_file:remove(Path, [recursive]),
     %% recreate the directory
-    filelib:ensure_dir(filename:join([Path, "dummy.beam"])).
+    ensure_dir(Path).
 
 
 %% Linux touch but using erlang functions to work in bot *nix os and
@@ -440,6 +441,10 @@ resolve_link(Path) ->
 split_dirname(Path) ->
     {filename:dirname(Path), filename:basename(Path)}.
 
+-spec ensure_dir(filelib:dirname_all()) -> ok | {error, file:posix()}.
+ensure_dir(Path) ->
+    filelib:ensure_dir(filename:join(Path, "fake_file")).
+
 %% ===================================================================
 %% Internal functions
 %% ===================================================================
@@ -505,7 +510,7 @@ cp_r_win32({true, SourceDir}, {false, DestDir}) ->
         false ->
             %% Specifying a target directory that doesn't currently exist.
             %% So let's attempt to create this directory
-            case filelib:ensure_dir(filename:join(DestDir, "dummy")) of
+            case ensure_dir(DestDir) of
                 ok ->
                     ok = xcopy_win32(SourceDir, DestDir);
                 {error, Reason} ->

src/rebar_otp_app.erl

@@ -60,7 +60,8 @@ compile(State, App) ->
 format_error({missing_app_file, Filename}) ->
     io_lib:format("App file is missing: ~ts", [Filename]);
 format_error({file_read, AppName, File, Reason}) ->
-    io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts", [File, AppName, file:format_error(Reason)]);
+    io_lib:format("Failed to read required ~ts file for processing the application '~ts': ~ts",
+                  [File, AppName, file:format_error(Reason)]);
 format_error({invalid_name, File, AppName}) ->
     io_lib:format("Invalid ~ts: name of application (~p) must match filename.", [File, AppName]).
 
@@ -125,7 +126,7 @@ preprocess(State, AppInfo, AppSrcFile) ->
 
             %% Setup file .app filename and write new contents
             EbinDir = rebar_app_info:ebin_dir(AppInfo),
-            filelib:ensure_dir(filename:join(EbinDir, "dummy.beam")),
+            rebar_file_utils:ensure_dir(EbinDir),
             AppFile = rebar_app_utils:app_src_to_app(OutDir, AppSrcFile),
             ok = rebar_file_utils:write_file_if_contents_differ(AppFile, Spec, utf8),
 
@@ -163,32 +164,17 @@ validate_name(AppName, File) ->
 
 ebin_modules(AppInfo, Dir) ->
     Beams = lists:sort(rebar_utils:beams(filename:join(Dir, "ebin"))),
-    ExtraDirs = extra_dirs(AppInfo),
-    F = fun(Beam) -> not in_extra_dir(AppInfo, Beam, ExtraDirs) end,
-    Filtered = lists:filter(F, Beams),
+    SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(AppInfo), ["src"]),
+    FindSourceRules = [{".beam", ".erl",
+                        [{"ebin", SrcDir} || SrcDir <- SrcDirs]}],
+    Filtered = lists:filter(fun(Beam) ->
+                                rebar_utils:find_source(filename:basename(Beam),
+                                                        filename:dirname(Beam),
+                                                        FindSourceRules)
+                                        =/= {error, not_found}
+                            end, Beams),
     [rebar_utils:beam_to_mod(N) || N <- Filtered].
 
-extra_dirs(State) ->
-    Extras = rebar_dir:extra_src_dirs(rebar_app_info:opts(State)),
-    SrcDirs = rebar_dir:src_dirs(rebar_app_info:opts(State), ["src"]),
-    %% remove any dirs that are defined in `src_dirs` from `extra_src_dirs`
-    Extras -- SrcDirs.
-
-in_extra_dir(AppInfo, Beam, Dirs) ->
-    lists:any(fun(Dir) -> lists:prefix(filename:join([rebar_app_info:out_dir(AppInfo), Dir]),
-                                       beam_src(Beam)) end,
-              Dirs).
-
-beam_src(Beam) ->
-    case beam_lib:chunks(Beam, [compile_info]) of
-        {ok, {_mod, Chunks}} ->
-            CompileInfo = proplists:get_value(compile_info, Chunks, []),
-            proplists:get_value(source, CompileInfo, []);
-        {error, beam_lib, Reason} ->
-            ?WARN("Couldn't read debug info from ~p for reason: ~p", [Beam, Reason]),
-            []
-    end.
-
 ensure_registered(AppData) ->
     case lists:keyfind(registered, 1, AppData) of
         false ->

src/rebar_packages.erl

@@ -60,7 +60,7 @@ get_all_names(State) ->
 get_package_versions(Dep, Repo, Table, State) ->
     ?MODULE:verify_table(State),
     AllowPreRelease = rebar_state:get(State, deps_allow_prerelease, false),
-    ets:select(Table, [{#package{key={Dep,{'$1', '$2'}, Repo},
+    ets:select(Table, [{#package{key={Dep, {'$1', '$2'}, Repo},
                                  _='_'},
                         [{'==', '$2', {{[],[]}}} || not AllowPreRelease], [{{'$1', '$2'}}]}]).
 

src/rebar_prv_clean.erl

@@ -67,11 +67,12 @@ format_error(Reason) ->
 %% ===================================================================
 
 clean_apps(State, Providers, Apps) ->
+    Compilers = rebar_state:compilers(State),
     [begin
          ?INFO("Cleaning out ~ts...", [rebar_app_info:name(AppInfo)]),
          AppDir = rebar_app_info:dir(AppInfo),
          AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State),
-         rebar_erlc_compiler:clean(AppInfo1),
+         rebar_compiler:clean(Compilers, AppInfo1),
          rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo1, State)
      end || AppInfo <- Apps].
 

src/rebar_prv_compile.erl

@@ -92,6 +92,12 @@ handle_project_apps(DepsPaths, Providers, State) ->
 -spec format_error(any()) -> iolist().
 format_error({missing_artifact, File}) ->
     io_lib:format("Missing artifact ~ts", [File]);
+format_error({bad_project_builder, Name, Type, Module}) ->
+    io_lib:format("Error building application ~s:~n     Required project builder ~s function "
+                  "~s:build/1 not found", [Name, Type, Module]);
+format_error({unknown_project_type, Name, Type}) ->
+    io_lib:format("Error building application ~s:~n     "
+                  "No project builder is configured for type ~s", [Name, Type]);
 format_error(Reason) ->
     io_lib:format("~p", [Reason]).
 
@@ -135,10 +141,19 @@ build_extra_dir(State, Dir) ->
         true ->
             BaseDir = filename:join([rebar_dir:base_dir(State), "extras"]),
             OutDir = filename:join([BaseDir, Dir]),
-            filelib:ensure_dir(filename:join([OutDir, "dummy.beam"])),
+            rebar_file_utils:ensure_dir(OutDir),
             copy(rebar_state:dir(State), BaseDir, Dir),
-            rebar_erlc_compiler:compile_dir(State, BaseDir, OutDir);
-        false -> ok
+
+            Compilers = rebar_state:compilers(State),
+            FakeApp = rebar_app_info:new(),
+            FakeApp1 = rebar_app_info:out_dir(FakeApp, BaseDir),
+            FakeApp2 = rebar_app_info:ebin_dir(FakeApp1, OutDir),
+            Opts = rebar_state:opts(State),
+            FakeApp3 = rebar_app_info:opts(FakeApp2, Opts),
+            FakeApp4 = rebar_app_info:set(FakeApp3, src_dirs, [OutDir]),
+            rebar_compiler:compile_all(Compilers, FakeApp4);
+        false ->
+            ok
     end.
 
 compile(State, AppInfo) ->
@@ -150,7 +165,9 @@ compile(State, Providers, AppInfo) ->
     AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER,  Providers, AppInfo, State),
 
     AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State),
-    rebar_erlc_compiler:compile(AppInfo2),
+
+    build_app(AppInfo2, State),
+
     AppInfo3 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo2, State),
 
     AppInfo4 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo3, State),
@@ -179,6 +196,24 @@ compile(State, Providers, AppInfo) ->
 %% Internal functions
 %% ===================================================================
 
+build_app(AppInfo, State) ->
+    case rebar_app_info:project_type(AppInfo) of
+        Type when Type =:= rebar3 ; Type =:= undefined ->
+            Compilers = rebar_state:compilers(State),
+            rebar_compiler:compile_all(Compilers, AppInfo);
+        Type ->
+            ProjectBuilders = rebar_state:project_builders(State),
+            case lists:keyfind(Type, 1, ProjectBuilders) of
+                {_, Module} ->
+                    %% load plugins since thats where project builders would be
+                    PluginDepsPaths = rebar_state:code_paths(State, all_plugin_deps),
+                    code:add_pathsa(PluginDepsPaths),
+                    Module:build(AppInfo),
+                    rebar_utils:remove_from_code_path(PluginDepsPaths);
+                _ ->
+                    throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type}))
+            end
+    end.
 update_code_paths(State, ProjectApps, DepsPaths) ->
     ProjAppsPaths = paths_for_apps(ProjectApps),
     ExtrasPaths = paths_for_extras(State, ProjectApps),

src/rebar_state.erl

@@ -38,6 +38,11 @@
 
          to_list/1,
 
+         compilers/1, compilers/2,
+         prepend_compilers/2, append_compilers/2,
+
+         project_builders/1, add_project_builder/3,
+
          create_resources/2,
          resources/1, resources/2, add_resource/2,
          providers/1, providers/2, add_provider/2,
@@ -66,6 +71,8 @@
                   all_plugin_deps     = []          :: [rebar_app_info:t()],
                   all_deps            = []          :: [rebar_app_info:t()],
 
+                  compilers           = []          :: [{compiler_type(), extension(), extension(), compile_fun()}],
+                  project_builders    = []          :: [{rebar_app_info:project_type(), module()}],
                   resources           = [],
                   providers           = [],
                   allow_provider_overrides = false  :: boolean()}).
@@ -74,6 +81,10 @@
 
 -type t() :: #state_t{}.
 
+-type compiler_type() :: atom().
+-type extension() :: string().
+-type compile_fun() :: fun(([file:filename()], rebar_app_info:t(), list()) -> ok).
+
 -spec new() -> t().
 new() ->
     BaseState = base_state(dict:new()),
@@ -392,6 +403,32 @@ warn_old_resource(ResourceModule) ->
     ?WARN("Using custom resource ~s that implements a deprecated api. "
           "It should be upgraded to rebar_resource_v2.", [ResourceModule]).
 
+compilers(#state_t{compilers=Compilers}) ->
+    Compilers.
+
+prepend_compilers(State=#state_t{compilers=Compilers}, NewCompilers) ->
+    State#state_t{compilers=NewCompilers++Compilers}.
+
+append_compilers(State=#state_t{compilers=Compilers}, NewCompilers) ->
+    State#state_t{compilers=Compilers++NewCompilers}.
+
+compilers(State, Compilers) ->
+    State#state_t{compilers=Compilers}.
+
+project_builders(#state_t{project_builders=ProjectBuilders}) ->
+    ProjectBuilders.
+
+add_project_builder(State=#state_t{project_builders=ProjectBuilders}, Type, Module) ->
+    _ = code:ensure_loaded(Module),
+    case erlang:function_exported(Module, build, 1) of
+        true ->
+            State#state_t{project_builders=[{Type, Module} | ProjectBuilders]};
+        false ->
+            ?WARN("Unable to add project builder for type ~s, required function ~s:build/1 not found.",
+                  [Type, Module]),
+            State
+    end.
+
 create_resources(Resources, State) ->
     lists:foldl(fun(R, StateAcc) ->
                         add_resource(StateAcc, R)

src/rebar_utils.erl

@@ -37,8 +37,9 @@
          escript_foldl/3,
          find_files/2,
          find_files/3,
+         find_files_in_dirs/3,
+         find_source/3,
          beam_to_mod/1,
-         beam_to_mod/2,
          erl_to_mod/1,
          beams/1,
          find_executable/1,
@@ -206,6 +207,12 @@ sh(Command0, Options0) ->
 find_files(Dir, Regex) ->
     find_files(Dir, Regex, true).
 
+find_files_in_dirs([], _Regex, _Recursive) ->
+    [];
+find_files_in_dirs([Dir | T], Regex, Recursive) ->
+    find_files(Dir, Regex, Recursive) ++ find_files_in_dirs(T, Regex, Recursive).
+
+
 find_files(Dir, Regex, Recursive) ->
     filelib:fold_files(Dir, Regex, Recursive,
                        fun(F, Acc) -> [F | Acc] end, []).
@@ -671,10 +678,6 @@ sh_loop(Port, Fun, Acc) ->
             end
     end.
 
-beam_to_mod(Dir, Filename) ->
-    [Dir | Rest] = filename:split(Filename),
-    list_to_atom(filename:basename(rebar_string:join(Rest, "."), ".beam")).
-
 beam_to_mod(Filename) ->
     list_to_atom(filename:basename(Filename, ".beam")).
 
@@ -1074,3 +1077,77 @@ version_pad([Major, Minor, Patch]) ->
     {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)};
 version_pad([Major, Minor, Patch | _]) ->
     {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}.
+
+
+-ifdef(filelib_find_source).
+find_source(Filename, Dir, Rules) ->
+    filelib:find_source(Filename, Dir, Rules).
+-else.
+%% Looks for a file relative to a given directory
+
+-type find_file_rule() :: {ObjDirSuffix::string(), SrcDirSuffix::string()}.
+
+%% Looks for a source file relative to the object file name and directory
+
+-type find_source_rule() :: {ObjExtension::string(), SrcExtension::string(),
+                             [find_file_rule()]}.
+
+keep_suffix_search_rules(Rules) ->
+    [T || {_,_,_}=T <- Rules].
+
+-spec find_source(file:filename(), file:filename(), [find_source_rule()]) ->
+        {ok, file:filename()} | {error, not_found}.
+find_source(Filename, Dir, Rules) ->
+    try_suffix_rules(keep_suffix_search_rules(Rules), Filename, Dir).
+
+try_suffix_rules(Rules, Filename, Dir) ->
+    Ext = filename:extension(Filename),
+    try_suffix_rules(Rules, filename:rootname(Filename, Ext), Dir, Ext).
+
+try_suffix_rules([{Ext,Src,Rules}|Rest], Root, Dir, Ext)
+  when is_list(Src), is_list(Rules) ->
+    case try_dir_rules(add_local_search(Rules), Root ++ Src, Dir) of
+        {ok, File} -> {ok, File};
+        _Other ->
+            try_suffix_rules(Rest, Root, Dir, Ext)
+    end;
+try_suffix_rules([_|Rest], Root, Dir, Ext) ->
+    try_suffix_rules(Rest, Root, Dir, Ext);
+try_suffix_rules([], _Root, _Dir, _Ext) ->
+    {error, not_found}.
+
+%% ensuring we check the directory of the object file before any other directory
+add_local_search(Rules) ->
+    Local = {"",""},
+    [Local] ++ lists:filter(fun (X) -> X =/= Local end, Rules).
+
+try_dir_rules([{From, To}|Rest], Filename, Dir)
+  when is_list(From), is_list(To) ->
+    case try_dir_rule(Dir, Filename, From, To) of
+	{ok, File} -> {ok, File};
+	error      -> try_dir_rules(Rest, Filename, Dir)
+    end;
+try_dir_rules([], _Filename, _Dir) ->
+    {error, not_found}.
+
+try_dir_rule(Dir, Filename, From, To) ->
+    case lists:suffix(From, Dir) of
+	true ->
+	    NewDir = lists:sublist(Dir, 1, length(Dir)-length(From))++To,
+	    Src = filename:join(NewDir, Filename),
+	    case filelib:is_regular(Src) of
+		true -> {ok, Src};
+		false -> find_regular_file(filelib:wildcard(Src))
+	    end;
+	false ->
+	    error
+    end.
+
+find_regular_file([]) ->
+    error;
+find_regular_file([File|Files]) ->
+    case filelib:is_regular(File) of
+        true -> {ok, File};
+        false -> find_regular_file(Files)
+    end.
+-endif.

test/rebar_eunit_SUITE.erl

@@ -605,4 +605,4 @@ alternate_test_regex(Config) ->
 
     Set = {ok, [{application, basic_app},
                 {module, basic_app_tests}]},
-    Set = rebar_prv_eunit:prepare_tests(S).
+    Set = rebar_prv_eunit:prepare_tests(S).