Introduce new DAG phases

While leaving the old one in place, prep the ground for new analysis
phases for the DAG work. The new DAG functions are added, but not hooked
in yet.

Fixes to the EPP handling have also been added due to issues in
resolving include_lib information

src/rebar_compiler.erl

@@ -1,6 +1,8 @@
 -module(rebar_compiler).
 
--export([compile_all/2,
+-export([analyze_all/2,
+         compile_analyzed/2,
+         compile_all/2,
          clean/2,
 
          needs_compile/3,
@@ -36,9 +38,21 @@
 
 -define(RE_PREFIX, "^(?!\\._)").
 
--spec compile_all([{module(), digraph:graph()}, ...], rebar_app_info:t()) -> ok
-      ;          ([module(), ...], rebar_app_info:t()) -> ok.
-compile_all(DAGs, AppInfo) when is_tuple(hd(DAGs)) -> % > 3.13.0
+
+%% @doc analysis by the caller, in order to let an OTP app
+%% find and resolve all its dependencies as part of compile_all's new
+%% API, which presumes a partial analysis is done ahead of time
+-spec analyze_all([DAG], [App, ...]) -> ok when
+      DAG :: {module(), digraph:graph()},
+      App :: rebar_app_info:t().
+analyze_all(_DAGs, _Apps) ->
+    %% Analyze apps one by one
+    %% then cover the include files in the digraph to update them
+    %% then propagate?
+    ok.
+
+-spec compile_analyzed([{module(), digraph:graph()}], rebar_app_info:t()) -> ok.
+compile_analyzed(DAGs, AppInfo) -> % > 3.13.0
     prepare_compiler_env(DAGs, AppInfo),
     lists:foreach(fun({Compiler, G}) ->
         run(G, Compiler, AppInfo),
@@ -49,14 +63,17 @@ compile_all(DAGs, AppInfo) when is_tuple(hd(DAGs)) -> % > 3.13.0
         [run(G, Compiler, ExtraAppInfo) || ExtraAppInfo <- ExtraApps],
         ok
     end,
-    DAGs);
+    DAGs).
+
+-spec compile_all([module(), ...], rebar_app_info:t()) -> ok.
 compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this!
     %% Support the old-style API by re-declaring a local DAG for the
     %% compile steps needed.
     lists:foreach(fun(Compiler) ->
         OutDir = rebar_app_info:out_dir(AppInfo),
         G = rebar_compiler_dag:init(OutDir, Compiler, undefined, []),
-        compile_all([{Compiler, G}], AppInfo),
+        analyze_all([{Compiler, G}], [AppInfo]),
+        compile_analyzed([{Compiler, G}], AppInfo),
         rebar_compiler_dag:maybe_store(G, OutDir, Compiler, undefined, []),
         rebar_compiler_dag:terminate(G)
      end, Compilers).

src/rebar_compiler_dag.erl

@@ -2,6 +2,8 @@
 %%% of all top-level applications by the various compiler plugins.
 -module(rebar_compiler_dag).
 -export([init/4, prune/4, update/5, maybe_store/5, terminate/1]).
+-export([populate_sources/5, populate_deps/3, propagate_stamps/1,
+         compile_order/2]).
 
 -include("rebar.hrl").
 
@@ -96,6 +98,180 @@ update(G, Compiler, InDirs, [Source|Erls], DepOpts) ->
             update(G, Compiler, InDirs, Erls, DepOpts)
     end.
 
+populate_sources(_G, _Compiler, _InDirs, [], _DepOpts) ->
+    ok;
+populate_sources(G, Compiler, InDirs, [Source|Erls], DepOpts) ->
+    case digraph:vertex(G, Source) of
+        {_, LastUpdated} ->
+            case filelib:last_modified(Source) of
+                0 ->
+                    %% The File doesn't exist anymore, delete
+                    %% from the graph.
+                    digraph:del_vertex(G, Source),
+                    mark_dirty(G),
+                    populate_sources(G, Compiler, InDirs, Erls, DepOpts);
+                LastModified when LastUpdated < LastModified ->
+                    digraph:add_vertex(G, Source, LastModified),
+                    prepopulate_deps(G, Compiler, InDirs, Source, DepOpts),
+                    mark_dirty(G);
+                _ -> % unchanged
+                    ok
+            end;
+        false ->
+            LastModified = filelib:last_modified(Source),
+            digraph:add_vertex(G, Source, LastModified),
+            prepopulate_deps(G, Compiler, InDirs, Source, DepOpts),
+            mark_dirty(G)
+    end,
+    populate_sources(G, Compiler, InDirs, Erls, DepOpts).
+
+prepopulate_deps(G, Compiler, InDirs, Source, DepOpts) ->
+    SourceDir = filename:dirname(Source),
+    AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of
+        false ->
+            Compiler:dependencies(Source, SourceDir, InDirs);
+        true ->
+            Compiler:dependencies(Source, SourceDir, InDirs, DepOpts)
+    end,
+    %% the file hasn't been visited yet; set it to existing, but with
+    %% a last modified value that's null so it gets updated to something new.
+    [digraph:add_vertex(G, Src, 0) || Src <- AbsIncls,
+                                      digraph:vertex(G, Src) =:= false],
+    [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls],
+    ok.
+
+populate_deps(G, SourceExt, ArtifactExts) ->
+    %% deps are files that are part of the digraph, but couldn't be scanned
+    %% because they are neither source files (`SourceExt') nor mappings
+    %% towards build artifacts (`ArtifactExts'); they will therefore never
+    %% be handled otherwise and need to be re-scanned for accuracy, even
+    %% if they are not being analyzed (we assume `Compiler:deps' did that
+    %% in depth already, and improvements should be driven at that level)
+    IgnoredExts = [SourceExt | ArtifactExts],
+    Vertices = digraph:vertices(G),
+    [refresh_dep(G, File)
+     || File <- Vertices,
+        Ext <- [filename:extension(File)],
+        not lists:member(Ext, IgnoredExts)],
+    ok.
+
+%% Take the timestamps/diff changes and propagate them from a dep to the parent;
+%% given:
+%%   A 0 -> B 1 -> C 3 -> D 2
+%% then we expect to get back:
+%%   A 3 -> B 3 -> C 3 -> D 2
+%% This is going to be safe for the current run of regeneration, but also for the
+%% next one; unless any file in the chain has changed, the stamp won't move up
+%% and there won't be a reason to recompile.
+%% The obvious caveat to this one is that a file changing by restoring an old version
+%% won't be picked up, but this weakness already existed in terms of timestamps.
+propagate_stamps(G) ->
+    case is_dirty(G) of
+        false ->
+            %% no change, no propagation to make
+            ok;
+        true ->
+            %% we can use a topsort, start at the end of it (files with no deps)
+            %% and update them all in order. By doing this, each file only needs to check
+            %% for one level of out-neighbours to set itself to the right appropriate time.
+            DepSort = lists:reverse(digraph_utils:topsort(G)),
+            propagate_stamps(G, DepSort)
+    end.
+
+
+propagate_stamps(_G, []) ->
+    ok;
+propagate_stamps(G, [File|Files]) ->
+    Stamps = [element(2, digraph:vertex(G, F))
+              || F <- digraph:out_neighbours(G, File)],
+    case Stamps of
+        [] ->
+            ok;
+        _ ->
+            Max = lists:max(Stamps),
+            case digraph:vertex(G, File) of
+                {_, Smaller} when Smaller < Max ->
+                    digraph:add_vertex(G, File, Max);
+                _ ->
+                    ok
+            end
+    end,
+    propagate_stamps(G, Files).
+
+
+compile_order(G, AppDefs) ->
+    %% Return the reverse sorting order to get dep-free apps first.
+    %% -- we would usually not need to consider the non-source files for the order to
+    %% be complete, but using them doesn't hurt.
+    Edges = [{V1,V2} || E <- digraph:edges(G),
+                        {_,V1,V2,_} <- [digraph:edge(G, E)]],
+    AppPaths = prepare_app_paths(AppDefs),
+    compile_order(Edges, AppPaths, #{}).
+
+compile_order([], _AppPaths, AppDeps) ->
+    %% use a digraph so we don't reimplement topsort by hand.
+    G = digraph:new([acyclic]), % ignore cycles and hope it works
+    Tups = maps:keys(AppDeps),
+    {Va,Vb} = lists:unzip(Tups),
+    [digraph:add_vertex(G, V) || V <- Va],
+    [digraph:add_vertex(G, V) || V <- Vb],
+    [digraph:add_edge(G, V1, V2) || {V1, V2} <- Tups],
+    Res = lists:reverse(digraph_utils:topsort(G)),
+    digraph:delete(G),
+    Res;
+compile_order([{P1,P2}|T], AppPaths, AppDeps) ->
+    %% Assume most dependencies are between files of the same app
+    %% so ask to see if it's the same before doing a deeper check:
+    {P1App, P1Path} = find_app(P1, AppPaths),
+    {P2App, _} = find_cached_app(P2, {P1App, P1Path}, AppPaths),
+    case {P1App, P2App} of
+        {A, A} ->
+            compile_order(T, AppPaths, AppDeps);
+        {V1, V2} ->
+            compile_order(T, AppPaths, AppDeps#{{V1, V2} => true})
+    end.
+
+prepare_app_paths(AppPaths) ->
+    lists:sort([{filename:split(Path), Name} || {Name, Path} <- AppPaths]).
+
+find_app(Path, AppPaths) ->
+    find_app_(filename:split(Path), AppPaths).
+
+find_cached_app(Path, {Name, AppPath}, AppPaths) ->
+    Split = filename:split(Path),
+    case find_app_(Split, [{AppPath, Name}]) of
+        not_found -> find_app_(Split, AppPaths);
+        LastEntry -> LastEntry
+    end.
+
+find_app_(_Path, []) ->
+    not_found;
+find_app_(Path, [{AppPath, AppName}|Rest]) ->
+    case lists:prefix(AppPath, Path) of
+        true ->
+            {AppName, AppPath};
+        false when AppPath > Path ->
+            not_found;
+        false ->
+            find_app_(Path, Rest)
+    end.
+
+
+refresh_dep(G, File) ->
+    {_, LastUpdated} = digraph:vertex(G, File),
+    case filelib:last_modified(File) of
+        0 ->
+            %% Gone! Erase from the graph
+            digraph:del_vertex(G, File),
+            mark_dirty(G);
+        LastModified when LastUpdated < LastModified ->
+            digraph:add_vertex(G, File, LastModified),
+            mark_dirty(G);
+        _ ->
+            % unchanged
+            ok
+    end.
+
 maybe_store(G, Dir, Compiler, Label, CritMeta) ->
     case is_dirty(G) of
         true ->

src/rebar_compiler_epp.erl

@@ -22,8 +22,9 @@
                       parse_transform := [atom()],
                       is_behaviour := boolean()}.
 deps(File, Opts) ->
-    {ok, Forms} = epp:parse_file(File, Opts),
-    normalize(handle_forms(Forms, default_attrs())).
+    {EppOpts, ExtraOpts} = split_opts(Opts),
+    {ok, Forms} = epp:parse_file(File, EppOpts),
+    normalize(handle_forms(Forms, default_attrs(), ExtraOpts)).
 
 %% Find the path matching a given erlang module
 resolve_module(Mod, Paths) ->
@@ -59,8 +60,9 @@ normalize(Map) ->
          behaviour => lists:usort(Behaviour),
          parse_transform => lists:usort(PTrans)}.
 
-handle_forms([File|Forms], Map) ->
-    lists:foldl(fun handle_form/2, Map, drop_self_file(File, Forms)).
+handle_forms([File|Forms], Map, Opts) ->
+    lists:foldl(fun(Form, M) -> handle_form(Form, M, Opts) end,
+                Map, drop_self_file(File, Forms)).
 
 drop_self_file(_, []) ->
     [];
@@ -74,7 +76,7 @@ drop_self_file(File, [Keep|Rest]) ->
 %% There are also references to the module's own file declaration
 %% in there, but this is dropped by `drop_self_file/2' and assumed
 %% to be gone here.
-handle_form({attribute, _Line, file, {Path, Ln}}, Map) ->
+handle_form({attribute, _Line, file, {Path, Ln}}, Map, Opts) ->
     %% Some people think they're funny and they go include attributes
     %% like:
     %%  -file("fake/file.hrl", Ln).
@@ -90,37 +92,70 @@ handle_form({attribute, _Line, file, {Path, Ln}}, Map) ->
         Path ->
             maps:update_with(include, fun(L) -> [Path|L] end, [Path], Map);
         _ -> % argh!
-            handle_form({error, {Ln, {epp, {include, file, Path}}}}, Map)
+            handle_form({error, {Ln, {epp, {include, file, Path}}}}, Map, Opts)
     end;
 %% Include files that EPP couldn't resolve
-handle_form({error, {_Line, epp, {include, file, Name}}}, Map) ->
+handle_form({error, {_Line, epp, {include, file, Name}}}, Map, _Opts) ->
     maps:update_with(missing_include_file, fun(L) -> [Name|L] end, [Name], Map);
-handle_form({error, {_Line, epp, {include, lib, Path}}}, Map) ->
-    maps:update_with(missing_include_lib, fun(L) -> [Path|L] end, [Path], Map);
+handle_form({error, {_Line, epp, {include, lib, Path}}}, Map, Opts) ->
+    %% This file might still exist in the regular paths not in
+    %% code:lib_dir, which depend on options we pass to this module.
+    %% recursively seek it, and add it to the paths to expand here.
+    case find_include_with_opts(Path, Opts) of
+        {ok, File} ->
+            %% we can't go and figure out the contents within that include
+            %% file because we'd need its own compiler opts and app opts
+            %% to do it safely. Tracking that file is still better
+            %% than nothing though.
+            maps:update_with(include, fun(L) -> [File|L] end, [File], Map);
+        {error, not_found} ->
+            maps:update_with(missing_include_lib, fun(L) -> [Path|L] end, [Path], Map)
+    end;
 %% Behaviour implementation declaration
-handle_form({attribute, _Line, behaviour, Name}, Map) ->
+handle_form({attribute, _Line, behaviour, Name}, Map, _Opts) ->
     maps:update_with(behaviour, fun(L) -> [Name|L] end, [Name], Map);
-handle_form({attribute, _Line, behavior, Name}, Map) ->
+handle_form({attribute, _Line, behavior, Name}, Map, _Opts) ->
     maps:update_with(behaviour, fun(L) -> [Name|L] end, [Name], Map);
 %% Extract parse transforms
-handle_form({attribute, Line, compile, Attr}, Map) when not is_list(Attr) ->
-    handle_form({attribute, Line, compile, [Attr]}, Map);
-handle_form({attribute, _Line, compile, Attrs}, Map) ->
+handle_form({attribute, Line, compile, Attr}, Map, _Opts) when not is_list(Attr) ->
+    handle_form({attribute, Line, compile, [Attr]}, Map, _Opts);
+handle_form({attribute, _Line, compile, Attrs}, Map, _Opts) ->
     Mods = [case T of
                 {_, {M,_}} -> M;
                 {_, M} -> M
             end || T <- proplists:lookup_all(parse_transform, Attrs)],
     maps:update_with(parse_transform, fun(L) -> Mods++L end, Mods, Map);
 %% Current style behaviour specification declaration
-handle_form({attribute, _Line, callback, _}, Map) ->
+handle_form({attribute, _Line, callback, _}, Map, _Opts) ->
     Map#{is_behaviour => true};
 %% Old style behaviour specification, both spellings supported
 %% The function needs to be exported, but we skip over that logic
 %% for now.
-handle_form({function, _Line, behaviour_info, 1, _}, Map) ->
+handle_form({function, _Line, behaviour_info, 1, _}, Map, _Opts) ->
     Map#{is_behaviour => true};
-handle_form({function, _Line, behavior_info, 1, _}, Map) ->
+handle_form({function, _Line, behavior_info, 1, _}, Map, _Opts) ->
     Map#{is_behaviour => true};
 %% Skip the rest
-handle_form(_, Map) ->
+handle_form(_, Map, _Opts) ->
     Map.
+
+split_opts(Opts) ->
+    %% Extra Opts are options we added to palliate to issues we had
+    %% with resolving include_libs and other things in EPP.
+    lists:partition(fun({OptName, _}) -> include_libs =/= OptName end,
+                    Opts).
+
+find_include_with_opts(Path, Opts) ->
+    InclPaths = proplists:get_value(include_libs, Opts, []),
+    find_include_lib(InclPaths, Path).
+
+find_include_lib([], _) ->
+    {error, not_found};
+find_include_lib([H|T], File) ->
+    Abs = filename:join([H, File]),
+    case filelib:is_regular(Abs) of
+        true -> {ok, Abs};
+        false -> find_include_lib(T, File)
+    end.
+
+

src/rebar_compiler_erl.erl

@@ -27,6 +27,8 @@ context(AppInfo) ->
     ErlOptIncludes = proplists:get_all_values(i, ErlOpts),
     InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes),
     AbsIncl = [filename:join([OutDir, "include"]) | InclDirs],
+    %% TODO: check that EPP can expand deps' include files
+    %% TODO: Ensure that EPP can expand other project apps' include files
     Macros = [case Tup of
                   {d,Name} -> Name;
                   {d,Name,Val} -> {Name,Val}
@@ -104,6 +106,8 @@ dependencies(Source, _SourceDir, Dirs, DepOpts) ->
             %% TODO: resolve parse_transforms cross-app
             %% TODO: report on missing files
             %% TODO: check for core transforms?
+            {_MissIncl, _MissInclLib} =/= {[],[]} andalso
+            ct:pal("Missing: ~p", [{_MissIncl, _MissInclLib}]),
             expand_file_names([module_to_erl(Mod) || Mod <- PTrans], Dirs) ++
             expand_file_names([module_to_erl(Mod) || Mod <- Behaviours], Dirs) ++
             AbsIncls
@@ -362,7 +366,14 @@ expand_file_names(Files, Dirs) ->
                   true ->
                       [Incl];
                   false ->
-                      rebar_utils:find_files_in_dirs(Dirs, [$^, Incl, $$], true)
+                      Res = rebar_utils:find_files_in_dirs(Dirs, [$^, Incl, $$], true),
+                      case Res of
+                          [] ->
+                              ?DEBUG("FILE ~p NOT FOUND", [Incl]),
+                              [];
+                          _ ->
+                              Res
+                      end
               end
       end, Files).
 

src/rebar_prv_compile.erl

@@ -169,9 +169,9 @@ run_compilers(State, _Providers, Apps, Tag) ->
     CritMeta = [], % used to be incldirs per app
     DAGs = [{Mod, rebar_compiler_dag:init(Dir, Mod, DAGLabel, CritMeta)}
             || Mod <- rebar_state:compilers(State)],
-    rebar_paths:set_paths([deps], State),
     %% Compile all the apps
-    [build_app(DAGs, AppInfo, State) || AppInfo <- Apps],
+    build_apps(DAGs, Apps, State),
+    %[build_app(DAGs, AppInfo, State) || AppInfo <- Apps],
     %% Potentially store shared compiler DAGs so next runs can easily
     %% share the base information for easy re-scans.
     lists:foreach(fun({Mod, G}) ->
@@ -260,31 +260,42 @@ extra_virtual_apps(State, VApp0, [Dir|Dirs]) ->
 %% Internal functions
 %% ===================================================================
 
-build_app(DAGs, AppInfo, State) ->
-    ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]),
-    case rebar_app_info:project_type(AppInfo) of
-        Type when Type =:= rebar3 ; Type =:= undefined ->
-            %% assume the deps paths are already set by the caller (run_compilers/3)
-            %% and shared for multiple apps to save work.
-            rebar_compiler:compile_all(DAGs, AppInfo);
-        Type ->
-            ProjectBuilders = rebar_state:project_builders(State),
-            case lists:keyfind(Type, 1, ProjectBuilders) of
-                {_, Module} ->
-                    %% load plugins since thats where project builders would be,
-                    %% prevents parallelism at this level.
-                    rebar_paths:set_paths([deps, plugins], State),
-                    Res = Module:build(AppInfo),
-                    rebar_paths:set_paths([deps], State),
-                    case Res of
-                        ok -> ok;
-                        {error, Reason} -> throw({error, {Module, Reason}})
-                    end;
-                _ ->
-                    throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type}))
-            end
+build_apps(DAGs, Apps, State) ->
+    {Rebar3, Custom} = lists:partition(
+        fun(AppInfo) ->
+            Type = rebar_app_info:project_type(AppInfo),
+            Type =:= rebar3 orelse Type =:= undefined
+        end,
+        Apps
+    ),
+    [build_custom_builder_app(AppInfo, State) || AppInfo <- Custom],
+    build_rebar3_apps(DAGs, Rebar3, State).
+
+build_custom_builder_app(AppInfo, State) ->
+    Type = rebar_app_info:project_type(AppInfo),
+    ProjectBuilders = rebar_state:project_builders(State),
+    case lists:keyfind(Type, 1, ProjectBuilders) of
+        {_, Module} ->
+            %% load plugins since thats where project builders would be,
+            %% prevents parallelism at this level.
+            rebar_paths:set_paths([deps, plugins], State),
+            Res = Module:build(AppInfo),
+            rebar_paths:set_paths([deps], State),
+            case Res of
+                ok -> ok;
+                {error, Reason} -> throw({error, {Module, Reason}})
+            end;
+        _ ->
+            throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type}))
     end.
 
+build_rebar3_apps(DAGs, Apps, State) ->
+    rebar_paths:set_paths([deps], State),
+    rebar_compiler:analyze_all(DAGs, Apps),
+    %% TODO: topsort apps here based on DAG data
+    [rebar_compiler:compile_analyzed(DAGs, AppInfo) || AppInfo <- Apps],
+    ok.
+
 update_code_paths(State, ProjectApps) ->
     ProjAppsPaths = paths_for_apps(ProjectApps),
     ExtrasPaths = paths_for_extras(State, ProjectApps),

test/rebar_compile_SUITE.erl

@@ -18,6 +18,8 @@ all() ->
      recompile_when_hrl_changes, recompile_when_included_hrl_changes,
      recompile_extra_when_hrl_in_src_changes,
      recompile_when_opts_included_hrl_changes,
+     recompile_when_foreign_included_hrl_changes,
+     recompile_when_foreign_behaviour_changes,
      recompile_when_opts_change,
      dont_recompile_when_opts_dont_change, dont_recompile_yrl_or_xrl,
      delete_beam_if_source_deleted,
@@ -801,6 +803,96 @@ recompile_when_opts_included_hrl_changes(Config) ->
 
     ?assert(ModTime =/= NewModTime).
 
+recompile_when_foreign_included_hrl_changes(Config) ->
+    AppDir = ?config(apps, Config),
+    AppsDir = filename:join([AppDir, "apps"]),
+
+    Name1 = rebar_test_utils:create_random_name("app1_"),
+    Name2 = rebar_test_utils:create_random_name("app2_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(filename:join(AppsDir, Name1),
+                                Name1, Vsn, [kernel, stdlib]),
+    rebar_test_utils:create_app(filename:join(AppsDir, Name2),
+                                Name2, Vsn, [kernel, stdlib]),
+
+    ExtraSrc = [<<"-module(test_header_include).\n"
+                  "-export([main/0]).\n"
+                  "-include_lib(\"">>, Name2, <<"/include/test_header_include.hrl\").\n"
+                  "main() -> ?SOME_DEFINE.\n">>],
+
+    ExtraHeader = <<"-define(SOME_DEFINE, true).\n">>,
+    ok = filelib:ensure_dir(filename:join([AppsDir, Name1, "src", "dummy"])),
+    ok = filelib:ensure_dir(filename:join([AppsDir, Name2, "include", "dummy"])),
+    HeaderFile = filename:join([AppsDir, Name2, "include", "test_header_include.hrl"]),
+    ok = file:write_file(filename:join([AppsDir, Name1, "src", "test_header_include.erl"]), ExtraSrc),
+    ok = file:write_file(HeaderFile, ExtraHeader),
+
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name1}]}),
+
+    EbinDir = filename:join([AppDir, "_build", "default", "lib", Name1, "ebin"]),
+    {ok, Files} = rebar_utils:list_dir(EbinDir),
+    ModTime = [filelib:last_modified(filename:join([EbinDir, F]))
+               || F <- Files, filename:extension(F) == ".beam"],
+
+    timer:sleep(1000),
+
+    NewExtraHeader = <<"-define(SOME_DEFINE, false).\n">>,
+    ok = file:write_file(HeaderFile, NewExtraHeader),
+
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name1}]}),
+
+    {ok, NewFiles} = rebar_utils:list_dir(EbinDir),
+    NewModTime = [filelib:last_modified(filename:join([EbinDir, F]))
+                  || F <- NewFiles, filename:extension(F) == ".beam"],
+
+    ?assert(ModTime =/= NewModTime).
+
+recompile_when_foreign_behaviour_changes(Config) ->
+    AppDir = ?config(apps, Config),
+    AppsDir = filename:join([AppDir, "apps"]),
+
+    Name1 = rebar_test_utils:create_random_name("app1_"),
+    Name2 = rebar_test_utils:create_random_name("app2_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(filename:join(AppsDir, Name1),
+                                Name1, Vsn, [kernel, stdlib]),
+    rebar_test_utils:create_app(filename:join(AppsDir, Name2),
+                                Name2, Vsn, [kernel, stdlib]),
+
+    ExtraSrc = <<"-module(test_behaviour_include).\n"
+                 "-export([main/0]).\n"
+                 "-behaviour(app2_behaviour).\n"
+                 "main() -> 1.\n">>,
+
+    Behaviour = <<"-module(app2_behaviour).\n"
+                  "-callback main() -> term().\n">>,
+    ok = filelib:ensure_dir(filename:join([AppsDir, Name1, "src", "dummy"])),
+    ok = filelib:ensure_dir(filename:join([AppsDir, Name2, "src", "dummy"])),
+    BehaviourFile = filename:join([AppsDir, Name2, "src", "app2_behaviour.erl"]),
+    ok = file:write_file(filename:join([AppsDir, Name1, "src", "test_behaviour_include.erl"]), ExtraSrc),
+    ok = file:write_file(BehaviourFile, Behaviour),
+
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name1}]}),
+
+    EbinDir = filename:join([AppDir, "_build", "default", "lib", Name1, "ebin"]),
+    {ok, Files} = rebar_utils:list_dir(EbinDir),
+    ModTime = [filelib:last_modified(filename:join([EbinDir, F]))
+               || F <- Files, filename:extension(F) == ".beam"],
+
+    timer:sleep(1000),
+
+    NewBehaviour = <<"-module(app2_behaviour).\n"
+                     "-callback main(_) -> term().\n">>,
+    ok = file:write_file(BehaviourFile, NewBehaviour),
+
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name1}]}),
+
+    {ok, NewFiles} = rebar_utils:list_dir(EbinDir),
+    NewModTime = [filelib:last_modified(filename:join([EbinDir, F]))
+                  || F <- NewFiles, filename:extension(F) == ".beam"],
+
+    ?assert(ModTime =/= NewModTime).
+
 recompile_when_opts_change(Config) ->
     AppDir = ?config(apps, Config),
 

test/rebar_compiler_dag_SUITE.erl

@@ -0,0 +1,453 @@
+-module(rebar_compiler_dag_SUITE).
+-compile([export_all, nowarn_export_all]).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("kernel/include/file.hrl").
+
+all() ->
+    [{group, with_project}].
+
+groups() ->
+    %% The tests in this group are dirty, the order is specific
+    %% and required across runs for tests to work.
+    [{with_project, [sequence], [
+        find_structure, app_sort,
+        propagate_include_app1a, propagate_include_app1b,
+        propagate_include_app2, propagate_behaviour,
+        propagate_app1_ptrans, propagate_app2_ptrans,
+        propagate_app2_ptrans_hrl
+      ]}
+    ].
+
+init_per_suite(Config) ->
+    rebar_compiler_erl:module_info(), % ensure it is loaded
+    Config.
+
+end_per_suite(Config) ->
+    Config.
+
+init_per_group(with_project, Config) ->
+    NewConfig = rebar_test_utils:init_rebar_state(Config, "apps"),
+    AppDir = ?config(apps, NewConfig),
+
+    Name1 = rebar_test_utils:create_random_name("app1_"),
+    Vsn1 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(filename:join([AppDir,"apps",Name1]), Name1, Vsn1, [kernel, stdlib]),
+
+    Name2 = rebar_test_utils:create_random_name("app2_"),
+    Vsn2 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(filename:join([AppDir,"apps",Name2]), Name2, Vsn2, [kernel, stdlib]),
+
+    Name3 = rebar_test_utils:create_random_name("app3_"),
+    Vsn3 = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(filename:join([AppDir,"apps",Name3]), Name3, Vsn3, [kernel, stdlib]),
+
+    apply_project(AppDir, [{app1, Name1}, {app2, Name2}, {app3, Name3}],
+                  project()),
+
+    [{app_names, [Name1, Name2, Name3]},
+     {vsns, [Vsn1, Vsn2, Vsn3]}
+     | NewConfig];
+init_per_group(_, Config) ->
+    Config.
+
+end_per_group(_, Config) ->
+    Config.
+
+
+project() ->
+    [{app1, [
+        {"src/app1.erl",
+         "-module(app1).\n"
+         "-include(\"app1_a.hrl\").\n"
+         "-include(\"app1_b.hrl\").\n"
+         "-include_lib(\"{{app2}}/include/app2.hrl\").\n"
+         "-compile({parse_transform, app1_trans}).\n"
+         "-compile({parse_transform, {app3, []}}).\n"
+         "-behaviour(app2).\n"
+         "-export([cb/0]).\n"
+         "cb() -> {?APP1A, ?APP1B, ?APP2}.\n"},
+        {"src/app1_trans.erl",
+         "-module(app1_trans).n"
+         "-export([parse_transform/2]).\n"
+         "parse_transform(Forms, _Opts) -> Forms.\n"},
+        {"src/app1_a.hrl",
+         "-define(APP1A, 1).\n"},
+        {"include/app1_b.hrl",
+         "-define(APP1B, 1).\n"}
+     ]},
+     {app2, [
+        {"src/app2.erl",
+         "-module(app2).\n"
+         "-callback cb() -> term().\n"},
+        {"include/app2.hrl",
+         "-include(\"app2_resolve.hrl\").\n"
+         "-define(APP2, 1).\n"},
+        {"src/app2_resolve.hrl",
+         "this file should be found but never is"},
+        {"include/never_found.hrl",
+         "%% just comments"}
+     ]},
+     {app3, [
+        {"src/app3.erl",
+         "-module(app3).\n"
+         "-include_lib(\"{{app2}}/include/app2.hrl\").\n"
+         "-include(\"app3_resolve.hrl\").\n"
+         "-export([parse_transform/2]).\n"
+         "parse_transform(Forms, _Opts) -> Forms.\n"},
+        {"src/app3_resolve.hrl",
+         "%% this file should be found"}
+     ]}
+    ].
+
+find_structure() ->
+    [{doc, "ensure a proper digraph is built with all files"}].
+find_structure(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    analyze_apps(G, AppNames, AppDir),
+    FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)],
+    Edges = [{V1,V2} || E <- digraph:edges(G),
+                        {_,V1,V2,_} <- [digraph:edge(G, E)]],
+    %% All timestamps are the same since we just created the thing
+    {_, Stamp} = hd(FileStamps),
+    Matches = [
+        {"/src/app1.erl", Stamp},
+        {"/src/app1_trans.erl", Stamp},
+        {"/src/app1_a.hrl", Stamp},
+        {"/include/app1_b.hrl", Stamp},
+        {"/src/app2.erl", Stamp},
+        {"/include/app2.hrl", Stamp},
+        {"/include/app2.hrl", Stamp},
+        {"/src/app3.erl", Stamp},
+        {"/src/app3_resolve.hrl", Stamp}
+    ],
+    matches(Matches, FileStamps),
+    ?assertEqual(undefined, find_match(".*/never_found.hrl", FileStamps)),
+    ?assertEqual(undefined, find_match(".*/app2_resolve.hrl", FileStamps)),
+    ct:pal("Edges: ~p", [Edges]),
+    edges([
+        {"/src/app1.erl", "/src/app1_a.hrl"},
+        {"/src/app1.erl", "/include/app1_b.hrl"},
+        {"/src/app1.erl", "/src/app2.erl"},
+        {"/src/app1.erl", "/include/app2.hrl"},
+        {"/src/app1.erl", "/src/app1_trans.erl"},
+        {"/src/app1.erl", "/src/app3.erl"},
+        {"/src/app3.erl", "/include/app2.hrl"},
+        {"/src/app3.erl", "/src/app3_resolve.hrl"}
+    ], Edges, FileStamps),
+    ok.
+
+app_sort() ->
+    [{doc, "once the digraph is complete, we can sort apps by dependency order"}].
+app_sort(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    analyze_apps(G, AppNames, AppDir),
+    AppPaths = [
+        {AppName, filename:join([AppDir, "apps", AppName])} || AppName <- AppNames
+    ],
+    ?assertEqual([lists:nth(2, AppNames),
+                  lists:nth(3, AppNames),
+                  lists:nth(1, AppNames)],
+                 rebar_compiler_dag:compile_order(G, AppPaths)),
+    ok.
+
+propagate_include_app1a() ->
+    [{doc, "changing the app1a header file propagates to its dependents"}].
+propagate_include_app1a(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    next_second(),
+    F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "src/app1_a.hrl"]),
+    bump_file(F),
+    analyze_apps(G, AppNames, AppDir),
+    FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)],
+    %% All timestamps are the same since we just created the thing
+    [Stamp1, Stamp2] = lists:usort([S || {_, S} <- FileStamps]),
+    Matches = [
+        {"/src/app1.erl", Stamp2},
+        {"/src/app1_trans.erl", Stamp1},
+        {"/src/app1_a.hrl", Stamp2},
+        {"/include/app1_b.hrl", Stamp1},
+        {"/src/app2.erl", Stamp1},
+        {"/include/app2.hrl", Stamp1},
+        {"/src/app3.erl", Stamp1},
+        {"/src/app3_resolve.hrl", Stamp1}
+    ],
+    matches(Matches, FileStamps),
+    ok.
+
+propagate_include_app1b() ->
+    [{doc, "changing the app1b header file propagates to its dependents"}].
+propagate_include_app1b(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    next_second(),
+    F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "include/app1_b.hrl"]),
+    bump_file(F),
+    analyze_apps(G, AppNames, AppDir),
+    FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)],
+    %% All timestamps are the same since we just created the thing
+    [Stamp1, Stamp2, Stamp3] = lists:usort([S || {_, S} <- FileStamps]),
+    Matches = [
+        {"/src/app1.erl", Stamp3},
+        {"/src/app1_trans.erl", Stamp1},
+        {"/src/app1_a.hrl", Stamp2},
+        {"/include/app1_b.hrl", Stamp3},
+        {"/src/app2.erl", Stamp1},
+        {"/include/app2.hrl", Stamp1},
+        {"/src/app3.erl", Stamp1},
+        {"/src/app3_resolve.hrl", Stamp1}
+    ],
+    matches(Matches, FileStamps),
+    ok.
+
+propagate_include_app2() ->
+    [{doc, "changing the app2 header file propagates to its dependents"}].
+propagate_include_app2(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    next_second(),
+    F = filename:join([AppDir, "apps", lists:nth(2, AppNames), "include/app2.hrl"]),
+    bump_file(F),
+    analyze_apps(G, AppNames, AppDir),
+    FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)],
+    %% All timestamps are the same since we just created the thing
+    [S1, S2, S3, S4] = lists:usort([S || {_, S} <- FileStamps]),
+    Matches = [
+        {"/src/app1.erl", S4},
+        {"/src/app1_trans.erl", S1},
+        {"/src/app1_a.hrl", S2},
+        {"/include/app1_b.hrl", S3},
+        {"/src/app2.erl", S1},
+        {"/include/app2.hrl", S4},
+        {"/src/app3.erl", S4},
+        {"/src/app3_resolve.hrl", S1}
+    ],
+    matches(Matches, FileStamps),
+    ok.
+
+propagate_behaviour() ->
+    [{doc, "changing the behaviour file propagates to its dependents"}].
+propagate_behaviour(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    next_second(),
+    F = filename:join([AppDir, "apps", lists:nth(2, AppNames), "src/app2.erl"]),
+    bump_file(F),
+    analyze_apps(G, AppNames, AppDir),
+    FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)],
+    %% All timestamps are the same since we just created the thing
+    [S1, S2, S3, S4, S5] = lists:usort([S || {_, S} <- FileStamps]),
+    Matches = [
+        {"/src/app1.erl", S5},
+        {"/src/app1_trans.erl", S1},
+        {"/src/app1_a.hrl", S2},
+        {"/include/app1_b.hrl", S3},
+        {"/src/app2.erl", S5},
+        {"/include/app2.hrl", S4},
+        {"/src/app3.erl", S4},
+        {"/src/app3_resolve.hrl", S1}
+    ],
+    matches(Matches, FileStamps),
+    ok.
+
+propagate_app1_ptrans() ->
+    [{doc, "changing an app-local parse transform propagates to its dependents"}].
+propagate_app1_ptrans(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    next_second(),
+    F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "src/app1_trans.erl"]),
+    bump_file(F),
+    analyze_apps(G, AppNames, AppDir),
+    FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)],
+    %% All timestamps are the same since we just created the thing
+    [S1, S2, S3, S4, S5, S6] = lists:usort([S || {_, S} <- FileStamps]),
+    Matches = [
+        {"/src/app1.erl", S6},
+        {"/src/app1_trans.erl", S6},
+        {"/src/app1_a.hrl", S2},
+        {"/include/app1_b.hrl", S3},
+        {"/src/app2.erl", S5},
+        {"/include/app2.hrl", S4},
+        {"/src/app3.erl", S4},
+        {"/src/app3_resolve.hrl", S1}
+    ],
+    matches(Matches, FileStamps),
+    ok.
+
+propagate_app2_ptrans() ->
+    [{doc, "changing an app-foreign parse transform propagates to its dependents"}].
+propagate_app2_ptrans(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    next_second(),
+    F = filename:join([AppDir, "apps", lists:nth(3, AppNames), "src/app3.erl"]),
+    bump_file(F),
+    analyze_apps(G, AppNames, AppDir),
+    FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)],
+    %% All timestamps are the same since we just created the thing
+    [S1, S2, S3, S4, S5, S6, S7] = lists:usort([S || {_, S} <- FileStamps]),
+    Matches = [
+        {"/src/app1.erl", S7},
+        {"/src/app1_trans.erl", S6},
+        {"/src/app1_a.hrl", S2},
+        {"/include/app1_b.hrl", S3},
+        {"/src/app2.erl", S5},
+        {"/include/app2.hrl", S4},
+        {"/src/app3.erl", S7},
+        {"/src/app3_resolve.hrl", S1}
+    ],
+    matches(Matches, FileStamps),
+    ok.
+
+propagate_app2_ptrans_hrl() ->
+    %% the app-foreign ptrans' foreign hrl dep is tested by propagate_include_app2 as well
+    [{doc, "changing an app-foreign parse transform's local hrl propagates to its dependents"}].
+propagate_app2_ptrans_hrl(Config) ->
+    AppDir = ?config(apps, Config),
+    AppNames = ?config(app_names, Config),
+    %% assume an empty graph
+    G = digraph:new([acyclic]),
+    next_second(),
+    F = filename:join([AppDir, "apps", lists:nth(3, AppNames), "src/app3_resolve.hrl"]),
+    bump_file(F),
+    analyze_apps(G, AppNames, AppDir),
+    FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)],
+    %% All timestamps are the same since we just created the thing
+    %% S1 and S7 are gone from the propagation now
+    [S2, S3, S4, S5, S6, S8] = lists:usort([S || {_, S} <- FileStamps]),
+    Matches = [
+        {"/src/app1.erl", S8},
+        {"/src/app1_trans.erl", S6},
+        {"/src/app1_a.hrl", S2},
+        {"/include/app1_b.hrl", S3},
+        {"/src/app2.erl", S5},
+        {"/include/app2.hrl", S4},
+        {"/src/app3.erl", S8},
+        {"/src/app3_resolve.hrl", S8}
+    ],
+    matches(Matches, FileStamps),
+    ok.
+
+%%%%%%%%%%%%%%%
+%%% HELPERS %%%
+%%%%%%%%%%%%%%%
+
+apply_project(_BaseDir, _Names, []) ->
+    ok;
+apply_project(BaseDir, Names, [{_AppName, []}|Rest]) ->
+    apply_project(BaseDir, Names, Rest);
+apply_project(BaseDir, Names, [{AppName, [File|Files]}|Rest]) ->
+    apply_file(BaseDir, Names, AppName, File),
+    apply_project(BaseDir, Names, [{AppName, Files}|Rest]).
+
+apply_file(BaseDir, Names, App, {FileName, Contents}) ->
+    AppName = proplists:get_value(App, Names),
+    FilePath = filename:join([BaseDir, "apps", AppName, FileName]),
+    ok = filelib:ensure_dir(FilePath),
+    file:write_file(FilePath, apply_template(Contents, Names)).
+
+apply_template("", _) -> "";
+apply_template("{{" ++ Text, Names) ->
+    {Var, Rest} = parse_to_var(Text),
+    App = list_to_atom(Var),
+    proplists:get_value(App, Names) ++ apply_template(Rest, Names);
+apply_template([H|T], Names) ->
+    [H|apply_template(T, Names)].
+
+parse_to_var(Str) -> parse_to_var(Str, []).
+
+parse_to_var("}}"++Rest, Acc) ->
+    {lists:reverse(Acc), Rest};
+parse_to_var([H|T], Acc) ->
+    parse_to_var(T, [H|Acc]).
+
+analyze_apps(G, AppNames, AppDir) ->
+    populate_app(G, lists:nth(1, AppNames), AppNames, AppDir, ["app1.erl", "app1_trans.erl"]),
+    populate_app(G, lists:nth(2, AppNames), AppNames, AppDir, ["app2.erl"]),
+    populate_app(G, lists:nth(3, AppNames), AppNames, AppDir, ["app3.erl"]),
+    rebar_compiler_dag:populate_deps(G, ".erl", [{".beam", "ebin/"}]),
+    rebar_compiler_dag:propagate_stamps(G),
+    %% manually clear the dirty bit for ease of validation
+    digraph:del_vertex(G, '$r3_dirty_bit').
+
+populate_app(G, Name, AppNames, AppDir, Sources) ->
+    InDirs = [filename:join([AppDir, "apps", AppName, "src"])
+              || AppName <- AppNames]
+          ++ [filename:join([AppDir, "apps", AppName, "include"])
+              || AppName <- AppNames],
+    AbsSources = [filename:join([AppDir, "apps", Name, "src", Src])
+                  || Src <- Sources],
+    DepOpts = [{includes,
+                [filename:join([AppDir, "apps", Name, "src"]),
+                 filename:join([AppDir, "apps", Name, "include"])
+                ]},
+               {include_libs, [filename:join([AppDir, "apps"])]}
+              ],
+    rebar_compiler_dag:populate_sources(
+        G, rebar_compiler_erl,
+        InDirs, AbsSources, DepOpts
+    ).
+
+find_match(Regex, FileStamps) ->
+    try
+        [throw(F) || {F, _} <- FileStamps, re:run(F, Regex) =/= nomatch],
+        undefined
+    catch
+        throw:F -> {ok, F}
+    end.
+
+matches([], _) ->
+    ok;
+matches([{R, Stamp} | T], FileStamps) ->
+    case find_match(R, FileStamps) of
+        {ok, F} ->
+            ?assertEqual(Stamp, proplists:get_value(F, FileStamps)),
+            matches(T, FileStamps);
+        undefined ->
+            ?assertEqual({R, Stamp}, FileStamps)
+    end.
+
+edges([], _, _) ->
+    ok;
+edges([{A,B}|T], Edges, Files) ->
+    {ok, AbsA} = find_match(A, Files),
+    {ok, AbsB} = find_match(B, Files),
+    ?assert(lists:member({AbsA, AbsB}, Edges)),
+    edges(T, Edges, Files).
+
+bump_file(F) ->
+    {ok, Bin} = file:read_file(F),
+    file:write_file(F, [Bin, "\n"]).
+
+next_second() ->
+    %% Sleep until the next second. Rather than just doing a
+    %% sleep(1000) call, sleep for the amount of time required
+    %% to reach the next second as seen by the OS; this can save us
+    %% a few hundred milliseconds per test by triggering shorter delays.
+    {Mega, Sec, Micro} = os:timestamp(),
+    Now = (Mega*1000000 + Sec)*1000 + round(Micro/1000),
+    Ms = (trunc(Now / 1000)*1000 + 1000) - Now,
+    %% add a 50ms for jitter since the exact amount sometimes causes failures
+    timer:sleep(max(Ms+50, 1000)).
+