Merge pull request #2246 from ferd/dag-artifact-tracking

Track build artifacts in DAG

src/rebar_compiler.erl

@@ -31,10 +31,14 @@
 -callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()].
 -callback dependencies(file:filename(), file:dirname(), [file:dirname()], term()) -> [file:filename()].
 -callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
-    ok | {ok, [string()]} | {ok, [string()], [string()]}.
+    ok | {ok, [string()]} | error | {error, [string()], [string()]} | skipped.
+-callback compile_and_track(file:filename(), out_mappings(), rebar_dict(), list()) ->
+    {ok, [{file:filename(), file:filename(), term()}]} |
+    {ok, [{file:filename(), file:filename(), term()}], [string()]} |
+    {error, [string()], [string()]} | error.
 -callback clean([file:filename()], rebar_app_info:t()) -> _.
 
--optional_callbacks([dependencies/4]).
+-optional_callbacks([dependencies/4, compile_and_track/4]).
 
 -define(RE_PREFIX, "^(?!\\._)").
 
@@ -180,17 +184,28 @@ run(G, CompilerMod, AppInfo, Contexts) ->
      {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo),
 
     compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
-    case RestFiles of
-        {Sequential, Parallel} -> % new parallelizable form
-            compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod),
+    Tracked = case RestFiles of
+        {Sequential, Parallel} -> % parallelizable form
+            compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod) ++
             compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod);
         _ when is_list(RestFiles) -> % traditional sequential build
             compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod)
-    end.
+    end,
+    store_artifacts(G, Tracked).
 
 compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
-    ok;
+    [];
 compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
+    case erlang:function_exported(CompilerMod, compile_and_track, 4) of
+        false ->
+            do_compile(CompilerMod, Source, Outs, Config, Opts),
+            compile_each(Rest, Opts, Config, Outs, CompilerMod);
+        true ->
+            do_compile_and_track(CompilerMod, Source, Outs, Config, Opts)
+            ++ compile_each(Rest, Opts, Config, Outs, CompilerMod)
+    end.
+
+do_compile(CompilerMod, Source, Outs, Config, Opts) ->
     case CompilerMod:compile(Source, Outs, Config, Opts) of
         ok ->
             ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
@@ -205,14 +220,47 @@ compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
             maybe_report(Error),
             ?DEBUG("Compilation failed: ~p", [Error]),
             ?FAIL
-    end,
-    compile_each(Rest, Opts, Config, Outs, CompilerMod).
+    end.
+
+do_compile_and_track(CompilerMod, Source, Outs, Config, Opts) ->
+    case CompilerMod:compile_and_track(Source, Outs, Config, Opts) of
+        {ok, Tracked} ->
+            ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
+            Tracked;
+        {ok, Tracked, Warnings} ->
+            report(Warnings),
+            ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
+            Tracked;
+        skipped ->
+            ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
+            [];
+        Error ->
+            NewSource = format_error_source(Source, Config),
+            ?ERROR("Compiling ~ts failed", [NewSource]),
+            maybe_report(Error),
+            ?DEBUG("Compilation failed: ~p", [Error]),
+            ?FAIL
+    end.
+
+store_artifacts(_G, []) ->
+    ok;
+store_artifacts(G, [{Source, Target, Meta}|Rest]) ->
+    %% Assume the source exists since it was tracked to be compiled
+    digraph:add_vertex(G, Target, {artifact, Meta}),
+    digraph:add_edge(G, Target, Source, artifact),
+    store_artifacts(G, Rest).
 
 compile_worker(QueuePid, Opts, Config, Outs, CompilerMod) ->
     QueuePid ! self(),
     receive
         {compile, Source} ->
-            Result = CompilerMod:compile(Source, Outs, Config, Opts),
+            Result =
+            case erlang:function_exported(CompilerMod, compile_and_track, 4) of
+                false ->
+                    CompilerMod:compile(Source, Outs, Config, Opts);
+                true ->
+                    CompilerMod:compile_and_track(Source, Outs, Config, Opts)
+            end,
             QueuePid ! {Result, Source},
             compile_worker(QueuePid, Opts, Config, Outs, CompilerMod);
         empty ->
@@ -220,7 +268,7 @@ compile_worker(QueuePid, Opts, Config, Outs, CompilerMod) ->
     end.
 
 compile_parallel([], _Opts, _BaseOpts, _Mappings, _CompilerMod) ->
-    ok;
+    [];
 compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) ->
     Self = self(),
     F = fun() -> compile_worker(Self, Opts, BaseOpts, Mappings, CompilerMod) end,
@@ -230,8 +278,9 @@ compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) ->
     compile_queue(Targets, Pids, Opts, BaseOpts, Mappings, CompilerMod).
 
 compile_queue([], [], _Opts, _Config, _Outs, _CompilerMod) ->
-    ok;
+    [];
 compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) ->
+    Tracking = erlang:function_exported(CompilerMod, compile_and_track, 4),
     receive
         Worker when is_pid(Worker), Targets =:= [] ->
             Worker ! empty,
@@ -242,10 +291,19 @@ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) ->
         {ok, Source} ->
             ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]),
             compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
-        {{ok, Warnings}, Source} ->
+        {{ok, Tracked}, Source} when Tracking ->
+            ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]),
+            Tracked ++
+              compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
+        {{ok, Warnings}, Source} when not Tracking ->
             report(Warnings),
             ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]),
             compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
+        {{ok, Tracked, Warnings}, Source} when Tracking ->
+            report(Warnings),
+            ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]),
+            Tracked ++
+              compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
         {skipped, Source} ->
             ?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), Source]),
             compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);

src/rebar_compiler_dag.erl

@@ -65,7 +65,7 @@ is_deleted_source(_G, _F, Extension, _SrcExt, Extension) ->
     %% artifact file - skip
     false;
 is_deleted_source(G, F, _Extension, _SrcExt, _ArtifactExt) ->
-    %% must be header file
+    %% must be header file or artifact
     digraph:in_edges(G, F) == [] andalso maybe_rm_vertex(G, F),
     false.
 
@@ -135,7 +135,7 @@ populate_deps(G, SourceExt, ArtifactExts) ->
     %% in depth already, and improvements should be driven at that level)
     IgnoredExts = [SourceExt | ArtifactExts],
     Vertices = digraph:vertices(G),
-    [refresh_dep(G, File)
+    [refresh_dep(G, digraph:vertex(G, File))
      || File <- Vertices,
         Ext <- [filename:extension(File)],
         not lists:member(Ext, IgnoredExts)],
@@ -169,6 +169,8 @@ propagate_stamps(G) ->
 %% @doc Return the reverse sorting order to get dep-free apps first.
 %% -- we would usually not need to consider the non-source files for the order to
 %% be complete, but using them doesn't hurt.
+compile_order(_, AppDefs) when length(AppDefs) =< 1 ->
+    [Name || {Name, _Path} <- AppDefs];
 compile_order(G, AppDefs) ->
     Edges = [{V1,V2} || E <- digraph:edges(G),
                         {_,V1,V2,_} <- [digraph:edge(G, E)]],
@@ -211,7 +213,7 @@ restore_dag(G, File, CritMeta) ->
             %% the whole restore operation.
             #dag{vsn=?DAG_VSN, info={Vs, Es, CritMeta}} = binary_to_term(Data),
             [digraph:add_vertex(G, V, LastUpdated) || {V, LastUpdated} <- Vs],
-            [digraph:add_edge(G, V1, V2) || {_, V1, V2, _} <- Es],
+            [digraph:add_edge(G, V1, V2, Label) || {_, V1, V2, Label} <- Es],
             ok;
         {error, _Err} ->
             ok
@@ -233,9 +235,21 @@ maybe_rm_artifact_and_edge(G, OutDir, SrcExt, Ext, Source) ->
             %% Actually exists, don't delete
             false;
         false ->
-            Target = target(OutDir, Source, SrcExt, Ext),
-            ?DEBUG("Source ~ts is gone, deleting previous ~ts file if it exists ~ts", [Source, Ext, Target]),
-            file:delete(Target),
+            Edges = digraph:in_edges(G, Source),
+            Targets = [V1 || Edge <- Edges,
+                             {_E, V1, _V2, artifact} <- [digraph:edge(G, Edge)]],
+            case Targets of
+                [] ->
+                    Target = target(OutDir, Source, SrcExt, Ext),
+                    ?DEBUG("Source ~ts is gone, deleting previous ~ts file if it exists ~ts", [Source, Ext, Target]),
+                    file:delete(Target);
+                [_|_] ->
+                    lists:foreach(fun(Target) ->
+                        ?DEBUG("Source ~ts is gone, deleting artifact ~ts "
+                                "if it exists", [Source, Target]),
+                        file:delete(Target)
+                    end, Targets)
+            end,
             digraph:del_vertex(G, Source),
             mark_dirty(G),
             true
@@ -269,15 +283,17 @@ prepopulate_deps(G, Compiler, InDirs, Source, DepOpts, Status) ->
     %% drop edges from deps that aren't included!
     [digraph:del_edge(G, Edge) || Status == old,
                                   Edge <- digraph:out_edges(G, Source),
-                                  {_, _Src, Path, _} <- [digraph:edge(G, Edge)],
+                                  {_, _Src, Path, _Label} <- [digraph:edge(G, Edge)],
                                   not lists:member(Path, AbsIncls)],
     %% Add the rest
     [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls],
     ok.
 
 %% check that a dep file is up to date
-refresh_dep(G, File) ->
-    {_, LastUpdated} = digraph:vertex(G, File),
+refresh_dep(_G, {artifact, _}) ->
+    %% ignore artifacts
+    ok;
+refresh_dep(G, {File, LastUpdated}) ->
     case filelib:last_modified(File) of
         0 ->
             %% Gone! Erase from the graph
@@ -287,7 +303,7 @@ refresh_dep(G, File) ->
             digraph:add_vertex(G, File, LastModified),
             mark_dirty(G);
         _ ->
-            % unchanged
+            %% unchanged
             ok
     end.
 
@@ -297,14 +313,18 @@ refresh_dep(G, File) ->
 propagate_stamps(_G, []) ->
     ok;
 propagate_stamps(G, [File|Files]) ->
-    Stamps = [element(2, digraph:vertex(G, F))
-              || F <- digraph:out_neighbours(G, File)],
+    Stamps = [Stamp
+              || F <- digraph:out_neighbours(G, File),
+                 {_, Stamp} <- [digraph:vertex(G, F)],
+                 is_tuple(Stamp) andalso element(1, Stamp) =/= artifact],
     case Stamps of
         [] ->
             ok;
         _ ->
             Max = lists:max(Stamps),
             case digraph:vertex(G, File) of
+                {_, {artifact, _}} ->
+                    ok;
                 {_, Smaller} when Smaller < Max ->
                     digraph:add_vertex(G, File, Max);
                 _ ->

src/rebar_compiler_erl.erl

@@ -5,7 +5,7 @@
 -export([context/1,
          needed_files/4,
          dependencies/3, dependencies/4,
-         compile/4,
+         compile/4, compile_and_track/4,
          clean/2,
          format_error/1]).
 
@@ -70,7 +70,11 @@ needed_files(Graph, FoundFiles, _, AppInfo) ->
     %% that none other depend of; the former must be sequentially
     %% built, the rest is parallelizable.
     OtherErls = lists:partition(
-        fun(Erl) -> digraph:in_degree(Graph, Erl) > 0 end,
+        fun(Erl) -> lists:any(
+            fun(Edge) ->
+                {_E, _V1, _V2, Kind} = digraph:edge(Graph, Edge),
+                Kind =/= artifact
+            end, digraph:in_edges(Graph, Erl)) end,
         lists:reverse([Dep || Dep <- DepErlsOrdered,
                               not lists:member(Dep, ErlFirstFiles)])
     ),
@@ -136,6 +140,29 @@ compile(Source, [{_, OutDir}], Config, ErlOpts) ->
             error
     end.
 
+compile_and_track(Source, [{Ext, OutDir}], Config, ErlOpts) ->
+    BuildOpts = [{outdir, OutDir} | ErlOpts],
+    Target = target_base(OutDir, Source) ++ Ext,
+    AllOpts = case erlang:function_exported(compile, env_compiler_options, 0) of
+        true  -> BuildOpts ++ compile:env_compiler_options();
+        false -> BuildOpts
+    end,
+    case compile:file(Source, BuildOpts) of
+        {ok, _Mod} ->
+            {ok, [{Source, Target, AllOpts}]};
+        {ok, _Mod, []} ->
+            {ok, [{Source, Target, AllOpts}]};
+        {ok, _Mod, Ws} ->
+            FormattedWs = format_error_sources(Ws, Config),
+            {ok, Warns} = rebar_compiler:ok_tuple(Source, FormattedWs),
+            {ok, [{Source, Target, AllOpts}], Warns};
+        {error, Es, Ws} ->
+            error_tuple(Source, Es, Ws, Config, ErlOpts);
+        error ->
+            error
+    end.
+
+
 clean(Files, AppInfo) ->
     EbinDir = rebar_app_info:ebin_dir(AppInfo),
     [begin
@@ -199,22 +226,29 @@ needed_files(Graph, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) ->
                                    ,{i, filename:join(Dir, "include")}
                                    ,{i, Dir}] ++ PrivIncludes ++ ErlOpts,
                          digraph:vertex(Graph, Source) > {Source, filelib:last_modified(Target)}
-                              orelse opts_changed(AllOpts, TargetBase)
+                              orelse opts_changed(Graph, AllOpts, Target, TargetBase)
                               orelse erl_compiler_opts_set()
                  end, SourceFiles).
 
 target_base(OutDir, Source) ->
     filename:join(OutDir, filename:basename(Source, ".erl")).
 
-opts_changed(NewOpts, Target) ->
+opts_changed(Graph, NewOpts, Target, TargetBase) ->
     TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of
         true  -> NewOpts ++ compile:env_compiler_options();
         false -> NewOpts
     end,
-    case compile_info(Target) of
-        {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts));
-        _          -> true
-    end.
+    TargetOpts = case digraph:vertex(Graph, Target) of
+        {_Target, {artifact, Opts}} -> % tracked dep is found
+            Opts;
+        false -> % not found; might be a non-tracked DAG
+            case compile_info(TargetBase) of
+                {ok, Opts} -> Opts;
+                _ -> []
+            end
+    end,
+    lists:any(fun effects_code_generation/1,
+              lists:usort(TotalOpts) -- lists:usort(TargetOpts)).
 
 effects_code_generation(Option) ->
     case Option of

src/rebar_erlc_compiler.erl

@@ -206,7 +206,13 @@ compile_dirs(RebarOpts, BaseDir, SrcDirs, OutDir, CompileOpts) ->
 
     {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, BaseDir, NeededErlFiles),
     {DepErls, OtherErls} = lists:partition(
-                             fun(Source) -> digraph:in_degree(G, Source) > 0 end,
+                             fun(Source) -> lists:any(
+                                 fun(Edge) ->
+                                     {_E, _V1, _V2, Kind} = digraph:edge(G, Edge),
+                                     Kind =/= artifact
+                                 end,
+                                 digraph:in_edges(G, Source))
+                             end,
                              [File || File <- NeededErlFiles, not lists:member(File, ErlFirstFiles)]),
     SubGraph = digraph_utils:subgraph(G, DepErls),
     DepErlsOrdered = digraph_utils:topsort(SubGraph),

test/rebar_compile_SUITE.erl

@@ -20,7 +20,7 @@ all() ->
      recompile_when_opts_included_hrl_changes,
      recompile_when_foreign_included_hrl_changes,
      recompile_when_foreign_behaviour_changes,
-     recompile_when_opts_change,
+     recompile_when_opts_change, recompile_when_dag_opts_change,
      dont_recompile_when_opts_dont_change, dont_recompile_yrl_or_xrl,
      delete_beam_if_source_deleted,
      deps_in_path, checkout_priority, highest_version_of_pkg_dep,
@@ -919,6 +919,41 @@ recompile_when_opts_change(Config) ->
 
     ?assert(ModTime =/= NewModTime).
 
+recompile_when_dag_opts_change(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),
+
+    EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]),
+    {ok, Files} = rebar_utils:list_dir(EbinDir),
+    Beams = [filename:join([EbinDir, F])
+             || F <- Files, filename:extension(F) == ".beam"],
+    ModTime = [filelib:last_modified(Beam) || Beam <- Beams],
+
+    timer:sleep(1000),
+
+    DepsDir = filename:join([AppDir, "_build", "default", "lib"]),
+    G = rebar_compiler_dag:init(DepsDir, rebar_compiler_erl, "project_apps", []),
+    %% change the config in the DAG...
+    [digraph:add_vertex(G, Beam, {artifact, [{d, some_define}]}) || Beam <- Beams],
+    digraph:add_vertex(G, '$r3_dirty_bit', true), % trigger a save
+    rebar_compiler_dag:maybe_store(G, DepsDir, rebar_compiler_erl, "project_apps", []),
+    rebar_compiler_dag:terminate(G),
+    %% ... but don't change the actual rebar3 config...
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),
+
+    %% ... and checks that it rebuilds anyway due to DAG changes
+    {ok, NewFiles} = rebar_utils:list_dir(EbinDir),
+    NewBeams = [filename:join([EbinDir, F])
+                || F <- NewFiles, filename:extension(F) == ".beam"],
+    NewModTime = [filelib:last_modified(Beam) || Beam <- NewBeams],
+
+    ?assert(ModTime =/= NewModTime).
+
 dont_recompile_when_opts_dont_change(Config) ->
     AppDir = ?config(apps, Config),