Track build artifacts in DAG

This allows to do quicker re-compile option validation by not requiring
to access the disk and check all candidate erlang files for option
changes. This also opens up the way to compilers that produce more than
one artifact per file.

src/rebar_compiler.erl

@@ -180,17 +180,28 @@ run(G, CompilerMod, AppInfo, Contexts) ->
      {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo),
 
     compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
-    case RestFiles of
-        {Sequential, Parallel} -> % new parallelizable form
-            compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod),
+    Tracked = case RestFiles of
+        {Sequential, Parallel} -> % parallelizable form
+            compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod) ++
             compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod);
         _ when is_list(RestFiles) -> % traditional sequential build
             compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod)
-    end.
+    end,
+    store_artifacts(G, Tracked).
 
 compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
-    ok;
+    [];
 compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
+    case erlang:function_exported(CompilerMod, compile_and_track, 4) of
+        false ->
+            do_compile(CompilerMod, Source, Outs, Config, Opts),
+            compile_each(Rest, Opts, Config, Outs, CompilerMod);
+        true ->
+            do_compile_and_track(CompilerMod, Source, Outs, Config, Opts)
+            ++ compile_each(Rest, Opts, Config, Outs, CompilerMod)
+    end.
+
+do_compile(CompilerMod, Source, Outs, Config, Opts) ->
     case CompilerMod:compile(Source, Outs, Config, Opts) of
         ok ->
             ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]);
@@ -205,14 +216,47 @@ compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) ->
             maybe_report(Error),
             ?DEBUG("Compilation failed: ~p", [Error]),
             ?FAIL
-    end,
-    compile_each(Rest, Opts, Config, Outs, CompilerMod).
+    end.
+
+do_compile_and_track(CompilerMod, Source, Outs, Config, Opts) ->
+    case CompilerMod:compile_and_track(Source, Outs, Config, Opts) of
+        {ok, Tracked} ->
+            ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
+            Tracked;
+        {ok, Tracked, Warnings} ->
+            report(Warnings),
+            ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
+            Tracked;
+        skipped ->
+            ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]),
+            [];
+        Error ->
+            NewSource = format_error_source(Source, Config),
+            ?ERROR("Compiling ~ts failed", [NewSource]),
+            maybe_report(Error),
+            ?DEBUG("Compilation failed: ~p", [Error]),
+            ?FAIL
+    end.
+
+store_artifacts(_G, []) ->
+    ok;
+store_artifacts(G, [{Source, Target, Meta}|Rest]) ->
+    %% Assume the source exists since it was tracked to be compiled
+    digraph:add_vertex(G, Target, Meta),
+    digraph:add_edge(G, Source, Target, artifact),
+    store_artifacts(G, Rest).
 
 compile_worker(QueuePid, Opts, Config, Outs, CompilerMod) ->
     QueuePid ! self(),
     receive
         {compile, Source} ->
-            Result = CompilerMod:compile(Source, Outs, Config, Opts),
+            Result =
+            case erlang:function_exported(CompilerMod, compile_and_track, 4) of
+                false ->
+                    CompilerMod:compile(Source, Outs, Config, Opts);
+                true ->
+                    CompilerMod:compile_and_track(Source, Outs, Config, Opts)
+            end,
             QueuePid ! {Result, Source},
             compile_worker(QueuePid, Opts, Config, Outs, CompilerMod);
         empty ->
@@ -220,7 +264,7 @@ compile_worker(QueuePid, Opts, Config, Outs, CompilerMod) ->
     end.
 
 compile_parallel([], _Opts, _BaseOpts, _Mappings, _CompilerMod) ->
-    ok;
+    [];
 compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) ->
     Self = self(),
     F = fun() -> compile_worker(Self, Opts, BaseOpts, Mappings, CompilerMod) end,
@@ -230,8 +274,9 @@ compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) ->
     compile_queue(Targets, Pids, Opts, BaseOpts, Mappings, CompilerMod).
 
 compile_queue([], [], _Opts, _Config, _Outs, _CompilerMod) ->
-    ok;
+    [];
 compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) ->
+    Tracking = erlang:function_exported(CompilerMod, compile_and_track, 4),
     receive
         Worker when is_pid(Worker), Targets =:= [] ->
             Worker ! empty,
@@ -242,10 +287,19 @@ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) ->
         {ok, Source} ->
             ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]),
             compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
-        {{ok, Warnings}, Source} ->
+        {{ok, Tracked}, Source} when Tracking ->
+            ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]),
+            Tracked ++
+              compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
+        {{ok, Warnings}, Source} when not Tracking ->
             report(Warnings),
             ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]),
             compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
+        {{ok, Tracked, Warnings}, Source} when Tracking ->
+            report(Warnings),
+            ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]),
+            Tracked ++
+              compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);
         {skipped, Source} ->
             ?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), Source]),
             compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod);

src/rebar_compiler_dag.erl

@@ -233,9 +233,21 @@ maybe_rm_artifact_and_edge(G, OutDir, SrcExt, Ext, Source) ->
             %% Actually exists, don't delete
             false;
         false ->
-            Target = target(OutDir, Source, SrcExt, Ext),
-            ?DEBUG("Source ~ts is gone, deleting previous ~ts file if it exists ~ts", [Source, Ext, Target]),
-            file:delete(Target),
+            Edges = digraph:out_edges(G, Source),
+            Targets = [V2 || Edge <- Edges,
+                             {_E, _V1, V2, artifact} <- [digraph:edge(G, Edge)]],
+            case Targets of
+                [] ->
+                    Target = target(OutDir, Source, SrcExt, Ext),
+                    ?DEBUG("Source ~ts is gone, deleting previous ~ts file if it exists ~ts", [Source, Ext, Target]),
+                    file:delete(Target);
+                [_|_] ->
+                    lists:foreach(fun(Target) ->
+                        ?DEBUG("Source ~ts is gone, deleting artiface ~ts "
+                                "if it exists ~ts", [Source, Target]),
+                        file:delete(Target)
+                    end, Targets)
+            end,
             digraph:del_vertex(G, Source),
             mark_dirty(G),
             true
@@ -269,7 +281,8 @@ prepopulate_deps(G, Compiler, InDirs, Source, DepOpts, Status) ->
     %% drop edges from deps that aren't included!
     [digraph:del_edge(G, Edge) || Status == old,
                                   Edge <- digraph:out_edges(G, Source),
-                                  {_, _Src, Path, _} <- [digraph:edge(G, Edge)],
+                                  {_, _Src, Path, Label} <- [digraph:edge(G, Edge)],
+                                  Label =/= artifact,
                                   not lists:member(Path, AbsIncls)],
     %% Add the rest
     [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls],

src/rebar_compiler_erl.erl

@@ -5,7 +5,7 @@
 -export([context/1,
          needed_files/4,
          dependencies/3, dependencies/4,
-         compile/4,
+         compile/4, compile_and_track/4,
          clean/2,
          format_error/1]).
 
@@ -136,6 +136,28 @@ compile(Source, [{_, OutDir}], Config, ErlOpts) ->
             error
     end.
 
+compile_and_track(Source, [{Ext, OutDir}], Config, ErlOpts) ->
+    BuildOpts = [{outdir, OutDir} | ErlOpts],
+    Target = target_base(OutDir, Source) ++ Ext,
+    case compile:file(Source, BuildOpts) of
+        {ok, _Mod} ->
+            AllOpts = BuildOpts ++ compile:env_compiler_options(),
+            {ok, [{Source, Target, AllOpts}]};
+        {ok, _Mod, []} ->
+            AllOpts = BuildOpts ++ compile:env_compiler_options(),
+            {ok, [{Source, Target, AllOpts}]};
+        {ok, _Mod, Ws} ->
+            FormattedWs = format_error_sources(Ws, Config),
+            {ok, Warns} = rebar_compiler:ok_tuple(Source, FormattedWs),
+            AllOpts = BuildOpts ++ compile:env_compiler_options(),
+            {ok, [{Source, Target, AllOpts}], Warns};
+        {error, Es, Ws} ->
+            error_tuple(Source, Es, Ws, Config, ErlOpts);
+        error ->
+            error
+    end.
+
+
 clean(Files, AppInfo) ->
     EbinDir = rebar_app_info:ebin_dir(AppInfo),
     [begin
@@ -199,22 +221,29 @@ needed_files(Graph, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) ->
                                    ,{i, filename:join(Dir, "include")}
                                    ,{i, Dir}] ++ PrivIncludes ++ ErlOpts,
                          digraph:vertex(Graph, Source) > {Source, filelib:last_modified(Target)}
-                              orelse opts_changed(AllOpts, TargetBase)
+                              orelse opts_changed(Graph, AllOpts, Target, TargetBase)
                               orelse erl_compiler_opts_set()
                  end, SourceFiles).
 
 target_base(OutDir, Source) ->
     filename:join(OutDir, filename:basename(Source, ".erl")).
 
-opts_changed(NewOpts, Target) ->
+opts_changed(Graph, NewOpts, Target, TargetBase) ->
     TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of
         true  -> NewOpts ++ compile:env_compiler_options();
         false -> NewOpts
     end,
-    case compile_info(Target) of
-        {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts));
-        _          -> true
-    end.
+    TargetOpts = case digraph:vertex(Graph, Target) of
+        {_Target, Opts} -> % tracked dep is found
+            Opts;
+        false -> % not found; might be a non-tracked DAG
+            case compile_info(TargetBase) of
+                {ok, Opts} -> Opts;
+                _ -> []
+            end
+    end,
+    lists:any(fun effects_code_generation/1,
+              lists:usort(TotalOpts) -- lists:usort(TargetOpts)).
 
 effects_code_generation(Option) ->
     case Option of

test/rebar_compile_SUITE.erl

@@ -20,7 +20,7 @@ all() ->
      recompile_when_opts_included_hrl_changes,
      recompile_when_foreign_included_hrl_changes,
      recompile_when_foreign_behaviour_changes,
-     recompile_when_opts_change,
+     recompile_when_opts_change, recompile_when_dag_opts_change,
      dont_recompile_when_opts_dont_change, dont_recompile_yrl_or_xrl,
      delete_beam_if_source_deleted,
      deps_in_path, checkout_priority, highest_version_of_pkg_dep,
@@ -919,6 +919,41 @@ recompile_when_opts_change(Config) ->
 
     ?assert(ModTime =/= NewModTime).
 
+recompile_when_dag_opts_change(Config) ->
+    AppDir = ?config(apps, Config),
+
+    Name = rebar_test_utils:create_random_name("app1_"),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]),
+
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),
+
+    EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]),
+    {ok, Files} = rebar_utils:list_dir(EbinDir),
+    Beams = [filename:join([EbinDir, F])
+             || F <- Files, filename:extension(F) == ".beam"],
+    ModTime = [filelib:last_modified(Beam) || Beam <- Beams],
+
+    timer:sleep(1000),
+
+    DepsDir = filename:join([AppDir, "_build", "default", "lib"]),
+    G = rebar_compiler_dag:init(DepsDir, rebar_compiler_erl, "project_apps", []),
+    %% change the config in the DAG...
+    [digraph:add_vertex(G, Beam, [{d, some_define}]) || Beam <- Beams],
+    digraph:add_vertex(G, '$r3_dirty_bit', true), % trigger a save
+    rebar_compiler_dag:maybe_store(G, DepsDir, rebar_compiler_erl, "project_apps", []),
+    rebar_compiler_dag:terminate(G),
+    %% ... but don't change the actual rebar3 config...
+    rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),
+
+    %% ... and checks that it rebuilds anyway due to DAG changes
+    {ok, NewFiles} = rebar_utils:list_dir(EbinDir),
+    NewBeams = [filename:join([EbinDir, F])
+                || F <- NewFiles, filename:extension(F) == ".beam"],
+    NewModTime = [filelib:last_modified(Beam) || Beam <- NewBeams],
+
+    ?assert(ModTime =/= NewModTime).
+
 dont_recompile_when_opts_dont_change(Config) ->
     AppDir = ?config(apps, Config),