Clear up old code and other shenanigns

src/rebar_compiler.erl

@@ -96,19 +96,22 @@ gather_in_dirs([{AppInfo, Ctx} | Rest], Acc) ->
 analyze_app({Compiler, G}, Contexts, AppInfo) ->
     AppName = rebar_app_info:name(AppInfo),
     BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
-    EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
+    OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)),
     BaseOpts = rebar_app_info:opts(AppInfo),
     #{src_dirs := SrcDirs,
       src_ext := SrcExt,
+      out_mappings := [{OutExt, OutPath}|_], % prune one dir for now (compat mode!)
       dependencies_opts := DepOpts} = maps:get(AppName, Contexts),
     %% Local resources
+    ArtifactDir = filename:join([OutDir, OutPath]),
     AbsSources = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
     LocalSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
     %% Multi-app resources
     InDirs = maps:get(in_dirs, Contexts),
     %% Prep the analysis
-    rebar_compiler_dag:prune(G, LocalSrcDirs, EbinDir, AbsSources),
-    rebar_compiler_dag:update(G, Compiler, InDirs, AbsSources, DepOpts),
+    rebar_compiler_dag:prune(
+        G, LocalSrcDirs, ArtifactDir, AbsSources, SrcExt, OutExt
+    ),
     %% Run the analysis
     rebar_compiler_dag:populate_sources(
         G, Compiler, InDirs, AbsSources, DepOpts

src/rebar_compiler_dag.erl

@@ -1,8 +1,8 @@
 %%% Module handling the directed graph required for the analysis
 %%% of all top-level applications by the various compiler plugins.
 -module(rebar_compiler_dag).
--export([init/4, prune/4, update/5, maybe_store/5, terminate/1]).
--export([populate_sources/5, populate_deps/3, propagate_stamps/1,
+-export([init/4, maybe_store/5, terminate/1]).
+-export([prune/6, populate_sources/5, populate_deps/3, propagate_stamps/1,
          compile_order/2]).
 
 -include("rebar.hrl").
@@ -19,7 +19,7 @@
 -record(dag, {vsn = ?DAG_VSN :: pos_integer(),
               info = {[], [], []} :: dag_rec()}).
 
-%% You should initialize one DAG per compiler module.
+%% @doc You should initialize one DAG per compiler module.
 %% `CritMeta' is any contextual information that, if it is found to change,
 %% must invalidate the DAG loaded from disk.
 -spec init(file:filename_all(), atom(), string() | undefined, critical_meta()) -> dag().
@@ -37,67 +37,28 @@ init(Dir, Compiler, Label, CritMeta) ->
     end,
     G.
 
--spec prune(dag(), file:filename_all(), file:filename_all(), [file:filename_all()]) -> ok.
-prune(G, SrcDirs, EbinDir, Erls) ->
+%% @doc Clear up inactive (deleted) source files from a given project.
+%% The `SrcDirs' must be all the directories that may contain source files
+%% for an OTP application; source files found in the DAG `G' that lie outside
+%% of this directory will be used.
+-spec prune(dag(), file:filename_all(), file:filename_all(),
+            [file:filename_all()], string(), string()) -> ok.
+prune(G, SrcDirs, OutDir, Erls, SrcExt, ArtifactExt) ->
     %% A source file may have been renamed or deleted. Remove it from the graph
     %% and remove any beam file for that source if it exists.
     Vertices = digraph:vertices(G),
     SrcParts = [filename:split(SrcDir) || SrcDir <- SrcDirs],
-    [maybe_rm_beam_and_edge(G, EbinDir, File)
+    [maybe_rm_artifact_and_edge(G, OutDir, ArtifactExt, File)
      || File <- lists:sort(Vertices) -- lists:sort(Erls),
-        filename:extension(File) =:= ".erl",
+        filename:extension(File) =:= SrcExt,
         lists:any(fun(Src) -> lists:prefix(Src, filename:split(File)) end,
                   SrcParts)],
     ok.
 
 %% @doc this function scans all the source files found and looks into
-%% all the `InDirs' for deps (other erl or .hrl files) that are related
-%% to them (by calling `CompileMod:dependencies()' on them).
-%%
-%% The trick here is that change detection, done with last_modified stamps,
-%% takes place at the same time as the graph propagation (finding deps)
-%% themselves. As such, this is a confusing mutually recursive depth-first
-%% search function that relies on side-effects and precise order-of-traversal
-%% to propagate file changes.
-%%
-%% To be replaced by a more declarative EPP-based flow.
--spec update(dag(), module(), [file:filename_all()], [file:filename_all()],
-             term()) -> ok.
-update(_, _, _, [], _) ->
-    ok;
-update(G, Compiler, InDirs, [Source|Erls], DepOpts) ->
-    case digraph:vertex(G, Source) of
-        {_, LastUpdated} ->
-            case filelib:last_modified(Source) of
-                0 ->
-                    %% The file doesn't exist anymore,
-                    %% erase it from the graph.
-                    %% All the edges will be erased automatically.
-                    digraph:del_vertex(G, Source),
-                    mark_dirty(G),
-                    update(G, Compiler, InDirs, Erls, DepOpts);
-                LastModified when LastUpdated < LastModified ->
-                    add_to_dag(G, Compiler, InDirs, Source, LastModified,
-                               filename:dirname(Source), DepOpts),
-                    update(G, Compiler, InDirs, Erls, DepOpts);
-                _ ->
-                    AltErls = digraph:out_neighbours(G, Source),
-                    %% Deps must be explored before the module itself
-                    update(G, Compiler, InDirs, AltErls, DepOpts),
-                    Modified = is_dirty(G),
-                    MaxModified = update_max_modified_deps(G, Source),
-                    case Modified orelse MaxModified > LastUpdated of
-                        true -> mark_dirty(G);
-                        false -> ok
-                    end,
-                    update(G, Compiler, InDirs, Erls, DepOpts)
-            end;
-        false ->
-            add_to_dag(G, Compiler, InDirs, Source, filelib:last_modified(Source),
-                       filename:dirname(Source), DepOpts),
-            update(G, Compiler, InDirs, Erls, DepOpts)
-    end.
-
+%% all the `InDirs' for deps (other source files, or files that aren't source
+%% but still returned by the compiler module) that are related
+%% to them.
 populate_sources(_G, _Compiler, _InDirs, [], _DepOpts) ->
     ok;
 populate_sources(G, Compiler, InDirs, [Source|Erls], DepOpts) ->
@@ -125,21 +86,9 @@ populate_sources(G, Compiler, InDirs, [Source|Erls], DepOpts) ->
     end,
     populate_sources(G, Compiler, InDirs, Erls, DepOpts).
 
-prepopulate_deps(G, Compiler, InDirs, Source, DepOpts) ->
-    SourceDir = filename:dirname(Source),
-    AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of
-        false ->
-            Compiler:dependencies(Source, SourceDir, InDirs);
-        true ->
-            Compiler:dependencies(Source, SourceDir, InDirs, DepOpts)
-    end,
-    %% the file hasn't been visited yet; set it to existing, but with
-    %% a last modified value that's null so it gets updated to something new.
-    [digraph:add_vertex(G, Src, 0) || Src <- AbsIncls,
-                                      digraph:vertex(G, Src) =:= false],
-    [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls],
-    ok.
-
+%% @doc Scan all files in the digraph that are seen as dependencies, but are
+%% neither source files nor artifacts (i.e. header files that don't produce
+%% artifacts of any kind).
 populate_deps(G, SourceExt, ArtifactExts) ->
     %% deps are files that are part of the digraph, but couldn't be scanned
     %% because they are neither source files (`SourceExt') nor mappings
@@ -155,8 +104,9 @@ populate_deps(G, SourceExt, ArtifactExts) ->
         not lists:member(Ext, IgnoredExts)],
     ok.
 
-%% Take the timestamps/diff changes and propagate them from a dep to the parent;
-%% given:
+
+%% @doc Take the timestamps/diff changes and propagate them from a dep to the
+%% parent; given:
 %%   A 0 -> B 1 -> C 3 -> D 2
 %% then we expect to get back:
 %%   A 3 -> B 3 -> C 3 -> D 2
@@ -179,6 +129,119 @@ propagate_stamps(G) ->
     end.
 
 
+%% @doc Return the reverse sorting order to get dep-free apps first.
+%% -- we would usually not need to consider the non-source files for the order to
+%% be complete, but using them doesn't hurt.
+compile_order(G, AppDefs) ->
+    Edges = [{V1,V2} || E <- digraph:edges(G),
+                        {_,V1,V2,_} <- [digraph:edge(G, E)]],
+    AppPaths = prepare_app_paths(AppDefs),
+    compile_order(Edges, AppPaths, #{}).
+
+%% @doc Store the DAG on disk if it was dirty
+maybe_store(G, Dir, Compiler, Label, CritMeta) ->
+    case is_dirty(G) of
+        true ->
+            clear_dirty(G),
+            File = dag_file(Dir, Compiler, Label),
+            store_dag(G, File, CritMeta);
+        false ->
+            ok
+    end.
+
+%% Get rid of the live state for the digraph; leave disk stuff in place.
+terminate(G) ->
+    true = digraph:delete(G).
+
+%%%%%%%%%%%%%%%
+%%% PRIVATE %%%
+%%%%%%%%%%%%%%%
+%% @private generate the name for the DAG based on the compiler module and
+%% a custom label, both of which are used to prevent various compiler runs
+%% from clobbering each other. The label `undefined' is kept for a default
+%% run of the compiler, to keep in line with previous versions of the file.
+dag_file(Dir, CompilerMod, undefined) ->
+    filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
+                   ?DAG_ROOT ++ ?DAG_EXT]);
+dag_file(Dir, CompilerMod, Label) ->
+    filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
+                   ?DAG_ROOT ++ "_" ++ Label ++ ?DAG_EXT]).
+
+restore_dag(G, File, CritMeta) ->
+    case file:read_file(File) of
+        {ok, Data} ->
+            %% The CritMeta value is checked and if it doesn't match, we fail
+            %% the whole restore operation.
+            #dag{vsn=?DAG_VSN, info={Vs, Es, CritMeta}} = binary_to_term(Data),
+            [digraph:add_vertex(G, V, LastUpdated) || {V, LastUpdated} <- Vs],
+            [digraph:add_edge(G, V1, V2) || {_, V1, V2, _} <- Es],
+            ok;
+        {error, _Err} ->
+            ok
+    end.
+
+store_dag(G, File, CritMeta) ->
+    ok = filelib:ensure_dir(File),
+    Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
+    Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
+    Data = term_to_binary(#dag{info={Vs, Es, CritMeta}}, [{compressed, 2}]),
+    file:write_file(File, Data).
+
+%% Drop a file from the digraph if it doesn't exist, and if so,
+%% delete its related build artifact
+maybe_rm_artifact_and_edge(G, OutDir, Ext, Source) ->
+    %% This is NOT a double check it is the only check that the source file is actually gone
+    case filelib:is_regular(Source) of
+        true ->
+            %% Actually exists, don't delete
+            false;
+        false ->
+            Target = target_base(OutDir, Source) ++ Ext,
+            ?DEBUG("Source ~ts is gone, deleting previous ~ts file if it exists ~ts", [Source, Ext, Target]),
+            file:delete(Target),
+            digraph:del_vertex(G, Source),
+            mark_dirty(G),
+            true
+    end.
+
+%% Add dependencies of a given file to the DAG. If the file is not found yet,
+%% mark its timestamp to 0, which means we have no info on it.
+%% Source files will be covered at a later point in their own scan, and
+%% non-source files are going to be covered by `populate_deps/3'.
+prepopulate_deps(G, Compiler, InDirs, Source, DepOpts) ->
+    SourceDir = filename:dirname(Source),
+    AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of
+        false ->
+            Compiler:dependencies(Source, SourceDir, InDirs);
+        true ->
+            Compiler:dependencies(Source, SourceDir, InDirs, DepOpts)
+    end,
+    %% the file hasn't been visited yet; set it to existing, but with
+    %% a last modified value that's null so it gets updated to something new.
+    [digraph:add_vertex(G, Src, 0) || Src <- AbsIncls,
+                                      digraph:vertex(G, Src) =:= false],
+    [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls],
+    ok.
+
+%% check that a dep file is up to date
+refresh_dep(G, File) ->
+    {_, LastUpdated} = digraph:vertex(G, File),
+    case filelib:last_modified(File) of
+        0 ->
+            %% Gone! Erase from the graph
+            digraph:del_vertex(G, File),
+            mark_dirty(G);
+        LastModified when LastUpdated < LastModified ->
+            digraph:add_vertex(G, File, LastModified),
+            mark_dirty(G);
+        _ ->
+            % unchanged
+            ok
+    end.
+
+%% Do the actual propagation of all files; the files are expected to be
+%% in a topological order such that we don't need to go more than a level
+%% deep in what we search.
 propagate_stamps(_G, []) ->
     ok;
 propagate_stamps(G, [File|Files]) ->
@@ -198,16 +261,10 @@ propagate_stamps(G, [File|Files]) ->
     end,
     propagate_stamps(G, Files).
 
-
-compile_order(G, AppDefs) ->
-    %% Return the reverse sorting order to get dep-free apps first.
-    %% -- we would usually not need to consider the non-source files for the order to
-    %% be complete, but using them doesn't hurt.
-    Edges = [{V1,V2} || E <- digraph:edges(G),
-                        {_,V1,V2,_} <- [digraph:edge(G, E)]],
-    AppPaths = prepare_app_paths(AppDefs),
-    compile_order(Edges, AppPaths, #{}).
-
+%% Do the actual reversal; be aware that only working from the edges
+%% may omit files, so we have to add all non-dependant apps manually
+%% to make sure we don't drop em. Since they have no deps, they're
+%% safer to put first (and compile first)
 compile_order([], AppPaths, AppDeps) ->
     %% use a digraph so we don't reimplement topsort by hand.
     G = digraph:new([acyclic]), % ignore cycles and hope it works
@@ -235,12 +292,23 @@ compile_order([{P1,P2}|T], AppPaths, AppDeps) ->
             end
     end.
 
+%% Swap app name with paths in the order, and sort there; this lets us
+%% bail out early in a search where a file won't be found.
 prepare_app_paths(AppPaths) ->
     lists:sort([{filename:split(Path), Name} || {Name, Path} <- AppPaths]).
 
+%% Look for the app to which the path belongs; needed to
+%% go from an edge between files in the DAG to building
+%% app-related orderings
 find_app(Path, AppPaths) ->
     find_app_(filename:split(Path), AppPaths).
 
+%% A cached search for the app to which a path belongs;
+%% the assumption is that sorted edges and common relationships
+%% are going to be between local files within an app most
+%% of the time; so we first look for the same path as a
+%% prior match to avoid searching _all_ potential candidates.
+%% If it doesn't work, go for the normal search.
 find_cached_app(Path, {Name, AppPath}, AppPaths) ->
     Split = filename:split(Path),
     case find_app_(Split, [{AppPath, Name}]) of
@@ -248,6 +316,7 @@ find_cached_app(Path, {Name, AppPath}, AppPaths) ->
         LastEntry -> LastEntry
     end.
 
+%% Do the actual recursive search
 find_app_(_Path, []) ->
     not_found;
 find_app_(Path, [{AppPath, AppName}|Rest]) ->
@@ -261,84 +330,6 @@ find_app_(Path, [{AppPath, AppName}|Rest]) ->
     end.
 
 
-refresh_dep(G, File) ->
-    {_, LastUpdated} = digraph:vertex(G, File),
-    case filelib:last_modified(File) of
-        0 ->
-            %% Gone! Erase from the graph
-            digraph:del_vertex(G, File),
-            mark_dirty(G);
-        LastModified when LastUpdated < LastModified ->
-            digraph:add_vertex(G, File, LastModified),
-            mark_dirty(G);
-        _ ->
-            % unchanged
-            ok
-    end.
-
-maybe_store(G, Dir, Compiler, Label, CritMeta) ->
-    case is_dirty(G) of
-        true ->
-            clear_dirty(G),
-            File = dag_file(Dir, Compiler, Label),
-            store_dag(G, File, CritMeta);
-        false ->
-            ok
-    end.
-
-terminate(G) ->
-    true = digraph:delete(G).
-
-%%%%%%%%%%%%%%%
-%%% PRIVATE %%%
-%%%%%%%%%%%%%%%
-%% @private generate the name for the DAG based on the compiler module and
-%% a custom label, both of which are used to prevent various compiler runs
-%% from clobbering each other. The label `undefined' is kept for a default
-%% run of the compiler, to keep in line with previous versions of the file.
-dag_file(Dir, CompilerMod, undefined) ->
-    filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
-                   ?DAG_ROOT ++ ?DAG_EXT]);
-dag_file(Dir, CompilerMod, Label) ->
-    filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
-                   ?DAG_ROOT ++ "_" ++ Label ++ ?DAG_EXT]).
-
-restore_dag(G, File, CritMeta) ->
-    case file:read_file(File) of
-        {ok, Data} ->
-            %% The CritMeta value is checked and if it doesn't match, we fail
-            %% the whole restore operation.
-            #dag{vsn=?DAG_VSN, info={Vs, Es, CritMeta}} = binary_to_term(Data),
-            [digraph:add_vertex(G, V, LastUpdated) || {V, LastUpdated} <- Vs],
-            [digraph:add_edge(G, V1, V2) || {_, V1, V2, _} <- Es],
-            ok;
-        {error, _Err} ->
-            ok
-    end.
-
-store_dag(G, File, CritMeta) ->
-    ok = filelib:ensure_dir(File),
-    Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
-    Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
-    Data = term_to_binary(#dag{info={Vs, Es, CritMeta}}, [{compressed, 2}]),
-    file:write_file(File, Data).
-
-%% Drop a file from the digraph if it doesn't exist, and if so,
-%% delete its related build artifact
-maybe_rm_beam_and_edge(G, OutDir, Source) ->
-    %% This is NOT a double check it is the only check that the source file is actually gone
-    case filelib:is_regular(Source) of
-        true ->
-            %% Actually exists, don't delete
-            false;
-        false ->
-            Target = target_base(OutDir, Source) ++ ".beam",
-            ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]),
-            file:delete(Target),
-            digraph:del_vertex(G, Source),
-            mark_dirty(G),
-            true
-    end.
 
 %% @private Return what should be the base name of an erl file, relocated to the
 %% target directory. For example:
@@ -346,47 +337,6 @@ maybe_rm_beam_and_edge(G, OutDir, Source) ->
 target_base(OutDir, Source) ->
     filename:join(OutDir, filename:basename(Source, ".erl")).
 
-%% @private a file has been found to change or wasn't part of the DAG before,
-%% and must be added, along with all its dependencies.
-add_to_dag(G, Compiler, InDirs, Source, LastModified, SourceDir, DepOpts) ->
-    AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of
-        false ->
-            Compiler:dependencies(Source, SourceDir, InDirs);
-        true ->
-            Compiler:dependencies(Source, SourceDir, InDirs, DepOpts)
-    end,
-    digraph:add_vertex(G, Source, LastModified),
-    digraph:del_edges(G, digraph:out_edges(G, Source)),
-    %% Deps must be explored before the module itself
-    [begin
-         update(G, Compiler, InDirs, [Incl], DepOpts),
-         digraph:add_edge(G, Source, Incl)
-     end || Incl <- AbsIncls],
-    mark_dirty(G),
-    AbsIncls.
-
-%% @private change status propagation: if the dependencies of a file have
-%% been updated, mark the last_modified time for that file to be equivalent
-%% to its most-recently-changed dependency; that way, nested header file
-%% change stamps are propagated to the final module.
-%% This is required because at some point the module is compared to its
-%% associated .beam file's last-generation stamp to know if it requires
-%% rebuilding.
-%% The responsibility for this is however diffuse across various modules.
-update_max_modified_deps(G, Source) ->
-    MaxModified = lists:foldl(
-        fun(File, Acc) ->
-            case digraph:vertex(G, File) of
-                {_, MaxModified} when MaxModified > Acc -> MaxModified;
-                _ -> Acc
-            end
-        end,
-        0,
-        [Source | digraph:out_neighbours(G, Source)]
-    ),
-    digraph:add_vertex(G, Source, MaxModified),
-    MaxModified.
-
 %% Mark the digraph as having been modified, which is required to
 %% save its updated form on disk after the compiling run.
 %% This uses a magic vertex to carry the dirty state. This is less