rebar3/src/rebar_compiler_dag.erl

%%% Module handling the directed graph required for the analysis
%%% of all top-level applications by the various compiler plugins.
-module(rebar_compiler_dag).
-export([init/4, prune/4, update/4, maybe_store/5, terminate/1]).

-include("rebar.hrl").

-define(DAG_VSN, 3).
-define(DAG_ROOT, "source").
-define(DAG_EXT, ".dag").

-type dag_v() :: {digraph:vertex(), term()} | 'false'.
-type dag_e() :: {digraph:vertex(), digraph:vertex()}.
-type critical_meta() :: term(). % if this changes, the DAG is invalid
-type dag_rec() :: {list(dag_v()), list(dag_e()), critical_meta()}.
-type dag() :: digraph:graph().
-record(dag, {vsn = ?DAG_VSN :: pos_integer(),
              info = {[], [], []} :: dag_rec()}).

%% You should initialize one DAG per compiler module.
%% `CritMeta' is any contextual information that, if it is found to change,
%% must invalidate the DAG loaded from disk.
-spec init(file:filename_all(), atom(), string() | undefined, critical_meta()) -> dag().
init(Dir, Compiler, Label, CritMeta) ->
    G = digraph:new([acyclic]),
    File = dag_file(Dir, Compiler, Label),
    try
        restore_dag(G, File, CritMeta)
    catch
        _:_ ->
            %% Don't mark as dirty yet to avoid creating compiler DAG files for
            %% compilers that are actually never used.
            ?WARN("Failed to restore ~ts file. Discarding it.~n", [File]),
            file:delete(File)
    end,
    G.

-spec prune(dag(), file:filename_all(), file:filename_all(), [file:filename_all()]) -> ok.
prune(G, SrcDirs, EbinDir, Erls) ->
    %% A source file may have been renamed or deleted. Remove it from the graph
    %% and remove any beam file for that source if it exists.
    Vertices = digraph:vertices(G),
    SrcParts = [filename:split(SrcDir) || SrcDir <- SrcDirs],
    [maybe_rm_beam_and_edge(G, EbinDir, File)
     || File <- lists:sort(Vertices) -- lists:sort(Erls),
        filename:extension(File) =:= ".erl",
        lists:any(fun(Src) -> lists:prefix(Src, filename:split(File)) end,
                  SrcParts)],
    ok.

%% @doc this function scans all the source files found and looks into
%% all the `InDirs' for deps (other erl or .hrl files) that are related
%% to them (by calling `CompileMod:dependencies()' on them).
%%
%% The trick here is that change detection, done with last_modified stamps,
%% takes place at the same time as the graph propagation (finding deps)
%% themselves. As such, this is a confusing mutually recursive depth-first
%% search function that relies on side-effects and precise order-of-traversal
%% to propagate file changes.
%%
%% To be replaced by a more declarative EPP-based flow.
-spec update(dag(), module(), [file:filename_all()], [file:filename_all()]) -> ok.
update(_, _, _, []) ->
    ok;
update(G, Compiler, InDirs, [Source|Erls]) ->
    case digraph:vertex(G, Source) of
        {_, LastUpdated} ->
            case filelib:last_modified(Source) of
                0 ->
                    %% The file doesn't exist anymore,
                    %% erase it from the graph.
                    %% All the edges will be erased automatically.
                    digraph:del_vertex(G, Source),
                    mark_dirty(G),
                    update(G, Compiler, InDirs, Erls);
                LastModified when LastUpdated < LastModified ->
                    add_to_dag(G, Compiler, InDirs, Source, LastModified, filename:dirname(Source)),
                    update(G, Compiler, InDirs, Erls);
                _ ->
                    AltErls = digraph:out_neighbours(G, Source),
                    %% Deps must be explored before the module itself
                    update(G, Compiler, InDirs, AltErls),
                    Modified = is_dirty(G),
                    MaxModified = update_max_modified_deps(G, Source),
                    case Modified orelse MaxModified > LastUpdated of
                        true -> mark_dirty(G);
                        false -> ok
                    end,
                    update(G, Compiler, InDirs, Erls)
            end;
        false ->
            add_to_dag(G, Compiler, InDirs, Source, filelib:last_modified(Source), filename:dirname(Source)),
            update(G, Compiler, InDirs, Erls)
    end.

maybe_store(G, Dir, Compiler, Label, CritMeta) ->
    case is_dirty(G) of
        true ->
            clear_dirty(G),
            File = dag_file(Dir, Compiler, Label),
            store_dag(G, File, CritMeta);
        false ->
            ok
    end.

terminate(G) ->
    true = digraph:delete(G).

%%%%%%%%%%%%%%%
%%% PRIVATE %%%
%%%%%%%%%%%%%%%
%% @private generate the name for the DAG based on the compiler module and
%% a custom label, both of which are used to prevent various compiler runs
%% from clobbering each other. The label `undefined' is kept for a default
%% run of the compiler, to keep in line with previous versions of the file.
dag_file(Dir, CompilerMod, undefined) ->
    filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
                   ?DAG_ROOT ++ ?DAG_EXT]);
dag_file(Dir, CompilerMod, Label) ->
    filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
                   ?DAG_ROOT ++ "_" ++ Label ++ ?DAG_EXT]).

restore_dag(G, File, CritMeta) ->
    case file:read_file(File) of
        {ok, Data} ->
            %% The CritMeta value is checked and if it doesn't match, we fail
            %% the whole restore operation.
            #dag{vsn=?DAG_VSN, info={Vs, Es, CritMeta}} = binary_to_term(Data),
            [digraph:add_vertex(G, V, LastUpdated) || {V, LastUpdated} <- Vs],
            [digraph:add_edge(G, V1, V2) || {_, V1, V2, _} <- Es],
            ok;
        {error, _Err} ->
            ok
    end.

store_dag(G, File, CritMeta) ->
    ok = filelib:ensure_dir(File),
    Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
    Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
    Data = term_to_binary(#dag{info={Vs, Es, CritMeta}}, [{compressed, 2}]),
    file:write_file(File, Data).

%% Drop a file from the digraph if it doesn't exist, and if so,
%% delete its related build artifact
maybe_rm_beam_and_edge(G, OutDir, Source) ->
    %% This is NOT a double check it is the only check that the source file is actually gone
    case filelib:is_regular(Source) of
        true ->
            %% Actually exists, don't delete
            false;
        false ->
            Target = target_base(OutDir, Source) ++ ".beam",
            ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]),
            file:delete(Target),
            digraph:del_vertex(G, Source),
            mark_dirty(G),
            true
    end.

%% @private Return what should be the base name of an erl file, relocated to the
%% target directory. For example:
%% target_base("ebin/", "src/my_module.erl") -> "ebin/my_module"
target_base(OutDir, Source) ->
    filename:join(OutDir, filename:basename(Source, ".erl")).

%% @private a file has been found to change or wasn't part of the DAG before,
%% and must be added, along with all its dependencies.
add_to_dag(G, Compiler, InDirs, Source, LastModified, SourceDir) ->
    AbsIncls = Compiler:dependencies(Source, SourceDir, InDirs),
    digraph:add_vertex(G, Source, LastModified),
    digraph:del_edges(G, digraph:out_edges(G, Source)),
    %% Deps must be explored before the module itself
    [begin
         update(G, Compiler, InDirs, [Incl]),
         digraph:add_edge(G, Source, Incl)
     end || Incl <- AbsIncls],
    mark_dirty(G),
    AbsIncls.

%% @private change status propagation: if the dependencies of a file have
%% been updated, mark the last_modified time for that file to be equivalent
%% to its most-recently-changed dependency; that way, nested header file
%% change stamps are propagated to the final module.
%% This is required because at some point the module is compared to its
%% associated .beam file's last-generation stamp to know if it requires
%% rebuilding.
%% The responsibility for this is however diffuse across various modules.
update_max_modified_deps(G, Source) ->
    MaxModified = lists:foldl(
        fun(File, Acc) ->
            case digraph:vertex(G, File) of
                {_, MaxModified} when MaxModified > Acc -> MaxModified;
                _ -> Acc
            end
        end,
        0,
        [Source | digraph:out_neighbours(G, Source)]
    ),
    digraph:add_vertex(G, Source, MaxModified),
    MaxModified.

%% Mark the digraph as having been modified, which is required to
%% save its updated form on disk after the compiling run.
%% This uses a magic vertex to carry the dirty state. This is less
%% than ideal because listing vertices may expect filenames and
%% instead there's going to be one trick atom through it.
mark_dirty(G) ->
    digraph:add_vertex(G, '$r3_dirty_bit', true),
    ok.

%% Check whether the digraph has been modified and is considered dirty.
is_dirty(G) ->
    case digraph:vertex(G, '$r3_dirty_bit') of
        {_, Bool} -> Bool;
        false -> false
    end.

%% Remove the dirty status. Because the saving of a digraph on disk saves all
%% vertices, clear the flag before serializing it.
clear_dirty(G) ->
    digraph:del_vertex(G, '$r3_dirty_bit').