From 1027f431cffb83b58cff2512e8da542c2e253a91 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 20 Feb 2020 17:10:08 +0000 Subject: [PATCH] Clear up old code and other shenanigns --- src/rebar_compiler.erl | 9 +- src/rebar_compiler_dag.erl | 350 ++++++++++++++++--------------------- 2 files changed, 156 insertions(+), 203 deletions(-) diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index 838e9282..c61bdeb4 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -96,19 +96,22 @@ gather_in_dirs([{AppInfo, Ctx} | Rest], Acc) -> analyze_app({Compiler, G}, Contexts, AppInfo) -> AppName = rebar_app_info:name(AppInfo), BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), - EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), + OutDir = rebar_utils:to_list(rebar_app_info:out_dir(AppInfo)), BaseOpts = rebar_app_info:opts(AppInfo), #{src_dirs := SrcDirs, src_ext := SrcExt, + out_mappings := [{OutExt, OutPath}|_], % prune one dir for now (compat mode!) dependencies_opts := DepOpts} = maps:get(AppName, Contexts), %% Local resources + ArtifactDir = filename:join([OutDir, OutPath]), AbsSources = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts), LocalSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs], %% Multi-app resources InDirs = maps:get(in_dirs, Contexts), %% Prep the analysis - rebar_compiler_dag:prune(G, LocalSrcDirs, EbinDir, AbsSources), - rebar_compiler_dag:update(G, Compiler, InDirs, AbsSources, DepOpts), + rebar_compiler_dag:prune( + G, LocalSrcDirs, ArtifactDir, AbsSources, SrcExt, OutExt + ), %% Run the analysis rebar_compiler_dag:populate_sources( G, Compiler, InDirs, AbsSources, DepOpts diff --git a/src/rebar_compiler_dag.erl b/src/rebar_compiler_dag.erl index d622881d..c16f4a5c 100644 --- a/src/rebar_compiler_dag.erl +++ b/src/rebar_compiler_dag.erl @@ -1,8 +1,8 @@ %%% Module handling the directed graph required for the analysis %%% of all top-level applications by the various compiler plugins. -module(rebar_compiler_dag). --export([init/4, prune/4, update/5, maybe_store/5, terminate/1]). --export([populate_sources/5, populate_deps/3, propagate_stamps/1, +-export([init/4, maybe_store/5, terminate/1]). +-export([prune/6, populate_sources/5, populate_deps/3, propagate_stamps/1, compile_order/2]). -include("rebar.hrl"). @@ -19,7 +19,7 @@ -record(dag, {vsn = ?DAG_VSN :: pos_integer(), info = {[], [], []} :: dag_rec()}). -%% You should initialize one DAG per compiler module. +%% @doc You should initialize one DAG per compiler module. %% `CritMeta' is any contextual information that, if it is found to change, %% must invalidate the DAG loaded from disk. -spec init(file:filename_all(), atom(), string() | undefined, critical_meta()) -> dag(). @@ -37,67 +37,28 @@ init(Dir, Compiler, Label, CritMeta) -> end, G. --spec prune(dag(), file:filename_all(), file:filename_all(), [file:filename_all()]) -> ok. -prune(G, SrcDirs, EbinDir, Erls) -> +%% @doc Clear up inactive (deleted) source files from a given project. +%% The `SrcDirs' must be all the directories that may contain source files +%% for an OTP application; source files found in the DAG `G' that lie outside +%% of this directory will be used. +-spec prune(dag(), file:filename_all(), file:filename_all(), + [file:filename_all()], string(), string()) -> ok. +prune(G, SrcDirs, OutDir, Erls, SrcExt, ArtifactExt) -> %% A source file may have been renamed or deleted. Remove it from the graph %% and remove any beam file for that source if it exists. Vertices = digraph:vertices(G), SrcParts = [filename:split(SrcDir) || SrcDir <- SrcDirs], - [maybe_rm_beam_and_edge(G, EbinDir, File) + [maybe_rm_artifact_and_edge(G, OutDir, ArtifactExt, File) || File <- lists:sort(Vertices) -- lists:sort(Erls), - filename:extension(File) =:= ".erl", + filename:extension(File) =:= SrcExt, lists:any(fun(Src) -> lists:prefix(Src, filename:split(File)) end, SrcParts)], ok. %% @doc this function scans all the source files found and looks into -%% all the `InDirs' for deps (other erl or .hrl files) that are related -%% to them (by calling `CompileMod:dependencies()' on them). -%% -%% The trick here is that change detection, done with last_modified stamps, -%% takes place at the same time as the graph propagation (finding deps) -%% themselves. As such, this is a confusing mutually recursive depth-first -%% search function that relies on side-effects and precise order-of-traversal -%% to propagate file changes. -%% -%% To be replaced by a more declarative EPP-based flow. --spec update(dag(), module(), [file:filename_all()], [file:filename_all()], - term()) -> ok. -update(_, _, _, [], _) -> - ok; -update(G, Compiler, InDirs, [Source|Erls], DepOpts) -> - case digraph:vertex(G, Source) of - {_, LastUpdated} -> - case filelib:last_modified(Source) of - 0 -> - %% The file doesn't exist anymore, - %% erase it from the graph. - %% All the edges will be erased automatically. - digraph:del_vertex(G, Source), - mark_dirty(G), - update(G, Compiler, InDirs, Erls, DepOpts); - LastModified when LastUpdated < LastModified -> - add_to_dag(G, Compiler, InDirs, Source, LastModified, - filename:dirname(Source), DepOpts), - update(G, Compiler, InDirs, Erls, DepOpts); - _ -> - AltErls = digraph:out_neighbours(G, Source), - %% Deps must be explored before the module itself - update(G, Compiler, InDirs, AltErls, DepOpts), - Modified = is_dirty(G), - MaxModified = update_max_modified_deps(G, Source), - case Modified orelse MaxModified > LastUpdated of - true -> mark_dirty(G); - false -> ok - end, - update(G, Compiler, InDirs, Erls, DepOpts) - end; - false -> - add_to_dag(G, Compiler, InDirs, Source, filelib:last_modified(Source), - filename:dirname(Source), DepOpts), - update(G, Compiler, InDirs, Erls, DepOpts) - end. - +%% all the `InDirs' for deps (other source files, or files that aren't source +%% but still returned by the compiler module) that are related +%% to them. populate_sources(_G, _Compiler, _InDirs, [], _DepOpts) -> ok; populate_sources(G, Compiler, InDirs, [Source|Erls], DepOpts) -> @@ -125,21 +86,9 @@ populate_sources(G, Compiler, InDirs, [Source|Erls], DepOpts) -> end, populate_sources(G, Compiler, InDirs, Erls, DepOpts). -prepopulate_deps(G, Compiler, InDirs, Source, DepOpts) -> - SourceDir = filename:dirname(Source), - AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of - false -> - Compiler:dependencies(Source, SourceDir, InDirs); - true -> - Compiler:dependencies(Source, SourceDir, InDirs, DepOpts) - end, - %% the file hasn't been visited yet; set it to existing, but with - %% a last modified value that's null so it gets updated to something new. - [digraph:add_vertex(G, Src, 0) || Src <- AbsIncls, - digraph:vertex(G, Src) =:= false], - [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls], - ok. - +%% @doc Scan all files in the digraph that are seen as dependencies, but are +%% neither source files nor artifacts (i.e. header files that don't produce +%% artifacts of any kind). populate_deps(G, SourceExt, ArtifactExts) -> %% deps are files that are part of the digraph, but couldn't be scanned %% because they are neither source files (`SourceExt') nor mappings @@ -155,8 +104,9 @@ populate_deps(G, SourceExt, ArtifactExts) -> not lists:member(Ext, IgnoredExts)], ok. -%% Take the timestamps/diff changes and propagate them from a dep to the parent; -%% given: + +%% @doc Take the timestamps/diff changes and propagate them from a dep to the +%% parent; given: %% A 0 -> B 1 -> C 3 -> D 2 %% then we expect to get back: %% A 3 -> B 3 -> C 3 -> D 2 @@ -179,6 +129,119 @@ propagate_stamps(G) -> end. +%% @doc Return the reverse sorting order to get dep-free apps first. +%% -- we would usually not need to consider the non-source files for the order to +%% be complete, but using them doesn't hurt. +compile_order(G, AppDefs) -> + Edges = [{V1,V2} || E <- digraph:edges(G), + {_,V1,V2,_} <- [digraph:edge(G, E)]], + AppPaths = prepare_app_paths(AppDefs), + compile_order(Edges, AppPaths, #{}). + +%% @doc Store the DAG on disk if it was dirty +maybe_store(G, Dir, Compiler, Label, CritMeta) -> + case is_dirty(G) of + true -> + clear_dirty(G), + File = dag_file(Dir, Compiler, Label), + store_dag(G, File, CritMeta); + false -> + ok + end. + +%% Get rid of the live state for the digraph; leave disk stuff in place. +terminate(G) -> + true = digraph:delete(G). + +%%%%%%%%%%%%%%% +%%% PRIVATE %%% +%%%%%%%%%%%%%%% +%% @private generate the name for the DAG based on the compiler module and +%% a custom label, both of which are used to prevent various compiler runs +%% from clobbering each other. The label `undefined' is kept for a default +%% run of the compiler, to keep in line with previous versions of the file. +dag_file(Dir, CompilerMod, undefined) -> + filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, + ?DAG_ROOT ++ ?DAG_EXT]); +dag_file(Dir, CompilerMod, Label) -> + filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, + ?DAG_ROOT ++ "_" ++ Label ++ ?DAG_EXT]). + +restore_dag(G, File, CritMeta) -> + case file:read_file(File) of + {ok, Data} -> + %% The CritMeta value is checked and if it doesn't match, we fail + %% the whole restore operation. + #dag{vsn=?DAG_VSN, info={Vs, Es, CritMeta}} = binary_to_term(Data), + [digraph:add_vertex(G, V, LastUpdated) || {V, LastUpdated} <- Vs], + [digraph:add_edge(G, V1, V2) || {_, V1, V2, _} <- Es], + ok; + {error, _Err} -> + ok + end. + +store_dag(G, File, CritMeta) -> + ok = filelib:ensure_dir(File), + Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), + Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), + Data = term_to_binary(#dag{info={Vs, Es, CritMeta}}, [{compressed, 2}]), + file:write_file(File, Data). + +%% Drop a file from the digraph if it doesn't exist, and if so, +%% delete its related build artifact +maybe_rm_artifact_and_edge(G, OutDir, Ext, Source) -> + %% This is NOT a double check it is the only check that the source file is actually gone + case filelib:is_regular(Source) of + true -> + %% Actually exists, don't delete + false; + false -> + Target = target_base(OutDir, Source) ++ Ext, + ?DEBUG("Source ~ts is gone, deleting previous ~ts file if it exists ~ts", [Source, Ext, Target]), + file:delete(Target), + digraph:del_vertex(G, Source), + mark_dirty(G), + true + end. + +%% Add dependencies of a given file to the DAG. If the file is not found yet, +%% mark its timestamp to 0, which means we have no info on it. +%% Source files will be covered at a later point in their own scan, and +%% non-source files are going to be covered by `populate_deps/3'. +prepopulate_deps(G, Compiler, InDirs, Source, DepOpts) -> + SourceDir = filename:dirname(Source), + AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of + false -> + Compiler:dependencies(Source, SourceDir, InDirs); + true -> + Compiler:dependencies(Source, SourceDir, InDirs, DepOpts) + end, + %% the file hasn't been visited yet; set it to existing, but with + %% a last modified value that's null so it gets updated to something new. + [digraph:add_vertex(G, Src, 0) || Src <- AbsIncls, + digraph:vertex(G, Src) =:= false], + [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls], + ok. + +%% check that a dep file is up to date +refresh_dep(G, File) -> + {_, LastUpdated} = digraph:vertex(G, File), + case filelib:last_modified(File) of + 0 -> + %% Gone! Erase from the graph + digraph:del_vertex(G, File), + mark_dirty(G); + LastModified when LastUpdated < LastModified -> + digraph:add_vertex(G, File, LastModified), + mark_dirty(G); + _ -> + % unchanged + ok + end. + +%% Do the actual propagation of all files; the files are expected to be +%% in a topological order such that we don't need to go more than a level +%% deep in what we search. propagate_stamps(_G, []) -> ok; propagate_stamps(G, [File|Files]) -> @@ -198,16 +261,10 @@ propagate_stamps(G, [File|Files]) -> end, propagate_stamps(G, Files). - -compile_order(G, AppDefs) -> - %% Return the reverse sorting order to get dep-free apps first. - %% -- we would usually not need to consider the non-source files for the order to - %% be complete, but using them doesn't hurt. - Edges = [{V1,V2} || E <- digraph:edges(G), - {_,V1,V2,_} <- [digraph:edge(G, E)]], - AppPaths = prepare_app_paths(AppDefs), - compile_order(Edges, AppPaths, #{}). - +%% Do the actual reversal; be aware that only working from the edges +%% may omit files, so we have to add all non-dependant apps manually +%% to make sure we don't drop em. Since they have no deps, they're +%% safer to put first (and compile first) compile_order([], AppPaths, AppDeps) -> %% use a digraph so we don't reimplement topsort by hand. G = digraph:new([acyclic]), % ignore cycles and hope it works @@ -235,12 +292,23 @@ compile_order([{P1,P2}|T], AppPaths, AppDeps) -> end end. +%% Swap app name with paths in the order, and sort there; this lets us +%% bail out early in a search where a file won't be found. prepare_app_paths(AppPaths) -> lists:sort([{filename:split(Path), Name} || {Name, Path} <- AppPaths]). +%% Look for the app to which the path belongs; needed to +%% go from an edge between files in the DAG to building +%% app-related orderings find_app(Path, AppPaths) -> find_app_(filename:split(Path), AppPaths). +%% A cached search for the app to which a path belongs; +%% the assumption is that sorted edges and common relationships +%% are going to be between local files within an app most +%% of the time; so we first look for the same path as a +%% prior match to avoid searching _all_ potential candidates. +%% If it doesn't work, go for the normal search. find_cached_app(Path, {Name, AppPath}, AppPaths) -> Split = filename:split(Path), case find_app_(Split, [{AppPath, Name}]) of @@ -248,6 +316,7 @@ find_cached_app(Path, {Name, AppPath}, AppPaths) -> LastEntry -> LastEntry end. +%% Do the actual recursive search find_app_(_Path, []) -> not_found; find_app_(Path, [{AppPath, AppName}|Rest]) -> @@ -261,84 +330,6 @@ find_app_(Path, [{AppPath, AppName}|Rest]) -> end. -refresh_dep(G, File) -> - {_, LastUpdated} = digraph:vertex(G, File), - case filelib:last_modified(File) of - 0 -> - %% Gone! Erase from the graph - digraph:del_vertex(G, File), - mark_dirty(G); - LastModified when LastUpdated < LastModified -> - digraph:add_vertex(G, File, LastModified), - mark_dirty(G); - _ -> - % unchanged - ok - end. - -maybe_store(G, Dir, Compiler, Label, CritMeta) -> - case is_dirty(G) of - true -> - clear_dirty(G), - File = dag_file(Dir, Compiler, Label), - store_dag(G, File, CritMeta); - false -> - ok - end. - -terminate(G) -> - true = digraph:delete(G). - -%%%%%%%%%%%%%%% -%%% PRIVATE %%% -%%%%%%%%%%%%%%% -%% @private generate the name for the DAG based on the compiler module and -%% a custom label, both of which are used to prevent various compiler runs -%% from clobbering each other. The label `undefined' is kept for a default -%% run of the compiler, to keep in line with previous versions of the file. -dag_file(Dir, CompilerMod, undefined) -> - filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, - ?DAG_ROOT ++ ?DAG_EXT]); -dag_file(Dir, CompilerMod, Label) -> - filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, - ?DAG_ROOT ++ "_" ++ Label ++ ?DAG_EXT]). - -restore_dag(G, File, CritMeta) -> - case file:read_file(File) of - {ok, Data} -> - %% The CritMeta value is checked and if it doesn't match, we fail - %% the whole restore operation. - #dag{vsn=?DAG_VSN, info={Vs, Es, CritMeta}} = binary_to_term(Data), - [digraph:add_vertex(G, V, LastUpdated) || {V, LastUpdated} <- Vs], - [digraph:add_edge(G, V1, V2) || {_, V1, V2, _} <- Es], - ok; - {error, _Err} -> - ok - end. - -store_dag(G, File, CritMeta) -> - ok = filelib:ensure_dir(File), - Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), - Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), - Data = term_to_binary(#dag{info={Vs, Es, CritMeta}}, [{compressed, 2}]), - file:write_file(File, Data). - -%% Drop a file from the digraph if it doesn't exist, and if so, -%% delete its related build artifact -maybe_rm_beam_and_edge(G, OutDir, Source) -> - %% This is NOT a double check it is the only check that the source file is actually gone - case filelib:is_regular(Source) of - true -> - %% Actually exists, don't delete - false; - false -> - Target = target_base(OutDir, Source) ++ ".beam", - ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), - file:delete(Target), - digraph:del_vertex(G, Source), - mark_dirty(G), - true - end. %% @private Return what should be the base name of an erl file, relocated to the %% target directory. For example: @@ -346,47 +337,6 @@ maybe_rm_beam_and_edge(G, OutDir, Source) -> target_base(OutDir, Source) -> filename:join(OutDir, filename:basename(Source, ".erl")). -%% @private a file has been found to change or wasn't part of the DAG before, -%% and must be added, along with all its dependencies. -add_to_dag(G, Compiler, InDirs, Source, LastModified, SourceDir, DepOpts) -> - AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of - false -> - Compiler:dependencies(Source, SourceDir, InDirs); - true -> - Compiler:dependencies(Source, SourceDir, InDirs, DepOpts) - end, - digraph:add_vertex(G, Source, LastModified), - digraph:del_edges(G, digraph:out_edges(G, Source)), - %% Deps must be explored before the module itself - [begin - update(G, Compiler, InDirs, [Incl], DepOpts), - digraph:add_edge(G, Source, Incl) - end || Incl <- AbsIncls], - mark_dirty(G), - AbsIncls. - -%% @private change status propagation: if the dependencies of a file have -%% been updated, mark the last_modified time for that file to be equivalent -%% to its most-recently-changed dependency; that way, nested header file -%% change stamps are propagated to the final module. -%% This is required because at some point the module is compared to its -%% associated .beam file's last-generation stamp to know if it requires -%% rebuilding. -%% The responsibility for this is however diffuse across various modules. -update_max_modified_deps(G, Source) -> - MaxModified = lists:foldl( - fun(File, Acc) -> - case digraph:vertex(G, File) of - {_, MaxModified} when MaxModified > Acc -> MaxModified; - _ -> Acc - end - end, - 0, - [Source | digraph:out_neighbours(G, Source)] - ), - digraph:add_vertex(G, Source, MaxModified), - MaxModified. - %% Mark the digraph as having been modified, which is required to %% save its updated form on disk after the compiling run. %% This uses a magic vertex to carry the dirty state. This is less