Bläddra i källkod
Merge pull request #2217 from ferd/merge-analysis-phases
Split up the compiler DAGpull/2213/head
Fred Hebert
5 år sedan
committed by
GitHub
GitHub
Ingen känd nyckel hittad för denna signaturen i databasen
GPG-nyckel ID: 4AEE18F83AFDEB23
3 ändrade filer med 344 tillägg och 238 borttagningar
Delad Vy
Diff Options
-
+54 -203src/rebar_compiler.erl
-
+221 -0src/rebar_compiler_dag.erl
-
+69 -35src/rebar_prv_compile.erl
+ 54
- 203
src/rebar_compiler.erl
Visa fil
+ 221
- 0
src/rebar_compiler_dag.erl
Visa fil
| @ -0,0 +1,221 @@ | |||
| %%% Module handling the directed graph required for the analysis | |||
| %%% of all top-level applications by the various compiler plugins. | |||
| -module(rebar_compiler_dag). | |||
| -export([init/4, prune/4, update/4, maybe_store/5, terminate/1]). | |||
| -include("rebar.hrl"). | |||
| -define(DAG_VSN, 3). | |||
| -define(DAG_ROOT, "source"). | |||
| -define(DAG_EXT, ".dag"). | |||
| -type dag_v() :: {digraph:vertex(), term()} | 'false'. | |||
| -type dag_e() :: {digraph:vertex(), digraph:vertex()}. | |||
| -type critical_meta() :: term(). % if this changes, the DAG is invalid | |||
| -type dag_rec() :: {list(dag_v()), list(dag_e()), critical_meta()}. | |||
| -type dag() :: digraph:graph(). | |||
| -record(dag, {vsn = ?DAG_VSN :: pos_integer(), | |||
| info = {[], [], []} :: dag_rec()}). | |||
| %% You should initialize one DAG per compiler module. | |||
| %% `CritMeta' is any contextual information that, if it is found to change, | |||
| %% must invalidate the DAG loaded from disk. | |||
| -spec init(file:filename_all(), atom(), string() | undefined, critical_meta()) -> dag(). | |||
| init(Dir, Compiler, Label, CritMeta) -> | |||
| G = digraph:new([acyclic]), | |||
| File = dag_file(Dir, Compiler, Label), | |||
| try | |||
| restore_dag(G, File, CritMeta) | |||
| catch | |||
| _:_ -> | |||
| %% Don't mark as dirty yet to avoid creating compiler DAG files for | |||
| %% compilers that are actually never used. | |||
| ?WARN("Failed to restore ~ts file. Discarding it.~n", [File]), | |||
| file:delete(File) | |||
| end, | |||
| G. | |||
| -spec prune(dag(), file:filename_all(), file:filename_all(), [file:filename_all()]) -> ok. | |||
| prune(G, SrcDirs, EbinDir, Erls) -> | |||
| %% A source file may have been renamed or deleted. Remove it from the graph | |||
| %% and remove any beam file for that source if it exists. | |||
| Vertices = digraph:vertices(G), | |||
| SrcParts = [filename:split(SrcDir) || SrcDir <- SrcDirs], | |||
| [maybe_rm_beam_and_edge(G, EbinDir, File) | |||
| || File <- lists:sort(Vertices) -- lists:sort(Erls), | |||
| filename:extension(File) =:= ".erl", | |||
| lists:any(fun(Src) -> lists:prefix(Src, filename:split(File)) end, | |||
| SrcParts)], | |||
| ok. | |||
| %% @doc this function scans all the source files found and looks into | |||
| %% all the `InDirs' for deps (other erl or .hrl files) that are related | |||
| %% to them (by calling `CompileMod:dependencies()' on them). | |||
| %% | |||
| %% The trick here is that change detection, done with last_modified stamps, | |||
| %% takes place at the same time as the graph propagation (finding deps) | |||
| %% themselves. As such, this is a confusing mutually recursive depth-first | |||
| %% search function that relies on side-effects and precise order-of-traversal | |||
| %% to propagate file changes. | |||
| %% | |||
| %% To be replaced by a more declarative EPP-based flow. | |||
| -spec update(dag(), module(), [file:filename_all()], [file:filename_all()]) -> ok. | |||
| update(_, _, _, []) -> | |||
| ok; | |||
| update(G, Compiler, InDirs, [Source|Erls]) -> | |||
| case digraph:vertex(G, Source) of | |||
| {_, LastUpdated} -> | |||
| case filelib:last_modified(Source) of | |||
| 0 -> | |||
| %% The file doesn't exist anymore, | |||
| %% erase it from the graph. | |||
| %% All the edges will be erased automatically. | |||
| digraph:del_vertex(G, Source), | |||
| mark_dirty(G), | |||
| update(G, Compiler, InDirs, Erls); | |||
| LastModified when LastUpdated < LastModified -> | |||
| add_to_dag(G, Compiler, InDirs, Source, LastModified, filename:dirname(Source)), | |||
| update(G, Compiler, InDirs, Erls); | |||
| _ -> | |||
| AltErls = digraph:out_neighbours(G, Source), | |||
| %% Deps must be explored before the module itself | |||
| update(G, Compiler, InDirs, AltErls), | |||
| Modified = is_dirty(G), | |||
| MaxModified = update_max_modified_deps(G, Source), | |||
| case Modified orelse MaxModified > LastUpdated of | |||
| true -> mark_dirty(G); | |||
| false -> ok | |||
| end, | |||
| update(G, Compiler, InDirs, Erls) | |||
| end; | |||
| false -> | |||
| add_to_dag(G, Compiler, InDirs, Source, filelib:last_modified(Source), filename:dirname(Source)), | |||
| update(G, Compiler, InDirs, Erls) | |||
| end. | |||
| maybe_store(G, Dir, Compiler, Label, CritMeta) -> | |||
| case is_dirty(G) of | |||
| true -> | |||
| clear_dirty(G), | |||
| File = dag_file(Dir, Compiler, Label), | |||
| store_dag(G, File, CritMeta); | |||
| false -> | |||
| ok | |||
| end. | |||
| terminate(G) -> | |||
| true = digraph:delete(G). | |||
| %%%%%%%%%%%%%%% | |||
| %%% PRIVATE %%% | |||
| %%%%%%%%%%%%%%% | |||
| %% @private generate the name for the DAG based on the compiler module and | |||
| %% a custom label, both of which are used to prevent various compiler runs | |||
| %% from clobbering each other. The label `undefined' is kept for a default | |||
| %% run of the compiler, to keep in line with previous versions of the file. | |||
| dag_file(Dir, CompilerMod, undefined) -> | |||
| filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, | |||
| ?DAG_ROOT ++ ?DAG_EXT]); | |||
| dag_file(Dir, CompilerMod, Label) -> | |||
| filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod, | |||
| ?DAG_ROOT ++ "_" ++ Label ++ ?DAG_EXT]). | |||
| restore_dag(G, File, CritMeta) -> | |||
| case file:read_file(File) of | |||
| {ok, Data} -> | |||
| %% The CritMeta value is checked and if it doesn't match, we fail | |||
| %% the whole restore operation. | |||
| #dag{vsn=?DAG_VSN, info={Vs, Es, CritMeta}} = binary_to_term(Data), | |||
| [digraph:add_vertex(G, V, LastUpdated) || {V, LastUpdated} <- Vs], | |||
| [digraph:add_edge(G, V1, V2) || {_, V1, V2, _} <- Es], | |||
| ok; | |||
| {error, _Err} -> | |||
| ok | |||
| end. | |||
| store_dag(G, File, CritMeta) -> | |||
| ok = filelib:ensure_dir(File), | |||
| Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)), | |||
| Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)), | |||
| Data = term_to_binary(#dag{info={Vs, Es, CritMeta}}, [{compressed, 2}]), | |||
| file:write_file(File, Data). | |||
| %% Drop a file from the digraph if it doesn't exist, and if so, | |||
| %% delete its related build artifact | |||
| maybe_rm_beam_and_edge(G, OutDir, Source) -> | |||
| %% This is NOT a double check it is the only check that the source file is actually gone | |||
| case filelib:is_regular(Source) of | |||
| true -> | |||
| %% Actually exists, don't delete | |||
| false; | |||
| false -> | |||
| Target = target_base(OutDir, Source) ++ ".beam", | |||
| ?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]), | |||
| file:delete(Target), | |||
| digraph:del_vertex(G, Source), | |||
| mark_dirty(G), | |||
| true | |||
| end. | |||
| %% @private Return what should be the base name of an erl file, relocated to the | |||
| %% target directory. For example: | |||
| %% target_base("ebin/", "src/my_module.erl") -> "ebin/my_module" | |||
| target_base(OutDir, Source) -> | |||
| filename:join(OutDir, filename:basename(Source, ".erl")). | |||
| %% @private a file has been found to change or wasn't part of the DAG before, | |||
| %% and must be added, along with all its dependencies. | |||
| add_to_dag(G, Compiler, InDirs, Source, LastModified, SourceDir) -> | |||
| AbsIncls = Compiler:dependencies(Source, SourceDir, InDirs), | |||
| digraph:add_vertex(G, Source, LastModified), | |||
| digraph:del_edges(G, digraph:out_edges(G, Source)), | |||
| %% Deps must be explored before the module itself | |||
| [begin | |||
| update(G, Compiler, InDirs, [Incl]), | |||
| digraph:add_edge(G, Source, Incl) | |||
| end || Incl <- AbsIncls], | |||
| mark_dirty(G), | |||
| AbsIncls. | |||
| %% @private change status propagation: if the dependencies of a file have | |||
| %% been updated, mark the last_modified time for that file to be equivalent | |||
| %% to its most-recently-changed dependency; that way, nested header file | |||
| %% change stamps are propagated to the final module. | |||
| %% This is required because at some point the module is compared to its | |||
| %% associated .beam file's last-generation stamp to know if it requires | |||
| %% rebuilding. | |||
| %% The responsibility for this is however diffuse across various modules. | |||
| update_max_modified_deps(G, Source) -> | |||
| MaxModified = lists:foldl( | |||
| fun(File, Acc) -> | |||
| case digraph:vertex(G, File) of | |||
| {_, MaxModified} when MaxModified > Acc -> MaxModified; | |||
| _ -> Acc | |||
| end | |||
| end, | |||
| 0, | |||
| [Source | digraph:out_neighbours(G, Source)] | |||
| ), | |||
| digraph:add_vertex(G, Source, MaxModified), | |||
| MaxModified. | |||
| %% Mark the digraph as having been modified, which is required to | |||
| %% save its updated form on disk after the compiling run. | |||
| %% This uses a magic vertex to carry the dirty state. This is less | |||
| %% than ideal because listing vertices may expect filenames and | |||
| %% instead there's going to be one trick atom through it. | |||
| mark_dirty(G) -> | |||
| digraph:add_vertex(G, '$r3_dirty_bit', true), | |||
| ok. | |||
| %% Check whether the digraph has been modified and is considered dirty. | |||
| is_dirty(G) -> | |||
| case digraph:vertex(G, '$r3_dirty_bit') of | |||
| {_, Bool} -> Bool; | |||
| false -> false | |||
| end. | |||
| %% Remove the dirty status. Because the saving of a digraph on disk saves all | |||
| %% vertices, clear the flag before serializing it. | |||
| clear_dirty(G) -> | |||
| digraph:del_vertex(G, '$r3_dirty_bit'). | |||