From f0d8cf16868a5fc057d540b3c46a14b88232360d Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 12 Mar 2020 00:40:01 +0000 Subject: [PATCH] Track build artifacts in DAG This allows to do quicker re-compile option validation by not requiring to access the disk and check all candidate erlang files for option changes. This also opens up the way to compilers that produce more than one artifact per file. --- src/rebar_compiler.erl | 76 ++++++++++++++++++++++++++++++------ src/rebar_compiler_dag.erl | 21 ++++++++-- src/rebar_compiler_erl.erl | 43 ++++++++++++++++---- test/rebar_compile_SUITE.erl | 37 +++++++++++++++++- 4 files changed, 154 insertions(+), 23 deletions(-) diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index 70d5378d..28e5f8df 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -180,17 +180,28 @@ run(G, CompilerMod, AppInfo, Contexts) -> {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo), compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod), - case RestFiles of - {Sequential, Parallel} -> % new parallelizable form - compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod), + Tracked = case RestFiles of + {Sequential, Parallel} -> % parallelizable form + compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod) ++ compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod); _ when is_list(RestFiles) -> % traditional sequential build compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod) - end. + end, + store_artifacts(G, Tracked). compile_each([], _Opts, _Config, _Outs, _CompilerMod) -> - ok; + []; compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) -> + case erlang:function_exported(CompilerMod, compile_and_track, 4) of + false -> + do_compile(CompilerMod, Source, Outs, Config, Opts), + compile_each(Rest, Opts, Config, Outs, CompilerMod); + true -> + do_compile_and_track(CompilerMod, Source, Outs, Config, Opts) + ++ compile_each(Rest, Opts, Config, Outs, CompilerMod) + end. + +do_compile(CompilerMod, Source, Outs, Config, Opts) -> case CompilerMod:compile(Source, Outs, Config, Opts) of ok -> ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]); @@ -205,14 +216,47 @@ compile_each([Source | Rest], Opts, Config, Outs, CompilerMod) -> maybe_report(Error), ?DEBUG("Compilation failed: ~p", [Error]), ?FAIL - end, - compile_each(Rest, Opts, Config, Outs, CompilerMod). + end. + +do_compile_and_track(CompilerMod, Source, Outs, Config, Opts) -> + case CompilerMod:compile_and_track(Source, Outs, Config, Opts) of + {ok, Tracked} -> + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]), + Tracked; + {ok, Tracked, Warnings} -> + report(Warnings), + ?DEBUG("~tsCompiled ~ts", [rebar_utils:indent(1), filename:basename(Source)]), + Tracked; + skipped -> + ?DEBUG("~tsSkipped ~ts", [rebar_utils:indent(1), filename:basename(Source)]), + []; + Error -> + NewSource = format_error_source(Source, Config), + ?ERROR("Compiling ~ts failed", [NewSource]), + maybe_report(Error), + ?DEBUG("Compilation failed: ~p", [Error]), + ?FAIL + end. + +store_artifacts(_G, []) -> + ok; +store_artifacts(G, [{Source, Target, Meta}|Rest]) -> + %% Assume the source exists since it was tracked to be compiled + digraph:add_vertex(G, Target, Meta), + digraph:add_edge(G, Source, Target, artifact), + store_artifacts(G, Rest). compile_worker(QueuePid, Opts, Config, Outs, CompilerMod) -> QueuePid ! self(), receive {compile, Source} -> - Result = CompilerMod:compile(Source, Outs, Config, Opts), + Result = + case erlang:function_exported(CompilerMod, compile_and_track, 4) of + false -> + CompilerMod:compile(Source, Outs, Config, Opts); + true -> + CompilerMod:compile_and_track(Source, Outs, Config, Opts) + end, QueuePid ! {Result, Source}, compile_worker(QueuePid, Opts, Config, Outs, CompilerMod); empty -> @@ -220,7 +264,7 @@ compile_worker(QueuePid, Opts, Config, Outs, CompilerMod) -> end. compile_parallel([], _Opts, _BaseOpts, _Mappings, _CompilerMod) -> - ok; + []; compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) -> Self = self(), F = fun() -> compile_worker(Self, Opts, BaseOpts, Mappings, CompilerMod) end, @@ -230,8 +274,9 @@ compile_parallel(Targets, Opts, BaseOpts, Mappings, CompilerMod) -> compile_queue(Targets, Pids, Opts, BaseOpts, Mappings, CompilerMod). compile_queue([], [], _Opts, _Config, _Outs, _CompilerMod) -> - ok; + []; compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) -> + Tracking = erlang:function_exported(CompilerMod, compile_and_track, 4), receive Worker when is_pid(Worker), Targets =:= [] -> Worker ! empty, @@ -242,10 +287,19 @@ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) -> {ok, Source} -> ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]), compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); - {{ok, Warnings}, Source} -> + {{ok, Tracked}, Source} when Tracking -> + ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]), + Tracked ++ + compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); + {{ok, Warnings}, Source} when not Tracking -> report(Warnings), ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]), compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); + {{ok, Tracked, Warnings}, Source} when Tracking -> + report(Warnings), + ?DEBUG("~sCompiled ~s", [rebar_utils:indent(1), Source]), + Tracked ++ + compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); {skipped, Source} -> ?DEBUG("~sSkipped ~s", [rebar_utils:indent(1), Source]), compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod); diff --git a/src/rebar_compiler_dag.erl b/src/rebar_compiler_dag.erl index 0a8f1faa..884bc847 100644 --- a/src/rebar_compiler_dag.erl +++ b/src/rebar_compiler_dag.erl @@ -233,9 +233,21 @@ maybe_rm_artifact_and_edge(G, OutDir, SrcExt, Ext, Source) -> %% Actually exists, don't delete false; false -> - Target = target(OutDir, Source, SrcExt, Ext), - ?DEBUG("Source ~ts is gone, deleting previous ~ts file if it exists ~ts", [Source, Ext, Target]), - file:delete(Target), + Edges = digraph:out_edges(G, Source), + Targets = [V2 || Edge <- Edges, + {_E, _V1, V2, artifact} <- [digraph:edge(G, Edge)]], + case Targets of + [] -> + Target = target(OutDir, Source, SrcExt, Ext), + ?DEBUG("Source ~ts is gone, deleting previous ~ts file if it exists ~ts", [Source, Ext, Target]), + file:delete(Target); + [_|_] -> + lists:foreach(fun(Target) -> + ?DEBUG("Source ~ts is gone, deleting artiface ~ts " + "if it exists ~ts", [Source, Target]), + file:delete(Target) + end, Targets) + end, digraph:del_vertex(G, Source), mark_dirty(G), true @@ -269,7 +281,8 @@ prepopulate_deps(G, Compiler, InDirs, Source, DepOpts, Status) -> %% drop edges from deps that aren't included! [digraph:del_edge(G, Edge) || Status == old, Edge <- digraph:out_edges(G, Source), - {_, _Src, Path, _} <- [digraph:edge(G, Edge)], + {_, _Src, Path, Label} <- [digraph:edge(G, Edge)], + Label =/= artifact, not lists:member(Path, AbsIncls)], %% Add the rest [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls], diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl index 75256169..920a1286 100644 --- a/src/rebar_compiler_erl.erl +++ b/src/rebar_compiler_erl.erl @@ -5,7 +5,7 @@ -export([context/1, needed_files/4, dependencies/3, dependencies/4, - compile/4, + compile/4, compile_and_track/4, clean/2, format_error/1]). @@ -136,6 +136,28 @@ compile(Source, [{_, OutDir}], Config, ErlOpts) -> error end. +compile_and_track(Source, [{Ext, OutDir}], Config, ErlOpts) -> + BuildOpts = [{outdir, OutDir} | ErlOpts], + Target = target_base(OutDir, Source) ++ Ext, + case compile:file(Source, BuildOpts) of + {ok, _Mod} -> + AllOpts = BuildOpts ++ compile:env_compiler_options(), + {ok, [{Source, Target, AllOpts}]}; + {ok, _Mod, []} -> + AllOpts = BuildOpts ++ compile:env_compiler_options(), + {ok, [{Source, Target, AllOpts}]}; + {ok, _Mod, Ws} -> + FormattedWs = format_error_sources(Ws, Config), + {ok, Warns} = rebar_compiler:ok_tuple(Source, FormattedWs), + AllOpts = BuildOpts ++ compile:env_compiler_options(), + {ok, [{Source, Target, AllOpts}], Warns}; + {error, Es, Ws} -> + error_tuple(Source, Es, Ws, Config, ErlOpts); + error -> + error + end. + + clean(Files, AppInfo) -> EbinDir = rebar_app_info:ebin_dir(AppInfo), [begin @@ -199,22 +221,29 @@ needed_files(Graph, ErlOpts, RebarOpts, Dir, OutDir, SourceFiles) -> ,{i, filename:join(Dir, "include")} ,{i, Dir}] ++ PrivIncludes ++ ErlOpts, digraph:vertex(Graph, Source) > {Source, filelib:last_modified(Target)} - orelse opts_changed(AllOpts, TargetBase) + orelse opts_changed(Graph, AllOpts, Target, TargetBase) orelse erl_compiler_opts_set() end, SourceFiles). target_base(OutDir, Source) -> filename:join(OutDir, filename:basename(Source, ".erl")). -opts_changed(NewOpts, Target) -> +opts_changed(Graph, NewOpts, Target, TargetBase) -> TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of true -> NewOpts ++ compile:env_compiler_options(); false -> NewOpts end, - case compile_info(Target) of - {ok, Opts} -> lists:any(fun effects_code_generation/1, lists:usort(TotalOpts) -- lists:usort(Opts)); - _ -> true - end. + TargetOpts = case digraph:vertex(Graph, Target) of + {_Target, Opts} -> % tracked dep is found + Opts; + false -> % not found; might be a non-tracked DAG + case compile_info(TargetBase) of + {ok, Opts} -> Opts; + _ -> [] + end + end, + lists:any(fun effects_code_generation/1, + lists:usort(TotalOpts) -- lists:usort(TargetOpts)). effects_code_generation(Option) -> case Option of diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index 9ad932a4..c121bd7e 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -20,7 +20,7 @@ all() -> recompile_when_opts_included_hrl_changes, recompile_when_foreign_included_hrl_changes, recompile_when_foreign_behaviour_changes, - recompile_when_opts_change, + recompile_when_opts_change, recompile_when_dag_opts_change, dont_recompile_when_opts_dont_change, dont_recompile_yrl_or_xrl, delete_beam_if_source_deleted, deps_in_path, checkout_priority, highest_version_of_pkg_dep, @@ -919,6 +919,41 @@ recompile_when_opts_change(Config) -> ?assert(ModTime =/= NewModTime). +recompile_when_dag_opts_change(Config) -> + AppDir = ?config(apps, Config), + + Name = rebar_test_utils:create_random_name("app1_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir, Name, Vsn, [kernel, stdlib]), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name, "ebin"]), + {ok, Files} = rebar_utils:list_dir(EbinDir), + Beams = [filename:join([EbinDir, F]) + || F <- Files, filename:extension(F) == ".beam"], + ModTime = [filelib:last_modified(Beam) || Beam <- Beams], + + timer:sleep(1000), + + DepsDir = filename:join([AppDir, "_build", "default", "lib"]), + G = rebar_compiler_dag:init(DepsDir, rebar_compiler_erl, "project_apps", []), + %% change the config in the DAG... + [digraph:add_vertex(G, Beam, [{d, some_define}]) || Beam <- Beams], + digraph:add_vertex(G, '$r3_dirty_bit', true), % trigger a save + rebar_compiler_dag:maybe_store(G, DepsDir, rebar_compiler_erl, "project_apps", []), + rebar_compiler_dag:terminate(G), + %% ... but don't change the actual rebar3 config... + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), + + %% ... and checks that it rebuilds anyway due to DAG changes + {ok, NewFiles} = rebar_utils:list_dir(EbinDir), + NewBeams = [filename:join([EbinDir, F]) + || F <- NewFiles, filename:extension(F) == ".beam"], + NewModTime = [filelib:last_modified(Beam) || Beam <- NewBeams], + + ?assert(ModTime =/= NewModTime). + dont_recompile_when_opts_dont_change(Config) -> AppDir = ?config(apps, Config),