From be1680e443113b0c8432d632c288c92fba428bf0 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 24 Jul 2020 13:01:26 +0000 Subject: [PATCH 1/4] Trigger rebuilds when OTP compiler version changes This fix is done in three parts: 1. Add the compiler version to the data tracked in the DAG, and extract it from source file (when the DAG isn't around) for comparisons. Then on each build job, check for the compiler version as a build option like every other one. A change in compiler version represents a change in build options, which triggers a rebuild 2. Make it work for plugins. This requires more work because plugins that can find their ebin/ dir are assumed to be pre-built. Rather than undoing this, poke at 1 single beam file in their ebin/ dir and check if their compiler version matches ours to do an additional filter round. This is likely to be faster than reanalyzing the whole app, but a bit more brittle in the long run. 3. Make it work for deps. This currently is done with copy/pasted plugin code that needs to be reworked later if this is shown to be an acceptable extra step. This patch also includes a fix on the path pruning for plugins, which mistakenly subtracted AppInfo records from file paths; the intent was to use the ebin dir as a comparison, which this patch fixes. No tests have been added at this point. --- src/rebar_compiler.erl | 1 + src/rebar_compiler_erl.erl | 14 +++++++---- src/rebar_plugins.erl | 45 ++++++++++++++++++++++++++++++++-- src/rebar_prv_install_deps.erl | 33 ++++++++++++++++++++++++- 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index d7081b00..c9d808e1 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -215,6 +215,7 @@ prepare_compiler_env(Compiler, Apps) -> %% necessary for erlang:function_exported/3 to work as expected %% called here for clarity as it's required by both opts_changed/2 %% and erl_compiler_opts_set/0 in needed_files + application:load(compiler), _ = code:ensure_loaded(compile), _ = code:ensure_loaded(Compiler), ok. diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl index 5c00ab8a..b366426d 100644 --- a/src/rebar_compiler_erl.erl +++ b/src/rebar_compiler_erl.erl @@ -145,9 +145,10 @@ compile_and_track(Source, [{Ext, OutDir}], Config, ErlOpts) -> rebar_compiler_epp:flush(), BuildOpts = [{outdir, OutDir} | ErlOpts], Target = target_base(OutDir, Source) ++ Ext, + {ok, CompileVsn} = application:get_key(compiler, vsn), AllOpts = case erlang:function_exported(compile, env_compiler_options, 0) of - true -> BuildOpts ++ compile:env_compiler_options(); - false -> BuildOpts + true -> [{compiler_version, CompileVsn}] ++ BuildOpts ++ compile:env_compiler_options(); + false -> [{compiler_version, CompileVsn}] ++ BuildOpts end, case compile:file(Source, BuildOpts) of {ok, _Mod} -> @@ -237,9 +238,10 @@ target_base(OutDir, Source) -> filename:join(OutDir, filename:basename(Source, ".erl")). opts_changed(Graph, NewOpts, Target, TargetBase) -> + {ok, CompileVsn} = application:get_key(compiler, vsn), TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of - true -> NewOpts ++ compile:env_compiler_options(); - false -> NewOpts + true -> [{compiler_version, CompileVsn}] ++ NewOpts ++ compile:env_compiler_options(); + false -> [{compiler_version, CompileVsn}] ++ NewOpts end, TargetOpts = case digraph:vertex(Graph, Target) of {_Target, {artifact, Opts}} -> % tracked dep is found @@ -273,7 +275,9 @@ compile_info(Target) -> case beam_lib:chunks(Target, [compile_info]) of {ok, {_mod, Chunks}} -> CompileInfo = proplists:get_value(compile_info, Chunks, []), - {ok, proplists:get_value(options, CompileInfo, [])}; + CompileVsn = proplists:get_value(version, CompileInfo, "unknown"), + {ok, [{compiler_version, CompileVsn} + | proplists:get_value(options, CompileInfo, [])]}; {error, beam_lib, Reason} -> ?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]), {error, Reason} diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 43b33db8..3bbd4a88 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -113,11 +113,16 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []), %% Add already built plugin deps to the code path - PreBuiltPaths = [rebar_app_info:ebin_dir(A) || A <- Apps] -- ToBuild, + ToBuildPaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild], + PreBuiltApps = [A || A <- Apps, + Ebin <- [rebar_app_info:ebin_dir(A)], + not lists:member(Ebin, ToBuildPaths)], + {PreUnsafe, PreSafe} = lists:partition(fun needs_rebuild/1, PreBuiltApps), + PreBuiltPaths = [rebar_app_info:ebin_dir(A) || A <- PreSafe], code:add_pathsa(PreBuiltPaths), %% Build plugin and its deps - [build_plugin(AppInfo, Apps, State2) || AppInfo <- ToBuild], + [build_plugin(AppInfo, Apps, State2) || AppInfo <- PreUnsafe++ToBuild], %% Add newly built deps and plugin to code path State3 = rebar_state:update_all_plugin_deps(State2, Apps), @@ -165,3 +170,39 @@ validate_plugin(Plugin) -> [Plugin] end end. + +%% @private do a quick validation of whether a plugin is expected to need +%% to be rebuilt; usually this is handled by the compiler, but since this +%% module does quick filtering by detection, we need to discriminate against +%% cases like the compiler version having changed that would otherwise +%% trigger a rebuild. +needs_rebuild(AppInfo) -> + Ebin = rebar_app_info:ebin_dir(AppInfo), + application:load(compiler), + {ok, CurrentCompileVsn} = application:get_key(compiler, vsn), + case find_some_beam(Ebin) of + {ok, Beam} -> + case beam_lib:chunks(Beam, [compile_info]) of + {ok, {_mod, Chunks}} -> + CompileInfo = proplists:get_value(compile_info, Chunks, []), + CompileVsn = proplists:get_value(version, CompileInfo, "unknown"), + CurrentCompileVsn =/= CompileVsn; + _ -> + %% could be built without debug_info + false + end; + _ -> + %% well we would expect a plugin to have a beam file + true + end. + +find_some_beam(Path) -> + case file:list_dir(Path) of + {error, Reason} -> + {error, Reason}; + {ok, Files} -> + case lists:dropwhile(fun(P) -> filename:extension(P) =/= ".beam" end, Files) of + [Beam|_] -> {ok, filename:join(Path, Beam)}; + _ -> {error, no_beam} + end + end. diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 34334d51..26caefbe 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -436,7 +436,8 @@ warn_skip_deps(AppInfo, State) -> not_needs_compile(App) -> not(rebar_app_info:is_checkout(App)) andalso rebar_app_info:valid(App) - andalso rebar_app_info:has_all_artifacts(App) =:= true. + andalso rebar_app_info:has_all_artifacts(App) =:= true + andalso not needs_rebuild(App). find_app_and_level_by_name([], _) -> false; @@ -446,3 +447,33 @@ find_app_and_level_by_name([App|Apps], Name) -> _ -> find_app_and_level_by_name(Apps, Name) end. +needs_rebuild(AppInfo) -> + Ebin = rebar_app_info:ebin_dir(AppInfo), + application:load(compiler), + {ok, CurrentCompileVsn} = application:get_key(compiler, vsn), + case find_some_beam(Ebin) of + {ok, Beam} -> + case beam_lib:chunks(Beam, [compile_info]) of + {ok, {_mod, Chunks}} -> + CompileInfo = proplists:get_value(compile_info, Chunks, []), + CompileVsn = proplists:get_value(version, CompileInfo, "unknown"), + CurrentCompileVsn =/= CompileVsn; + _ -> + %% could be built without debug_info + false + end; + _ -> + %% well we would expect a plugin to have a beam file + true + end. + +find_some_beam(Path) -> + case file:list_dir(Path) of + {error, Reason} -> + {error, Reason}; + {ok, Files} -> + case lists:dropwhile(fun(P) -> filename:extension(P) =/= ".beam" end, Files) of + [Beam|_] -> {ok, filename:join(Path, Beam)}; + _ -> {error, no_beam} + end + end. From 3e30b824ef74b1f1e49327721cc08e1a364c82fc Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 28 Oct 2020 14:26:31 +0000 Subject: [PATCH 2/4] Move dep compiler version change to DAG check Rather than doing a check on each single file in a dep set (which can grow quite large), do a dynamic rebuild of deps based on whether the DAG mentions the files have been modified. To avoid reloading the whole DAG more than once and then having to still map files, the compiler version is added to rebar_compiler_erl's CritMeta for the DAG. This comparison is only set in place to impact the rebar_prv_install_deps call that handles pruncing and culling deps by the compiler provider. Intra-DAG comparisons still need to be done by rebar_compiler_erl for things to work, and the hope is that since most built-in compiler mods end up generating erl files in deps, that will be transparent enough to work well. A DAG call to check the status of the dag (vsn, critmeta, existing, etc.) has been added to do the faster check there and allowing proper dispatch. Plugins couldn't be updated because they don't really use the standard dep-style facilities for things due to an early filtering. --- src/rebar_compiler_dag.erl | 26 ++++++++- src/rebar_prv_compile.erl | 87 ++++++++++++++++++++++++------- src/rebar_prv_install_deps.erl | 34 +----------- test/rebar_compile_SUITE.erl | 7 ++- test/rebar_compiler_dag_SUITE.erl | 15 +++++- 5 files changed, 113 insertions(+), 56 deletions(-) diff --git a/src/rebar_compiler_dag.erl b/src/rebar_compiler_dag.erl index 6fcc3e1a..d6038e83 100644 --- a/src/rebar_compiler_dag.erl +++ b/src/rebar_compiler_dag.erl @@ -1,7 +1,7 @@ %%% Module handling the directed graph required for the analysis %%% of all top-level applications by the various compiler plugins. -module(rebar_compiler_dag). --export([init/4, maybe_store/5, terminate/1]). +-export([init/4, status/4, maybe_store/5, terminate/1]). -export([prune/5, populate_sources/5, populate_deps/3, propagate_stamps/1, compile_order/2, store_artifact/4]). @@ -39,6 +39,29 @@ init(Dir, Compiler, Label, CritMeta) -> end, G. +%% @doc Quickly validate whether a DAG exists by validating its file name, +%% version, and CritMeta data, without attempting to actually build it. +-spec status(file:filename_all(), atom(), string() | undefined, critical_meta()) -> + valid | bad_format | bad_vsn | bad_meta | not_found. +status(Dir, Compiler, Label, CritMeta) -> + File = dag_file(Dir, Compiler, Label), + case file:read_file(File) of + {ok, Data} -> + %% The CritMeta value is checked and if it doesn't match, we + %% consider things invalid. Same for the version. + try binary_to_term(Data) of + #dag{vsn = ?DAG_VSN, meta = CritMeta} -> valid; + #dag{vsn = ?DAG_VSN} -> bad_meta; + #dag{meta = CritMeta} -> bad_vsn; + _ -> bad_format + catch + _:_ -> + bad_format + end; + {error, _Err} -> + not_found + end. + %% @doc Clear up inactive (deleted) source files from a given project. %% The file must be in one of the directories that may contain source files %% for an OTP application; source files found in the DAG `G' that lie outside @@ -240,6 +263,7 @@ terminate(G) -> true = digraph:delete(G). store_artifact(G, Source, Target, Meta) -> + mark_dirty(G), digraph:add_vertex(G, Target, {artifact, Meta}), digraph:add_edge(G, Target, Source, artifact). diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index f799cbd0..d4d268d7 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -39,8 +39,9 @@ do(State) -> rebar_paths:set_paths([deps], State), Providers = rebar_state:providers(State), - Deps = rebar_state:deps_to_build(State), - CompiledDeps = copy_and_build_apps(State, Providers, Deps), + MustBuildDeps = rebar_state:deps_to_build(State), + Deps = rebar_state:all_deps(State), + CompiledDeps = copy_and_build_deps(State, Providers, MustBuildDeps, Deps), State0 = rebar_state:merge_all_deps(State, CompiledDeps), State1 = case IsDepsOnly of @@ -97,7 +98,30 @@ format_error({unknown_project_type, Name, Type}) -> format_error(Reason) -> io_lib:format("~p", [Reason]). -copy_and_build_apps(State, Providers, Apps) -> +pick_deps_to_build(State, MustBuild, All, Tag) -> + InvalidDags = lists:any( + fun({_Mod, Status}) -> + %% A DAG that is not found can be valid, it just has no matching + %% source file. However, bad_vsn, bad_format, and bad_meta should + %% all trigger rebuilds. + %% Good: + %% lists:member(Status, [valid, not_found]) + %% Bad: + %% lists:member(Status, [bad_vsn, bad_format, bad_meta]) + %% + %% Since the fastest check is done on smaller lists handling + %% the common case first: + not lists:member(Status, [valid, not_found]) + end, + check_dags(State, Tag) + ), + case InvalidDags of + true -> All; + false -> MustBuild + end. + +copy_and_build_deps(State, Providers, MustBuildApps, AllApps) -> + Apps = pick_deps_to_build(State, MustBuildApps, AllApps, apps), Apps0 = [prepare_app(State, Providers, App) || App <- Apps], compile(State, Providers, Apps0, apps). @@ -155,31 +179,54 @@ prepare_compilers(State, Providers, AppInfo) -> rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo, State). run_compilers(State, _Providers, Apps, Tag) -> - %% Prepare a compiler digraph to be shared by all compiled applications - %% in a given run, providing the ability to combine their dependency - %% ordering and resources. - %% The Tag allows to create a Label when someone cares about a specific - %% run for compilation; - DAGLabel = case Tag of - undefined -> undefined; - _ -> atom_to_list(Tag) - end, - %% The Dir for the DAG is set to deps_dir so builds taking place - %% in different contexts (i.e. plugins) don't risk clobbering regular deps. - Dir = rebar_dir:deps_dir(State), - CritMeta = [], % used to be incldirs per app - DAGs = [{Mod, rebar_compiler_dag:init(Dir, Mod, DAGLabel, CritMeta)} - || Mod <- rebar_state:compilers(State)], + %% Get the compiler graphs for all modules and artifacts, for each + %% compiler we run + DAGsInfo = load_dags(State, Tag), + DAGs = [{Mod, DAG} || {Mod, {DAG, _Meta}} <- DAGsInfo], %% Compile all the apps build_apps(DAGs, Apps, State), %% Potentially store shared compiler DAGs so next runs can easily %% share the base information for easy re-scans. - lists:foreach(fun({Mod, G}) -> + lists:foreach(fun({Mod, {G, {Dir, DAGLabel, CritMeta}}}) -> rebar_compiler_dag:maybe_store(G, Dir, Mod, DAGLabel, CritMeta), rebar_compiler_dag:terminate(G) - end, DAGs), + end, DAGsInfo), Apps. +load_dags(State, Tag) -> + F = fun(Dir, Mod, DAGLabel, CritMeta) -> + {rebar_compiler_dag:init(Dir, Mod, DAGLabel, CritMeta), + {Dir, DAGLabel, CritMeta}} + end, + map_dags(F, State, Tag). + +check_dags(State, Tag) -> + map_dags(fun rebar_compiler_dag:status/4, State, Tag). + +map_dags(F, State, Tag) -> + DAGLabel = format_label(Tag), + %% The Dir for the DAG is set to deps_dir so builds taking place + %% in different contexts (i.e. plugins) don't risk clobbering regular deps. + Dir = rebar_dir:deps_dir(State), + SharedCritMeta = [], + [{Mod, F(Dir, Mod, DAGLabel, mod_meta(Mod, SharedCritMeta))} + || Mod <- rebar_state:compilers(State)]. + +mod_meta(rebar_compiler_erl, CritMeta) -> + application:load(compiler), + {ok, CompileVsn} = application:get_key(compiler, vsn), + [{compiler, CompileVsn} | CritMeta]; +mod_meta(_, CritMeta) -> + CritMeta. + +format_label(Tag) -> + %% The Tag allows to create a Label when someone cares about a specific + %% run for compilation; + case Tag of + undefined -> undefined; + _ -> atom_to_list(Tag) + end. + finalize_compilers(State, Providers, AppInfo) -> AppDir = rebar_app_info:dir(AppInfo), rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo, State). diff --git a/src/rebar_prv_install_deps.erl b/src/rebar_prv_install_deps.erl index 26caefbe..5331c4cb 100644 --- a/src/rebar_prv_install_deps.erl +++ b/src/rebar_prv_install_deps.erl @@ -436,8 +436,7 @@ warn_skip_deps(AppInfo, State) -> not_needs_compile(App) -> not(rebar_app_info:is_checkout(App)) andalso rebar_app_info:valid(App) - andalso rebar_app_info:has_all_artifacts(App) =:= true - andalso not needs_rebuild(App). + andalso rebar_app_info:has_all_artifacts(App) =:= true. find_app_and_level_by_name([], _) -> false; @@ -446,34 +445,3 @@ find_app_and_level_by_name([App|Apps], Name) -> Name -> {ok, App, rebar_app_info:dep_level(App)}; _ -> find_app_and_level_by_name(Apps, Name) end. - -needs_rebuild(AppInfo) -> - Ebin = rebar_app_info:ebin_dir(AppInfo), - application:load(compiler), - {ok, CurrentCompileVsn} = application:get_key(compiler, vsn), - case find_some_beam(Ebin) of - {ok, Beam} -> - case beam_lib:chunks(Beam, [compile_info]) of - {ok, {_mod, Chunks}} -> - CompileInfo = proplists:get_value(compile_info, Chunks, []), - CompileVsn = proplists:get_value(version, CompileInfo, "unknown"), - CurrentCompileVsn =/= CompileVsn; - _ -> - %% could be built without debug_info - false - end; - _ -> - %% well we would expect a plugin to have a beam file - true - end. - -find_some_beam(Path) -> - case file:list_dir(Path) of - {error, Reason} -> - {error, Reason}; - {ok, Files} -> - case lists:dropwhile(fun(P) -> filename:extension(P) =/= ".beam" end, Files) of - [Beam|_] -> {ok, filename:join(Path, Beam)}; - _ -> {error, no_beam} - end - end. diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index 3252a977..feff69ee 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -987,7 +987,12 @@ recompile_when_dag_opts_change(Config) -> %% change the config in the DAG... [digraph:add_vertex(G, Beam, {artifact, [{d, some_define}]}) || Beam <- Beams], digraph:add_vertex(G, '$r3_dirty_bit', true), % trigger a save - rebar_compiler_dag:maybe_store(G, DepsDir, rebar_compiler_erl, "project_apps", []), + %% the rebar_compiler_erl module is annotated with a compiler version + %% to help rebuild deps + {ok, CompileVsn} = application:get_key(compiler, vsn), + CritMeta = [{compiler, CompileVsn}], + + rebar_compiler_dag:maybe_store(G, DepsDir, rebar_compiler_erl, "project_apps", CritMeta), rebar_compiler_dag:terminate(G), %% ... but don't change the actual rebar3 config... rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}), diff --git a/test/rebar_compiler_dag_SUITE.erl b/test/rebar_compiler_dag_SUITE.erl index ebd90ff8..0fac5b3a 100644 --- a/test/rebar_compiler_dag_SUITE.erl +++ b/test/rebar_compiler_dag_SUITE.erl @@ -6,7 +6,7 @@ -include_lib("kernel/include/file.hrl"). all() -> - [{group, with_project}]. + [exists, {group, with_project}]. groups() -> %% The tests in this group are dirty, the order is specific @@ -55,6 +55,19 @@ init_per_group(_, Config) -> end_per_group(_, Config) -> Config. +exists(Config) -> + %% Create a DAG + Priv = ?config(priv_dir, Config), + G = rebar_compiler_dag:init(Priv, compilermod, "label", [crit_meta]), + rebar_compiler_dag:store_artifact(G, "somefile", "someartifact", [written]), + rebar_compiler_dag:maybe_store(G, Priv, compilermod, "label", [crit_meta]), + rebar_compiler_dag:terminate(G), + + ?assertEqual(valid, rebar_compiler_dag:status(Priv, compilermod, "label", [crit_meta])), + ?assertEqual(not_found, rebar_compiler_dag:status(Priv, compilermad, "label", [crit_meta])), + ?assertEqual(not_found, rebar_compiler_dag:status(Priv, compilermod, "lobel", [crit_meta])), + ?assertEqual(bad_meta, rebar_compiler_dag:status(Priv, compilermod, "label", [crit_zeta])), + ok. project() -> [{app1, [ From 54396dbfc041e9f66b7ca9f7623661789313f728 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 29 Oct 2020 17:57:03 +0000 Subject: [PATCH 3/4] Fold plugin build into main dependency workflow This was a long-standing pending issue with plugins having a weird code path handling. The work done in the previous commit to do version detection for the compiler in dependencies along with prior PRs that added a deps-only mode to the `compile` provider means we can now re-introduce plugins into the 3.14 compilation mechanism. This makes compiler version detection a compiler task concern and removes annoying manual checks, but also solves an unreported issue (brought up on slack only) where a plugin would no longer have its priv/ or include/ dir included when used as a checkout dependency. Essentially the latter bug was caused by being out-of-sync with application preparation for a build task, but by re-folding it into the main flow, we know we have the proper support as used for deps. --- bootstrap | 5 ++-- src/rebar_plugins.erl | 53 ++++++++----------------------------------- 2 files changed, 13 insertions(+), 45 deletions(-) diff --git a/bootstrap b/bootstrap index c338624f..b58f1e52 100755 --- a/bootstrap +++ b/bootstrap @@ -20,10 +20,11 @@ main(_) -> %% cause weird failures when compilers get modified between releases. rm_rf("_build/prod"), %% The same pattern happens with default/ as well, particularly when - %% developing new things. - %% Keep other deps in default/lib for build environments like Nix + %% developing new things. + %% Keep other deps in /lib for build environments like Nix %% where internet access is disabled that deps are not downloadable. rm_rf("_build/default/lib/rebar"), + rm_rf("_build/test/lib/rebar"), %% We fetch a few deps from hex for boostraping, %% so we must compile r3_safe_erl_term.xrl which diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 3bbd4a88..7fe56b69 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -117,12 +117,11 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> PreBuiltApps = [A || A <- Apps, Ebin <- [rebar_app_info:ebin_dir(A)], not lists:member(Ebin, ToBuildPaths)], - {PreUnsafe, PreSafe} = lists:partition(fun needs_rebuild/1, PreBuiltApps), - PreBuiltPaths = [rebar_app_info:ebin_dir(A) || A <- PreSafe], + PreBuiltPaths = [rebar_app_info:ebin_dir(A) || A <- PreBuiltApps], code:add_pathsa(PreBuiltPaths), %% Build plugin and its deps - [build_plugin(AppInfo, Apps, State2) || AppInfo <- PreUnsafe++ToBuild], + build_plugins(ToBuild -- PreBuiltApps, Apps, State2), %% Add newly built deps and plugin to code path State3 = rebar_state:update_all_plugin_deps(State2, Apps), @@ -140,11 +139,14 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> {[], State} end. -build_plugin(AppInfo, Apps, State) -> - Providers = rebar_state:providers(State), - S = rebar_state:all_deps(State, Apps), - S1 = rebar_state:set(S, deps_dir, ?DEFAULT_PLUGINS_DIR), - rebar_prv_compile:compile(S1, Providers, AppInfo). +build_plugins(MustBuildApps, AllApps, State) -> + State1 = rebar_state:deps_to_build(State, MustBuildApps), + State2 = rebar_state:all_deps(State1, AllApps), + State3 = rebar_state:set(State2, deps_dir, ?DEFAULT_PLUGINS_DIR), + {Args, Extra} = rebar_state:command_parsed_args(State), + State4 = rebar_state:command_parsed_args(State3, {[{deps_only, true}|Args], Extra}), + rebar_prv_compile:do(State4), + ok. plugin_providers({Plugin, _, _, _}) when is_atom(Plugin) -> validate_plugin(Plugin); @@ -171,38 +173,3 @@ validate_plugin(Plugin) -> end end. -%% @private do a quick validation of whether a plugin is expected to need -%% to be rebuilt; usually this is handled by the compiler, but since this -%% module does quick filtering by detection, we need to discriminate against -%% cases like the compiler version having changed that would otherwise -%% trigger a rebuild. -needs_rebuild(AppInfo) -> - Ebin = rebar_app_info:ebin_dir(AppInfo), - application:load(compiler), - {ok, CurrentCompileVsn} = application:get_key(compiler, vsn), - case find_some_beam(Ebin) of - {ok, Beam} -> - case beam_lib:chunks(Beam, [compile_info]) of - {ok, {_mod, Chunks}} -> - CompileInfo = proplists:get_value(compile_info, Chunks, []), - CompileVsn = proplists:get_value(version, CompileInfo, "unknown"), - CurrentCompileVsn =/= CompileVsn; - _ -> - %% could be built without debug_info - false - end; - _ -> - %% well we would expect a plugin to have a beam file - true - end. - -find_some_beam(Path) -> - case file:list_dir(Path) of - {error, Reason} -> - {error, Reason}; - {ok, Files} -> - case lists:dropwhile(fun(P) -> filename:extension(P) =/= ".beam" end, Files) of - [Beam|_] -> {ok, filename:join(Path, Beam)}; - _ -> {error, no_beam} - end - end. From aa95b79020875a48234d8b92c713c928d65a5e1f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 29 Oct 2020 18:16:47 +0000 Subject: [PATCH 4/4] Keep the prebuilt app set broader Closer to what it was initially, and reduces some extra operations too. --- src/rebar_plugins.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/rebar_plugins.erl b/src/rebar_plugins.erl index 7fe56b69..f2d22233 100644 --- a/src/rebar_plugins.erl +++ b/src/rebar_plugins.erl @@ -114,14 +114,13 @@ handle_plugin(Profile, Plugin, State, Upgrade) -> %% Add already built plugin deps to the code path ToBuildPaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild], - PreBuiltApps = [A || A <- Apps, - Ebin <- [rebar_app_info:ebin_dir(A)], - not lists:member(Ebin, ToBuildPaths)], - PreBuiltPaths = [rebar_app_info:ebin_dir(A) || A <- PreBuiltApps], + PreBuiltPaths = [Ebin || A <- Apps, + Ebin <- [rebar_app_info:ebin_dir(A)], + not lists:member(Ebin, ToBuildPaths)], code:add_pathsa(PreBuiltPaths), %% Build plugin and its deps - build_plugins(ToBuild -- PreBuiltApps, Apps, State2), + build_plugins(ToBuild, Apps, State2), %% Add newly built deps and plugin to code path State3 = rebar_state:update_all_plugin_deps(State2, Apps),