Browse Source

Merge pull request #2317 from ferd/rebuild-on-compiler-change

Trigger rebuilds when OTP compiler version changes
pull/2424/head
Fred Hebert 4 years ago
committed by GitHub
parent
commit
5e22387074
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 139 additions and 38 deletions
  1. +3
    -2
      bootstrap
  2. +1
    -0
      src/rebar_compiler.erl
  3. +25
    -1
      src/rebar_compiler_dag.erl
  4. +9
    -5
      src/rebar_compiler_erl.erl
  5. +14
    -7
      src/rebar_plugins.erl
  6. +67
    -20
      src/rebar_prv_compile.erl
  7. +0
    -1
      src/rebar_prv_install_deps.erl
  8. +6
    -1
      test/rebar_compile_SUITE.erl
  9. +14
    -1
      test/rebar_compiler_dag_SUITE.erl

+ 3
- 2
bootstrap View File

@ -20,10 +20,11 @@ main(_) ->
%% cause weird failures when compilers get modified between releases.
rm_rf("_build/prod"),
%% The same pattern happens with default/ as well, particularly when
%% developing new things.
%% Keep other deps in default/lib for build environments like Nix
%% developing new things.
%% Keep other deps in <profile>/lib for build environments like Nix
%% where internet access is disabled that deps are not downloadable.
rm_rf("_build/default/lib/rebar"),
rm_rf("_build/test/lib/rebar"),
%% We fetch a few deps from hex for boostraping,
%% so we must compile r3_safe_erl_term.xrl which

+ 1
- 0
src/rebar_compiler.erl View File

@ -212,6 +212,7 @@ prepare_compiler_env(Compiler, Apps) ->
%% necessary for erlang:function_exported/3 to work as expected
%% called here for clarity as it's required by both opts_changed/2
%% and erl_compiler_opts_set/0 in needed_files
application:load(compiler),
_ = code:ensure_loaded(compile),
_ = code:ensure_loaded(Compiler),
ok.

+ 25
- 1
src/rebar_compiler_dag.erl View File

@ -1,7 +1,7 @@
%%% Module handling the directed graph required for the analysis
%%% of all top-level applications by the various compiler plugins.
-module(rebar_compiler_dag).
-export([init/4, maybe_store/5, terminate/1]).
-export([init/4, status/4, maybe_store/5, terminate/1]).
-export([prune/5, populate_sources/5, populate_deps/3, propagate_stamps/1,
compile_order/2, store_artifact/4]).
@ -39,6 +39,29 @@ init(Dir, Compiler, Label, CritMeta) ->
end,
G.
%% @doc Quickly validate whether a DAG exists by validating its file name,
%% version, and CritMeta data, without attempting to actually build it.
-spec status(file:filename_all(), atom(), string() | undefined, critical_meta()) ->
valid | bad_format | bad_vsn | bad_meta | not_found.
status(Dir, Compiler, Label, CritMeta) ->
File = dag_file(Dir, Compiler, Label),
case file:read_file(File) of
{ok, Data} ->
%% The CritMeta value is checked and if it doesn't match, we
%% consider things invalid. Same for the version.
try binary_to_term(Data) of
#dag{vsn = ?DAG_VSN, meta = CritMeta} -> valid;
#dag{vsn = ?DAG_VSN} -> bad_meta;
#dag{meta = CritMeta} -> bad_vsn;
_ -> bad_format
catch
_:_ ->
bad_format
end;
{error, _Err} ->
not_found
end.
%% @doc Clear up inactive (deleted) source files from a given project.
%% The file must be in one of the directories that may contain source files
%% for an OTP application; source files found in the DAG `G' that lie outside
@ -240,6 +263,7 @@ terminate(G) ->
true = digraph:delete(G).
store_artifact(G, Source, Target, Meta) ->
mark_dirty(G),
digraph:add_vertex(G, Target, {artifact, Meta}),
digraph:add_edge(G, Target, Source, artifact).

+ 9
- 5
src/rebar_compiler_erl.erl View File

@ -145,9 +145,10 @@ compile_and_track(Source, [{Ext, OutDir}], Config, ErlOpts) ->
rebar_compiler_epp:flush(),
BuildOpts = [{outdir, OutDir} | ErlOpts],
Target = target_base(OutDir, Source) ++ Ext,
{ok, CompileVsn} = application:get_key(compiler, vsn),
AllOpts = case erlang:function_exported(compile, env_compiler_options, 0) of
true -> BuildOpts ++ compile:env_compiler_options();
false -> BuildOpts
true -> [{compiler_version, CompileVsn}] ++ BuildOpts ++ compile:env_compiler_options();
false -> [{compiler_version, CompileVsn}] ++ BuildOpts
end,
case compile:file(Source, BuildOpts) of
{ok, _Mod} ->
@ -237,9 +238,10 @@ target_base(OutDir, Source) ->
filename:join(OutDir, filename:basename(Source, ".erl")).
opts_changed(Graph, NewOpts, Target, TargetBase) ->
{ok, CompileVsn} = application:get_key(compiler, vsn),
TotalOpts = case erlang:function_exported(compile, env_compiler_options, 0) of
true -> NewOpts ++ compile:env_compiler_options();
false -> NewOpts
true -> [{compiler_version, CompileVsn}] ++ NewOpts ++ compile:env_compiler_options();
false -> [{compiler_version, CompileVsn}] ++ NewOpts
end,
TargetOpts = case digraph:vertex(Graph, Target) of
{_Target, {artifact, Opts}} -> % tracked dep is found
@ -273,7 +275,9 @@ compile_info(Target) ->
case beam_lib:chunks(Target, [compile_info]) of
{ok, {_mod, Chunks}} ->
CompileInfo = proplists:get_value(compile_info, Chunks, []),
{ok, proplists:get_value(options, CompileInfo, [])};
CompileVsn = proplists:get_value(version, CompileInfo, "unknown"),
{ok, [{compiler_version, CompileVsn}
| proplists:get_value(options, CompileInfo, [])]};
{error, beam_lib, Reason} ->
?WARN("Couldn't read debug info from ~p for reason: ~p", [Target, Reason]),
{error, Reason}

+ 14
- 7
src/rebar_plugins.erl View File

@ -113,11 +113,14 @@ handle_plugin(Profile, Plugin, State, Upgrade) ->
ToBuild = rebar_prv_install_deps:cull_compile(Sorted, []),
%% Add already built plugin deps to the code path
PreBuiltPaths = [rebar_app_info:ebin_dir(A) || A <- Apps] -- ToBuild,
ToBuildPaths = [rebar_app_info:ebin_dir(A) || A <- ToBuild],
PreBuiltPaths = [Ebin || A <- Apps,
Ebin <- [rebar_app_info:ebin_dir(A)],
not lists:member(Ebin, ToBuildPaths)],
code:add_pathsa(PreBuiltPaths),
%% Build plugin and its deps
[build_plugin(AppInfo, Apps, State2) || AppInfo <- ToBuild],
build_plugins(ToBuild, Apps, State2),
%% Add newly built deps and plugin to code path
State3 = rebar_state:update_all_plugin_deps(State2, Apps),
@ -135,11 +138,14 @@ handle_plugin(Profile, Plugin, State, Upgrade) ->
{[], State}
end.
build_plugin(AppInfo, Apps, State) ->
Providers = rebar_state:providers(State),
S = rebar_state:all_deps(State, Apps),
S1 = rebar_state:set(S, deps_dir, ?DEFAULT_PLUGINS_DIR),
rebar_prv_compile:compile(S1, Providers, AppInfo).
build_plugins(MustBuildApps, AllApps, State) ->
State1 = rebar_state:deps_to_build(State, MustBuildApps),
State2 = rebar_state:all_deps(State1, AllApps),
State3 = rebar_state:set(State2, deps_dir, ?DEFAULT_PLUGINS_DIR),
{Args, Extra} = rebar_state:command_parsed_args(State),
State4 = rebar_state:command_parsed_args(State3, {[{deps_only, true}|Args], Extra}),
rebar_prv_compile:do(State4),
ok.
plugin_providers({Plugin, _, _, _}) when is_atom(Plugin) ->
validate_plugin(Plugin);
@ -165,3 +171,4 @@ validate_plugin(Plugin) ->
[Plugin]
end
end.

+ 67
- 20
src/rebar_prv_compile.erl View File

@ -39,8 +39,9 @@ do(State) ->
rebar_paths:set_paths([deps], State),
Providers = rebar_state:providers(State),
Deps = rebar_state:deps_to_build(State),
CompiledDeps = copy_and_build_apps(State, Providers, Deps),
MustBuildDeps = rebar_state:deps_to_build(State),
Deps = rebar_state:all_deps(State),
CompiledDeps = copy_and_build_deps(State, Providers, MustBuildDeps, Deps),
State0 = rebar_state:merge_all_deps(State, CompiledDeps),
State1 = case IsDepsOnly of
@ -97,7 +98,30 @@ format_error({unknown_project_type, Name, Type}) ->
format_error(Reason) ->
io_lib:format("~p", [Reason]).
copy_and_build_apps(State, Providers, Apps) ->
pick_deps_to_build(State, MustBuild, All, Tag) ->
InvalidDags = lists:any(
fun({_Mod, Status}) ->
%% A DAG that is not found can be valid, it just has no matching
%% source file. However, bad_vsn, bad_format, and bad_meta should
%% all trigger rebuilds.
%% Good:
%% lists:member(Status, [valid, not_found])
%% Bad:
%% lists:member(Status, [bad_vsn, bad_format, bad_meta])
%%
%% Since the fastest check is done on smaller lists handling
%% the common case first:
not lists:member(Status, [valid, not_found])
end,
check_dags(State, Tag)
),
case InvalidDags of
true -> All;
false -> MustBuild
end.
copy_and_build_deps(State, Providers, MustBuildApps, AllApps) ->
Apps = pick_deps_to_build(State, MustBuildApps, AllApps, apps),
Apps0 = [prepare_app(State, Providers, App) || App <- Apps],
compile(State, Providers, Apps0, apps).
@ -155,31 +179,54 @@ prepare_compilers(State, Providers, AppInfo) ->
rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo, State).
run_compilers(State, _Providers, Apps, Tag) ->
%% Prepare a compiler digraph to be shared by all compiled applications
%% in a given run, providing the ability to combine their dependency
%% ordering and resources.
%% The Tag allows to create a Label when someone cares about a specific
%% run for compilation;
DAGLabel = case Tag of
undefined -> undefined;
_ -> atom_to_list(Tag)
end,
%% The Dir for the DAG is set to deps_dir so builds taking place
%% in different contexts (i.e. plugins) don't risk clobbering regular deps.
Dir = rebar_dir:deps_dir(State),
CritMeta = [], % used to be incldirs per app
DAGs = [{Mod, rebar_compiler_dag:init(Dir, Mod, DAGLabel, CritMeta)}
|| Mod <- rebar_state:compilers(State)],
%% Get the compiler graphs for all modules and artifacts, for each
%% compiler we run
DAGsInfo = load_dags(State, Tag),
DAGs = [{Mod, DAG} || {Mod, {DAG, _Meta}} <- DAGsInfo],
%% Compile all the apps
build_apps(DAGs, Apps, State),
%% Potentially store shared compiler DAGs so next runs can easily
%% share the base information for easy re-scans.
lists:foreach(fun({Mod, G}) ->
lists:foreach(fun({Mod, {G, {Dir, DAGLabel, CritMeta}}}) ->
rebar_compiler_dag:maybe_store(G, Dir, Mod, DAGLabel, CritMeta),
rebar_compiler_dag:terminate(G)
end, DAGs),
end, DAGsInfo),
Apps.
load_dags(State, Tag) ->
F = fun(Dir, Mod, DAGLabel, CritMeta) ->
{rebar_compiler_dag:init(Dir, Mod, DAGLabel, CritMeta),
{Dir, DAGLabel, CritMeta}}
end,
map_dags(F, State, Tag).
check_dags(State, Tag) ->
map_dags(fun rebar_compiler_dag:status/4, State, Tag).
map_dags(F, State, Tag) ->
DAGLabel = format_label(Tag),
%% The Dir for the DAG is set to deps_dir so builds taking place
%% in different contexts (i.e. plugins) don't risk clobbering regular deps.
Dir = rebar_dir:deps_dir(State),
SharedCritMeta = [],
[{Mod, F(Dir, Mod, DAGLabel, mod_meta(Mod, SharedCritMeta))}
|| Mod <- rebar_state:compilers(State)].
mod_meta(rebar_compiler_erl, CritMeta) ->
application:load(compiler),
{ok, CompileVsn} = application:get_key(compiler, vsn),
[{compiler, CompileVsn} | CritMeta];
mod_meta(_, CritMeta) ->
CritMeta.
format_label(Tag) ->
%% The Tag allows to create a Label when someone cares about a specific
%% run for compilation;
case Tag of
undefined -> undefined;
_ -> atom_to_list(Tag)
end.
finalize_compilers(State, Providers, AppInfo) ->
AppDir = rebar_app_info:dir(AppInfo),
rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo, State).

+ 0
- 1
src/rebar_prv_install_deps.erl View File

@ -445,4 +445,3 @@ find_app_and_level_by_name([App|Apps], Name) ->
Name -> {ok, App, rebar_app_info:dep_level(App)};
_ -> find_app_and_level_by_name(Apps, Name)
end.

+ 6
- 1
test/rebar_compile_SUITE.erl View File

@ -987,7 +987,12 @@ recompile_when_dag_opts_change(Config) ->
%% change the config in the DAG...
[digraph:add_vertex(G, Beam, {artifact, [{d, some_define}]}) || Beam <- Beams],
digraph:add_vertex(G, '$r3_dirty_bit', true), % trigger a save
rebar_compiler_dag:maybe_store(G, DepsDir, rebar_compiler_erl, "project_apps", []),
%% the rebar_compiler_erl module is annotated with a compiler version
%% to help rebuild deps
{ok, CompileVsn} = application:get_key(compiler, vsn),
CritMeta = [{compiler, CompileVsn}],
rebar_compiler_dag:maybe_store(G, DepsDir, rebar_compiler_erl, "project_apps", CritMeta),
rebar_compiler_dag:terminate(G),
%% ... but don't change the actual rebar3 config...
rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name}]}),

+ 14
- 1
test/rebar_compiler_dag_SUITE.erl View File

@ -6,7 +6,7 @@
-include_lib("kernel/include/file.hrl").
all() ->
[{group, with_project}].
[exists, {group, with_project}].
groups() ->
%% The tests in this group are dirty, the order is specific
@ -55,6 +55,19 @@ init_per_group(_, Config) ->
end_per_group(_, Config) ->
Config.
exists(Config) ->
%% Create a DAG
Priv = ?config(priv_dir, Config),
G = rebar_compiler_dag:init(Priv, compilermod, "label", [crit_meta]),
rebar_compiler_dag:store_artifact(G, "somefile", "someartifact", [written]),
rebar_compiler_dag:maybe_store(G, Priv, compilermod, "label", [crit_meta]),
rebar_compiler_dag:terminate(G),
?assertEqual(valid, rebar_compiler_dag:status(Priv, compilermod, "label", [crit_meta])),
?assertEqual(not_found, rebar_compiler_dag:status(Priv, compilermad, "label", [crit_meta])),
?assertEqual(not_found, rebar_compiler_dag:status(Priv, compilermod, "lobel", [crit_meta])),
?assertEqual(bad_meta, rebar_compiler_dag:status(Priv, compilermod, "label", [crit_zeta])),
ok.
project() ->
[{app1, [

Loading…
Cancel
Save