From 394307bac8f0209e89bc86135451e33c6b00d621 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 22 Jan 2020 21:48:27 -0500 Subject: [PATCH] Initial split of pre-hooks for compiler This allows breaking apart the pre-hooks from the rest of the compilation steps, as a preliminary step towards being able to do some analysis on all project apps at once before actually compiling them. --- src/rebar_prv_compile.erl | 47 ++++---- test/rebar_compile_SUITE.erl | 203 ++++++++++++++++++++++++++++++++++- 2 files changed, 230 insertions(+), 20 deletions(-) diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 2919c40c..56fc08bf 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -6,8 +6,7 @@ do/1, format_error/1]). --export([compile/2, - compile/3]). +-export([compile/2, compile/3]). -include_lib("providers/include/providers.hrl"). -include("rebar.hrl"). @@ -98,15 +97,22 @@ format_error(Reason) -> io_lib:format("~p", [Reason]). copy_and_build_apps(State, Providers, Apps) -> - [build_app(State, Providers, AppInfo) || AppInfo <- Apps]. + PrepApps = [prepare_app(State, Providers, AppInfo) || AppInfo <- Apps], + [compile_prepared(State, Providers, AppInfo) || AppInfo <- PrepApps]. -build_app(State, Providers, AppInfo) -> +prepare_app(State, Providers, AppInfo) -> AppDir = rebar_app_info:dir(AppInfo), OutDir = rebar_app_info:out_dir(AppInfo), copy_app_dirs(AppInfo, AppDir, OutDir), - compile(State, Providers, AppInfo). + AppDir = rebar_app_info:dir(AppInfo), + AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), + rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State). copy_and_build_project_apps(State, Providers, Apps) -> + [compile_prepared(State, Providers, AppInfo) + || AppInfo <- prepare_project_apps(State, Providers, Apps)]. + +prepare_project_apps(State, Providers, Apps) -> %% Top-level apps, because of profile usage and specific orderings (i.e. %% may require an include file from a profile-specific app for an extra_dirs %% entry that only exists in a test context), need to be @@ -116,8 +122,12 @@ copy_and_build_project_apps(State, Providers, Apps) -> rebar_app_info:out_dir(AppInfo)) || AppInfo <- Apps], code:add_pathsa([rebar_app_info:ebin_dir(AppInfo) || AppInfo <- Apps]), - [compile(State, Providers, AppInfo) || AppInfo <- Apps]. + [pre_hooks(State, Providers, AppInfo) || AppInfo <- Apps]. +pre_hooks(State, Providers, AppInfo) -> + AppDir = rebar_app_info:dir(AppInfo), + AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), + rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State). build_extra_dirs(State, Apps) -> BaseDir = rebar_state:dir(State), @@ -156,33 +166,32 @@ compile(State, AppInfo) -> compile(State, rebar_state:providers(State), AppInfo). compile(State, Providers, AppInfo) -> + compile_prepared(State, Providers, pre_hooks(State, Providers, AppInfo)). + +compile_prepared(State, Providers, AppInfo) -> ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]), AppDir = rebar_app_info:dir(AppInfo), - AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER, Providers, AppInfo, State), - - AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State), - - build_app(AppInfo2, State), - AppInfo3 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo2, State), + build_app(AppInfo, State), - AppInfo4 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo3, State), + AppInfo1 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo, State), + AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo1, State), %% Load plugins back for make_vsn calls in custom resources. %% The rebar_otp_app compilation step is safe regarding the %% overall path management, so we can just load all plugins back %% in memory. rebar_paths:set_paths([plugins], State), - AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4), + AppFileCompileResult = rebar_otp_app:compile(State, AppInfo2), %% Clean up after ourselves, leave things as they were with deps first rebar_paths:set_paths([deps], State), case AppFileCompileResult of - {ok, AppInfo5} -> - AppInfo6 = rebar_hooks:run_all_hooks(AppDir, post, ?APP_HOOK, Providers, AppInfo5, State), - AppInfo7 = rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo6, State), - has_all_artifacts(AppInfo5), - AppInfo7; + {ok, AppInfo3} -> + AppInfo4 = rebar_hooks:run_all_hooks(AppDir, post, ?APP_HOOK, Providers, AppInfo3, State), + AppInfo5 = rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo4, State), + has_all_artifacts(AppInfo3), + AppInfo5; Error -> throw(Error) end. diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index bb7f9e8b..cb151485 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -40,7 +40,7 @@ all() -> always_recompile_when_erl_compiler_options_set, dont_recompile_when_erl_compiler_options_env_does_not_change, recompile_when_erl_compiler_options_env_changes, - rebar_config_os_var, + rebar_config_os_var, split_project_apps_hooks, app_file_linting]. groups() -> @@ -2375,6 +2375,207 @@ regex_filter_regression(Config) -> {ok, [{file, Expected}]}), ok. +%% This test could also have existed in rebar_hooks_SUITE but it's more +%% about compiler implementation details than the hook logic itself, +%% so it was located here. +split_project_apps_hooks() -> + [{doc, "Ensure that a project with multiple project apps runs the " + "pre-hooks before all the apps are compiled, and the post " + "hooks after they are all compiled."}]. +split_project_apps_hooks(Config) -> + BaseDir = ?config(apps, Config), + Name1 = rebar_test_utils:create_random_name("app2_"), + Name2 = rebar_test_utils:create_random_name("app1_"), + AppDir1 = filename:join([BaseDir, "lib", Name1]), + AppDir2 = filename:join([BaseDir, "lib", Name2]), + HookDir = filename:join([BaseDir, "hooks"]), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(AppDir1, Name1, Vsn, [kernel, stdlib, list_to_atom(Name2)]), + rebar_test_utils:create_app(AppDir2, Name2, Vsn, [kernel, stdlib]), + + ok = filelib:ensure_dir(filename:join([AppDir1, "src", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppDir1, "test", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppDir2, "src", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppDir2, "include", "dummy"])), + ok = filelib:ensure_dir(filename:join([HookDir, "dummy"])), + Cfg = fun(Name) -> + [{pre_hooks, [{compile, "ls "++HookDir++" > "++filename:join(HookDir, "pre-compile-"++Name)}, + {erlc_compile, "ls "++HookDir++" > "++filename:join(HookDir, "pre-erlc-"++Name)}, + {app_compile, "ls "++HookDir++" > "++filename:join(HookDir, "pre-app-"++Name)}]}, + {post_hooks, [{compile, "ls "++HookDir++" > "++filename:join(HookDir, "post-compile-"++Name)}, + {erlc_compile, "ls "++HookDir++" > "++filename:join(HookDir, "post-erlc-"++Name)}, + {app_compile, "ls "++HookDir++" > "++filename:join(HookDir, "post-app-"++Name)}]} + ] + end, + ok = file:write_file(filename:join(AppDir1, "rebar.config"), + io_lib:format("~p.~n~p.", Cfg("app1"))), + ok = file:write_file(filename:join(AppDir2, "rebar.config"), + io_lib:format("~p.~n~p.", Cfg("app2"))), + RebarConfig = Cfg("all"), + ct:pal("RebarConfig: ~p~n", [RebarConfig]), + rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"], + {ok, [{app, Name1}, {app, Name2}]}), + %% Now for the big party: + %% - we expect whole pre-hooks to run before either app is compiled + %% - we don't expect app and erlc hooks on the total run + %% - we expect app2 to be compiled before app1 (rev alphabetical order) + %% - we expect all pre-hooks to show up before post-hooks + %% - the pre-order is: compile->erlc, taking place before any app + %% is actually compiled, so that analysis can be done on all apps. + %% - the post-order is more as expected: + %% - erlc post hook runs right with the app + %% - app pre hook runs right with the app + %% - app post hook runs right with the app + %% - compile post hook runs for each app individually + %% - we expect app compile post-hooks to show up in order + %% - we expect whole post-hooks to run last + ?assertEqual({ok, <<"pre-compile-all\n">>}, + file:read_file(filename:join(HookDir, "pre-compile-all"))), + ?assertEqual({ok, << + "pre-compile-all\n" + "pre-compile-app2\n" + >>}, + file:read_file(filename:join(HookDir, "pre-compile-app2"))), + ?assertEqual({ok, << + "pre-compile-all\n" + "pre-compile-app2\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "pre-erlc-app2"))), + ?assertEqual({ok, << + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "pre-compile-app1"))), + ?assertEqual({ok, << + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "pre-erlc-app1"))), + ?assertEqual({ok, << + "post-erlc-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "post-erlc-app2"))), + ?assertEqual({ok, << + "post-erlc-app2\n" + "pre-app-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "pre-app-app2"))), + ?assertEqual({ok, << + "post-app-app2\n" + "post-erlc-app2\n" + "pre-app-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "post-app-app2"))), + ?assertEqual({ok, << + "post-app-app2\n" + "post-compile-app2\n" + "post-erlc-app2\n" + "pre-app-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "post-compile-app2"))), + ?assertEqual({ok, << + "post-app-app2\n" + "post-compile-app2\n" + "post-erlc-app1\n" + "post-erlc-app2\n" + "pre-app-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "post-erlc-app1"))), + ?assertEqual({ok, << + "post-app-app2\n" + "post-compile-app2\n" + "post-erlc-app1\n" + "post-erlc-app2\n" + "pre-app-app1\n" + "pre-app-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "pre-app-app1"))), + ?assertEqual({ok, << + "post-app-app1\n" + "post-app-app2\n" + "post-compile-app2\n" + "post-erlc-app1\n" + "post-erlc-app2\n" + "pre-app-app1\n" + "pre-app-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "post-app-app1"))), + ?assertEqual({ok, << + "post-app-app1\n" + "post-app-app2\n" + "post-compile-app1\n" + "post-compile-app2\n" + "post-erlc-app1\n" + "post-erlc-app2\n" + "pre-app-app1\n" + "pre-app-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "post-compile-app1"))), + ?assertEqual({ok, << + "post-app-app1\n" + "post-app-app2\n" + "post-compile-all\n" + "post-compile-app1\n" + "post-compile-app2\n" + "post-erlc-app1\n" + "post-erlc-app2\n" + "pre-app-app1\n" + "pre-app-app2\n" + "pre-compile-all\n" + "pre-compile-app1\n" + "pre-compile-app2\n" + "pre-erlc-app1\n" + "pre-erlc-app2\n" + >>}, + file:read_file(filename:join(HookDir, "post-compile-all"))), + ok. + app_file_linting(Config) -> meck:new(rebar_log, [no_link, passthrough]), AppDir = ?config(apps, Config),