Initial split of pre-hooks for compiler

This allows breaking apart the pre-hooks from the rest of the
compilation steps, as a preliminary step towards being able to do some
analysis on all project apps at once before actually compiling them.

src/rebar_prv_compile.erl

@@ -6,8 +6,7 @@
          do/1,
          format_error/1]).
 
--export([compile/2,
-         compile/3]).
+-export([compile/2, compile/3]).
 
 -include_lib("providers/include/providers.hrl").
 -include("rebar.hrl").
@@ -98,15 +97,22 @@ format_error(Reason) ->
     io_lib:format("~p", [Reason]).
 
 copy_and_build_apps(State, Providers, Apps) ->
-    [build_app(State, Providers, AppInfo) || AppInfo <- Apps].
+    PrepApps = [prepare_app(State, Providers, AppInfo) || AppInfo <- Apps],
+    [compile_prepared(State, Providers, AppInfo) || AppInfo <- PrepApps].
 
-build_app(State, Providers, AppInfo) ->
+prepare_app(State, Providers, AppInfo) ->
     AppDir = rebar_app_info:dir(AppInfo),
     OutDir = rebar_app_info:out_dir(AppInfo),
     copy_app_dirs(AppInfo, AppDir, OutDir),
-    compile(State, Providers, AppInfo).
+    AppDir = rebar_app_info:dir(AppInfo),
+    AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER,  Providers, AppInfo, State),
+    rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State).
 
 copy_and_build_project_apps(State, Providers, Apps) ->
+    [compile_prepared(State, Providers, AppInfo)
+     || AppInfo <- prepare_project_apps(State, Providers, Apps)].
+
+prepare_project_apps(State, Providers, Apps) ->
     %% Top-level apps, because of profile usage and specific orderings (i.e.
     %% may require an include file from a profile-specific app for an extra_dirs
     %% entry that only exists in a test context), need to be
@@ -116,8 +122,12 @@ copy_and_build_project_apps(State, Providers, Apps) ->
                    rebar_app_info:out_dir(AppInfo))
      || AppInfo <- Apps],
     code:add_pathsa([rebar_app_info:ebin_dir(AppInfo) || AppInfo <- Apps]),
-    [compile(State, Providers, AppInfo) || AppInfo <- Apps].
+    [pre_hooks(State, Providers, AppInfo) || AppInfo <- Apps].
 
+pre_hooks(State, Providers, AppInfo) ->
+    AppDir = rebar_app_info:dir(AppInfo),
+    AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER,  Providers, AppInfo, State),
+    rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State).
 
 build_extra_dirs(State, Apps) ->
     BaseDir = rebar_state:dir(State),
@@ -156,33 +166,32 @@ compile(State, AppInfo) ->
     compile(State, rebar_state:providers(State), AppInfo).
 
 compile(State, Providers, AppInfo) ->
+    compile_prepared(State, Providers, pre_hooks(State, Providers, AppInfo)).
+
+compile_prepared(State, Providers, AppInfo) ->
     ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]),
     AppDir = rebar_app_info:dir(AppInfo),
-    AppInfo1 = rebar_hooks:run_all_hooks(AppDir, pre, ?PROVIDER,  Providers, AppInfo, State),
-
-    AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?ERLC_HOOK, Providers, AppInfo1, State),
-
-    build_app(AppInfo2, State),
 
-    AppInfo3 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo2, State),
+    build_app(AppInfo, State),
 
-    AppInfo4 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo3, State),
+    AppInfo1 = rebar_hooks:run_all_hooks(AppDir, post, ?ERLC_HOOK, Providers, AppInfo, State),
+    AppInfo2 = rebar_hooks:run_all_hooks(AppDir, pre, ?APP_HOOK, Providers, AppInfo1, State),
 
     %% Load plugins back for make_vsn calls in custom resources.
     %% The rebar_otp_app compilation step is safe regarding the
     %% overall path management, so we can just load all plugins back
     %% in memory.
     rebar_paths:set_paths([plugins], State),
-    AppFileCompileResult = rebar_otp_app:compile(State, AppInfo4),
+    AppFileCompileResult = rebar_otp_app:compile(State, AppInfo2),
     %% Clean up after ourselves, leave things as they were with deps first
     rebar_paths:set_paths([deps], State),
 
     case AppFileCompileResult of
-        {ok, AppInfo5} ->
-            AppInfo6 = rebar_hooks:run_all_hooks(AppDir, post, ?APP_HOOK, Providers, AppInfo5, State),
-            AppInfo7 = rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo6, State),
-            has_all_artifacts(AppInfo5),
-            AppInfo7;
+        {ok, AppInfo3} ->
+            AppInfo4 = rebar_hooks:run_all_hooks(AppDir, post, ?APP_HOOK, Providers, AppInfo3, State),
+            AppInfo5 = rebar_hooks:run_all_hooks(AppDir, post, ?PROVIDER, Providers, AppInfo4, State),
+            has_all_artifacts(AppInfo3),
+            AppInfo5;
         Error ->
             throw(Error)
     end.

test/rebar_compile_SUITE.erl

@@ -40,7 +40,7 @@ all() ->
      always_recompile_when_erl_compiler_options_set,
      dont_recompile_when_erl_compiler_options_env_does_not_change,
      recompile_when_erl_compiler_options_env_changes,
-     rebar_config_os_var,
+     rebar_config_os_var, split_project_apps_hooks,
      app_file_linting].
 
 groups() ->
@@ -2375,6 +2375,207 @@ regex_filter_regression(Config) ->
                                    {ok, [{file, Expected}]}),
     ok.
 
+%% This test could also have existed in rebar_hooks_SUITE but it's more
+%% about compiler implementation details than the hook logic itself,
+%% so it was located here.
+split_project_apps_hooks() ->
+    [{doc, "Ensure that a project with multiple project apps runs the "
+           "pre-hooks before all the apps are compiled, and the post "
+           "hooks after they are all compiled."}].
+split_project_apps_hooks(Config) ->
+    BaseDir = ?config(apps, Config),
+    Name1 = rebar_test_utils:create_random_name("app2_"),
+    Name2 = rebar_test_utils:create_random_name("app1_"),
+    AppDir1 = filename:join([BaseDir, "lib", Name1]),
+    AppDir2 = filename:join([BaseDir, "lib", Name2]),
+    HookDir = filename:join([BaseDir, "hooks"]),
+    Vsn = rebar_test_utils:create_random_vsn(),
+    rebar_test_utils:create_app(AppDir1, Name1, Vsn, [kernel, stdlib, list_to_atom(Name2)]),
+    rebar_test_utils:create_app(AppDir2, Name2, Vsn, [kernel, stdlib]),
+
+    ok = filelib:ensure_dir(filename:join([AppDir1, "src", "dummy"])),
+    ok = filelib:ensure_dir(filename:join([AppDir1, "test", "dummy"])),
+    ok = filelib:ensure_dir(filename:join([AppDir2, "src", "dummy"])),
+    ok = filelib:ensure_dir(filename:join([AppDir2, "include", "dummy"])),
+    ok = filelib:ensure_dir(filename:join([HookDir, "dummy"])),
+    Cfg = fun(Name) ->
+        [{pre_hooks, [{compile, "ls "++HookDir++" > "++filename:join(HookDir, "pre-compile-"++Name)},
+                      {erlc_compile, "ls "++HookDir++" > "++filename:join(HookDir, "pre-erlc-"++Name)},
+                      {app_compile, "ls "++HookDir++" > "++filename:join(HookDir, "pre-app-"++Name)}]},
+         {post_hooks, [{compile, "ls "++HookDir++" > "++filename:join(HookDir, "post-compile-"++Name)},
+                       {erlc_compile, "ls "++HookDir++" > "++filename:join(HookDir, "post-erlc-"++Name)},
+                       {app_compile, "ls "++HookDir++" > "++filename:join(HookDir, "post-app-"++Name)}]}
+        ]
+    end,
+    ok = file:write_file(filename:join(AppDir1, "rebar.config"),
+                         io_lib:format("~p.~n~p.", Cfg("app1"))),
+    ok = file:write_file(filename:join(AppDir2, "rebar.config"),
+                         io_lib:format("~p.~n~p.", Cfg("app2"))),
+    RebarConfig = Cfg("all"),
+    ct:pal("RebarConfig: ~p~n", [RebarConfig]),
+    rebar_test_utils:run_and_check(Config, RebarConfig, ["compile"],
+                                   {ok, [{app, Name1}, {app, Name2}]}),
+    %% Now for the big party:
+    %% - we expect whole pre-hooks to run before either app is compiled
+    %% - we don't expect app and erlc hooks on the total run
+    %% - we expect app2 to be compiled before app1 (rev alphabetical order)
+    %% - we expect all pre-hooks to show up before post-hooks
+    %% - the pre-order is: compile->erlc, taking place before any app
+    %%   is actually compiled, so that analysis can be done on all apps.
+    %% - the post-order is more as expected:
+    %%   - erlc post hook runs right with the app
+    %%   - app pre hook runs right with the app
+    %%   - app post hook runs right with the app
+    %%   - compile post hook runs for each app individually
+    %% - we expect app compile post-hooks to show up in order
+    %% - we expect whole post-hooks to run last
+    ?assertEqual({ok, <<"pre-compile-all\n">>},
+                 file:read_file(filename:join(HookDir, "pre-compile-all"))),
+    ?assertEqual({ok, <<
+                        "pre-compile-all\n"
+                        "pre-compile-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "pre-compile-app2"))),
+    ?assertEqual({ok, <<
+                        "pre-compile-all\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "pre-erlc-app2"))),
+    ?assertEqual({ok, <<
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "pre-compile-app1"))),
+    ?assertEqual({ok, <<
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "pre-erlc-app1"))),
+    ?assertEqual({ok, <<
+                        "post-erlc-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "post-erlc-app2"))),
+    ?assertEqual({ok, <<
+                        "post-erlc-app2\n"
+                        "pre-app-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "pre-app-app2"))),
+    ?assertEqual({ok, <<
+                        "post-app-app2\n"
+                        "post-erlc-app2\n"
+                        "pre-app-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "post-app-app2"))),
+    ?assertEqual({ok, <<
+                        "post-app-app2\n"
+                        "post-compile-app2\n"
+                        "post-erlc-app2\n"
+                        "pre-app-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "post-compile-app2"))),
+    ?assertEqual({ok, <<
+                        "post-app-app2\n"
+                        "post-compile-app2\n"
+                        "post-erlc-app1\n"
+                        "post-erlc-app2\n"
+                        "pre-app-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "post-erlc-app1"))),
+    ?assertEqual({ok, <<
+                        "post-app-app2\n"
+                        "post-compile-app2\n"
+                        "post-erlc-app1\n"
+                        "post-erlc-app2\n"
+                        "pre-app-app1\n"
+                        "pre-app-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "pre-app-app1"))),
+    ?assertEqual({ok, <<
+                        "post-app-app1\n"
+                        "post-app-app2\n"
+                        "post-compile-app2\n"
+                        "post-erlc-app1\n"
+                        "post-erlc-app2\n"
+                        "pre-app-app1\n"
+                        "pre-app-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "post-app-app1"))),
+    ?assertEqual({ok, <<
+                        "post-app-app1\n"
+                        "post-app-app2\n"
+                        "post-compile-app1\n"
+                        "post-compile-app2\n"
+                        "post-erlc-app1\n"
+                        "post-erlc-app2\n"
+                        "pre-app-app1\n"
+                        "pre-app-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "post-compile-app1"))),
+    ?assertEqual({ok, <<
+                        "post-app-app1\n"
+                        "post-app-app2\n"
+                        "post-compile-all\n"
+                        "post-compile-app1\n"
+                        "post-compile-app2\n"
+                        "post-erlc-app1\n"
+                        "post-erlc-app2\n"
+                        "pre-app-app1\n"
+                        "pre-app-app2\n"
+                        "pre-compile-all\n"
+                        "pre-compile-app1\n"
+                        "pre-compile-app2\n"
+                        "pre-erlc-app1\n"
+                        "pre-erlc-app2\n"
+                      >>},
+                 file:read_file(filename:join(HookDir, "post-compile-all"))),
+    ok.
+
 app_file_linting(Config) ->
     meck:new(rebar_log, [no_link, passthrough]),
     AppDir = ?config(apps, Config),