integrating new dag functionality with EPP stuff

This moves the old DAG stuff out from the main code paths and
substitutes it with the new epp-based DAG work.

Also fixes previous work for integration tests:

- consider ptrans from erl_opts in app ordering
- display proper app compilation info messages due to reordering

src/rebar_compiler.erl

@@ -1,7 +1,7 @@
 -module(rebar_compiler).
 
 -export([analyze_all/2,
-         compile_analyzed/2,
+         compile_analyzed/3,
          compile_all/2,
          clean/2,
 
@@ -42,28 +42,93 @@
 %% @doc analysis by the caller, in order to let an OTP app
 %% find and resolve all its dependencies as part of compile_all's new
 %% API, which presumes a partial analysis is done ahead of time
--spec analyze_all([DAG], [App, ...]) -> ok when
+-spec analyze_all(DAG, [App, ...]) -> ok when
       DAG :: {module(), digraph:graph()},
       App :: rebar_app_info:t().
-analyze_all(_DAGs, _Apps) ->
+analyze_all({Compiler, G}, Apps) ->
+    prepare_compiler_env(Compiler, Apps),
     %% Analyze apps one by one
     %% then cover the include files in the digraph to update them
     %% then propagate?
-    ok.
+    Contexts = gather_contexts(Compiler, Apps),
+    [analyze_app({Compiler, G}, Contexts, AppInfo) || AppInfo <- Apps],
+    rebar_compiler_dag:populate_deps(G, maps:get(src_ext, Contexts),
+                                        maps:get(artifact_exts, Contexts)),
+    rebar_compiler_dag:propagate_stamps(G),
+
+    AppPaths = [{rebar_app_info:name(AppInfo),
+                 rebar_utils:to_list(rebar_app_info:dir(AppInfo))}
+                || AppInfo <- Apps],
+    AppNames = rebar_compiler_dag:compile_order(G, AppPaths),
+    {Contexts, sort_apps(AppNames, Apps)}.
+
+gather_contexts(Compiler, Apps) ->
+    Default = default_ctx(),
+    Contexts = [{rebar_app_info:name(AppInfo),
+                 maps:merge(Default, Compiler:context(AppInfo))}
+                || AppInfo <- Apps],
+    ContextMap = maps:from_list(Contexts),
+    %% only support one extension type at once for now
+    [{_, #{src_ext := SrcExt}} | _] = Contexts,
+    %% gather multi-app stuff once to avoid recomputing it
+    ArtifactExts = lists:usort(
+        [Ext || {_, #{out_mappings := Mappings}} <- Contexts,
+                {Ext, _Dir} <- Mappings]
+    ),
+    InDirs = gather_in_dirs(lists:zip(Apps, [Context || {_, Context} <- Contexts])),
+    ContextMap#{src_ext => SrcExt,
+                artifact_exts => ArtifactExts,
+                in_dirs => InDirs}.
+
+gather_in_dirs(AppCtx) ->
+    gather_in_dirs(AppCtx, []).
+
+gather_in_dirs([], Paths) ->
+    lists:usort(Paths);
+gather_in_dirs([{AppInfo, Ctx} | Rest], Acc) ->
+    #{include_dirs := InclDirs,
+      src_dirs := SrcDirs} = Ctx,
+    BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
+    AbsIncl = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs],
+    AbsSrc = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
+    gather_in_dirs(Rest, AbsSrc ++ AbsIncl ++ Acc).
 
--spec compile_analyzed([{module(), digraph:graph()}], rebar_app_info:t()) -> ok.
-compile_analyzed(DAGs, AppInfo) -> % > 3.13.0
-    prepare_compiler_env(DAGs, AppInfo),
-    lists:foreach(fun({Compiler, G}) ->
-        run(G, Compiler, AppInfo),
-        %% TODO: disable default recursivity in extra_src_dirs compiling to
-        %% prevent compiling sample modules in _SUITE_data/ directories
-        %% in CT.
-        ExtraApps = annotate_extras(AppInfo),
-        [run(G, Compiler, ExtraAppInfo) || ExtraAppInfo <- ExtraApps],
-        ok
-    end,
-    DAGs).
+analyze_app({Compiler, G}, Contexts, AppInfo) ->
+    AppName = rebar_app_info:name(AppInfo),
+    BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
+    EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
+    BaseOpts = rebar_app_info:opts(AppInfo),
+    #{src_dirs := SrcDirs,
+      src_ext := SrcExt,
+      dependencies_opts := DepOpts} = maps:get(AppName, Contexts),
+    %% Local resources
+    AbsSources = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
+    LocalSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
+    %% Multi-app resources
+    InDirs = maps:get(in_dirs, Contexts),
+    %% Prep the analysis
+    rebar_compiler_dag:prune(G, LocalSrcDirs, EbinDir, AbsSources),
+    rebar_compiler_dag:update(G, Compiler, InDirs, AbsSources, DepOpts),
+    %% Run the analysis
+    rebar_compiler_dag:populate_sources(
+        G, Compiler, InDirs, AbsSources, DepOpts
+    ).
+
+sort_apps(Names, Apps) ->
+    NamedApps = [{rebar_app_info:name(App), App} || App <- Apps],
+    [App || Name <- Names,
+            {_, App} <- [lists:keyfind(Name, 1, NamedApps)]].
+
+-spec compile_analyzed({module(), digraph:graph()}, rebar_app_info:t(), map()) -> ok.
+compile_analyzed({Compiler, G}, AppInfo, Contexts) -> % > 3.13.0
+    run(G, Compiler, AppInfo, Contexts),
+    %% Extras are tricky and get their own mini-analysis
+    ExtraApps = annotate_extras(AppInfo),
+    [begin
+         {ExtraCtx, [SortedExtra]} = analyze_all({Compiler, G}, [ExtraAppInfo]),
+         run(G, Compiler, SortedExtra, ExtraCtx)
+     end || ExtraAppInfo <- ExtraApps],
+    ok.
 
 -spec compile_all([module(), ...], rebar_app_info:t()) -> ok.
 compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this!
@@ -72,46 +137,40 @@ compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this!
     lists:foreach(fun(Compiler) ->
         OutDir = rebar_app_info:out_dir(AppInfo),
         G = rebar_compiler_dag:init(OutDir, Compiler, undefined, []),
-        analyze_all([{Compiler, G}], [AppInfo]),
-        compile_analyzed([{Compiler, G}], AppInfo),
+        Ctx = analyze_all({Compiler, G}, [AppInfo]),
+        compile_analyzed({Compiler, G}, AppInfo, Ctx),
         rebar_compiler_dag:maybe_store(G, OutDir, Compiler, undefined, []),
         rebar_compiler_dag:terminate(G)
      end, Compilers).
 
-prepare_compiler_env(DAGs, AppInfo) ->
-    EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
-    %% Make sure that outdir is on the path
-    ok = rebar_file_utils:ensure_dir(EbinDir),
-    true = code:add_patha(filename:absname(EbinDir)),
-
+prepare_compiler_env(Compiler, Apps) ->
+    lists:foreach(
+        fun(AppInfo) ->
+            EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
+            %% Make sure that outdir is on the path
+            ok = rebar_file_utils:ensure_dir(EbinDir),
+            true = code:add_patha(filename:absname(EbinDir))
+        end,
+        Apps
+    ),
     %% necessary for erlang:function_exported/3 to work as expected
     %% called here for clarity as it's required by both opts_changed/2
     %% and erl_compiler_opts_set/0 in needed_files
     _ = code:ensure_loaded(compile),
-    [code:ensure_loaded(Mod) || {Mod, _} <- DAGs],
+    _ = code:ensure_loaded(Compiler),
     ok.
 
-run(G, CompilerMod, AppInfo) ->
-    Ctx = CompilerMod:context(AppInfo),
+run(G, CompilerMod, AppInfo, Contexts) ->
+    Name = rebar_app_info:name(AppInfo),
     #{src_dirs := SrcDirs,
-      include_dirs := InclDirs,
       src_ext := SrcExt,
-      out_mappings := Mappings,
-      dependencies_opts := DepOpts} = maps:merge(default_ctx(), Ctx),
+      out_mappings := Mappings} = maps:get(Name, Contexts),
 
     BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)),
-    EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
 
     BaseOpts = rebar_app_info:opts(AppInfo),
-    AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs],
     FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),
 
-    AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
-
-    InDirs = lists:usort(AbsInclDirs ++ AbsSrcDirs),
-
-    rebar_compiler_dag:prune(G, AbsSrcDirs, EbinDir, FoundFiles),
-    rebar_compiler_dag:update(G, CompilerMod, InDirs, FoundFiles, DepOpts),
     {{FirstFiles, FirstFileOpts},
      {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo),
 

src/rebar_compiler_dag.erl

@@ -208,7 +208,7 @@ compile_order(G, AppDefs) ->
     AppPaths = prepare_app_paths(AppDefs),
     compile_order(Edges, AppPaths, #{}).
 
-compile_order([], _AppPaths, AppDeps) ->
+compile_order([], AppPaths, AppDeps) ->
     %% use a digraph so we don't reimplement topsort by hand.
     G = digraph:new([acyclic]), % ignore cycles and hope it works
     Tups = maps:keys(AppDeps),
@@ -216,19 +216,23 @@ compile_order([], _AppPaths, AppDeps) ->
     [digraph:add_vertex(G, V) || V <- Va],
     [digraph:add_vertex(G, V) || V <- Vb],
     [digraph:add_edge(G, V1, V2) || {V1, V2} <- Tups],
-    Res = lists:reverse(digraph_utils:topsort(G)),
+    Sorted = lists:reverse(digraph_utils:topsort(G)),
     digraph:delete(G),
-    Res;
+    Standalone = [Name || {_, Name} <- AppPaths] -- Sorted,
+    Standalone ++ Sorted;
 compile_order([{P1,P2}|T], AppPaths, AppDeps) ->
     %% Assume most dependencies are between files of the same app
     %% so ask to see if it's the same before doing a deeper check:
-    {P1App, P1Path} = find_app(P1, AppPaths),
-    {P2App, _} = find_cached_app(P2, {P1App, P1Path}, AppPaths),
-    case {P1App, P2App} of
-        {A, A} ->
+    case find_app(P1, AppPaths) of
+        not_found -> % system lib probably! not in the repo
             compile_order(T, AppPaths, AppDeps);
-        {V1, V2} ->
-            compile_order(T, AppPaths, AppDeps#{{V1, V2} => true})
+        {P1App, P1Path} ->
+            case find_cached_app(P2, {P1App, P1Path}, AppPaths) of
+                {P2App, _} when P2App =/= P1App ->
+                    compile_order(T, AppPaths, AppDeps#{{P1App,P2App} => true});
+                _ ->
+                    compile_order(T, AppPaths, AppDeps)
+            end
     end.
 
 prepare_app_paths(AppPaths) ->

src/rebar_compiler_epp.erl

@@ -142,8 +142,12 @@ handle_form(_, Map, _Opts) ->
 split_opts(Opts) ->
     %% Extra Opts are options we added to palliate to issues we had
     %% with resolving include_libs and other things in EPP.
-    lists:partition(fun({OptName, _}) -> include_libs =/= OptName end,
-                    Opts).
+    lists:partition(
+        fun({OptName, _}) ->
+            not lists:member(OptName, [include_libs, parse_transforms])
+        end,
+        Opts
+    ).
 
 find_include_with_opts(Path, Opts) ->
     InclPaths = proplists:get_value(include_libs, Opts, []),

src/rebar_compiler_erl.erl

@@ -27,8 +27,7 @@ context(AppInfo) ->
     ErlOptIncludes = proplists:get_all_values(i, ErlOpts),
     InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes),
     AbsIncl = [filename:join([OutDir, "include"]) | InclDirs],
-    %% TODO: check that EPP can expand deps' include files
-    %% TODO: Ensure that EPP can expand other project apps' include files
+    PTrans = proplists:get_all_values(parse_transform, ErlOpts),
     Macros = [case Tup of
                   {d,Name} -> Name;
                   {d,Name,Val} -> {Name,Val}
@@ -39,7 +38,8 @@ context(AppInfo) ->
       include_dirs => AbsIncl,
       src_ext => ".erl",
       out_mappings => Mappings,
-      dependencies_opts => [{includes, AbsIncl}, {macros, Macros}]}.
+      dependencies_opts => [{includes, AbsIncl}, {macros, Macros},
+                            {parse_transforms, PTrans}]}.
 
 
 needed_files(Graph, FoundFiles, _, AppInfo) ->
@@ -96,19 +96,17 @@ dependencies(Source, SourceDir, Dirs) ->
     end.
 
 dependencies(Source, _SourceDir, Dirs, DepOpts) ->
+    OptPTrans = proplists:get_value(parse_transforms, DepOpts, []),
     try rebar_compiler_epp:deps(Source, DepOpts) of
         #{include := AbsIncls,
           missing_include_file := _MissIncl,
           missing_include_lib := _MissInclLib,
           parse_transform := PTrans,
           behaviour := Behaviours} ->
-            %% TODO: resolve behaviours cross-app
-            %% TODO: resolve parse_transforms cross-app
-            %% TODO: report on missing files
             %% TODO: check for core transforms?
             {_MissIncl, _MissInclLib} =/= {[],[]} andalso
-            ct:pal("Missing: ~p", [{_MissIncl, _MissInclLib}]),
-            expand_file_names([module_to_erl(Mod) || Mod <- PTrans], Dirs) ++
+            ?DEBUG("Missing: ~p", [{_MissIncl, _MissInclLib}]),
+            expand_file_names([module_to_erl(Mod) || Mod <- OptPTrans ++ PTrans], Dirs) ++
             expand_file_names([module_to_erl(Mod) || Mod <- Behaviours], Dirs) ++
             AbsIncls
     catch

src/rebar_prv_compile.erl

@@ -171,7 +171,6 @@ run_compilers(State, _Providers, Apps, Tag) ->
             || Mod <- rebar_state:compilers(State)],
     %% Compile all the apps
     build_apps(DAGs, Apps, State),
-    %[build_app(DAGs, AppInfo, State) || AppInfo <- Apps],
     %% Potentially store shared compiler DAGs so next runs can easily
     %% share the base information for easy re-scans.
     lists:foreach(fun({Mod, G}) ->
@@ -272,6 +271,7 @@ build_apps(DAGs, Apps, State) ->
     build_rebar3_apps(DAGs, Rebar3, State).
 
 build_custom_builder_app(AppInfo, State) ->
+    ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]),
     Type = rebar_app_info:project_type(AppInfo),
     ProjectBuilders = rebar_state:project_builders(State),
     case lists:keyfind(Type, 1, ProjectBuilders) of
@@ -289,11 +289,31 @@ build_custom_builder_app(AppInfo, State) ->
             throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type}))
     end.
 
+build_rebar3_apps(DAGs, Apps, _State) when DAGs =:= []; Apps =:= [] ->
+    %% No apps to actually build, probably just other compile phases
+    %% to run for non-rebar3 apps, someone wanting .app files built,
+    %% or just needing the hooks to run maybe.
+    ok;
 build_rebar3_apps(DAGs, Apps, State) ->
     rebar_paths:set_paths([deps], State),
-    rebar_compiler:analyze_all(DAGs, Apps),
-    %% TODO: topsort apps here based on DAG data
-    [rebar_compiler:compile_analyzed(DAGs, AppInfo) || AppInfo <- Apps],
+    %% To maintain output order, we need to mention each app being compiled
+    %% in order, even if the order isn't really there anymore due to each
+    %% compiler being run in broken sequence. The last compiler tends to be
+    %% the big ERLC one so we use the last compiler for the output.
+    LastDAG = lists:last(DAGs),
+    %% we actually need to compile each DAG one after the other to prevent
+    %% issues where a .yrl file that generates a .erl file gets to be seen.
+    [begin
+         {Ctx, ReorderedApps} = rebar_compiler:analyze_all(DAG, Apps),
+         lists:foreach(
+             fun(AppInfo) ->
+                DAG =:= LastDAG andalso
+                  ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]),
+                rebar_compiler:compile_analyzed(DAG, AppInfo, Ctx)
+             end,
+             ReorderedApps
+         )
+     end || DAG <- DAGs],
     ok.
 
 update_code_paths(State, ProjectApps) ->