diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index a0bc128a..838e9282 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -1,7 +1,7 @@ -module(rebar_compiler). -export([analyze_all/2, - compile_analyzed/2, + compile_analyzed/3, compile_all/2, clean/2, @@ -42,28 +42,93 @@ %% @doc analysis by the caller, in order to let an OTP app %% find and resolve all its dependencies as part of compile_all's new %% API, which presumes a partial analysis is done ahead of time --spec analyze_all([DAG], [App, ...]) -> ok when +-spec analyze_all(DAG, [App, ...]) -> ok when DAG :: {module(), digraph:graph()}, App :: rebar_app_info:t(). -analyze_all(_DAGs, _Apps) -> +analyze_all({Compiler, G}, Apps) -> + prepare_compiler_env(Compiler, Apps), %% Analyze apps one by one %% then cover the include files in the digraph to update them %% then propagate? - ok. + Contexts = gather_contexts(Compiler, Apps), + [analyze_app({Compiler, G}, Contexts, AppInfo) || AppInfo <- Apps], + rebar_compiler_dag:populate_deps(G, maps:get(src_ext, Contexts), + maps:get(artifact_exts, Contexts)), + rebar_compiler_dag:propagate_stamps(G), + + AppPaths = [{rebar_app_info:name(AppInfo), + rebar_utils:to_list(rebar_app_info:dir(AppInfo))} + || AppInfo <- Apps], + AppNames = rebar_compiler_dag:compile_order(G, AppPaths), + {Contexts, sort_apps(AppNames, Apps)}. + +gather_contexts(Compiler, Apps) -> + Default = default_ctx(), + Contexts = [{rebar_app_info:name(AppInfo), + maps:merge(Default, Compiler:context(AppInfo))} + || AppInfo <- Apps], + ContextMap = maps:from_list(Contexts), + %% only support one extension type at once for now + [{_, #{src_ext := SrcExt}} | _] = Contexts, + %% gather multi-app stuff once to avoid recomputing it + ArtifactExts = lists:usort( + [Ext || {_, #{out_mappings := Mappings}} <- Contexts, + {Ext, _Dir} <- Mappings] + ), + InDirs = gather_in_dirs(lists:zip(Apps, [Context || {_, Context} <- Contexts])), + ContextMap#{src_ext => SrcExt, + artifact_exts => ArtifactExts, + in_dirs => InDirs}. + +gather_in_dirs(AppCtx) -> + gather_in_dirs(AppCtx, []). + +gather_in_dirs([], Paths) -> + lists:usort(Paths); +gather_in_dirs([{AppInfo, Ctx} | Rest], Acc) -> + #{include_dirs := InclDirs, + src_dirs := SrcDirs} = Ctx, + BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), + AbsIncl = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs], + AbsSrc = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs], + gather_in_dirs(Rest, AbsSrc ++ AbsIncl ++ Acc). --spec compile_analyzed([{module(), digraph:graph()}], rebar_app_info:t()) -> ok. -compile_analyzed(DAGs, AppInfo) -> % > 3.13.0 - prepare_compiler_env(DAGs, AppInfo), - lists:foreach(fun({Compiler, G}) -> - run(G, Compiler, AppInfo), - %% TODO: disable default recursivity in extra_src_dirs compiling to - %% prevent compiling sample modules in _SUITE_data/ directories - %% in CT. - ExtraApps = annotate_extras(AppInfo), - [run(G, Compiler, ExtraAppInfo) || ExtraAppInfo <- ExtraApps], - ok - end, - DAGs). +analyze_app({Compiler, G}, Contexts, AppInfo) -> + AppName = rebar_app_info:name(AppInfo), + BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), + EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), + BaseOpts = rebar_app_info:opts(AppInfo), + #{src_dirs := SrcDirs, + src_ext := SrcExt, + dependencies_opts := DepOpts} = maps:get(AppName, Contexts), + %% Local resources + AbsSources = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts), + LocalSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs], + %% Multi-app resources + InDirs = maps:get(in_dirs, Contexts), + %% Prep the analysis + rebar_compiler_dag:prune(G, LocalSrcDirs, EbinDir, AbsSources), + rebar_compiler_dag:update(G, Compiler, InDirs, AbsSources, DepOpts), + %% Run the analysis + rebar_compiler_dag:populate_sources( + G, Compiler, InDirs, AbsSources, DepOpts + ). + +sort_apps(Names, Apps) -> + NamedApps = [{rebar_app_info:name(App), App} || App <- Apps], + [App || Name <- Names, + {_, App} <- [lists:keyfind(Name, 1, NamedApps)]]. + +-spec compile_analyzed({module(), digraph:graph()}, rebar_app_info:t(), map()) -> ok. +compile_analyzed({Compiler, G}, AppInfo, Contexts) -> % > 3.13.0 + run(G, Compiler, AppInfo, Contexts), + %% Extras are tricky and get their own mini-analysis + ExtraApps = annotate_extras(AppInfo), + [begin + {ExtraCtx, [SortedExtra]} = analyze_all({Compiler, G}, [ExtraAppInfo]), + run(G, Compiler, SortedExtra, ExtraCtx) + end || ExtraAppInfo <- ExtraApps], + ok. -spec compile_all([module(), ...], rebar_app_info:t()) -> ok. compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this! @@ -72,46 +137,40 @@ compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this! lists:foreach(fun(Compiler) -> OutDir = rebar_app_info:out_dir(AppInfo), G = rebar_compiler_dag:init(OutDir, Compiler, undefined, []), - analyze_all([{Compiler, G}], [AppInfo]), - compile_analyzed([{Compiler, G}], AppInfo), + Ctx = analyze_all({Compiler, G}, [AppInfo]), + compile_analyzed({Compiler, G}, AppInfo, Ctx), rebar_compiler_dag:maybe_store(G, OutDir, Compiler, undefined, []), rebar_compiler_dag:terminate(G) end, Compilers). -prepare_compiler_env(DAGs, AppInfo) -> - EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), - %% Make sure that outdir is on the path - ok = rebar_file_utils:ensure_dir(EbinDir), - true = code:add_patha(filename:absname(EbinDir)), - +prepare_compiler_env(Compiler, Apps) -> + lists:foreach( + fun(AppInfo) -> + EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), + %% Make sure that outdir is on the path + ok = rebar_file_utils:ensure_dir(EbinDir), + true = code:add_patha(filename:absname(EbinDir)) + end, + Apps + ), %% necessary for erlang:function_exported/3 to work as expected %% called here for clarity as it's required by both opts_changed/2 %% and erl_compiler_opts_set/0 in needed_files _ = code:ensure_loaded(compile), - [code:ensure_loaded(Mod) || {Mod, _} <- DAGs], + _ = code:ensure_loaded(Compiler), ok. -run(G, CompilerMod, AppInfo) -> - Ctx = CompilerMod:context(AppInfo), +run(G, CompilerMod, AppInfo, Contexts) -> + Name = rebar_app_info:name(AppInfo), #{src_dirs := SrcDirs, - include_dirs := InclDirs, src_ext := SrcExt, - out_mappings := Mappings, - dependencies_opts := DepOpts} = maps:merge(default_ctx(), Ctx), + out_mappings := Mappings} = maps:get(Name, Contexts), BaseDir = rebar_utils:to_list(rebar_app_info:dir(AppInfo)), - EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)), BaseOpts = rebar_app_info:opts(AppInfo), - AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs], FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts), - AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs], - - InDirs = lists:usort(AbsInclDirs ++ AbsSrcDirs), - - rebar_compiler_dag:prune(G, AbsSrcDirs, EbinDir, FoundFiles), - rebar_compiler_dag:update(G, CompilerMod, InDirs, FoundFiles, DepOpts), {{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo), diff --git a/src/rebar_compiler_dag.erl b/src/rebar_compiler_dag.erl index b74ad3ba..d622881d 100644 --- a/src/rebar_compiler_dag.erl +++ b/src/rebar_compiler_dag.erl @@ -208,7 +208,7 @@ compile_order(G, AppDefs) -> AppPaths = prepare_app_paths(AppDefs), compile_order(Edges, AppPaths, #{}). -compile_order([], _AppPaths, AppDeps) -> +compile_order([], AppPaths, AppDeps) -> %% use a digraph so we don't reimplement topsort by hand. G = digraph:new([acyclic]), % ignore cycles and hope it works Tups = maps:keys(AppDeps), @@ -216,19 +216,23 @@ compile_order([], _AppPaths, AppDeps) -> [digraph:add_vertex(G, V) || V <- Va], [digraph:add_vertex(G, V) || V <- Vb], [digraph:add_edge(G, V1, V2) || {V1, V2} <- Tups], - Res = lists:reverse(digraph_utils:topsort(G)), + Sorted = lists:reverse(digraph_utils:topsort(G)), digraph:delete(G), - Res; + Standalone = [Name || {_, Name} <- AppPaths] -- Sorted, + Standalone ++ Sorted; compile_order([{P1,P2}|T], AppPaths, AppDeps) -> %% Assume most dependencies are between files of the same app %% so ask to see if it's the same before doing a deeper check: - {P1App, P1Path} = find_app(P1, AppPaths), - {P2App, _} = find_cached_app(P2, {P1App, P1Path}, AppPaths), - case {P1App, P2App} of - {A, A} -> + case find_app(P1, AppPaths) of + not_found -> % system lib probably! not in the repo compile_order(T, AppPaths, AppDeps); - {V1, V2} -> - compile_order(T, AppPaths, AppDeps#{{V1, V2} => true}) + {P1App, P1Path} -> + case find_cached_app(P2, {P1App, P1Path}, AppPaths) of + {P2App, _} when P2App =/= P1App -> + compile_order(T, AppPaths, AppDeps#{{P1App,P2App} => true}); + _ -> + compile_order(T, AppPaths, AppDeps) + end end. prepare_app_paths(AppPaths) -> diff --git a/src/rebar_compiler_epp.erl b/src/rebar_compiler_epp.erl index 54e64836..8be954fa 100644 --- a/src/rebar_compiler_epp.erl +++ b/src/rebar_compiler_epp.erl @@ -142,8 +142,12 @@ handle_form(_, Map, _Opts) -> split_opts(Opts) -> %% Extra Opts are options we added to palliate to issues we had %% with resolving include_libs and other things in EPP. - lists:partition(fun({OptName, _}) -> include_libs =/= OptName end, - Opts). + lists:partition( + fun({OptName, _}) -> + not lists:member(OptName, [include_libs, parse_transforms]) + end, + Opts + ). find_include_with_opts(Path, Opts) -> InclPaths = proplists:get_value(include_libs, Opts, []), diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl index 528c9cba..75256169 100644 --- a/src/rebar_compiler_erl.erl +++ b/src/rebar_compiler_erl.erl @@ -27,8 +27,7 @@ context(AppInfo) -> ErlOptIncludes = proplists:get_all_values(i, ErlOpts), InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), AbsIncl = [filename:join([OutDir, "include"]) | InclDirs], - %% TODO: check that EPP can expand deps' include files - %% TODO: Ensure that EPP can expand other project apps' include files + PTrans = proplists:get_all_values(parse_transform, ErlOpts), Macros = [case Tup of {d,Name} -> Name; {d,Name,Val} -> {Name,Val} @@ -39,7 +38,8 @@ context(AppInfo) -> include_dirs => AbsIncl, src_ext => ".erl", out_mappings => Mappings, - dependencies_opts => [{includes, AbsIncl}, {macros, Macros}]}. + dependencies_opts => [{includes, AbsIncl}, {macros, Macros}, + {parse_transforms, PTrans}]}. needed_files(Graph, FoundFiles, _, AppInfo) -> @@ -96,19 +96,17 @@ dependencies(Source, SourceDir, Dirs) -> end. dependencies(Source, _SourceDir, Dirs, DepOpts) -> + OptPTrans = proplists:get_value(parse_transforms, DepOpts, []), try rebar_compiler_epp:deps(Source, DepOpts) of #{include := AbsIncls, missing_include_file := _MissIncl, missing_include_lib := _MissInclLib, parse_transform := PTrans, behaviour := Behaviours} -> - %% TODO: resolve behaviours cross-app - %% TODO: resolve parse_transforms cross-app - %% TODO: report on missing files %% TODO: check for core transforms? {_MissIncl, _MissInclLib} =/= {[],[]} andalso - ct:pal("Missing: ~p", [{_MissIncl, _MissInclLib}]), - expand_file_names([module_to_erl(Mod) || Mod <- PTrans], Dirs) ++ + ?DEBUG("Missing: ~p", [{_MissIncl, _MissInclLib}]), + expand_file_names([module_to_erl(Mod) || Mod <- OptPTrans ++ PTrans], Dirs) ++ expand_file_names([module_to_erl(Mod) || Mod <- Behaviours], Dirs) ++ AbsIncls catch diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 9c3c9668..f8974466 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -171,7 +171,6 @@ run_compilers(State, _Providers, Apps, Tag) -> || Mod <- rebar_state:compilers(State)], %% Compile all the apps build_apps(DAGs, Apps, State), - %[build_app(DAGs, AppInfo, State) || AppInfo <- Apps], %% Potentially store shared compiler DAGs so next runs can easily %% share the base information for easy re-scans. lists:foreach(fun({Mod, G}) -> @@ -272,6 +271,7 @@ build_apps(DAGs, Apps, State) -> build_rebar3_apps(DAGs, Rebar3, State). build_custom_builder_app(AppInfo, State) -> + ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]), Type = rebar_app_info:project_type(AppInfo), ProjectBuilders = rebar_state:project_builders(State), case lists:keyfind(Type, 1, ProjectBuilders) of @@ -289,11 +289,31 @@ build_custom_builder_app(AppInfo, State) -> throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) end. +build_rebar3_apps(DAGs, Apps, _State) when DAGs =:= []; Apps =:= [] -> + %% No apps to actually build, probably just other compile phases + %% to run for non-rebar3 apps, someone wanting .app files built, + %% or just needing the hooks to run maybe. + ok; build_rebar3_apps(DAGs, Apps, State) -> rebar_paths:set_paths([deps], State), - rebar_compiler:analyze_all(DAGs, Apps), - %% TODO: topsort apps here based on DAG data - [rebar_compiler:compile_analyzed(DAGs, AppInfo) || AppInfo <- Apps], + %% To maintain output order, we need to mention each app being compiled + %% in order, even if the order isn't really there anymore due to each + %% compiler being run in broken sequence. The last compiler tends to be + %% the big ERLC one so we use the last compiler for the output. + LastDAG = lists:last(DAGs), + %% we actually need to compile each DAG one after the other to prevent + %% issues where a .yrl file that generates a .erl file gets to be seen. + [begin + {Ctx, ReorderedApps} = rebar_compiler:analyze_all(DAG, Apps), + lists:foreach( + fun(AppInfo) -> + DAG =:= LastDAG andalso + ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]), + rebar_compiler:compile_analyzed(DAG, AppInfo, Ctx) + end, + ReorderedApps + ) + end || DAG <- DAGs], ok. update_code_paths(State, ProjectApps) ->