diff --git a/src/rebar_compiler.erl b/src/rebar_compiler.erl index d810996d..a0bc128a 100644 --- a/src/rebar_compiler.erl +++ b/src/rebar_compiler.erl @@ -1,6 +1,8 @@ -module(rebar_compiler). --export([compile_all/2, +-export([analyze_all/2, + compile_analyzed/2, + compile_all/2, clean/2, needs_compile/3, @@ -36,9 +38,21 @@ -define(RE_PREFIX, "^(?!\\._)"). --spec compile_all([{module(), digraph:graph()}, ...], rebar_app_info:t()) -> ok - ; ([module(), ...], rebar_app_info:t()) -> ok. -compile_all(DAGs, AppInfo) when is_tuple(hd(DAGs)) -> % > 3.13.0 + +%% @doc analysis by the caller, in order to let an OTP app +%% find and resolve all its dependencies as part of compile_all's new +%% API, which presumes a partial analysis is done ahead of time +-spec analyze_all([DAG], [App, ...]) -> ok when + DAG :: {module(), digraph:graph()}, + App :: rebar_app_info:t(). +analyze_all(_DAGs, _Apps) -> + %% Analyze apps one by one + %% then cover the include files in the digraph to update them + %% then propagate? + ok. + +-spec compile_analyzed([{module(), digraph:graph()}], rebar_app_info:t()) -> ok. +compile_analyzed(DAGs, AppInfo) -> % > 3.13.0 prepare_compiler_env(DAGs, AppInfo), lists:foreach(fun({Compiler, G}) -> run(G, Compiler, AppInfo), @@ -49,14 +63,17 @@ compile_all(DAGs, AppInfo) when is_tuple(hd(DAGs)) -> % > 3.13.0 [run(G, Compiler, ExtraAppInfo) || ExtraAppInfo <- ExtraApps], ok end, - DAGs); + DAGs). + +-spec compile_all([module(), ...], rebar_app_info:t()) -> ok. compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this! %% Support the old-style API by re-declaring a local DAG for the %% compile steps needed. lists:foreach(fun(Compiler) -> OutDir = rebar_app_info:out_dir(AppInfo), G = rebar_compiler_dag:init(OutDir, Compiler, undefined, []), - compile_all([{Compiler, G}], AppInfo), + analyze_all([{Compiler, G}], [AppInfo]), + compile_analyzed([{Compiler, G}], AppInfo), rebar_compiler_dag:maybe_store(G, OutDir, Compiler, undefined, []), rebar_compiler_dag:terminate(G) end, Compilers). diff --git a/src/rebar_compiler_dag.erl b/src/rebar_compiler_dag.erl index 1a7af5bf..b74ad3ba 100644 --- a/src/rebar_compiler_dag.erl +++ b/src/rebar_compiler_dag.erl @@ -2,6 +2,8 @@ %%% of all top-level applications by the various compiler plugins. -module(rebar_compiler_dag). -export([init/4, prune/4, update/5, maybe_store/5, terminate/1]). +-export([populate_sources/5, populate_deps/3, propagate_stamps/1, + compile_order/2]). -include("rebar.hrl"). @@ -96,6 +98,180 @@ update(G, Compiler, InDirs, [Source|Erls], DepOpts) -> update(G, Compiler, InDirs, Erls, DepOpts) end. +populate_sources(_G, _Compiler, _InDirs, [], _DepOpts) -> + ok; +populate_sources(G, Compiler, InDirs, [Source|Erls], DepOpts) -> + case digraph:vertex(G, Source) of + {_, LastUpdated} -> + case filelib:last_modified(Source) of + 0 -> + %% The File doesn't exist anymore, delete + %% from the graph. + digraph:del_vertex(G, Source), + mark_dirty(G), + populate_sources(G, Compiler, InDirs, Erls, DepOpts); + LastModified when LastUpdated < LastModified -> + digraph:add_vertex(G, Source, LastModified), + prepopulate_deps(G, Compiler, InDirs, Source, DepOpts), + mark_dirty(G); + _ -> % unchanged + ok + end; + false -> + LastModified = filelib:last_modified(Source), + digraph:add_vertex(G, Source, LastModified), + prepopulate_deps(G, Compiler, InDirs, Source, DepOpts), + mark_dirty(G) + end, + populate_sources(G, Compiler, InDirs, Erls, DepOpts). + +prepopulate_deps(G, Compiler, InDirs, Source, DepOpts) -> + SourceDir = filename:dirname(Source), + AbsIncls = case erlang:function_exported(Compiler, dependencies, 4) of + false -> + Compiler:dependencies(Source, SourceDir, InDirs); + true -> + Compiler:dependencies(Source, SourceDir, InDirs, DepOpts) + end, + %% the file hasn't been visited yet; set it to existing, but with + %% a last modified value that's null so it gets updated to something new. + [digraph:add_vertex(G, Src, 0) || Src <- AbsIncls, + digraph:vertex(G, Src) =:= false], + [digraph:add_edge(G, Source, Incl) || Incl <- AbsIncls], + ok. + +populate_deps(G, SourceExt, ArtifactExts) -> + %% deps are files that are part of the digraph, but couldn't be scanned + %% because they are neither source files (`SourceExt') nor mappings + %% towards build artifacts (`ArtifactExts'); they will therefore never + %% be handled otherwise and need to be re-scanned for accuracy, even + %% if they are not being analyzed (we assume `Compiler:deps' did that + %% in depth already, and improvements should be driven at that level) + IgnoredExts = [SourceExt | ArtifactExts], + Vertices = digraph:vertices(G), + [refresh_dep(G, File) + || File <- Vertices, + Ext <- [filename:extension(File)], + not lists:member(Ext, IgnoredExts)], + ok. + +%% Take the timestamps/diff changes and propagate them from a dep to the parent; +%% given: +%% A 0 -> B 1 -> C 3 -> D 2 +%% then we expect to get back: +%% A 3 -> B 3 -> C 3 -> D 2 +%% This is going to be safe for the current run of regeneration, but also for the +%% next one; unless any file in the chain has changed, the stamp won't move up +%% and there won't be a reason to recompile. +%% The obvious caveat to this one is that a file changing by restoring an old version +%% won't be picked up, but this weakness already existed in terms of timestamps. +propagate_stamps(G) -> + case is_dirty(G) of + false -> + %% no change, no propagation to make + ok; + true -> + %% we can use a topsort, start at the end of it (files with no deps) + %% and update them all in order. By doing this, each file only needs to check + %% for one level of out-neighbours to set itself to the right appropriate time. + DepSort = lists:reverse(digraph_utils:topsort(G)), + propagate_stamps(G, DepSort) + end. + + +propagate_stamps(_G, []) -> + ok; +propagate_stamps(G, [File|Files]) -> + Stamps = [element(2, digraph:vertex(G, F)) + || F <- digraph:out_neighbours(G, File)], + case Stamps of + [] -> + ok; + _ -> + Max = lists:max(Stamps), + case digraph:vertex(G, File) of + {_, Smaller} when Smaller < Max -> + digraph:add_vertex(G, File, Max); + _ -> + ok + end + end, + propagate_stamps(G, Files). + + +compile_order(G, AppDefs) -> + %% Return the reverse sorting order to get dep-free apps first. + %% -- we would usually not need to consider the non-source files for the order to + %% be complete, but using them doesn't hurt. + Edges = [{V1,V2} || E <- digraph:edges(G), + {_,V1,V2,_} <- [digraph:edge(G, E)]], + AppPaths = prepare_app_paths(AppDefs), + compile_order(Edges, AppPaths, #{}). + +compile_order([], _AppPaths, AppDeps) -> + %% use a digraph so we don't reimplement topsort by hand. + G = digraph:new([acyclic]), % ignore cycles and hope it works + Tups = maps:keys(AppDeps), + {Va,Vb} = lists:unzip(Tups), + [digraph:add_vertex(G, V) || V <- Va], + [digraph:add_vertex(G, V) || V <- Vb], + [digraph:add_edge(G, V1, V2) || {V1, V2} <- Tups], + Res = lists:reverse(digraph_utils:topsort(G)), + digraph:delete(G), + Res; +compile_order([{P1,P2}|T], AppPaths, AppDeps) -> + %% Assume most dependencies are between files of the same app + %% so ask to see if it's the same before doing a deeper check: + {P1App, P1Path} = find_app(P1, AppPaths), + {P2App, _} = find_cached_app(P2, {P1App, P1Path}, AppPaths), + case {P1App, P2App} of + {A, A} -> + compile_order(T, AppPaths, AppDeps); + {V1, V2} -> + compile_order(T, AppPaths, AppDeps#{{V1, V2} => true}) + end. + +prepare_app_paths(AppPaths) -> + lists:sort([{filename:split(Path), Name} || {Name, Path} <- AppPaths]). + +find_app(Path, AppPaths) -> + find_app_(filename:split(Path), AppPaths). + +find_cached_app(Path, {Name, AppPath}, AppPaths) -> + Split = filename:split(Path), + case find_app_(Split, [{AppPath, Name}]) of + not_found -> find_app_(Split, AppPaths); + LastEntry -> LastEntry + end. + +find_app_(_Path, []) -> + not_found; +find_app_(Path, [{AppPath, AppName}|Rest]) -> + case lists:prefix(AppPath, Path) of + true -> + {AppName, AppPath}; + false when AppPath > Path -> + not_found; + false -> + find_app_(Path, Rest) + end. + + +refresh_dep(G, File) -> + {_, LastUpdated} = digraph:vertex(G, File), + case filelib:last_modified(File) of + 0 -> + %% Gone! Erase from the graph + digraph:del_vertex(G, File), + mark_dirty(G); + LastModified when LastUpdated < LastModified -> + digraph:add_vertex(G, File, LastModified), + mark_dirty(G); + _ -> + % unchanged + ok + end. + maybe_store(G, Dir, Compiler, Label, CritMeta) -> case is_dirty(G) of true -> diff --git a/src/rebar_compiler_epp.erl b/src/rebar_compiler_epp.erl index 24dcdfe2..54e64836 100644 --- a/src/rebar_compiler_epp.erl +++ b/src/rebar_compiler_epp.erl @@ -22,8 +22,9 @@ parse_transform := [atom()], is_behaviour := boolean()}. deps(File, Opts) -> - {ok, Forms} = epp:parse_file(File, Opts), - normalize(handle_forms(Forms, default_attrs())). + {EppOpts, ExtraOpts} = split_opts(Opts), + {ok, Forms} = epp:parse_file(File, EppOpts), + normalize(handle_forms(Forms, default_attrs(), ExtraOpts)). %% Find the path matching a given erlang module resolve_module(Mod, Paths) -> @@ -59,8 +60,9 @@ normalize(Map) -> behaviour => lists:usort(Behaviour), parse_transform => lists:usort(PTrans)}. -handle_forms([File|Forms], Map) -> - lists:foldl(fun handle_form/2, Map, drop_self_file(File, Forms)). +handle_forms([File|Forms], Map, Opts) -> + lists:foldl(fun(Form, M) -> handle_form(Form, M, Opts) end, + Map, drop_self_file(File, Forms)). drop_self_file(_, []) -> []; @@ -74,7 +76,7 @@ drop_self_file(File, [Keep|Rest]) -> %% There are also references to the module's own file declaration %% in there, but this is dropped by `drop_self_file/2' and assumed %% to be gone here. -handle_form({attribute, _Line, file, {Path, Ln}}, Map) -> +handle_form({attribute, _Line, file, {Path, Ln}}, Map, Opts) -> %% Some people think they're funny and they go include attributes %% like: %% -file("fake/file.hrl", Ln). @@ -90,37 +92,70 @@ handle_form({attribute, _Line, file, {Path, Ln}}, Map) -> Path -> maps:update_with(include, fun(L) -> [Path|L] end, [Path], Map); _ -> % argh! - handle_form({error, {Ln, {epp, {include, file, Path}}}}, Map) + handle_form({error, {Ln, {epp, {include, file, Path}}}}, Map, Opts) end; %% Include files that EPP couldn't resolve -handle_form({error, {_Line, epp, {include, file, Name}}}, Map) -> +handle_form({error, {_Line, epp, {include, file, Name}}}, Map, _Opts) -> maps:update_with(missing_include_file, fun(L) -> [Name|L] end, [Name], Map); -handle_form({error, {_Line, epp, {include, lib, Path}}}, Map) -> - maps:update_with(missing_include_lib, fun(L) -> [Path|L] end, [Path], Map); +handle_form({error, {_Line, epp, {include, lib, Path}}}, Map, Opts) -> + %% This file might still exist in the regular paths not in + %% code:lib_dir, which depend on options we pass to this module. + %% recursively seek it, and add it to the paths to expand here. + case find_include_with_opts(Path, Opts) of + {ok, File} -> + %% we can't go and figure out the contents within that include + %% file because we'd need its own compiler opts and app opts + %% to do it safely. Tracking that file is still better + %% than nothing though. + maps:update_with(include, fun(L) -> [File|L] end, [File], Map); + {error, not_found} -> + maps:update_with(missing_include_lib, fun(L) -> [Path|L] end, [Path], Map) + end; %% Behaviour implementation declaration -handle_form({attribute, _Line, behaviour, Name}, Map) -> +handle_form({attribute, _Line, behaviour, Name}, Map, _Opts) -> maps:update_with(behaviour, fun(L) -> [Name|L] end, [Name], Map); -handle_form({attribute, _Line, behavior, Name}, Map) -> +handle_form({attribute, _Line, behavior, Name}, Map, _Opts) -> maps:update_with(behaviour, fun(L) -> [Name|L] end, [Name], Map); %% Extract parse transforms -handle_form({attribute, Line, compile, Attr}, Map) when not is_list(Attr) -> - handle_form({attribute, Line, compile, [Attr]}, Map); -handle_form({attribute, _Line, compile, Attrs}, Map) -> +handle_form({attribute, Line, compile, Attr}, Map, _Opts) when not is_list(Attr) -> + handle_form({attribute, Line, compile, [Attr]}, Map, _Opts); +handle_form({attribute, _Line, compile, Attrs}, Map, _Opts) -> Mods = [case T of {_, {M,_}} -> M; {_, M} -> M end || T <- proplists:lookup_all(parse_transform, Attrs)], maps:update_with(parse_transform, fun(L) -> Mods++L end, Mods, Map); %% Current style behaviour specification declaration -handle_form({attribute, _Line, callback, _}, Map) -> +handle_form({attribute, _Line, callback, _}, Map, _Opts) -> Map#{is_behaviour => true}; %% Old style behaviour specification, both spellings supported %% The function needs to be exported, but we skip over that logic %% for now. -handle_form({function, _Line, behaviour_info, 1, _}, Map) -> +handle_form({function, _Line, behaviour_info, 1, _}, Map, _Opts) -> Map#{is_behaviour => true}; -handle_form({function, _Line, behavior_info, 1, _}, Map) -> +handle_form({function, _Line, behavior_info, 1, _}, Map, _Opts) -> Map#{is_behaviour => true}; %% Skip the rest -handle_form(_, Map) -> +handle_form(_, Map, _Opts) -> Map. + +split_opts(Opts) -> + %% Extra Opts are options we added to palliate to issues we had + %% with resolving include_libs and other things in EPP. + lists:partition(fun({OptName, _}) -> include_libs =/= OptName end, + Opts). + +find_include_with_opts(Path, Opts) -> + InclPaths = proplists:get_value(include_libs, Opts, []), + find_include_lib(InclPaths, Path). + +find_include_lib([], _) -> + {error, not_found}; +find_include_lib([H|T], File) -> + Abs = filename:join([H, File]), + case filelib:is_regular(Abs) of + true -> {ok, Abs}; + false -> find_include_lib(T, File) + end. + + diff --git a/src/rebar_compiler_erl.erl b/src/rebar_compiler_erl.erl index bf812666..528c9cba 100644 --- a/src/rebar_compiler_erl.erl +++ b/src/rebar_compiler_erl.erl @@ -27,6 +27,8 @@ context(AppInfo) -> ErlOptIncludes = proplists:get_all_values(i, ErlOpts), InclDirs = lists:map(fun(Incl) -> filename:absname(Incl) end, ErlOptIncludes), AbsIncl = [filename:join([OutDir, "include"]) | InclDirs], + %% TODO: check that EPP can expand deps' include files + %% TODO: Ensure that EPP can expand other project apps' include files Macros = [case Tup of {d,Name} -> Name; {d,Name,Val} -> {Name,Val} @@ -104,6 +106,8 @@ dependencies(Source, _SourceDir, Dirs, DepOpts) -> %% TODO: resolve parse_transforms cross-app %% TODO: report on missing files %% TODO: check for core transforms? + {_MissIncl, _MissInclLib} =/= {[],[]} andalso + ct:pal("Missing: ~p", [{_MissIncl, _MissInclLib}]), expand_file_names([module_to_erl(Mod) || Mod <- PTrans], Dirs) ++ expand_file_names([module_to_erl(Mod) || Mod <- Behaviours], Dirs) ++ AbsIncls @@ -362,7 +366,14 @@ expand_file_names(Files, Dirs) -> true -> [Incl]; false -> - rebar_utils:find_files_in_dirs(Dirs, [$^, Incl, $$], true) + Res = rebar_utils:find_files_in_dirs(Dirs, [$^, Incl, $$], true), + case Res of + [] -> + ?DEBUG("FILE ~p NOT FOUND", [Incl]), + []; + _ -> + Res + end end end, Files). diff --git a/src/rebar_prv_compile.erl b/src/rebar_prv_compile.erl index 03d0b878..9c3c9668 100644 --- a/src/rebar_prv_compile.erl +++ b/src/rebar_prv_compile.erl @@ -169,9 +169,9 @@ run_compilers(State, _Providers, Apps, Tag) -> CritMeta = [], % used to be incldirs per app DAGs = [{Mod, rebar_compiler_dag:init(Dir, Mod, DAGLabel, CritMeta)} || Mod <- rebar_state:compilers(State)], - rebar_paths:set_paths([deps], State), %% Compile all the apps - [build_app(DAGs, AppInfo, State) || AppInfo <- Apps], + build_apps(DAGs, Apps, State), + %[build_app(DAGs, AppInfo, State) || AppInfo <- Apps], %% Potentially store shared compiler DAGs so next runs can easily %% share the base information for easy re-scans. lists:foreach(fun({Mod, G}) -> @@ -260,31 +260,42 @@ extra_virtual_apps(State, VApp0, [Dir|Dirs]) -> %% Internal functions %% =================================================================== -build_app(DAGs, AppInfo, State) -> - ?INFO("Compiling ~ts", [rebar_app_info:name(AppInfo)]), - case rebar_app_info:project_type(AppInfo) of - Type when Type =:= rebar3 ; Type =:= undefined -> - %% assume the deps paths are already set by the caller (run_compilers/3) - %% and shared for multiple apps to save work. - rebar_compiler:compile_all(DAGs, AppInfo); - Type -> - ProjectBuilders = rebar_state:project_builders(State), - case lists:keyfind(Type, 1, ProjectBuilders) of - {_, Module} -> - %% load plugins since thats where project builders would be, - %% prevents parallelism at this level. - rebar_paths:set_paths([deps, plugins], State), - Res = Module:build(AppInfo), - rebar_paths:set_paths([deps], State), - case Res of - ok -> ok; - {error, Reason} -> throw({error, {Module, Reason}}) - end; - _ -> - throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) - end +build_apps(DAGs, Apps, State) -> + {Rebar3, Custom} = lists:partition( + fun(AppInfo) -> + Type = rebar_app_info:project_type(AppInfo), + Type =:= rebar3 orelse Type =:= undefined + end, + Apps + ), + [build_custom_builder_app(AppInfo, State) || AppInfo <- Custom], + build_rebar3_apps(DAGs, Rebar3, State). + +build_custom_builder_app(AppInfo, State) -> + Type = rebar_app_info:project_type(AppInfo), + ProjectBuilders = rebar_state:project_builders(State), + case lists:keyfind(Type, 1, ProjectBuilders) of + {_, Module} -> + %% load plugins since thats where project builders would be, + %% prevents parallelism at this level. + rebar_paths:set_paths([deps, plugins], State), + Res = Module:build(AppInfo), + rebar_paths:set_paths([deps], State), + case Res of + ok -> ok; + {error, Reason} -> throw({error, {Module, Reason}}) + end; + _ -> + throw(?PRV_ERROR({unknown_project_type, rebar_app_info:name(AppInfo), Type})) end. +build_rebar3_apps(DAGs, Apps, State) -> + rebar_paths:set_paths([deps], State), + rebar_compiler:analyze_all(DAGs, Apps), + %% TODO: topsort apps here based on DAG data + [rebar_compiler:compile_analyzed(DAGs, AppInfo) || AppInfo <- Apps], + ok. + update_code_paths(State, ProjectApps) -> ProjAppsPaths = paths_for_apps(ProjectApps), ExtrasPaths = paths_for_extras(State, ProjectApps), diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index 9aacd944..9ad932a4 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -18,6 +18,8 @@ all() -> recompile_when_hrl_changes, recompile_when_included_hrl_changes, recompile_extra_when_hrl_in_src_changes, recompile_when_opts_included_hrl_changes, + recompile_when_foreign_included_hrl_changes, + recompile_when_foreign_behaviour_changes, recompile_when_opts_change, dont_recompile_when_opts_dont_change, dont_recompile_yrl_or_xrl, delete_beam_if_source_deleted, @@ -801,6 +803,96 @@ recompile_when_opts_included_hrl_changes(Config) -> ?assert(ModTime =/= NewModTime). +recompile_when_foreign_included_hrl_changes(Config) -> + AppDir = ?config(apps, Config), + AppsDir = filename:join([AppDir, "apps"]), + + Name1 = rebar_test_utils:create_random_name("app1_"), + Name2 = rebar_test_utils:create_random_name("app2_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(filename:join(AppsDir, Name1), + Name1, Vsn, [kernel, stdlib]), + rebar_test_utils:create_app(filename:join(AppsDir, Name2), + Name2, Vsn, [kernel, stdlib]), + + ExtraSrc = [<<"-module(test_header_include).\n" + "-export([main/0]).\n" + "-include_lib(\"">>, Name2, <<"/include/test_header_include.hrl\").\n" + "main() -> ?SOME_DEFINE.\n">>], + + ExtraHeader = <<"-define(SOME_DEFINE, true).\n">>, + ok = filelib:ensure_dir(filename:join([AppsDir, Name1, "src", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppsDir, Name2, "include", "dummy"])), + HeaderFile = filename:join([AppsDir, Name2, "include", "test_header_include.hrl"]), + ok = file:write_file(filename:join([AppsDir, Name1, "src", "test_header_include.erl"]), ExtraSrc), + ok = file:write_file(HeaderFile, ExtraHeader), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name1}]}), + + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name1, "ebin"]), + {ok, Files} = rebar_utils:list_dir(EbinDir), + ModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- Files, filename:extension(F) == ".beam"], + + timer:sleep(1000), + + NewExtraHeader = <<"-define(SOME_DEFINE, false).\n">>, + ok = file:write_file(HeaderFile, NewExtraHeader), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name1}]}), + + {ok, NewFiles} = rebar_utils:list_dir(EbinDir), + NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- NewFiles, filename:extension(F) == ".beam"], + + ?assert(ModTime =/= NewModTime). + +recompile_when_foreign_behaviour_changes(Config) -> + AppDir = ?config(apps, Config), + AppsDir = filename:join([AppDir, "apps"]), + + Name1 = rebar_test_utils:create_random_name("app1_"), + Name2 = rebar_test_utils:create_random_name("app2_"), + Vsn = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(filename:join(AppsDir, Name1), + Name1, Vsn, [kernel, stdlib]), + rebar_test_utils:create_app(filename:join(AppsDir, Name2), + Name2, Vsn, [kernel, stdlib]), + + ExtraSrc = <<"-module(test_behaviour_include).\n" + "-export([main/0]).\n" + "-behaviour(app2_behaviour).\n" + "main() -> 1.\n">>, + + Behaviour = <<"-module(app2_behaviour).\n" + "-callback main() -> term().\n">>, + ok = filelib:ensure_dir(filename:join([AppsDir, Name1, "src", "dummy"])), + ok = filelib:ensure_dir(filename:join([AppsDir, Name2, "src", "dummy"])), + BehaviourFile = filename:join([AppsDir, Name2, "src", "app2_behaviour.erl"]), + ok = file:write_file(filename:join([AppsDir, Name1, "src", "test_behaviour_include.erl"]), ExtraSrc), + ok = file:write_file(BehaviourFile, Behaviour), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name1}]}), + + EbinDir = filename:join([AppDir, "_build", "default", "lib", Name1, "ebin"]), + {ok, Files} = rebar_utils:list_dir(EbinDir), + ModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- Files, filename:extension(F) == ".beam"], + + timer:sleep(1000), + + NewBehaviour = <<"-module(app2_behaviour).\n" + "-callback main(_) -> term().\n">>, + ok = file:write_file(BehaviourFile, NewBehaviour), + + rebar_test_utils:run_and_check(Config, [], ["compile"], {ok, [{app, Name1}]}), + + {ok, NewFiles} = rebar_utils:list_dir(EbinDir), + NewModTime = [filelib:last_modified(filename:join([EbinDir, F])) + || F <- NewFiles, filename:extension(F) == ".beam"], + + ?assert(ModTime =/= NewModTime). + recompile_when_opts_change(Config) -> AppDir = ?config(apps, Config), diff --git a/test/rebar_compiler_dag_SUITE.erl b/test/rebar_compiler_dag_SUITE.erl new file mode 100644 index 00000000..ebd90ff8 --- /dev/null +++ b/test/rebar_compiler_dag_SUITE.erl @@ -0,0 +1,453 @@ +-module(rebar_compiler_dag_SUITE). +-compile([export_all, nowarn_export_all]). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/file.hrl"). + +all() -> + [{group, with_project}]. + +groups() -> + %% The tests in this group are dirty, the order is specific + %% and required across runs for tests to work. + [{with_project, [sequence], [ + find_structure, app_sort, + propagate_include_app1a, propagate_include_app1b, + propagate_include_app2, propagate_behaviour, + propagate_app1_ptrans, propagate_app2_ptrans, + propagate_app2_ptrans_hrl + ]} + ]. + +init_per_suite(Config) -> + rebar_compiler_erl:module_info(), % ensure it is loaded + Config. + +end_per_suite(Config) -> + Config. + +init_per_group(with_project, Config) -> + NewConfig = rebar_test_utils:init_rebar_state(Config, "apps"), + AppDir = ?config(apps, NewConfig), + + Name1 = rebar_test_utils:create_random_name("app1_"), + Vsn1 = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(filename:join([AppDir,"apps",Name1]), Name1, Vsn1, [kernel, stdlib]), + + Name2 = rebar_test_utils:create_random_name("app2_"), + Vsn2 = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(filename:join([AppDir,"apps",Name2]), Name2, Vsn2, [kernel, stdlib]), + + Name3 = rebar_test_utils:create_random_name("app3_"), + Vsn3 = rebar_test_utils:create_random_vsn(), + rebar_test_utils:create_app(filename:join([AppDir,"apps",Name3]), Name3, Vsn3, [kernel, stdlib]), + + apply_project(AppDir, [{app1, Name1}, {app2, Name2}, {app3, Name3}], + project()), + + [{app_names, [Name1, Name2, Name3]}, + {vsns, [Vsn1, Vsn2, Vsn3]} + | NewConfig]; +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + + +project() -> + [{app1, [ + {"src/app1.erl", + "-module(app1).\n" + "-include(\"app1_a.hrl\").\n" + "-include(\"app1_b.hrl\").\n" + "-include_lib(\"{{app2}}/include/app2.hrl\").\n" + "-compile({parse_transform, app1_trans}).\n" + "-compile({parse_transform, {app3, []}}).\n" + "-behaviour(app2).\n" + "-export([cb/0]).\n" + "cb() -> {?APP1A, ?APP1B, ?APP2}.\n"}, + {"src/app1_trans.erl", + "-module(app1_trans).n" + "-export([parse_transform/2]).\n" + "parse_transform(Forms, _Opts) -> Forms.\n"}, + {"src/app1_a.hrl", + "-define(APP1A, 1).\n"}, + {"include/app1_b.hrl", + "-define(APP1B, 1).\n"} + ]}, + {app2, [ + {"src/app2.erl", + "-module(app2).\n" + "-callback cb() -> term().\n"}, + {"include/app2.hrl", + "-include(\"app2_resolve.hrl\").\n" + "-define(APP2, 1).\n"}, + {"src/app2_resolve.hrl", + "this file should be found but never is"}, + {"include/never_found.hrl", + "%% just comments"} + ]}, + {app3, [ + {"src/app3.erl", + "-module(app3).\n" + "-include_lib(\"{{app2}}/include/app2.hrl\").\n" + "-include(\"app3_resolve.hrl\").\n" + "-export([parse_transform/2]).\n" + "parse_transform(Forms, _Opts) -> Forms.\n"}, + {"src/app3_resolve.hrl", + "%% this file should be found"} + ]} + ]. + +find_structure() -> + [{doc, "ensure a proper digraph is built with all files"}]. +find_structure(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + analyze_apps(G, AppNames, AppDir), + FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], + Edges = [{V1,V2} || E <- digraph:edges(G), + {_,V1,V2,_} <- [digraph:edge(G, E)]], + %% All timestamps are the same since we just created the thing + {_, Stamp} = hd(FileStamps), + Matches = [ + {"/src/app1.erl", Stamp}, + {"/src/app1_trans.erl", Stamp}, + {"/src/app1_a.hrl", Stamp}, + {"/include/app1_b.hrl", Stamp}, + {"/src/app2.erl", Stamp}, + {"/include/app2.hrl", Stamp}, + {"/include/app2.hrl", Stamp}, + {"/src/app3.erl", Stamp}, + {"/src/app3_resolve.hrl", Stamp} + ], + matches(Matches, FileStamps), + ?assertEqual(undefined, find_match(".*/never_found.hrl", FileStamps)), + ?assertEqual(undefined, find_match(".*/app2_resolve.hrl", FileStamps)), + ct:pal("Edges: ~p", [Edges]), + edges([ + {"/src/app1.erl", "/src/app1_a.hrl"}, + {"/src/app1.erl", "/include/app1_b.hrl"}, + {"/src/app1.erl", "/src/app2.erl"}, + {"/src/app1.erl", "/include/app2.hrl"}, + {"/src/app1.erl", "/src/app1_trans.erl"}, + {"/src/app1.erl", "/src/app3.erl"}, + {"/src/app3.erl", "/include/app2.hrl"}, + {"/src/app3.erl", "/src/app3_resolve.hrl"} + ], Edges, FileStamps), + ok. + +app_sort() -> + [{doc, "once the digraph is complete, we can sort apps by dependency order"}]. +app_sort(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + analyze_apps(G, AppNames, AppDir), + AppPaths = [ + {AppName, filename:join([AppDir, "apps", AppName])} || AppName <- AppNames + ], + ?assertEqual([lists:nth(2, AppNames), + lists:nth(3, AppNames), + lists:nth(1, AppNames)], + rebar_compiler_dag:compile_order(G, AppPaths)), + ok. + +propagate_include_app1a() -> + [{doc, "changing the app1a header file propagates to its dependents"}]. +propagate_include_app1a(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + next_second(), + F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "src/app1_a.hrl"]), + bump_file(F), + analyze_apps(G, AppNames, AppDir), + FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], + %% All timestamps are the same since we just created the thing + [Stamp1, Stamp2] = lists:usort([S || {_, S} <- FileStamps]), + Matches = [ + {"/src/app1.erl", Stamp2}, + {"/src/app1_trans.erl", Stamp1}, + {"/src/app1_a.hrl", Stamp2}, + {"/include/app1_b.hrl", Stamp1}, + {"/src/app2.erl", Stamp1}, + {"/include/app2.hrl", Stamp1}, + {"/src/app3.erl", Stamp1}, + {"/src/app3_resolve.hrl", Stamp1} + ], + matches(Matches, FileStamps), + ok. + +propagate_include_app1b() -> + [{doc, "changing the app1b header file propagates to its dependents"}]. +propagate_include_app1b(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + next_second(), + F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "include/app1_b.hrl"]), + bump_file(F), + analyze_apps(G, AppNames, AppDir), + FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], + %% All timestamps are the same since we just created the thing + [Stamp1, Stamp2, Stamp3] = lists:usort([S || {_, S} <- FileStamps]), + Matches = [ + {"/src/app1.erl", Stamp3}, + {"/src/app1_trans.erl", Stamp1}, + {"/src/app1_a.hrl", Stamp2}, + {"/include/app1_b.hrl", Stamp3}, + {"/src/app2.erl", Stamp1}, + {"/include/app2.hrl", Stamp1}, + {"/src/app3.erl", Stamp1}, + {"/src/app3_resolve.hrl", Stamp1} + ], + matches(Matches, FileStamps), + ok. + +propagate_include_app2() -> + [{doc, "changing the app2 header file propagates to its dependents"}]. +propagate_include_app2(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + next_second(), + F = filename:join([AppDir, "apps", lists:nth(2, AppNames), "include/app2.hrl"]), + bump_file(F), + analyze_apps(G, AppNames, AppDir), + FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], + %% All timestamps are the same since we just created the thing + [S1, S2, S3, S4] = lists:usort([S || {_, S} <- FileStamps]), + Matches = [ + {"/src/app1.erl", S4}, + {"/src/app1_trans.erl", S1}, + {"/src/app1_a.hrl", S2}, + {"/include/app1_b.hrl", S3}, + {"/src/app2.erl", S1}, + {"/include/app2.hrl", S4}, + {"/src/app3.erl", S4}, + {"/src/app3_resolve.hrl", S1} + ], + matches(Matches, FileStamps), + ok. + +propagate_behaviour() -> + [{doc, "changing the behaviour file propagates to its dependents"}]. +propagate_behaviour(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + next_second(), + F = filename:join([AppDir, "apps", lists:nth(2, AppNames), "src/app2.erl"]), + bump_file(F), + analyze_apps(G, AppNames, AppDir), + FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], + %% All timestamps are the same since we just created the thing + [S1, S2, S3, S4, S5] = lists:usort([S || {_, S} <- FileStamps]), + Matches = [ + {"/src/app1.erl", S5}, + {"/src/app1_trans.erl", S1}, + {"/src/app1_a.hrl", S2}, + {"/include/app1_b.hrl", S3}, + {"/src/app2.erl", S5}, + {"/include/app2.hrl", S4}, + {"/src/app3.erl", S4}, + {"/src/app3_resolve.hrl", S1} + ], + matches(Matches, FileStamps), + ok. + +propagate_app1_ptrans() -> + [{doc, "changing an app-local parse transform propagates to its dependents"}]. +propagate_app1_ptrans(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + next_second(), + F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "src/app1_trans.erl"]), + bump_file(F), + analyze_apps(G, AppNames, AppDir), + FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], + %% All timestamps are the same since we just created the thing + [S1, S2, S3, S4, S5, S6] = lists:usort([S || {_, S} <- FileStamps]), + Matches = [ + {"/src/app1.erl", S6}, + {"/src/app1_trans.erl", S6}, + {"/src/app1_a.hrl", S2}, + {"/include/app1_b.hrl", S3}, + {"/src/app2.erl", S5}, + {"/include/app2.hrl", S4}, + {"/src/app3.erl", S4}, + {"/src/app3_resolve.hrl", S1} + ], + matches(Matches, FileStamps), + ok. + +propagate_app2_ptrans() -> + [{doc, "changing an app-foreign parse transform propagates to its dependents"}]. +propagate_app2_ptrans(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + next_second(), + F = filename:join([AppDir, "apps", lists:nth(3, AppNames), "src/app3.erl"]), + bump_file(F), + analyze_apps(G, AppNames, AppDir), + FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], + %% All timestamps are the same since we just created the thing + [S1, S2, S3, S4, S5, S6, S7] = lists:usort([S || {_, S} <- FileStamps]), + Matches = [ + {"/src/app1.erl", S7}, + {"/src/app1_trans.erl", S6}, + {"/src/app1_a.hrl", S2}, + {"/include/app1_b.hrl", S3}, + {"/src/app2.erl", S5}, + {"/include/app2.hrl", S4}, + {"/src/app3.erl", S7}, + {"/src/app3_resolve.hrl", S1} + ], + matches(Matches, FileStamps), + ok. + +propagate_app2_ptrans_hrl() -> + %% the app-foreign ptrans' foreign hrl dep is tested by propagate_include_app2 as well + [{doc, "changing an app-foreign parse transform's local hrl propagates to its dependents"}]. +propagate_app2_ptrans_hrl(Config) -> + AppDir = ?config(apps, Config), + AppNames = ?config(app_names, Config), + %% assume an empty graph + G = digraph:new([acyclic]), + next_second(), + F = filename:join([AppDir, "apps", lists:nth(3, AppNames), "src/app3_resolve.hrl"]), + bump_file(F), + analyze_apps(G, AppNames, AppDir), + FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], + %% All timestamps are the same since we just created the thing + %% S1 and S7 are gone from the propagation now + [S2, S3, S4, S5, S6, S8] = lists:usort([S || {_, S} <- FileStamps]), + Matches = [ + {"/src/app1.erl", S8}, + {"/src/app1_trans.erl", S6}, + {"/src/app1_a.hrl", S2}, + {"/include/app1_b.hrl", S3}, + {"/src/app2.erl", S5}, + {"/include/app2.hrl", S4}, + {"/src/app3.erl", S8}, + {"/src/app3_resolve.hrl", S8} + ], + matches(Matches, FileStamps), + ok. + +%%%%%%%%%%%%%%% +%%% HELPERS %%% +%%%%%%%%%%%%%%% + +apply_project(_BaseDir, _Names, []) -> + ok; +apply_project(BaseDir, Names, [{_AppName, []}|Rest]) -> + apply_project(BaseDir, Names, Rest); +apply_project(BaseDir, Names, [{AppName, [File|Files]}|Rest]) -> + apply_file(BaseDir, Names, AppName, File), + apply_project(BaseDir, Names, [{AppName, Files}|Rest]). + +apply_file(BaseDir, Names, App, {FileName, Contents}) -> + AppName = proplists:get_value(App, Names), + FilePath = filename:join([BaseDir, "apps", AppName, FileName]), + ok = filelib:ensure_dir(FilePath), + file:write_file(FilePath, apply_template(Contents, Names)). + +apply_template("", _) -> ""; +apply_template("{{" ++ Text, Names) -> + {Var, Rest} = parse_to_var(Text), + App = list_to_atom(Var), + proplists:get_value(App, Names) ++ apply_template(Rest, Names); +apply_template([H|T], Names) -> + [H|apply_template(T, Names)]. + +parse_to_var(Str) -> parse_to_var(Str, []). + +parse_to_var("}}"++Rest, Acc) -> + {lists:reverse(Acc), Rest}; +parse_to_var([H|T], Acc) -> + parse_to_var(T, [H|Acc]). + +analyze_apps(G, AppNames, AppDir) -> + populate_app(G, lists:nth(1, AppNames), AppNames, AppDir, ["app1.erl", "app1_trans.erl"]), + populate_app(G, lists:nth(2, AppNames), AppNames, AppDir, ["app2.erl"]), + populate_app(G, lists:nth(3, AppNames), AppNames, AppDir, ["app3.erl"]), + rebar_compiler_dag:populate_deps(G, ".erl", [{".beam", "ebin/"}]), + rebar_compiler_dag:propagate_stamps(G), + %% manually clear the dirty bit for ease of validation + digraph:del_vertex(G, '$r3_dirty_bit'). + +populate_app(G, Name, AppNames, AppDir, Sources) -> + InDirs = [filename:join([AppDir, "apps", AppName, "src"]) + || AppName <- AppNames] + ++ [filename:join([AppDir, "apps", AppName, "include"]) + || AppName <- AppNames], + AbsSources = [filename:join([AppDir, "apps", Name, "src", Src]) + || Src <- Sources], + DepOpts = [{includes, + [filename:join([AppDir, "apps", Name, "src"]), + filename:join([AppDir, "apps", Name, "include"]) + ]}, + {include_libs, [filename:join([AppDir, "apps"])]} + ], + rebar_compiler_dag:populate_sources( + G, rebar_compiler_erl, + InDirs, AbsSources, DepOpts + ). + +find_match(Regex, FileStamps) -> + try + [throw(F) || {F, _} <- FileStamps, re:run(F, Regex) =/= nomatch], + undefined + catch + throw:F -> {ok, F} + end. + +matches([], _) -> + ok; +matches([{R, Stamp} | T], FileStamps) -> + case find_match(R, FileStamps) of + {ok, F} -> + ?assertEqual(Stamp, proplists:get_value(F, FileStamps)), + matches(T, FileStamps); + undefined -> + ?assertEqual({R, Stamp}, FileStamps) + end. + +edges([], _, _) -> + ok; +edges([{A,B}|T], Edges, Files) -> + {ok, AbsA} = find_match(A, Files), + {ok, AbsB} = find_match(B, Files), + ?assert(lists:member({AbsA, AbsB}, Edges)), + edges(T, Edges, Files). + +bump_file(F) -> + {ok, Bin} = file:read_file(F), + file:write_file(F, [Bin, "\n"]). + +next_second() -> + %% Sleep until the next second. Rather than just doing a + %% sleep(1000) call, sleep for the amount of time required + %% to reach the next second as seen by the OS; this can save us + %% a few hundred milliseconds per test by triggering shorter delays. + {Mega, Sec, Micro} = os:timestamp(), + Now = (Mega*1000000 + Sec)*1000 + round(Micro/1000), + Ms = (trunc(Now / 1000)*1000 + 1000) - Now, + %% add a 50ms for jitter since the exact amount sometimes causes failures + timer:sleep(max(Ms+50, 1000)). +