瀏覽代碼
Merge pull request #2236 from ferd/epp-analysis
Switch to EPP analysis covering all apps at once when compilingpull/2245/head
Fred Hebert
5 年之前
committed by
GitHub
GitHub
沒有發現已知的金鑰在資料庫的簽署中
GPG 金鑰 ID: 4AEE18F83AFDEB23
共有 10 個檔案被更改，包括 1532 行新增 和 153 行删除
統一視圖
Diff Options
-
+4 -0bootstrap
-
+1 -0rebar.config
-
+127 -37src/rebar_compiler.erl
-
+278 -88src/rebar_compiler_dag.erl
-
+176 -0src/rebar_compiler_epp.erl
-
+46 -4src/rebar_compiler_erl.erl
-
+55 -24src/rebar_prv_compile.erl
-
+92 -0test/rebar_compile_SUITE.erl
-
+453 -0test/rebar_compiler_dag_SUITE.erl
-
+300 -0test/rebar_compiler_epp_SUITE.erl
+ 4
- 0
bootstrap
查看文件
+ 1
- 0
rebar.config
查看文件
+ 127
- 37
src/rebar_compiler.erl
查看文件
+ 278
- 88
src/rebar_compiler_dag.erl
查看文件
+ 176
- 0
src/rebar_compiler_epp.erl
查看文件
| @ -0,0 +1,176 @@ | |||||
| %%% @doc | |||||
| %%% Analyze erlang-related files and compilation data using EPP, in order to | |||||
| %%% build complete and accurate DAGs | |||||
| %%% @end | |||||
| -module(rebar_compiler_epp). | |||||
| -export([deps/2, resolve_module/2]). | |||||
| -include_lib("kernel/include/file.hrl"). | |||||
| %%%%%%%%%%%%%%%%%%%%%%%%%%% | |||||
| %%% Basic File Handling %%% | |||||
| %%%%%%%%%%%%%%%%%%%%%%%%%%% | |||||
| %% Find all Erlang code dependencies for a given file | |||||
| -spec deps(file:filename_all(), Opts) -> Attributes when | |||||
| Opts :: [Opt, ...], | |||||
| Opt :: {includes, [file:filename_all()]} | |||||
| | {macros, [file:filename_all()]}, | |||||
| %% following are all required, OTP-18 don't like it though | |||||
| Attributes :: #{include => [file:filename_all()], | |||||
| missing_include_file => [file:filename_all()], | |||||
| missing_include_lib => [file:filename_all()], | |||||
| behaviour => [atom()], | |||||
| parse_transform => [atom()], | |||||
| is_behaviour => boolean()}. | |||||
| deps(File, Opts) -> | |||||
| {EppOpts, ExtraOpts} = split_opts(Opts), | |||||
| {ok, Forms} = epp:parse_file(File, EppOpts), | |||||
| normalize(handle_forms(Forms, default_attrs(), ExtraOpts)). | |||||
| %% Find the path matching a given erlang module | |||||
| resolve_module(Mod, Paths) -> | |||||
| ModStr = atom_to_list(Mod), | |||||
| try | |||||
| [throw(P) || P <- Paths, ModStr =:= filename:basename(P, ".erl")], | |||||
| {error, not_found} | |||||
| catch | |||||
| Path -> {ok, Path} | |||||
| end. | |||||
| %%%%%%%%%%%%%%% | |||||
| %%% PRIVATE %%% | |||||
| %%%%%%%%%%%%%%% | |||||
| default_attrs() -> | |||||
| #{include => [], | |||||
| missing_include_file => [], | |||||
| missing_include_lib => [], | |||||
| behaviour => [], | |||||
| parse_transform => [], | |||||
| is_behaviour => false}. | |||||
| normalize(Map) -> | |||||
| #{include := Incl, | |||||
| missing_include_file := InclF, | |||||
| missing_include_lib := InclL, | |||||
| behaviour := Behaviour, | |||||
| parse_transform := PTrans} = Map, | |||||
| Map#{include => lists:usort(Incl), | |||||
| missing_include_file => lists:usort(InclF), | |||||
| missing_include_lib => lists:usort(InclL), | |||||
| behaviour => lists:usort(Behaviour), | |||||
| parse_transform => lists:usort(PTrans)}. | |||||
| handle_forms([File|Forms], Map, Opts) -> | |||||
| lists:foldl(fun(Form, M) -> handle_form(Form, M, Opts) end, | |||||
| Map, drop_self_file(File, Forms)). | |||||
| drop_self_file(_, []) -> | |||||
| []; | |||||
| drop_self_file({attribute, _, file, {Path,_}} = File, | |||||
| [{attribute,_, file, {Path,_}} | Rest]) -> | |||||
| drop_self_file(File, Rest); | |||||
| drop_self_file(File, [Keep|Rest]) -> | |||||
| [Keep | drop_self_file(File, Rest)]. | |||||
| %% Included files (both libs and direct includes); | |||||
| %% There are also references to the module's own file declaration | |||||
| %% in there, but this is dropped by `drop_self_file/2' and assumed | |||||
| %% to be gone here. | |||||
| handle_form({attribute, _Line, file, {Path, Ln}}, Map, Opts) -> | |||||
| %% Some people think they're funny and they go include attributes | |||||
| %% like: | |||||
| %% -file("fake/file.hrl", Ln). | |||||
| %% Which are expanded to the very clause we have here, which in | |||||
| %% turn is impossible to distinguish from actual included files | |||||
| %% once checked through epp. The way we work around that here | |||||
| %% is to check if the path is absolute, and if so, keep it in since | |||||
| %% epp has expanded it; otherwise consider it to be a failed include. | |||||
| %% This is not perfect but we can't do much more without touching the | |||||
| %% disk and hopefully nobody else in the community has relied on this | |||||
| %% thing. | |||||
| case filename:absname(Path) of | |||||
| Path -> | |||||
| update_with(include, fun(L) -> [Path|L] end, [Path], Map); | |||||
| _ -> % argh! | |||||
| handle_form({error, {Ln, {epp, {include, file, Path}}}}, Map, Opts) | |||||
| end; | |||||
| %% Include files that EPP couldn't resolve | |||||
| handle_form({error, {_Line, epp, {include, file, Name}}}, Map, _Opts) -> | |||||
| update_with(missing_include_file, fun(L) -> [Name|L] end, [Name], Map); | |||||
| handle_form({error, {_Line, epp, {include, lib, Path}}}, Map, Opts) -> | |||||
| %% This file might still exist in the regular paths not in | |||||
| %% code:lib_dir, which depend on options we pass to this module. | |||||
| %% recursively seek it, and add it to the paths to expand here. | |||||
| case find_include_with_opts(Path, Opts) of | |||||
| {ok, File} -> | |||||
| %% we can't go and figure out the contents within that include | |||||
| %% file because we'd need its own compiler opts and app opts | |||||
| %% to do it safely. Tracking that file is still better | |||||
| %% than nothing though. | |||||
| update_with(include, fun(L) -> [File|L] end, [File], Map); | |||||
| {error, not_found} -> | |||||
| update_with(missing_include_lib, fun(L) -> [Path|L] end, [Path], Map) | |||||
| end; | |||||
| %% Behaviour implementation declaration | |||||
| handle_form({attribute, _Line, behaviour, Name}, Map, _Opts) -> | |||||
| update_with(behaviour, fun(L) -> [Name|L] end, [Name], Map); | |||||
| handle_form({attribute, _Line, behavior, Name}, Map, _Opts) -> | |||||
| update_with(behaviour, fun(L) -> [Name|L] end, [Name], Map); | |||||
| %% Extract parse transforms | |||||
| handle_form({attribute, Line, compile, Attr}, Map, _Opts) when not is_list(Attr) -> | |||||
| handle_form({attribute, Line, compile, [Attr]}, Map, _Opts); | |||||
| handle_form({attribute, _Line, compile, Attrs}, Map, _Opts) -> | |||||
| Mods = [case T of | |||||
| {_, {M,_}} -> M; | |||||
| {_, M} -> M | |||||
| end || T <- proplists:lookup_all(parse_transform, Attrs)], | |||||
| update_with(parse_transform, fun(L) -> Mods++L end, Mods, Map); | |||||
| %% Current style behaviour specification declaration | |||||
| handle_form({attribute, _Line, callback, _}, Map, _Opts) -> | |||||
| Map#{is_behaviour => true}; | |||||
| %% Old style behaviour specification, both spellings supported | |||||
| %% The function needs to be exported, but we skip over that logic | |||||
| %% for now. | |||||
| handle_form({function, _Line, behaviour_info, 1, _}, Map, _Opts) -> | |||||
| Map#{is_behaviour => true}; | |||||
| handle_form({function, _Line, behavior_info, 1, _}, Map, _Opts) -> | |||||
| Map#{is_behaviour => true}; | |||||
| %% Skip the rest | |||||
| handle_form(_, Map, _Opts) -> | |||||
| Map. | |||||
| split_opts(Opts) -> | |||||
| %% Extra Opts are options we added to palliate to issues we had | |||||
| %% with resolving include_libs and other things in EPP. | |||||
| lists:partition( | |||||
| fun({OptName, _}) -> | |||||
| not lists:member(OptName, [include_libs, parse_transforms]) | |||||
| end, | |||||
| Opts | |||||
| ). | |||||
| find_include_with_opts(Path, Opts) -> | |||||
| InclPaths = proplists:get_value(include_libs, Opts, []), | |||||
| find_include_lib(InclPaths, Path). | |||||
| find_include_lib([], _) -> | |||||
| {error, not_found}; | |||||
| find_include_lib([H|T], File) -> | |||||
| Abs = filename:join([H, File]), | |||||
| case filelib:is_regular(Abs) of | |||||
| true -> {ok, Abs}; | |||||
| false -> find_include_lib(T, File) | |||||
| end. | |||||
| -ifdef(no_maps_update_with). | |||||
| update_with(Key, Fun, Default, Map) -> | |||||
| case Map of | |||||
| #{Key := Value} -> Map#{Key := Fun(Value)}; | |||||
| _ -> Map#{Key => Default} | |||||
| end. | |||||
| -else. | |||||
| update_with(Key, Fun, Default, Map) -> | |||||
| maps:update_with(Key, Fun, Default, Map). | |||||
| -endif. | |||||
+ 46
- 4
src/rebar_compiler_erl.erl
查看文件
+ 55
- 24
src/rebar_prv_compile.erl
查看文件
+ 92
- 0
test/rebar_compile_SUITE.erl
查看文件
+ 453
- 0
test/rebar_compiler_dag_SUITE.erl
查看文件
| @ -0,0 +1,453 @@ | |||||
| -module(rebar_compiler_dag_SUITE). | |||||
| -compile([export_all, nowarn_export_all]). | |||||
| -include_lib("common_test/include/ct.hrl"). | |||||
| -include_lib("eunit/include/eunit.hrl"). | |||||
| -include_lib("kernel/include/file.hrl"). | |||||
| all() -> | |||||
| [{group, with_project}]. | |||||
| groups() -> | |||||
| %% The tests in this group are dirty, the order is specific | |||||
| %% and required across runs for tests to work. | |||||
| [{with_project, [sequence], [ | |||||
| find_structure, app_sort, | |||||
| propagate_include_app1a, propagate_include_app1b, | |||||
| propagate_include_app2, propagate_behaviour, | |||||
| propagate_app1_ptrans, propagate_app2_ptrans, | |||||
| propagate_app2_ptrans_hrl | |||||
| ]} | |||||
| ]. | |||||
| init_per_suite(Config) -> | |||||
| rebar_compiler_erl:module_info(), % ensure it is loaded | |||||
| Config. | |||||
| end_per_suite(Config) -> | |||||
| Config. | |||||
| init_per_group(with_project, Config) -> | |||||
| NewConfig = rebar_test_utils:init_rebar_state(Config, "apps"), | |||||
| AppDir = ?config(apps, NewConfig), | |||||
| Name1 = rebar_test_utils:create_random_name("app1_"), | |||||
| Vsn1 = rebar_test_utils:create_random_vsn(), | |||||
| rebar_test_utils:create_app(filename:join([AppDir,"apps",Name1]), Name1, Vsn1, [kernel, stdlib]), | |||||
| Name2 = rebar_test_utils:create_random_name("app2_"), | |||||
| Vsn2 = rebar_test_utils:create_random_vsn(), | |||||
| rebar_test_utils:create_app(filename:join([AppDir,"apps",Name2]), Name2, Vsn2, [kernel, stdlib]), | |||||
| Name3 = rebar_test_utils:create_random_name("app3_"), | |||||
| Vsn3 = rebar_test_utils:create_random_vsn(), | |||||
| rebar_test_utils:create_app(filename:join([AppDir,"apps",Name3]), Name3, Vsn3, [kernel, stdlib]), | |||||
| apply_project(AppDir, [{app1, Name1}, {app2, Name2}, {app3, Name3}], | |||||
| project()), | |||||
| [{app_names, [Name1, Name2, Name3]}, | |||||
| {vsns, [Vsn1, Vsn2, Vsn3]} | |||||
| | NewConfig]; | |||||
| init_per_group(_, Config) -> | |||||
| Config. | |||||
| end_per_group(_, Config) -> | |||||
| Config. | |||||
| project() -> | |||||
| [{app1, [ | |||||
| {"src/app1.erl", | |||||
| "-module(app1).\n" | |||||
| "-include(\"app1_a.hrl\").\n" | |||||
| "-include(\"app1_b.hrl\").\n" | |||||
| "-include_lib(\"{{app2}}/include/app2.hrl\").\n" | |||||
| "-compile({parse_transform, app1_trans}).\n" | |||||
| "-compile({parse_transform, {app3, []}}).\n" | |||||
| "-behaviour(app2).\n" | |||||
| "-export([cb/0]).\n" | |||||
| "cb() -> {?APP1A, ?APP1B, ?APP2}.\n"}, | |||||
| {"src/app1_trans.erl", | |||||
| "-module(app1_trans).n" | |||||
| "-export([parse_transform/2]).\n" | |||||
| "parse_transform(Forms, _Opts) -> Forms.\n"}, | |||||
| {"src/app1_a.hrl", | |||||
| "-define(APP1A, 1).\n"}, | |||||
| {"include/app1_b.hrl", | |||||
| "-define(APP1B, 1).\n"} | |||||
| ]}, | |||||
| {app2, [ | |||||
| {"src/app2.erl", | |||||
| "-module(app2).\n" | |||||
| "-callback cb() -> term().\n"}, | |||||
| {"include/app2.hrl", | |||||
| "-include(\"app2_resolve.hrl\").\n" | |||||
| "-define(APP2, 1).\n"}, | |||||
| {"src/app2_resolve.hrl", | |||||
| "this file should be found but never is"}, | |||||
| {"include/never_found.hrl", | |||||
| "%% just comments"} | |||||
| ]}, | |||||
| {app3, [ | |||||
| {"src/app3.erl", | |||||
| "-module(app3).\n" | |||||
| "-include_lib(\"{{app2}}/include/app2.hrl\").\n" | |||||
| "-include(\"app3_resolve.hrl\").\n" | |||||
| "-export([parse_transform/2]).\n" | |||||
| "parse_transform(Forms, _Opts) -> Forms.\n"}, | |||||
| {"src/app3_resolve.hrl", | |||||
| "%% this file should be found"} | |||||
| ]} | |||||
| ]. | |||||
| find_structure() -> | |||||
| [{doc, "ensure a proper digraph is built with all files"}]. | |||||
| find_structure(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], | |||||
| Edges = [{V1,V2} || E <- digraph:edges(G), | |||||
| {_,V1,V2,_} <- [digraph:edge(G, E)]], | |||||
| %% All timestamps are the same since we just created the thing | |||||
| {_, Stamp} = hd(FileStamps), | |||||
| Matches = [ | |||||
| {"/src/app1.erl", Stamp}, | |||||
| {"/src/app1_trans.erl", Stamp}, | |||||
| {"/src/app1_a.hrl", Stamp}, | |||||
| {"/include/app1_b.hrl", Stamp}, | |||||
| {"/src/app2.erl", Stamp}, | |||||
| {"/include/app2.hrl", Stamp}, | |||||
| {"/include/app2.hrl", Stamp}, | |||||
| {"/src/app3.erl", Stamp}, | |||||
| {"/src/app3_resolve.hrl", Stamp} | |||||
| ], | |||||
| matches(Matches, FileStamps), | |||||
| ?assertEqual(undefined, find_match(".*/never_found.hrl", FileStamps)), | |||||
| ?assertEqual(undefined, find_match(".*/app2_resolve.hrl", FileStamps)), | |||||
| ct:pal("Edges: ~p", [Edges]), | |||||
| edges([ | |||||
| {"/src/app1.erl", "/src/app1_a.hrl"}, | |||||
| {"/src/app1.erl", "/include/app1_b.hrl"}, | |||||
| {"/src/app1.erl", "/src/app2.erl"}, | |||||
| {"/src/app1.erl", "/include/app2.hrl"}, | |||||
| {"/src/app1.erl", "/src/app1_trans.erl"}, | |||||
| {"/src/app1.erl", "/src/app3.erl"}, | |||||
| {"/src/app3.erl", "/include/app2.hrl"}, | |||||
| {"/src/app3.erl", "/src/app3_resolve.hrl"} | |||||
| ], Edges, FileStamps), | |||||
| ok. | |||||
| app_sort() -> | |||||
| [{doc, "once the digraph is complete, we can sort apps by dependency order"}]. | |||||
| app_sort(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| AppPaths = [ | |||||
| {AppName, filename:join([AppDir, "apps", AppName])} || AppName <- AppNames | |||||
| ], | |||||
| ?assertEqual([lists:nth(2, AppNames), | |||||
| lists:nth(3, AppNames), | |||||
| lists:nth(1, AppNames)], | |||||
| rebar_compiler_dag:compile_order(G, AppPaths)), | |||||
| ok. | |||||
| propagate_include_app1a() -> | |||||
| [{doc, "changing the app1a header file propagates to its dependents"}]. | |||||
| propagate_include_app1a(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| next_second(), | |||||
| F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "src/app1_a.hrl"]), | |||||
| bump_file(F), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], | |||||
| %% All timestamps are the same since we just created the thing | |||||
| [Stamp1, Stamp2] = lists:usort([S || {_, S} <- FileStamps]), | |||||
| Matches = [ | |||||
| {"/src/app1.erl", Stamp2}, | |||||
| {"/src/app1_trans.erl", Stamp1}, | |||||
| {"/src/app1_a.hrl", Stamp2}, | |||||
| {"/include/app1_b.hrl", Stamp1}, | |||||
| {"/src/app2.erl", Stamp1}, | |||||
| {"/include/app2.hrl", Stamp1}, | |||||
| {"/src/app3.erl", Stamp1}, | |||||
| {"/src/app3_resolve.hrl", Stamp1} | |||||
| ], | |||||
| matches(Matches, FileStamps), | |||||
| ok. | |||||
| propagate_include_app1b() -> | |||||
| [{doc, "changing the app1b header file propagates to its dependents"}]. | |||||
| propagate_include_app1b(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| next_second(), | |||||
| F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "include/app1_b.hrl"]), | |||||
| bump_file(F), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], | |||||
| %% All timestamps are the same since we just created the thing | |||||
| [Stamp1, Stamp2, Stamp3] = lists:usort([S || {_, S} <- FileStamps]), | |||||
| Matches = [ | |||||
| {"/src/app1.erl", Stamp3}, | |||||
| {"/src/app1_trans.erl", Stamp1}, | |||||
| {"/src/app1_a.hrl", Stamp2}, | |||||
| {"/include/app1_b.hrl", Stamp3}, | |||||
| {"/src/app2.erl", Stamp1}, | |||||
| {"/include/app2.hrl", Stamp1}, | |||||
| {"/src/app3.erl", Stamp1}, | |||||
| {"/src/app3_resolve.hrl", Stamp1} | |||||
| ], | |||||
| matches(Matches, FileStamps), | |||||
| ok. | |||||
| propagate_include_app2() -> | |||||
| [{doc, "changing the app2 header file propagates to its dependents"}]. | |||||
| propagate_include_app2(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| next_second(), | |||||
| F = filename:join([AppDir, "apps", lists:nth(2, AppNames), "include/app2.hrl"]), | |||||
| bump_file(F), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], | |||||
| %% All timestamps are the same since we just created the thing | |||||
| [S1, S2, S3, S4] = lists:usort([S || {_, S} <- FileStamps]), | |||||
| Matches = [ | |||||
| {"/src/app1.erl", S4}, | |||||
| {"/src/app1_trans.erl", S1}, | |||||
| {"/src/app1_a.hrl", S2}, | |||||
| {"/include/app1_b.hrl", S3}, | |||||
| {"/src/app2.erl", S1}, | |||||
| {"/include/app2.hrl", S4}, | |||||
| {"/src/app3.erl", S4}, | |||||
| {"/src/app3_resolve.hrl", S1} | |||||
| ], | |||||
| matches(Matches, FileStamps), | |||||
| ok. | |||||
| propagate_behaviour() -> | |||||
| [{doc, "changing the behaviour file propagates to its dependents"}]. | |||||
| propagate_behaviour(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| next_second(), | |||||
| F = filename:join([AppDir, "apps", lists:nth(2, AppNames), "src/app2.erl"]), | |||||
| bump_file(F), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], | |||||
| %% All timestamps are the same since we just created the thing | |||||
| [S1, S2, S3, S4, S5] = lists:usort([S || {_, S} <- FileStamps]), | |||||
| Matches = [ | |||||
| {"/src/app1.erl", S5}, | |||||
| {"/src/app1_trans.erl", S1}, | |||||
| {"/src/app1_a.hrl", S2}, | |||||
| {"/include/app1_b.hrl", S3}, | |||||
| {"/src/app2.erl", S5}, | |||||
| {"/include/app2.hrl", S4}, | |||||
| {"/src/app3.erl", S4}, | |||||
| {"/src/app3_resolve.hrl", S1} | |||||
| ], | |||||
| matches(Matches, FileStamps), | |||||
| ok. | |||||
| propagate_app1_ptrans() -> | |||||
| [{doc, "changing an app-local parse transform propagates to its dependents"}]. | |||||
| propagate_app1_ptrans(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| next_second(), | |||||
| F = filename:join([AppDir, "apps", lists:nth(1, AppNames), "src/app1_trans.erl"]), | |||||
| bump_file(F), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], | |||||
| %% All timestamps are the same since we just created the thing | |||||
| [S1, S2, S3, S4, S5, S6] = lists:usort([S || {_, S} <- FileStamps]), | |||||
| Matches = [ | |||||
| {"/src/app1.erl", S6}, | |||||
| {"/src/app1_trans.erl", S6}, | |||||
| {"/src/app1_a.hrl", S2}, | |||||
| {"/include/app1_b.hrl", S3}, | |||||
| {"/src/app2.erl", S5}, | |||||
| {"/include/app2.hrl", S4}, | |||||
| {"/src/app3.erl", S4}, | |||||
| {"/src/app3_resolve.hrl", S1} | |||||
| ], | |||||
| matches(Matches, FileStamps), | |||||
| ok. | |||||
| propagate_app2_ptrans() -> | |||||
| [{doc, "changing an app-foreign parse transform propagates to its dependents"}]. | |||||
| propagate_app2_ptrans(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| next_second(), | |||||
| F = filename:join([AppDir, "apps", lists:nth(3, AppNames), "src/app3.erl"]), | |||||
| bump_file(F), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], | |||||
| %% All timestamps are the same since we just created the thing | |||||
| [S1, S2, S3, S4, S5, S6, S7] = lists:usort([S || {_, S} <- FileStamps]), | |||||
| Matches = [ | |||||
| {"/src/app1.erl", S7}, | |||||
| {"/src/app1_trans.erl", S6}, | |||||
| {"/src/app1_a.hrl", S2}, | |||||
| {"/include/app1_b.hrl", S3}, | |||||
| {"/src/app2.erl", S5}, | |||||
| {"/include/app2.hrl", S4}, | |||||
| {"/src/app3.erl", S7}, | |||||
| {"/src/app3_resolve.hrl", S1} | |||||
| ], | |||||
| matches(Matches, FileStamps), | |||||
| ok. | |||||
| propagate_app2_ptrans_hrl() -> | |||||
| %% the app-foreign ptrans' foreign hrl dep is tested by propagate_include_app2 as well | |||||
| [{doc, "changing an app-foreign parse transform's local hrl propagates to its dependents"}]. | |||||
| propagate_app2_ptrans_hrl(Config) -> | |||||
| AppDir = ?config(apps, Config), | |||||
| AppNames = ?config(app_names, Config), | |||||
| %% assume an empty graph | |||||
| G = digraph:new([acyclic]), | |||||
| next_second(), | |||||
| F = filename:join([AppDir, "apps", lists:nth(3, AppNames), "src/app3_resolve.hrl"]), | |||||
| bump_file(F), | |||||
| analyze_apps(G, AppNames, AppDir), | |||||
| FileStamps = [digraph:vertex(G, V) || V <- digraph:vertices(G)], | |||||
| %% All timestamps are the same since we just created the thing | |||||
| %% S1 and S7 are gone from the propagation now | |||||
| [S2, S3, S4, S5, S6, S8] = lists:usort([S || {_, S} <- FileStamps]), | |||||
| Matches = [ | |||||
| {"/src/app1.erl", S8}, | |||||
| {"/src/app1_trans.erl", S6}, | |||||
| {"/src/app1_a.hrl", S2}, | |||||
| {"/include/app1_b.hrl", S3}, | |||||
| {"/src/app2.erl", S5}, | |||||
| {"/include/app2.hrl", S4}, | |||||
| {"/src/app3.erl", S8}, | |||||
| {"/src/app3_resolve.hrl", S8} | |||||
| ], | |||||
| matches(Matches, FileStamps), | |||||
| ok. | |||||
| %%%%%%%%%%%%%%% | |||||
| %%% HELPERS %%% | |||||
| %%%%%%%%%%%%%%% | |||||
| apply_project(_BaseDir, _Names, []) -> | |||||
| ok; | |||||
| apply_project(BaseDir, Names, [{_AppName, []}|Rest]) -> | |||||
| apply_project(BaseDir, Names, Rest); | |||||
| apply_project(BaseDir, Names, [{AppName, [File|Files]}|Rest]) -> | |||||
| apply_file(BaseDir, Names, AppName, File), | |||||
| apply_project(BaseDir, Names, [{AppName, Files}|Rest]). | |||||
| apply_file(BaseDir, Names, App, {FileName, Contents}) -> | |||||
| AppName = proplists:get_value(App, Names), | |||||
| FilePath = filename:join([BaseDir, "apps", AppName, FileName]), | |||||
| ok = filelib:ensure_dir(FilePath), | |||||
| file:write_file(FilePath, apply_template(Contents, Names)). | |||||
| apply_template("", _) -> ""; | |||||
| apply_template("{{" ++ Text, Names) -> | |||||
| {Var, Rest} = parse_to_var(Text), | |||||
| App = list_to_atom(Var), | |||||
| proplists:get_value(App, Names) ++ apply_template(Rest, Names); | |||||
| apply_template([H|T], Names) -> | |||||
| [H|apply_template(T, Names)]. | |||||
| parse_to_var(Str) -> parse_to_var(Str, []). | |||||
| parse_to_var("}}"++Rest, Acc) -> | |||||
| {lists:reverse(Acc), Rest}; | |||||
| parse_to_var([H|T], Acc) -> | |||||
| parse_to_var(T, [H|Acc]). | |||||
| analyze_apps(G, AppNames, AppDir) -> | |||||
| populate_app(G, lists:nth(1, AppNames), AppNames, AppDir, ["app1.erl", "app1_trans.erl"]), | |||||
| populate_app(G, lists:nth(2, AppNames), AppNames, AppDir, ["app2.erl"]), | |||||
| populate_app(G, lists:nth(3, AppNames), AppNames, AppDir, ["app3.erl"]), | |||||
| rebar_compiler_dag:populate_deps(G, ".erl", [{".beam", "ebin/"}]), | |||||
| rebar_compiler_dag:propagate_stamps(G), | |||||
| %% manually clear the dirty bit for ease of validation | |||||
| digraph:del_vertex(G, '$r3_dirty_bit'). | |||||
| populate_app(G, Name, AppNames, AppDir, Sources) -> | |||||
| InDirs = [filename:join([AppDir, "apps", AppName, "src"]) | |||||
| || AppName <- AppNames] | |||||
| ++ [filename:join([AppDir, "apps", AppName, "include"]) | |||||
| || AppName <- AppNames], | |||||
| AbsSources = [filename:join([AppDir, "apps", Name, "src", Src]) | |||||
| || Src <- Sources], | |||||
| DepOpts = [{includes, | |||||
| [filename:join([AppDir, "apps", Name, "src"]), | |||||
| filename:join([AppDir, "apps", Name, "include"]) | |||||
| ]}, | |||||
| {include_libs, [filename:join([AppDir, "apps"])]} | |||||
| ], | |||||
| rebar_compiler_dag:populate_sources( | |||||
| G, rebar_compiler_erl, | |||||
| InDirs, AbsSources, DepOpts | |||||
| ). | |||||
| find_match(Regex, FileStamps) -> | |||||
| try | |||||
| [throw(F) || {F, _} <- FileStamps, re:run(F, Regex) =/= nomatch], | |||||
| undefined | |||||
| catch | |||||
| throw:F -> {ok, F} | |||||
| end. | |||||
| matches([], _) -> | |||||
| ok; | |||||
| matches([{R, Stamp} | T], FileStamps) -> | |||||
| case find_match(R, FileStamps) of | |||||
| {ok, F} -> | |||||
| ?assertEqual(Stamp, proplists:get_value(F, FileStamps)), | |||||
| matches(T, FileStamps); | |||||
| undefined -> | |||||
| ?assertEqual({R, Stamp}, FileStamps) | |||||
| end. | |||||
| edges([], _, _) -> | |||||
| ok; | |||||
| edges([{A,B}|T], Edges, Files) -> | |||||
| {ok, AbsA} = find_match(A, Files), | |||||
| {ok, AbsB} = find_match(B, Files), | |||||
| ?assert(lists:member({AbsA, AbsB}, Edges)), | |||||
| edges(T, Edges, Files). | |||||
| bump_file(F) -> | |||||
| {ok, Bin} = file:read_file(F), | |||||
| file:write_file(F, [Bin, "\n"]). | |||||
| next_second() -> | |||||
| %% Sleep until the next second. Rather than just doing a | |||||
| %% sleep(1000) call, sleep for the amount of time required | |||||
| %% to reach the next second as seen by the OS; this can save us | |||||
| %% a few hundred milliseconds per test by triggering shorter delays. | |||||
| {Mega, Sec, Micro} = os:timestamp(), | |||||
| Now = (Mega*1000000 + Sec)*1000 + round(Micro/1000), | |||||
| Ms = (trunc(Now / 1000)*1000 + 1000) - Now, | |||||
| %% add a 50ms for jitter since the exact amount sometimes causes failures | |||||
| timer:sleep(max(Ms+50, 1000)). | |||||
+ 300
- 0
test/rebar_compiler_epp_SUITE.erl
查看文件
| @ -0,0 +1,300 @@ | |||||
| %%% @doc | |||||
| %%% Unit tests for epp-related compiler utils. | |||||
| %%% Make it easier to validate internal behaviour of compiler data and | |||||
| %%% handling of module parsing without having to actually set up | |||||
| %%% entire projects. | |||||
| %%% @end | |||||
| -module(rebar_compiler_epp_SUITE). | |||||
| -include_lib("common_test/include/ct.hrl"). | |||||
| -include_lib("eunit/include/eunit.hrl"). | |||||
| -compile([export_all, nowarn_export_all]). | |||||
| all() -> | |||||
| [{group, module}]. | |||||
| groups() -> | |||||
| [{module, [], [ | |||||
| analyze, analyze_old_behaviour, analyze_old_behavior, | |||||
| analyze_empty, analyze_bad_mod, | |||||
| resolve_module | |||||
| ]} | |||||
| ]. | |||||
| init_per_group(module, Config) -> | |||||
| to_file(Config, {"direct.hrl", "-direct(val). "}), | |||||
| Config; | |||||
| init_per_group(_, Config) -> | |||||
| Config. | |||||
| end_per_group(_, Config) -> | |||||
| Config. | |||||
| init_per_testcase(_, Config) -> | |||||
| Config. | |||||
| end_per_testcase(_, Config) -> | |||||
| Config. | |||||
| %%%%%%%%%%%%%%%%%%%%%%%%%%%%% | |||||
| %%% module analysis group %%% | |||||
| %%%%%%%%%%%%%%%%%%%%%%%%%%%%% | |||||
| analyze() -> | |||||
| [{docs, "Analyzing a module returns all the " | |||||
| "parseable dependencies for it in a map."}]. | |||||
| analyze(Config) -> | |||||
| ?assert(check_analyze( | |||||
| #{include => [ | |||||
| "eunit-[0-9.]+/include/eunit.hrl$", | |||||
| "stdlib-[0-9.]+/include/assert.hrl$", | |||||
| "/direct.hrl$" | |||||
| ], | |||||
| %% missing includes | |||||
| missing_include_file => [ | |||||
| "^false.hrl$" | |||||
| ], | |||||
| missing_include_lib => [ | |||||
| "^some_app/include/lib.hrl$" | |||||
| ], | |||||
| parse_transform => [ | |||||
| erl_id_trans, | |||||
| eunit_autoexport, % added by include file! | |||||
| missing_parse_trans1, | |||||
| missing_parse_trans2 | |||||
| ], | |||||
| behaviour => [gen_server, gen_statem], | |||||
| is_behaviour => true | |||||
| }, | |||||
| rebar_compiler_epp:deps( | |||||
| to_file(Config, fake_mod()), | |||||
| [{includes, []}, {macros, []}] | |||||
| ) | |||||
| )), | |||||
| ok. | |||||
| analyze_old_behaviour() -> | |||||
| [{docs, "Analyzing old-style behaviour annotation"}]. | |||||
| analyze_old_behaviour(Config) -> | |||||
| ?assert(check_analyze( | |||||
| #{include => [], | |||||
| missing_include_file => [], | |||||
| missing_include_lib => [], | |||||
| parse_transform => [], | |||||
| behaviour => [], | |||||
| is_behaviour => true | |||||
| }, | |||||
| rebar_compiler_epp:deps( | |||||
| to_file(Config, old_behaviour_mod()), | |||||
| [{includes, []}, {macros, []}] | |||||
| ) | |||||
| )), | |||||
| ok. | |||||
| analyze_old_behavior() -> | |||||
| [{docs, "Analyzing old-style behavior annotation"}]. | |||||
| analyze_old_behavior(Config) -> | |||||
| ?assert(check_analyze( | |||||
| #{include => [], | |||||
| missing_include_file => [], | |||||
| missing_include_lib => [], | |||||
| parse_transform => [], | |||||
| behaviour => [], | |||||
| is_behaviour => true | |||||
| }, | |||||
| rebar_compiler_epp:deps( | |||||
| to_file(Config, old_behavior_mod()), | |||||
| [{includes, []}, {macros, []}] | |||||
| ) | |||||
| )), | |||||
| ok. | |||||
| analyze_empty() -> | |||||
| [{docs, "Making sure empty files are properly handled as valid but null " | |||||
| "and let some other compiler phase handle this. We follow " | |||||
| "what EPP handles."}]. | |||||
| analyze_empty(Config) -> | |||||
| ?assert(check_analyze( | |||||
| #{include => [], | |||||
| missing_include_file => [], | |||||
| missing_include_lib => [], | |||||
| parse_transform => [], | |||||
| behaviour => [], | |||||
| is_behaviour => false | |||||
| }, | |||||
| rebar_compiler_epp:deps( | |||||
| to_file(Config, empty_mod()), | |||||
| [{includes, []}, {macros, []}] | |||||
| ) | |||||
| )), | |||||
| ok. | |||||
| analyze_bad_mod() -> | |||||
| [{docs, "Errors for bad modules that don't compile are skipped " | |||||
| "by EPP and so we defer that to a later phase of the " | |||||
| "compilation process"}]. | |||||
| analyze_bad_mod(Config) -> | |||||
| ?assert(check_analyze( | |||||
| #{include => [], | |||||
| missing_include_file => [], | |||||
| missing_include_lib => [], | |||||
| parse_transform => [], | |||||
| behaviour => [], | |||||
| is_behaviour => false | |||||
| }, | |||||
| rebar_compiler_epp:deps( | |||||
| to_file(Config, bad_mod()), | |||||
| [{includes, []}, {macros, []}] | |||||
| ) | |||||
| )), | |||||
| ok. | |||||
| resolve_module() -> | |||||
| [{doc, "given a module name and a bunch of paths, find " | |||||
| "the first path that matches the module"}]. | |||||
| resolve_module(Config) -> | |||||
| Path1 = to_file(Config, fake_mod()), | |||||
| Path2 = to_file(Config, old_behaviour_mod()), | |||||
| Path3 = to_file(Config, empty_mod()), | |||||
| ?assertEqual( | |||||
| {ok, Path2}, | |||||
| rebar_compiler_epp:resolve_module( | |||||
| old_behaviour, | |||||
| [Path1, Path2, Path3] | |||||
| ) | |||||
| ), | |||||
| ok. | |||||
| %%%%%%%%%%%%%%% | |||||
| %%% HELPERS %%% | |||||
| %%%%%%%%%%%%%%% | |||||
| %% check each field of `Map' and validate them against `CheckMap'. | |||||
| %% This allows to check each value in the map has a matching assertion. | |||||
| %% Then check each field of `CheckMap' against `Map' to find if | |||||
| %% any missing value exists. | |||||
| check_analyze(CheckMap, Map) -> | |||||
| ct:pal("check_analyze:~n~p~n~p", [CheckMap, Map]), | |||||
| maps:fold(fun(K,V,Acc) -> check(CheckMap, K, V) and Acc end, | |||||
| true, Map) | |||||
| andalso | |||||
| maps:fold( | |||||
| fun(K,_,Acc) -> | |||||
| check(CheckMap, K, maps:get(K, Map, make_ref())) and Acc | |||||
| end, | |||||
| true, | |||||
| Map | |||||
| ). | |||||
| check(Map, K, V) -> | |||||
| case maps:is_key(K, Map) of | |||||
| false -> false; | |||||
| true -> | |||||
| #{K := Val} = Map, | |||||
| compare_val(Val, V) | |||||
| end. | |||||
| %% two identical values always works | |||||
| compare_val(V, V) -> | |||||
| true; | |||||
| %% compare lists of strings; each string must be checked individually | |||||
| %% because they are assumed to be regexes. | |||||
| compare_val(V1, V2) when is_list(hd(V1)) -> | |||||
| match_regexes(V1, V2); | |||||
| compare_val(V1, _V2) when not is_integer(hd(V1)) -> | |||||
| %% failing list of some sort, but not a string | |||||
| false; | |||||
| %% strings as regexes | |||||
| compare_val(V1, V2) when is_list(V1) -> | |||||
| match_regex(V1, [V2]) =/= nomatch; | |||||
| %% anything else is not literally the same and is bad | |||||
| compare_val(_, _) -> | |||||
| false. | |||||
| match_regexes([], List) -> | |||||
| List == []; % no extra patterns, that would be weird | |||||
| match_regexes([H|T], List) -> | |||||
| case match_regex(H, List) of | |||||
| nomatch -> | |||||
| false; | |||||
| {ok, Entry} -> | |||||
| match_regexes(T, List -- [Entry]) | |||||
| end. | |||||
| match_regex(_Pattern, []) -> | |||||
| nomatch; | |||||
| match_regex(Pattern, [H|T]) -> | |||||
| case re:run(H, Pattern) of | |||||
| nomatch -> match_regex(Pattern, T); | |||||
| _ -> {ok, H} | |||||
| end. | |||||
| %% custom zip function that causes value failures (by using make_ref() | |||||
| %% that will never match in compare_val/2) rather than crashing because | |||||
| %% of lists of different lengths. | |||||
| zip([], []) -> []; | |||||
| zip([], [H|T]) -> [{make_ref(),H} | zip([], T)]; | |||||
| zip([H|T], []) -> [{H,make_ref()} | zip(T, [])]; | |||||
| zip([X|Xs], [Y|Ys]) -> [{X,Y} | zip(Xs, Ys)]. | |||||
| %%%%%%%%%%%%%%%%%%%%%%%%%%%%% | |||||
| %%% Module specifications %%% | |||||
| %%%%%%%%%%%%%%%%%%%%%%%%%%%%% | |||||
| %% turn a module string to a file that will live in CT's scratch dir | |||||
| to_file(Config, {Name,Contents}) -> | |||||
| Path = filename:join([?config(priv_dir, Config), Name]), | |||||
| file:write_file(Path, Contents, [sync]), | |||||
| Path. | |||||
| %% base module with all the interesting includes and attributes | |||||
| %% we want to track | |||||
| fake_mod() -> | |||||
| {"somemod.erl", " | |||||
| -module(somemod). | |||||
| -export([f/1]). | |||||
| -include(\"direct.hrl\"). | |||||
| -include(\"direct.hrl\"). | |||||
| -include_lib(\"some_app/include/lib.hrl\"). | |||||
| -include_lib(\"eunit/include/eunit.hrl\"). | |||||
| -compile({parse_transform, {erl_id_trans, []}}). | |||||
| -compile({parse_transform, missing_parse_trans1}). | |||||
| -compile([{parse_transform, {missing_parse_trans2, []}}]). | |||||
| -behaviour(gen_server). | |||||
| -behavior(gen_statem). | |||||
| -callback f() -> ok. | |||||
| -ifdef(OPT). | |||||
| -include(\"true.hrl\"). | |||||
| -else. | |||||
| -include(\"false.hrl\"). | |||||
| -endif. | |||||
| f(X) -> X. | |||||
| "}. | |||||
| %% variations for attributes that can't be checked in the | |||||
| %% same base module | |||||
| old_behaviour_mod() -> | |||||
| {"old_behaviour.erl", " | |||||
| -module(old_behaviour). | |||||
| -export([f/1, behaviour_info/1]). | |||||
| f(X) -> X. | |||||
| behaviour_info(callbacks) -> [{f,1}]. | |||||
| "}. | |||||
| old_behavior_mod() -> | |||||
| {"old_behaviour.erl", " | |||||
| -module(old_behaviour). | |||||
| -export([f/1, behaviour_info/1]). | |||||
| f(X) -> X. | |||||
| behavior_info(callbacks) -> [{f,1}]. | |||||
| "}. | |||||
| empty_mod() -> | |||||
| {"empty.erl", ""}. | |||||
| bad_mod() -> | |||||
| {"badmod.erl", " | |||||
| -module(bad_mod). % wrong name! | |||||
| f(x) -> X+1. % bad vars | |||||
| f((x)cv) -> bad syntax. | |||||
| "}. | |||||